Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
  jakman85andClaude Opus 5
|
9137bbd0c3 |
No files matched your search
@@ -1,57 +0,0 @@
|
||||
A|*.keithjackson.dev|136.51.115.252
|
||||
A|auth.home.keithjackson.dev|192.168.1.53
|
||||
A|brickbreak.keithjackson.dev|20.236.242.14
|
||||
A|crafty-mc.home.keithjackson.dev|192.168.1.55
|
||||
A|crafty.home.keithjackson.dev|192.168.1.53
|
||||
A|dev.brickbreak.keithjackson.dev|20.236.242.14
|
||||
A|gitea.home.keithjackson.dev|192.168.1.53
|
||||
A|grafana.home.keithjackson.dev|192.168.1.53
|
||||
A|headlamp.home.keithjackson.dev|192.168.1.53
|
||||
A|home.keithjackson.dev|192.168.1.53
|
||||
A|jackett.home.keithjackson.dev|192.168.1.53
|
||||
A|keycloak.home.keithjackson.dev|192.168.1.53
|
||||
A|nextcloud.home.keithjackson.dev|192.168.1.53
|
||||
A|pihole.home.keithjackson.dev|192.168.1.53
|
||||
A|plex.home.keithjackson.dev|192.168.1.53
|
||||
A|qbittorrent.home.keithjackson.dev|192.168.1.53
|
||||
A|radarr.home.keithjackson.dev|192.168.1.53
|
||||
A|seerr.home.keithjackson.dev|192.168.1.53
|
||||
A|semaphore.home.keithjackson.dev|192.168.1.53
|
||||
A|sonarr.home.keithjackson.dev|192.168.1.53
|
||||
A|unifi.home.keithjackson.dev|192.168.1.54
|
||||
TXT|a-auth.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/oauth2-proxy/oauth2-proxy
|
||||
TXT|a-crafty-mc.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/crafty/crafty-l4-tcp
|
||||
TXT|a-crafty.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/crafty/crafty
|
||||
TXT|a-gitea.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/gitea/gitea-3000
|
||||
TXT|a-grafana.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/monitoring/grafana
|
||||
TXT|a-headlamp.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/headlamp/headlamp
|
||||
TXT|a-home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/homepage/homepage
|
||||
TXT|a-jackett.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/jackett/jackett-9117
|
||||
TXT|a-keycloak.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/keycloak/keycloak
|
||||
TXT|a-nextcloud.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/nextcloud/nextcloud-80
|
||||
TXT|a-pihole.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/pihole/pihole-80
|
||||
TXT|a-plex.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/plex/plex-32400
|
||||
TXT|a-qbittorrent.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/qbittorrent/qbittorrent-8080
|
||||
TXT|a-radarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/radarr/radarr-7878
|
||||
TXT|a-seerr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/seerr/seerr-5055
|
||||
TXT|a-semaphore.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/semaphore/semaphore-3000
|
||||
TXT|a-sonarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/sonarr/sonarr-8989
|
||||
TXT|a-unifi.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/unifi/unifi-l4-tcp
|
||||
TXT|auth.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/oauth2-proxy/oauth2-proxy
|
||||
TXT|crafty-mc.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/crafty/crafty-l4-tcp
|
||||
TXT|crafty.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/crafty/crafty
|
||||
TXT|gitea.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/gitea/gitea-3000
|
||||
TXT|grafana.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/monitoring/grafana
|
||||
TXT|headlamp.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/headlamp/headlamp
|
||||
TXT|home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/homepage/homepage
|
||||
TXT|jackett.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/jackett/jackett-9117
|
||||
TXT|keycloak.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/keycloak/keycloak
|
||||
TXT|nextcloud.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/nextcloud/nextcloud-80
|
||||
TXT|pihole.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/pihole/pihole-80
|
||||
TXT|plex.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/plex/plex-32400
|
||||
TXT|qbittorrent.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/qbittorrent/qbittorrent-8080
|
||||
TXT|radarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/radarr/radarr-7878
|
||||
TXT|seerr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/seerr/seerr-5055
|
||||
TXT|semaphore.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/semaphore/semaphore-3000
|
||||
TXT|sonarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/sonarr/sonarr-8989
|
||||
TXT|unifi.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/unifi/unifi-l4-tcp
|
||||
@@ -1,95 +0,0 @@
|
||||
A|*.keithjackson.dev|136.51.115.252
|
||||
A|auth.home.keithjackson.dev|192.168.1.53
|
||||
A|auth.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|brickbreak.keithjackson.dev|20.236.242.14
|
||||
A|conformance-dns.home.keithjackson.dev|192.168.1.56
|
||||
A|crafty-mc.home.keithjackson.dev|192.168.1.55
|
||||
A|crafty-mc.prod.home.keithjackson.dev|192.168.2.22
|
||||
A|crafty.home.keithjackson.dev|192.168.1.53
|
||||
A|crafty.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|dev.brickbreak.keithjackson.dev|20.236.242.14
|
||||
A|gitea.home.keithjackson.dev|192.168.1.53
|
||||
A|gitea.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|grafana.home.keithjackson.dev|192.168.1.53
|
||||
A|grafana.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|headlamp.home.keithjackson.dev|192.168.1.53
|
||||
A|headlamp.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|home.keithjackson.dev|192.168.1.53
|
||||
A|jackett.home.keithjackson.dev|192.168.1.53
|
||||
A|jackett.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|keycloak.home.keithjackson.dev|192.168.1.53
|
||||
A|keycloak.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|nextcloud.home.keithjackson.dev|192.168.1.53
|
||||
A|nextcloud.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|pihole.home.keithjackson.dev|192.168.1.53
|
||||
A|pihole.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|plex.home.keithjackson.dev|192.168.1.53
|
||||
A|plex.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|prod.home.keithjackson.dev|192.168.2.20
|
||||
A|qbittorrent.home.keithjackson.dev|192.168.1.53
|
||||
A|qbittorrent.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|radarr.home.keithjackson.dev|192.168.1.53
|
||||
A|radarr.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|seerr.home.keithjackson.dev|192.168.1.53
|
||||
A|seerr.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|semaphore.home.keithjackson.dev|192.168.1.53
|
||||
A|semaphore.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|sonarr.home.keithjackson.dev|192.168.1.53
|
||||
A|sonarr.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|unifi.home.keithjackson.dev|192.168.1.54
|
||||
A|unifi.prod.home.keithjackson.dev|192.168.2.21
|
||||
TXT|a-auth.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/oauth2-proxy/oauth2-proxy
|
||||
TXT|a-auth.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/oauth2-proxy/oauth2-proxy
|
||||
TXT|a-conformance-dns.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/chainsaw-growing-grouse/dns-probe-svc
|
||||
TXT|a-crafty-mc.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/crafty/crafty-l4-tcp
|
||||
TXT|a-crafty-mc.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=service/crafty/crafty-l4-tcp
|
||||
TXT|a-crafty.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/crafty/crafty
|
||||
TXT|a-crafty.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/crafty/crafty
|
||||
TXT|a-gitea.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/gitea/gitea-3000
|
||||
TXT|a-gitea.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/gitea/gitea-3000
|
||||
TXT|a-grafana.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/monitoring/grafana
|
||||
TXT|a-grafana.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/monitoring/grafana
|
||||
TXT|a-headlamp.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/headlamp/headlamp
|
||||
TXT|a-headlamp.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/headlamp/headlamp
|
||||
TXT|a-home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/homepage/homepage
|
||||
TXT|a-jackett.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/jackett/jackett-9117
|
||||
TXT|a-jackett.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/jackett/jackett-9117
|
||||
TXT|a-keycloak.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/keycloak/keycloak
|
||||
TXT|a-keycloak.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/keycloak/keycloak
|
||||
TXT|a-nextcloud.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/nextcloud/nextcloud-80
|
||||
TXT|a-nextcloud.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/nextcloud/nextcloud-80
|
||||
TXT|a-pihole.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/pihole/pihole-80
|
||||
TXT|a-pihole.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/pihole/pihole-80
|
||||
TXT|a-plex.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/plex/plex-32400
|
||||
TXT|a-plex.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/plex/plex-32400
|
||||
TXT|a-prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/homepage/homepage
|
||||
TXT|a-qbittorrent.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/qbittorrent/qbittorrent-8080
|
||||
TXT|a-qbittorrent.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/qbittorrent/qbittorrent-8080
|
||||
TXT|a-radarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/radarr/radarr-7878
|
||||
TXT|a-radarr.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/radarr/radarr-7878
|
||||
TXT|a-seerr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/seerr/seerr-5055
|
||||
TXT|a-seerr.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/seerr/seerr-5055
|
||||
TXT|a-semaphore.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/semaphore/semaphore-3000
|
||||
TXT|a-semaphore.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/semaphore/semaphore-3000
|
||||
TXT|a-sonarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/sonarr/sonarr-8989
|
||||
TXT|a-sonarr.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/sonarr/sonarr-8989
|
||||
TXT|a-unifi.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/unifi/unifi-l4-tcp
|
||||
TXT|a-unifi.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=service/unifi/unifi-l4-tcp
|
||||
TXT|auth.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/oauth2-proxy/oauth2-proxy
|
||||
TXT|crafty-mc.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/crafty/crafty-l4-tcp
|
||||
TXT|crafty.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/crafty/crafty
|
||||
TXT|gitea.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/gitea/gitea-3000
|
||||
TXT|grafana.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/monitoring/grafana
|
||||
TXT|headlamp.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/headlamp/headlamp
|
||||
TXT|home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/homepage/homepage
|
||||
TXT|jackett.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/jackett/jackett-9117
|
||||
TXT|keycloak.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/keycloak/keycloak
|
||||
TXT|nextcloud.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/nextcloud/nextcloud-80
|
||||
TXT|pihole.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/pihole/pihole-80
|
||||
TXT|plex.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/plex/plex-32400
|
||||
TXT|qbittorrent.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/qbittorrent/qbittorrent-8080
|
||||
TXT|radarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/radarr/radarr-7878
|
||||
TXT|seerr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/seerr/seerr-5055
|
||||
TXT|semaphore.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/semaphore/semaphore-3000
|
||||
TXT|sonarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/sonarr/sonarr-8989
|
||||
TXT|unifi.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/unifi/unifi-l4-tcp
|
||||
@@ -1,93 +0,0 @@
|
||||
A|*.keithjackson.dev|136.51.115.252
|
||||
A|auth.home.keithjackson.dev|192.168.1.53
|
||||
A|auth.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|brickbreak.keithjackson.dev|20.236.242.14
|
||||
A|crafty-mc.home.keithjackson.dev|192.168.1.55
|
||||
A|crafty-mc.prod.home.keithjackson.dev|192.168.2.22
|
||||
A|crafty.home.keithjackson.dev|192.168.1.53
|
||||
A|crafty.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|dev.brickbreak.keithjackson.dev|20.236.242.14
|
||||
A|gitea.home.keithjackson.dev|192.168.1.53
|
||||
A|gitea.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|grafana.home.keithjackson.dev|192.168.1.53
|
||||
A|grafana.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|headlamp.home.keithjackson.dev|192.168.1.53
|
||||
A|headlamp.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|home.keithjackson.dev|192.168.1.53
|
||||
A|jackett.home.keithjackson.dev|192.168.1.53
|
||||
A|jackett.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|keycloak.home.keithjackson.dev|192.168.1.53
|
||||
A|keycloak.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|nextcloud.home.keithjackson.dev|192.168.1.53
|
||||
A|nextcloud.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|pihole.home.keithjackson.dev|192.168.1.53
|
||||
A|pihole.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|plex.home.keithjackson.dev|192.168.1.53
|
||||
A|plex.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|prod.home.keithjackson.dev|192.168.2.20
|
||||
A|qbittorrent.home.keithjackson.dev|192.168.1.53
|
||||
A|qbittorrent.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|radarr.home.keithjackson.dev|192.168.1.53
|
||||
A|radarr.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|seerr.home.keithjackson.dev|192.168.1.53
|
||||
A|seerr.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|semaphore.home.keithjackson.dev|192.168.1.53
|
||||
A|semaphore.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|sonarr.home.keithjackson.dev|192.168.1.53
|
||||
A|sonarr.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|unifi.home.keithjackson.dev|192.168.1.54
|
||||
A|unifi.prod.home.keithjackson.dev|192.168.2.21
|
||||
TXT|a-auth.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/oauth2-proxy/oauth2-proxy
|
||||
TXT|a-auth.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/oauth2-proxy/oauth2-proxy
|
||||
TXT|a-crafty-mc.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/crafty/crafty-l4-tcp
|
||||
TXT|a-crafty-mc.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=service/crafty/crafty-l4-tcp
|
||||
TXT|a-crafty.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/crafty/crafty
|
||||
TXT|a-crafty.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/crafty/crafty
|
||||
TXT|a-gitea.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/gitea/gitea-3000
|
||||
TXT|a-gitea.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/gitea/gitea-3000
|
||||
TXT|a-grafana.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/monitoring/grafana
|
||||
TXT|a-grafana.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/monitoring/grafana
|
||||
TXT|a-headlamp.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/headlamp/headlamp
|
||||
TXT|a-headlamp.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/headlamp/headlamp
|
||||
TXT|a-home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/homepage/homepage
|
||||
TXT|a-jackett.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/jackett/jackett-9117
|
||||
TXT|a-jackett.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/jackett/jackett-9117
|
||||
TXT|a-keycloak.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/keycloak/keycloak
|
||||
TXT|a-keycloak.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/keycloak/keycloak
|
||||
TXT|a-nextcloud.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/nextcloud/nextcloud-80
|
||||
TXT|a-nextcloud.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/nextcloud/nextcloud-80
|
||||
TXT|a-pihole.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/pihole/pihole-80
|
||||
TXT|a-pihole.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/pihole/pihole-80
|
||||
TXT|a-plex.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/plex/plex-32400
|
||||
TXT|a-plex.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/plex/plex-32400
|
||||
TXT|a-prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/homepage/homepage
|
||||
TXT|a-qbittorrent.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/qbittorrent/qbittorrent-8080
|
||||
TXT|a-qbittorrent.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/qbittorrent/qbittorrent-8080
|
||||
TXT|a-radarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/radarr/radarr-7878
|
||||
TXT|a-radarr.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/radarr/radarr-7878
|
||||
TXT|a-seerr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/seerr/seerr-5055
|
||||
TXT|a-seerr.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/seerr/seerr-5055
|
||||
TXT|a-semaphore.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/semaphore/semaphore-3000
|
||||
TXT|a-semaphore.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/semaphore/semaphore-3000
|
||||
TXT|a-sonarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/sonarr/sonarr-8989
|
||||
TXT|a-sonarr.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/sonarr/sonarr-8989
|
||||
TXT|a-unifi.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/unifi/unifi-l4-tcp
|
||||
TXT|a-unifi.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=service/unifi/unifi-l4-tcp
|
||||
TXT|auth.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/oauth2-proxy/oauth2-proxy
|
||||
TXT|crafty-mc.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/crafty/crafty-l4-tcp
|
||||
TXT|crafty.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/crafty/crafty
|
||||
TXT|gitea.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/gitea/gitea-3000
|
||||
TXT|grafana.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/monitoring/grafana
|
||||
TXT|headlamp.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/headlamp/headlamp
|
||||
TXT|home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/homepage/homepage
|
||||
TXT|jackett.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/jackett/jackett-9117
|
||||
TXT|keycloak.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/keycloak/keycloak
|
||||
TXT|nextcloud.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/nextcloud/nextcloud-80
|
||||
TXT|pihole.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/pihole/pihole-80
|
||||
TXT|plex.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/plex/plex-32400
|
||||
TXT|qbittorrent.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/qbittorrent/qbittorrent-8080
|
||||
TXT|radarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/radarr/radarr-7878
|
||||
TXT|seerr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/seerr/seerr-5055
|
||||
TXT|semaphore.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/semaphore/semaphore-3000
|
||||
TXT|sonarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/sonarr/sonarr-8989
|
||||
TXT|unifi.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/unifi/unifi-l4-tcp
|
||||
@@ -1758,334 +1758,3 @@ into a transient hard Kustomization failure.
|
||||
- **Prod's source URL cannot work on a cold prod cluster** — split-horizon answers the whole zone
|
||||
with `${GATEWAY_VIP}`, so prod would resolve `gitea.home.keithjackson.dev` to its own gateway.
|
||||
Bring-up needs a temporary source and a cutover, as `docs/05` describes for dev's own Phase-6.
|
||||
|
||||
## 2026-09-07 — LIVE BUG: prod's OIDC clients were pointed at DEV's Keycloak. Secrets are a substitution blind spot.
|
||||
|
||||
The operator could not log in to Headlamp on prod:
|
||||
|
||||
```
|
||||
Get "https://keycloak.home.keithjackson.dev/realms/homelab/.well-known/openid-configuration":
|
||||
dial tcp 192.168.1.53:443: i/o timeout
|
||||
```
|
||||
|
||||
`keycloak.home.keithjackson.dev` is **dev's** Keycloak. It resolves to `192.168.1.53`, on the dev
|
||||
LAN behind pfSense; prod lives on `192.168.2.0/24` (pfSense's WAN) and cannot route there, so the
|
||||
discovery fetch simply hung until it timed out.
|
||||
|
||||
### Root cause — the class, not the instance
|
||||
|
||||
**Flux's `${BASE_DOMAIN}` postBuild substitution rewrites the RENDERED MANIFEST TEXT. It can never
|
||||
reach inside a SOPS-encrypted Secret.** Environment-specific URLs had been stored *inside* two
|
||||
encrypted Secrets purely because it was convenient to keep one blob per app:
|
||||
|
||||
| File | Buried, un-substitutable values |
|
||||
|---|---|
|
||||
| `platform/headlamp/headlamp-oidc-credentials.sops.yaml` | `OIDC_ISSUER_URL`, `OIDC_CALLBACK_URL` |
|
||||
| `apps/wave-1/semaphore-bootstrap-secret.sops.yaml` (`oidc-providers-json`) | the realm issuer + the OIDC redirect URI |
|
||||
|
||||
Both were frozen at the apex `home.keithjackson.dev`. **Dev IS the apex, so dev was correct by
|
||||
coincidence** — the values were never per-environment, they just happened to match one environment.
|
||||
Prod was wrong from the first reconcile. Semaphore on prod was broken the same way and nobody had
|
||||
noticed, because nobody had tried prod SSO yet.
|
||||
|
||||
This is the same failure shape as the 2026-09-01 nextcloud guard: **a value that silently resolves
|
||||
to something plausible is worse than one that fails.** `StrictPostBuildSubstitutions` cannot catch
|
||||
it either — there is no `${...}` left unresolved to complain about; the ciphertext never contained
|
||||
one.
|
||||
|
||||
### The rule this establishes
|
||||
|
||||
> **Non-secret, environment-specific values MUST NOT live inside an encrypted Secret.** Issuer URLs,
|
||||
> callback URLs, redirect URIs and hostnames are public. Encrypt the secret half only, and template
|
||||
> the rest where `${BASE_DOMAIN}` can reach it.
|
||||
|
||||
### The fix
|
||||
|
||||
- **Headlamp** — `OIDC_ISSUER_URL` / `OIDC_CALLBACK_URL` moved to the HelmRelease's
|
||||
`spec.values.env`, templated `${BASE_DOMAIN}`. The Secret keeps `OIDC_CLIENT_SECRET` +
|
||||
the environment-invariant `OIDC_CLIENT_ID` / `OIDC_SCOPES`, still `envFrom`'d.
|
||||
**Exactly one source per variable** — an explicit `env:` entry silently outranks the same name
|
||||
arriving via `envFrom`, so the two sets deliberately do not overlap.
|
||||
Side effect, intended: naming `OIDC_CALLBACK_URL` in `env:` makes chart 0.41.0 finally emit
|
||||
`-oidc-callback-url=$(OIDC_CALLBACK_URL)`; it gates that arg on `env:`/`config.oidc`, which a
|
||||
Secret key alone never tripped — so the Secret's callback URL had been dead weight all along.
|
||||
- **Semaphore** — the `SEMAPHORE_OIDC_PROVIDERS` JSON is now a plaintext literal in
|
||||
`catalog/semaphore.yaml` (re-rendered to the `apps/wave-1` golden). Only the client secret stays
|
||||
encrypted, in its own key `oidc-client-secret`, and the **kubelet** splices it back via
|
||||
dependent-env-var expansion: `$(SEMAPHORE_OIDC_CLIENT_SECRET)` resolves against the `secretKeyRef`
|
||||
env var declared immediately above it. **Order is load-bearing** — the kubelet resolves `env` in
|
||||
list order and leaves a forward reference as literal text. `flux envsubst` ignores `$(...)`, so
|
||||
the two expansion layers do not collide (the `$${VAR}` escape in `apps/wave-1/nextcloud.yaml`
|
||||
documents the one form that DOES need escaping).
|
||||
|
||||
### Proof
|
||||
|
||||
Rendered per overlay (`kubectl kustomize --load-restrictor LoadRestrictionsNone`, then
|
||||
`flux envsubst --strict` with that overlay's `cluster-vars`), plus `helm template` of headlamp
|
||||
0.41.0 against the substituted values:
|
||||
|
||||
| | dev | prod |
|
||||
|---|---|---|
|
||||
| Headlamp `OIDC_ISSUER_URL` | `https://keycloak.home.keithjackson.dev/realms/homelab` | `https://keycloak.prod.home.keithjackson.dev/realms/homelab` |
|
||||
| Headlamp `OIDC_CALLBACK_URL` | `https://headlamp.home.keithjackson.dev/oidc-callback` | `https://headlamp.prod.home.keithjackson.dev/oidc-callback` |
|
||||
| Semaphore `provider_url` | `https://keycloak.home.keithjackson.dev/realms/homelab` | `https://keycloak.prod.home.keithjackson.dev/realms/homelab` |
|
||||
| Semaphore `redirect_url` | `https://semaphore.home.keithjackson.dev/api/...` | `https://semaphore.prod.home.keithjackson.dev/api/...` |
|
||||
|
||||
Dev's values are byte-identical to what it runs today — this change moves prod only.
|
||||
|
||||
### [parity] the apiserver invariant HOLDS
|
||||
|
||||
Prod's kube-apiserver `oidc-issuer-url` is an **install-time k3s flag** derived from
|
||||
`k3s_base_domain` (`infra/ansible/inventory/group_vars/k3s_prod/vars.yml` = `prod.home.keithjackson.dev`),
|
||||
not from `cluster-vars` — it cannot read the Flux variable. Headlamp forwards the user's `id_token`
|
||||
to the apiserver, so the two must name the same issuer. After this change they do, in both
|
||||
environments, because both derive from the same per-environment domain. Unchanged risk, now written
|
||||
down in `platform/headlamp/README.md`: moving one without the other 401s every request.
|
||||
|
||||
### Also fixed for the next person
|
||||
|
||||
`headlamp-oidc-credentials.sops.yaml.example` used to *teach* the broken pattern — it listed
|
||||
`OIDC_ISSUER_URL` and `OIDC_CALLBACK_URL` as keys to fill in. It now says, in the file, that adding
|
||||
either key back re-breaks prod.
|
||||
## 2026-09-07 — k3s-prod-1 NotReady: etcd starved of disk I/O by Longhorn (branch `fix/etcd-io-priority`)
|
||||
|
||||
**06:05:21Z, `k3s-prod-1` went NotReady** (`Kubelet stopped posting node status`) with the host
|
||||
up, pingable and SSH-able. The embedded etcd could not complete a WAL `fdatasync`:
|
||||
|
||||
```
|
||||
slow fdatasync took=3.56s / 6.71s / 4.95s / 3.40s (etcd expects 1s)
|
||||
apply request took too long took=8.30s (expects 100ms)
|
||||
etcdserver: too many requests
|
||||
Failed to check local etcd status for learner management: context deadline exceeded
|
||||
```
|
||||
|
||||
No OOM, no MCE, no hung task — **this is not the §0b memattr bug and not memory**. It is the §2
|
||||
shared-device risk, which §2 recorded as "accepted to be MEASURED, not assumed". The measurement
|
||||
came back, in production, 2 d 7 h after bring-up. `/proc/pressure/io` **full avg300**: node 1
|
||||
**11.36**, node 2 (NVMe) 0.75, node 3 (WD Blue SATA) 1.29. Node 1's drive is a 2011
|
||||
`C400-MTFDDAC512MAM` with 61,440 reallocated sectors and no power-loss protection. §2's premise —
|
||||
"every prod node is SSD or NVMe, where rebuild I/O costs latency rather than collapsing it" — was
|
||||
right about node 2 and wrong about node 1. §2 now records this under "THE MEASUREMENT CAME BACK".
|
||||
|
||||
**The exit condition is OPEN and the fix is hardware: replace node 1's drive, or give etcd its own
|
||||
device (`node_baseline_etcd_source: device`, the path the role still supports). That is the
|
||||
operator's.** Three software mitigations shipped meanwhile, none of which is the fix.
|
||||
|
||||
### 1. The outage was invisible to the monitoring stack — fixed first
|
||||
|
||||
There were alerts for **CPU** PSI and memory and **none for I/O**, so the first signal was a node
|
||||
already out of the cluster. Added `NodeIOPressureHigh` (full IO PSI > 5% for 10m) and
|
||||
`NodeIOPressureCritical` (> 10% for 5m), thresholds taken from the numbers above against a ~1%
|
||||
steady state, plus an I/O row on the node-pressure dashboard. Verified live that node-exporter
|
||||
already exports `node_pressure_io_{stalled,waiting}_seconds_total` on all three nodes.
|
||||
|
||||
**Gap left open for the operator:** etcd's own `etcd_disk_wal_fsync_duration_seconds` — the metric
|
||||
§2's exit condition literally asks for — is **not collected**. `kubeEtcd` is deliberately disabled
|
||||
in `platform/monitoring/kube-prometheus-stack.yaml` for k3s. Enabling it (k3s
|
||||
`--etcd-expose-metrics` + a scrape target) would make §2 directly measurable instead of inferred
|
||||
from PSI. Not done here: it is a k3s server-flag change requiring a rolling restart, which is not
|
||||
something to bundle into an incident fix.
|
||||
|
||||
### 2. Longhorn was already throttled as far as Longhorn goes
|
||||
|
||||
`concurrentReplicaRebuildPerNodeLimit: 1` was **in force during the outage and it held** — the 18
|
||||
volumes rebuilt one at a time and the node stalled anyway. Recorded plainly because it would be
|
||||
easy to "fix" this incident by setting a knob that was already set. For the v1 data engine there is
|
||||
no further rebuild lever: `replicaRebuildingBandwidthLimit` is v2/SPDK only (live Setting reads
|
||||
`{"v2":"0"}`, no v1 key) and `rebuildConcurrentSyncLimit` is already at its floor of 1.
|
||||
|
||||
So what was tightened is every *other* Longhorn I/O source, each defaulting to 5 concurrent
|
||||
operations per node: `concurrentVolumeBackupRestorePerNodeLimit` 5→1 (an R-STATE-03 restore of 18
|
||||
volumes would otherwise run five parallel full-volume writes onto the disk etcd fsyncs to),
|
||||
`snapshotHeavyTaskConcurrentLimit` 5→1, plus pins on `replicaReplenishmentWaitInterval`,
|
||||
`backup`/`restoreConcurrentLimit` and `concurrentAutomaticEngineUpgradePerNodeLimit`.
|
||||
|
||||
**And one setting in that file was inert.** `orphanAutoDeletion: true` has been silently dropped
|
||||
since the 1.11→1.12 upgrade — Longhorn 1.9 renamed it `orphanResourceAutoDeletion` (a list, not a
|
||||
bool) and the chart drops unknown `defaultSettings` keys with no error. Confirmed from both ends:
|
||||
absent from chart v1.12.1's `templates/default-setting.yaml`, and absent from the rendered
|
||||
`longhorn-default-setting` ConfigMap in the live cluster. Removed, deliberately not replaced. That
|
||||
is the **third** time this repo has shipped config that looked applied and was not.
|
||||
|
||||
### 3. Block-layer arbitration — and four mechanisms that are inert on these nodes
|
||||
|
||||
`roles/etcd_io_priority` + `playbooks/etcd-io-priority.yml`: BFQ on the shared disk, `nr_requests`
|
||||
64→32, `IOWeight=10000` on `system.slice` **and** `k3s.service`, then every value read back out of
|
||||
the kernel and asserted. Enabled for `[k3s_prod]`; **applies nothing until the operator runs it**.
|
||||
|
||||
The investigation's main value is the negative results, all checked on the live nodes:
|
||||
|
||||
- **`IOSchedulingClass=realtime` on `k3s.service` is a footgun, not a fix.** `mq-deadline` does
|
||||
honour the priority class, so it looks correct — but ioprio is inherited across `fork` and kept
|
||||
across `exec` (`block/blk-ioc.c` `__copy_io` copies `ioc->ioprio`), and **containerd is a child
|
||||
of the k3s server process** (verified: containerd pid 84553, ppid 84333 = `k3s-server`; and
|
||||
`ionice -c2 -n0 -- sh -c 'ionice -p $$; sh -c "ionice -p \$\$"'` reports best-effort prio 0 at
|
||||
both levels). Realtime for k3s would be realtime for every container including Longhorn.
|
||||
- **`IOSchedulingPriority` (level within a class) is a no-op** — `mq-deadline` buckets by class only.
|
||||
- **`io.latency` is unavailable** — `CONFIG_BLK_CGROUP_IOLATENCY` is not built in Debian trixie's
|
||||
7.1.8 kernel (only `IOCOST` is), and no `io.latency` exists under `/sys/fs/cgroup`.
|
||||
- **A bare `IOWeight=` drop-in is decoration.** systemd writes it to `io.weight` (blk-iocost —
|
||||
compiled in but disabled; `io.cost.qos` is empty) and `io.bfq.weight` (BFQ — not loaded). Neither
|
||||
reader exists on the stock nodes. And `IOWeight=1000` would map to `io.bfq.weight` ≈ 101 against
|
||||
BFQ's unweighted default of 100 — hence 10000, and an asserted floor of 500.
|
||||
|
||||
**What none of it fixes, stated so it is not forgotten:** etcd and Longhorn still share one ext4
|
||||
filesystem, and no cgroup or scheduler control arbitrates the **jbd2 journal** that etcd's
|
||||
`fdatasync` waits on and Longhorn's writes dirty. Software raises the load level at which this
|
||||
happens; it does not remove it.
|
||||
|
||||
**Unproven.** The role fails loudly if its settings do not *land*; it cannot tell you they helped.
|
||||
Apply to `k3s-prod-3` first (WD Blue, PSI 1.29 — same code path, far lower stakes), watch
|
||||
`NodeIOPressureHigh` for a day, then node 1.
|
||||
|
||||
### Doc correction found on the way
|
||||
|
||||
`infra/ansible/README.md` still claimed `k3s-prod-2`/`-3` "stay commented out" of the inventory.
|
||||
They have been live in `[k3s_prod_secondary]` since 2026-09-03. Corrected, because that sentence is
|
||||
exactly the kind used to reason about blast radius: **`-e k3s_hosts=k3s_prod` targets three hosts,
|
||||
not one.** Confirmed by `--list-hosts` (dev default → 3 dev hosts, where the new role is a no-op;
|
||||
`-e k3s_hosts=k3s_prod` → 3 prod hosts; `--limit k3s-prod-3` → 1).
|
||||
|
||||
### Still requires the operator
|
||||
|
||||
1. **The hardware fix.** Replace node 1's C400, or add an M.2 per node and set
|
||||
`node_baseline_etcd_source: device`. Everything above is a delay, not a repair.
|
||||
2. **Run `playbooks/etcd-io-priority.yml`** — node 3 first, `--limit`, `--list-hosts` confirmed.
|
||||
Nothing was run against either cluster for this change.
|
||||
3. **Decide on `kubeEtcd` metrics**, so §2's exit condition can be measured with etcd's own fsync
|
||||
histograms rather than inferred from node PSI.
|
||||
|
||||
## 2026-09-07 — R-ENV-03's L0 check was RED on `main`, and the requirement's literal wording is why (branch `fix/r-env-03-overlay-selected-credentials`)
|
||||
|
||||
`main`'s `conformance/tests/l0-static/prod-baremetal-not-dev-surface` failed 4 of 96 assertions.
|
||||
All four are the per-environment backup credentials merged in `e7c4a95`:
|
||||
|
||||
```
|
||||
apps/wave-0/nextcloud-db-backup-credentials-prod.sops.yaml
|
||||
apps/wave-0/semaphore-db-backup-credentials-prod.sops.yaml
|
||||
platform/keycloak/keycloak-db-backup-credentials-prod.sops.yaml
|
||||
platform/velero/velero-cloud-prod.sops.yaml
|
||||
... is a prod-specific file inside the shared configuration
|
||||
```
|
||||
|
||||
**The guard was a filename regex.** Any tracked file under `platform/ platform-config apps/
|
||||
catalog/ clusters/base/` whose name contained a delimited `prod` token failed, with no allowlist
|
||||
and no exception mechanism. Nothing about the four files' CONTENT was inspected.
|
||||
|
||||
**Renaming them to dodge the regex was rejected** (operator, explicitly): the guard exists for a
|
||||
reason and a rename would leave the reason unserved while making the check lie. So the check was
|
||||
refined to test the harm instead of the spelling, and the requirement wording is escalated below.
|
||||
|
||||
### Why these four files are compliant with R-ENV-02, and why the regex could not see it
|
||||
|
||||
Backup credentials became per-environment because dev backs up to MinIO and prod to Cloudflare
|
||||
R2, and ONE shared Secret meant either environment's key rotation broke the other (worse for
|
||||
CNPG: `archive_mode=on` turns a 403 on WAL archive into a monotonically filling pgdata PVC).
|
||||
|
||||
The mechanism is a single cluster-vars knob, `BACKUP_CREDENTIALS_SUFFIX` (base `""`, dev `""`,
|
||||
prod `"-prod"`). **Both Secret objects are applied to BOTH clusters** as base resources; the
|
||||
overlay changes only which one the shared consumer REFERENCES
|
||||
(`<name>${BACKUP_CREDENTIALS_SUFFIX}`). Every environment renders the identical manifests and the
|
||||
identical object set. That is exactly the "same manifests, overlay-scoped values" contract of
|
||||
R-ENV-02 — nothing is forked, and there is no prod-only content for anyone to develop against.
|
||||
The `-prod` in a filename is not a prod-only manifest; it is one half of an environment-selected
|
||||
pair, present everywhere.
|
||||
|
||||
### What the check now asserts (committed on the branch; not a path allowlist)
|
||||
|
||||
A file whose name carries the environment token is excused ONLY when the tree proves all of:
|
||||
|
||||
- **(A)** it declares nothing but Secrets whose names end in the environment suffix — an overlay
|
||||
can select which SECRET a shared manifest references and nothing else, so any other
|
||||
prod-suffixed object in a shared path is prod-only content and still fails;
|
||||
- **(1)** its own directory's `kustomization.yaml` lists it in `resources` (so EVERY environment
|
||||
applies it), and EVERY environment's value of the selector resolves to a Secret that some
|
||||
resource of that same kustomization declares, in the same namespace — no environment is left
|
||||
selecting an object that does not exist;
|
||||
- **(2)** nothing names the suffixed Secret — or the suffixed FILE — literally: every reference is
|
||||
`<base-name>${VAR}`, and only the applying kustomization may name the file;
|
||||
- **(3)** that `VAR` is defined for `clusters/base` AND for every `clusters/<env>` overlay
|
||||
(StrictPostBuildSubstitutions fails on an UNDEFINED variable, so an overlay that forgot it
|
||||
would stop that cluster reconciling), and prod's value is the one that selects THIS file.
|
||||
|
||||
References are read from PARSED YAML VALUES, never raw text — the mechanism is documented by name
|
||||
in comments beside the resources, and a name in a comment references nothing.
|
||||
|
||||
Mutation-tested: eleven mutations, each restored afterwards, each of which must FAIL and does — a
|
||||
prod-only ConfigMap in a shared path, a prod-only Secret with no counterpart, a consumer
|
||||
hardcoding prod's Secret name, a kustomization naming prod's file from another directory, the
|
||||
knob missing from dev, the knob missing from base, prod's value drifted to a suffix nothing
|
||||
declares, the prod half dropped from `resources`, the base half deleted (with and without its
|
||||
`resources` entry), the halves split across namespaces, and one of two consumer references moved
|
||||
to a second knob. The legitimate four pass, with a `note:` per file saying why they were excused.
|
||||
|
||||
### PROPOSED R-ENV-03 AMENDMENT — for the System Architect (spec repo is READ-ONLY here)
|
||||
|
||||
**What the requirement says today** (`docs/00-vision-and-scope.md`, detail in `docs/02`): prod is
|
||||
a 3x mini-PC bare-metal k3s cluster with local storage and **"MUST NOT be an active development
|
||||
surface until the roadmap declares it ready"**; `docs/02` adds **"Never develop against prod.
|
||||
Prod hardware is a target, not a sandbox (R-ENV-03)."**
|
||||
|
||||
**The problem with the literal reading.** Taken literally against the repository, "no prod-only
|
||||
content in the shared configuration" reads as "no artifact in a shared path may be specific to
|
||||
prod". That forbids the environment-selected PAIR, which is the very pattern R-ENV-02 prescribes
|
||||
for a value that must differ per environment while the manifests stay identical. The two
|
||||
requirements contradict each other on this case: R-ENV-02 says put the difference in an
|
||||
overlay-scoped value, and a Secret's difference cannot live in cluster-vars (it is ciphertext, per
|
||||
SOPS file, R-SEC-01), so it must live in a per-environment FILE that every environment applies.
|
||||
The requirement should forbid what is actually harmful — configuration only prod runs — not the
|
||||
appearance of an environment name in a path.
|
||||
|
||||
**Proposed wording** (add as a clarifying clause to R-ENV-03 in `docs/02`, and mirror the phrase
|
||||
in `docs/00`):
|
||||
|
||||
> Prod MUST NOT be an active development surface. Concretely: no artifact in the shared
|
||||
> configuration may be reachable by prod alone. An artifact whose NAME carries an environment
|
||||
> token is not by itself a violation — provided every environment APPLIES it, no shared manifest
|
||||
> references it by its environment-qualified name, and the selection between environment
|
||||
> counterparts is made solely by an overlay-scoped cluster-vars value that is DEFINED for every
|
||||
> environment. An artifact that only prod applies, or that a shared manifest names literally, IS
|
||||
> a violation, whatever it is called.
|
||||
|
||||
That wording keeps the guard's teeth (an artifact only prod applies still fails), removes the
|
||||
false positive, and states the property the check now enforces so the test and the spec agree.
|
||||
|
||||
**Related spec ask, same clause.** R-ENV-02 should say out loud that a per-environment SECRET is
|
||||
expressed as counterpart SOPS files applied everywhere plus a cluster-vars selector, because that
|
||||
is the one value class that cannot be expressed as a cluster-vars value itself. Without it, the
|
||||
next person hits the same contradiction.
|
||||
|
||||
### How this reached `main` green — the gate hole, and whether the fix in flight closes it
|
||||
|
||||
`ci/l0.sh` Stage 3b scopes the L0 conformance run to the TOUCHED requirement subset
|
||||
(`load_touched_requirements` -> `run.sh --requirements ...`). The delivery `e7c4a95` touched
|
||||
`apps/wave-0/`, `catalog/render.py`, `platform/keycloak/`, `platform/velero/` and declared no
|
||||
R-ID on a non-comment line, so the path map attributed R-GITOPS-02 / R-IDP-01..06 / R-STATE-03 —
|
||||
**and never R-ENV-03**. `prod-baremetal-not-dev-surface` was therefore FILTERED OUT of the run
|
||||
that gated the merge. The gate printed PASS having never executed the check that was already red.
|
||||
|
||||
**Landing `fix/touched-requirements-fail-closed` would NOT have caught this.** Checked, not
|
||||
assumed: that branch's `ci/requirement-paths.map` classifies every path (so nothing is silently
|
||||
unmapped any more), but it attributes R-ENV-03 only to `infra/ansible/*` and `infra/tofu/*`.
|
||||
Replaying `e7c4a95`'s twelve paths through that map yields
|
||||
`R-GITOPS-01 R-GITOPS-02 R-IDP-01..06 R-PLAT-02 R-STATE-03` — still no R-ENV-03. Its `ci/l0.sh`
|
||||
change also leaves Stage 3b scoped (only the substitution stage stays unconditional). The
|
||||
fail-closed branch fixes the "unmapped path contributes nothing" hole; it does not fix this one.
|
||||
|
||||
**Two things that would have caught it, for the operator/architect to choose between:**
|
||||
|
||||
1. **Whole-repo L0 invariants should be unconditional.** `flux-postbuild-substitution`,
|
||||
`keycloak-client-secret-reconciler` and `spec-immutability` already run as their own
|
||||
unconditional stages for exactly this reason. `prod-baremetal-not-dev-surface`,
|
||||
`no-plaintext-secrets` and `least-privilege-rbac` are the same shape — they assert over the
|
||||
WHOLE tracked tree, so scoping them to a changed path is meaningless. Cheapest correct fix.
|
||||
2. **A changed test should attribute the requirements it declares.** `conformance/*` maps to
|
||||
R-TEST-01..04 only, so editing `prod-baremetal-not-dev-surface/check.sh` does not select
|
||||
R-ENV-03 — this very branch's gate run reports `touched requirements: R-ENV-02 R-TEST-01
|
||||
R-TEST-02 R-TEST-03 R-TEST-04` and ran 2 of 12 L0 checks, NOT including the one it changed.
|
||||
A changed `conformance/tests/<test>/` should contribute that test's own
|
||||
`conformance.home.k3s/requirements` label.
|
||||
|
||||
Both belong to `ci/`, so they are left to that branch rather than smuggled in here. The full L0
|
||||
layer was run unscoped on this branch to compensate: **12 offline checks, 11 PASS, 1 SKIP**
|
||||
(`placeholder-guard-coupling`, pre-existing), 0 FAIL.
|
||||
@@ -83,7 +83,7 @@ spec:
|
||||
- containerPort: 9117
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /config
|
||||
mountPath: /data/config
|
||||
- name: oauth2-proxy
|
||||
image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1
|
||||
imagePullPolicy: IfNotPresent
|
||||
|
||||
@@ -11,7 +11,6 @@ resources:
|
||||
- jackett-oauth2-proxy-credentials.sops.yaml
|
||||
- jackett.yaml
|
||||
- nextcloud-db-app.sops.yaml
|
||||
- nextcloud-db-backup-credentials-prod.sops.yaml
|
||||
- nextcloud-db-backup-credentials.sops.yaml
|
||||
- nextcloud-db.yaml
|
||||
- pihole-oauth2-proxy-credentials.sops.yaml
|
||||
@@ -20,6 +19,5 @@ resources:
|
||||
- qbittorrent-oauth2-proxy-credentials.sops.yaml
|
||||
- qbittorrent.yaml
|
||||
- semaphore-db-app.sops.yaml
|
||||
- semaphore-db-backup-credentials-prod.sops.yaml
|
||||
- semaphore-db-backup-credentials.sops.yaml
|
||||
- semaphore-db.yaml
|
||||
@@ -1,29 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: nextcloud-db-backup-credentials-prod
|
||||
namespace: nextcloud-db
|
||||
type: Opaque
|
||||
stringData:
|
||||
ACCESS_KEY_ID: ENC[AES256_GCM,data:4lx6et4HcXDegulyIoa7jZR/lVgapGPEkukajfCKjvk=,iv:p6V7tizbikk55tOvpPOsPHTe/p52QLX5A5xrwuB14rA=,tag:oNJuipT/Lb6zynJ2K2Q71A==,type:str]
|
||||
ACCESS_SECRET_KEY: ENC[AES256_GCM,data:V5bHPoPeBOVyLp1n1Jilyso4i5P2aPSH0hrryfweg2q/SctWZxRbLBQ/FLzOigbJsAhlwOGu069mQPma6xjfdw==,iv:8u4UOvI8dC/cm6mEWDlyN5E3SfRZmwKY9uZY/DjPEsQ=,tag:tw7tGs29TmUvMPBCtCszAA==,type:str]
|
||||
sops:
|
||||
kms: []
|
||||
gcp_kms: []
|
||||
azure_kv: []
|
||||
hc_vault: []
|
||||
age:
|
||||
- recipient: age175l8mdpdfu7hq0aml3exs3gykarwjmuvr23sq56s3wjejnyk44wsp4mfzg
|
||||
enc: |
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBiSUt5N2k2N2lDU2ZsMy9m
|
||||
M1NFS0M2QjFsNFhnR0hCMzFyaTdiQWFZRFJZCmhBTWRXNmxLMGtsc3Rxd2FSMDZp
|
||||
aktjcTE2c1hmU1EzM3RWNksyb3Q1YVUKLS0tIDBFS1I5aDBER2cvcWtLZ3BXTEp5
|
||||
ZGI5ZmZIcStheUNQamxIVVVpc0ZMOHMKlE5aHewtHJpeU24F5sodc9CEm9KTjHQQ
|
||||
8ThEQ/8vyazL9V4ojlICxSHU/SHkHpD1eBaTKcAQd/LZzujIcJ+QfA==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
lastmodified: "2026-09-07T06:36:33Z"
|
||||
mac: ENC[AES256_GCM,data:8awz1Jh8t4zjDMyd93/v+6on9iiIaSi01293A7uYhxj9L1+g63zeO+DPmSn6+KbZvHdqU2A7up09H1NzfXAEdGuvVbB36cdGq0zKank3PUi8c7IT6qNuWydxRAcqLnbbD+c30Z6GB+gAuZHstNOe4Rgc4aJvMN6FuP6x6Z0870c=,iv:KGGKJq2QA5xztvoHs/SViXW72DYwT16i+W+dJBUeIJQ=,tag:3UpYqx2+RIK0QdMH/BSIpA==,type:str]
|
||||
pgp: []
|
||||
encrypted_regex: ^(data|stringData)$
|
||||
version: 3.9.4
|
||||
@@ -37,53 +37,16 @@ metadata:
|
||||
name: nextcloud-db
|
||||
namespace: nextcloud-db
|
||||
spec:
|
||||
# THREE instances — one primary + two hot standbys (2026-09-07), same as keycloak-db; see
|
||||
# platform/keycloak/postgres-cluster.yaml for the full rationale. Short version: Longhorn's 3-way
|
||||
# replication saves the DATA on node loss, not the SERVICE, and with `instances: 1` all three CNPG
|
||||
# primaries had drifted onto k3s-prod-2, so one node took Nextcloud down alongside SSO and
|
||||
# Semaphore. A streaming standby lets CNPG promote in seconds and repoint the -rw Service.
|
||||
# Gated on the Barman archive below actually working (ContinuousArchiving=True with a real
|
||||
# firstRecoverabilityPoint) — standbys replicate corruption, they are not a backup.
|
||||
#
|
||||
# Scaling out is non-disruptive: CNPG creates -2 then -3 sequentially, each seeded by
|
||||
# pg_basebackup from the running primary, which is neither restarted nor disconnected.
|
||||
instances: 3
|
||||
# Single Postgres instance on 3-replica Longhorn is acceptable at home scale (ADR-0014, same as
|
||||
# keycloak-db). The storage is already 3-way replicated by Longhorn; raise to 3 for Postgres-level
|
||||
# HA as a future hardening.
|
||||
instances: 1
|
||||
imageName: ghcr.io/cloudnative-pg/postgresql:16.4
|
||||
# Stamp app.kubernetes.io/name=nextcloud-db onto every generated pod/service so the rendered
|
||||
# nextcloud egress CiliumNetworkPolicy's toEndpoints selector matches the primary pod (R-NET-02).
|
||||
inheritedMetadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: nextcloud-db
|
||||
# HARD anti-affinity — the three instances MUST land on three DIFFERENT nodes; and roll the
|
||||
# primary by SWITCHOVER rather than CNPG's default in-place `restart`, which would delete and
|
||||
# recreate the primary pod. Full rationale in platform/keycloak/postgres-cluster.yaml. Short
|
||||
# version: CNPG's default `preferred` is a scheduler hint, so when CNPG recreates an instance
|
||||
# after a node loss it may stack two instances on one node — which is how `instances: 3` decays
|
||||
# back into the single-node failure domain this change removes. `required` makes it a constraint;
|
||||
# the cost is that an instance whose node is gone stays Pending, which is correct (2 of 3 still
|
||||
# serve). Satisfiable on both clusters: prod and dev each have 3 schedulable, untainted nodes.
|
||||
#
|
||||
# APPLYING THIS CHANGES THE POD SPEC, so CNPG rolls the instances: standbys first, then one
|
||||
# switchover. Expect a single brief connection reset for Nextcloud.
|
||||
affinity:
|
||||
enablePodAntiAffinity: true
|
||||
podAntiAffinityType: required
|
||||
primaryUpdateMethod: switchover
|
||||
# SYNCHRONOUS REPLICATION — quorum of ONE standby, PREFERRED durability (2026-09-07). Full
|
||||
# rationale in platform/keycloak/postgres-cluster.yaml. Short version: async replication loses the
|
||||
# last few transactions when a primary dies before its standbys catch up, and `method: any` +
|
||||
# `number: 1` makes COMMIT wait for one standby to confirm. dataDurability MUST stay `preferred`,
|
||||
# never `required` — `required` blocks writes outright when fewer than `number` healthy standbys
|
||||
# are available, which on a 3-node cluster sharing nodes with etcd and the CNI turns one degraded
|
||||
# standby into a total write outage. `preferred` relaxes to async instead.
|
||||
#
|
||||
# NON-DISRUPTIVE: compiles to `synchronous_standby_names`, a reload-only PostgreSQL parameter —
|
||||
# no restart, no switchover, no client disconnect. Land it after the anti-affinity roll.
|
||||
postgresql:
|
||||
synchronous:
|
||||
method: any
|
||||
number: 1
|
||||
dataDurability: preferred
|
||||
storage:
|
||||
storageClass: longhorn
|
||||
size: 5Gi
|
||||
@@ -105,38 +68,25 @@ spec:
|
||||
memory: 256Mi
|
||||
limits:
|
||||
memory: 512Mi
|
||||
# --- Off-cluster DR backup (R-STATE-03) — CNPG-native Barman → object storage ---
|
||||
# --- Off-cluster DR backup (R-STATE-03) — CNPG-native Barman → the on-Proxmox MinIO `cnpg` bucket ---
|
||||
# Continuous WAL archiving + periodic base backups = proper Postgres PITR (realises the SD's
|
||||
# backup.policy daily-offcluster). CNPG appends serverName (defaults to the Cluster name) to
|
||||
# destinationPath, so objects land under ${CNPG_BACKUP_PATH}/nextcloud-db/ — dev
|
||||
# s3://cnpg/nextcloud-db/ on the on-Proxmox MinIO, prod
|
||||
# s3://k3s-prod-backup/barman/nextcloud-db/ on Cloudflare R2. Both endpointURL and
|
||||
# destinationPath are cluster-vars knobs (R-ENV-02), substituted by `apps-wave-0`.
|
||||
# Credentials: PER-ENVIRONMENT (2026-09-07). dev reads nextcloud-db-backup-credentials (the
|
||||
# scoped MinIO key), prod reads nextcloud-db-backup-credentials-prod (the Cloudflare R2
|
||||
# token). BOTH Secrets are applied to BOTH clusters by apps/wave-0/kustomization.yaml (a
|
||||
# generated golden — render.py places the -prod sidecar in this DB's wave); the
|
||||
# cluster-vars knob ${BACKUP_CREDENTIALS_SUFFIX} (dev "" | prod "-prod") chooses which
|
||||
# one this Cluster reads. Both are in-namespace, which is all a pod can read. They were
|
||||
# ONE shared Secret until the split, so giving prod its R2 token would have left dev
|
||||
# signing MinIO requests with an R2 key id — 403 on every archive command, i.e. the
|
||||
# disk-full clock below rather than a merely missing backup.
|
||||
#
|
||||
# archive_mode is ON, so a FAILING archive command is a disk-full clock, not a degraded backup:
|
||||
# Postgres refuses to recycle a WAL segment it has not archived, and the pgdata PVC fills until
|
||||
# the primary PANICs. Prod demonstrated this from bootstrap to 2026-09-07 with a placeholder
|
||||
# endpoint. Treat ContinuousArchiving=False here as an availability incident.
|
||||
# backup.policy daily-offcluster). destinationPath is the bucket ROOT; CNPG appends serverName
|
||||
# (defaults to the Cluster name), so objects land under s3://cnpg/nextcloud-db/. Path-style HTTP.
|
||||
# Credentials: the scoped key from the in-namespace SOPS Secret nextcloud-db-backup-credentials
|
||||
# (same-ns only), kept in sync with platform/backup/minio-credentials.sops.yaml.
|
||||
# endpointURL is ${S3_BACKUP_ENDPOINT} (cluster-vars, R-ENV-02; dev http://192.168.1.9:9000),
|
||||
# substituted by the `apps-wave-0` Flux Kustomization. Prod supplies its own off-cluster target.
|
||||
backup:
|
||||
retentionPolicy: "30d"
|
||||
barmanObjectStore:
|
||||
destinationPath: ${CNPG_BACKUP_PATH}
|
||||
destinationPath: s3://cnpg
|
||||
endpointURL: ${S3_BACKUP_ENDPOINT}
|
||||
s3Credentials:
|
||||
accessKeyId:
|
||||
name: nextcloud-db-backup-credentials${BACKUP_CREDENTIALS_SUFFIX}
|
||||
name: nextcloud-db-backup-credentials
|
||||
key: ACCESS_KEY_ID
|
||||
secretAccessKey:
|
||||
name: nextcloud-db-backup-credentials${BACKUP_CREDENTIALS_SUFFIX}
|
||||
name: nextcloud-db-backup-credentials
|
||||
key: ACCESS_SECRET_KEY
|
||||
wal:
|
||||
compression: gzip
|
||||
|
||||
@@ -35,34 +35,6 @@ spec:
|
||||
- protocol: TCP
|
||||
port: 53
|
||||
---
|
||||
# PersistentVolume plex-media - static external NFS (bulk tier)
|
||||
apiVersion: v1
|
||||
kind: PersistentVolume
|
||||
metadata:
|
||||
name: plex-media
|
||||
labels:
|
||||
app.kubernetes.io/name: plex
|
||||
spec:
|
||||
capacity:
|
||||
storage: 9Ti
|
||||
accessModes:
|
||||
- ReadOnlyMany
|
||||
persistentVolumeReclaimPolicy: Retain
|
||||
storageClassName: ''
|
||||
mountOptions:
|
||||
- nfsvers=4.1
|
||||
- hard
|
||||
- noatime
|
||||
- timeo=600
|
||||
- retrans=2
|
||||
nfs:
|
||||
server: ${BULK_NFS_SERVER}
|
||||
path: /mnt/Storage/plex
|
||||
readOnly: true
|
||||
claimRef:
|
||||
namespace: plex
|
||||
name: plex-media
|
||||
---
|
||||
# PVC plex-config (R-STATE)
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
@@ -79,23 +51,6 @@ spec:
|
||||
requests:
|
||||
storage: 8Gi
|
||||
---
|
||||
# PVC plex-media (R-STATE)
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: plex-media
|
||||
namespace: plex
|
||||
labels:
|
||||
app.kubernetes.io/name: plex
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadOnlyMany
|
||||
storageClassName: ''
|
||||
volumeName: plex-media
|
||||
resources:
|
||||
requests:
|
||||
storage: 9Ti
|
||||
---
|
||||
# Deployment (R-PLAT-02)
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
@@ -135,16 +90,10 @@ spec:
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /config
|
||||
- name: media
|
||||
mountPath: /media
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: config
|
||||
persistentVolumeClaim:
|
||||
claimName: plex-config
|
||||
- name: media
|
||||
persistentVolumeClaim:
|
||||
claimName: plex-media
|
||||
---
|
||||
# Service
|
||||
apiVersion: v1
|
||||
|
||||
@@ -83,7 +83,7 @@ spec:
|
||||
- containerPort: 8080
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /config
|
||||
mountPath: /data/config
|
||||
- name: oauth2-proxy
|
||||
image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1
|
||||
imagePullPolicy: IfNotPresent
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: semaphore-db-backup-credentials-prod
|
||||
namespace: semaphore-db
|
||||
type: Opaque
|
||||
stringData:
|
||||
ACCESS_KEY_ID: ENC[AES256_GCM,data:OhWEEYqrvmDB40NmiETgeUisBvaO9ocevCZAsurNijw=,iv:kEjOIg9NHbRYm+Ul1WrBAzb3gLbIv+ipiYCxFK0QoVc=,tag:KAB1JAIw0ATwbjDEKZVT9A==,type:str]
|
||||
ACCESS_SECRET_KEY: ENC[AES256_GCM,data:C0PNKv07LCYpFONLqd55sz+ioyU/i2YJUkbZ8OX/dCErM/DilqchyUAT5EqnT/r3YJmdaA1pX85S3fxlVVHkbg==,iv:4ha28b+xIebvX2JPITmtTovIe4vdmj2XfI3cgkUHH4c=,tag:xbMyflcHojytPzgaoOYcXQ==,type:str]
|
||||
sops:
|
||||
kms: []
|
||||
gcp_kms: []
|
||||
azure_kv: []
|
||||
hc_vault: []
|
||||
age:
|
||||
- recipient: age175l8mdpdfu7hq0aml3exs3gykarwjmuvr23sq56s3wjejnyk44wsp4mfzg
|
||||
enc: |
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBZVTJ3cXR6cks3Y0RNS2hx
|
||||
K3pjZEduYUxmL0V3SmFUcU9XbGVHSVFITWh3Cjl1eHlTLytoOW5PaVNFVkJoeTBo
|
||||
T1FnRHNDVzJ6dWpuR09WaElvbWF4YU0KLS0tIEJoQ1VsM1cxRlg0R1dtNGdOenNn
|
||||
aVNjMjVBTytLdHlFdmZCS2tPelo1Y2MKYt1G5t+sDiHgcg+Q9ECBzJn2gS/paSEC
|
||||
7aoX60SBrPgHvnGls5lzc3Tin9nH4jWYl6Nu8DAvIE0YPXDFr5a0UA==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
lastmodified: "2026-09-07T06:36:33Z"
|
||||
mac: ENC[AES256_GCM,data:mhVP7tjdZK2FR2imvpJbTyT7BigvcIKEpLSnBS1oSITUirB8xZC7aHBfyFuIoKOrpOJgEe40nHUdDYCHRJtb4dLeVja8+7meVaQWzxC8BWVHsksW+73hsqhjAzlHeobx6Irbk4PAMqIY0/jnf4MHryjnRn3dUuwu9U1CqL30JbE=,iv:57oCcp6FW9EAIVKnYGtkaTrnbijyv54T+vmZxMbMqnw=,tag:agrRP/JLypU60KOeN0s/5w==,type:str]
|
||||
pgp: []
|
||||
encrypted_regex: ^(data|stringData)$
|
||||
version: 3.9.4
|
||||
@@ -38,53 +38,16 @@ metadata:
|
||||
name: semaphore-db
|
||||
namespace: semaphore-db
|
||||
spec:
|
||||
# THREE instances — one primary + two hot standbys (2026-09-07), same as keycloak-db /
|
||||
# nextcloud-db; see platform/keycloak/postgres-cluster.yaml for the full rationale. Short version:
|
||||
# Longhorn's 3-way replication saves the DATA on node loss, not the SERVICE, and with
|
||||
# `instances: 1` all three CNPG primaries had drifted onto k3s-prod-2, so one node took Semaphore
|
||||
# down alongside SSO and Nextcloud. A streaming standby lets CNPG promote in seconds and repoint
|
||||
# the -rw Service. Gated on the Barman archive below actually working (ContinuousArchiving=True
|
||||
# with a real firstRecoverabilityPoint) — standbys replicate corruption, they are not a backup.
|
||||
#
|
||||
# Scaling out is non-disruptive: CNPG creates -2 then -3 sequentially, each seeded by
|
||||
# pg_basebackup from the running primary, which is neither restarted nor disconnected.
|
||||
instances: 3
|
||||
# Single Postgres instance on 3-replica Longhorn is acceptable at home scale (ADR-0014, same as
|
||||
# keycloak-db / nextcloud-db). The storage is already 3-way replicated by Longhorn; raise to 3 for
|
||||
# Postgres-level HA as a future hardening.
|
||||
instances: 1
|
||||
imageName: ghcr.io/cloudnative-pg/postgresql:16.4
|
||||
# Stamp app.kubernetes.io/name=semaphore-db onto every generated pod/service so the rendered
|
||||
# semaphore egress CiliumNetworkPolicy's toEndpoints selector matches the primary pod (R-NET-02).
|
||||
inheritedMetadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: semaphore-db
|
||||
# HARD anti-affinity — the three instances MUST land on three DIFFERENT nodes; and roll the
|
||||
# primary by SWITCHOVER rather than CNPG's default in-place `restart`, which would delete and
|
||||
# recreate the primary pod. Full rationale in platform/keycloak/postgres-cluster.yaml. Short
|
||||
# version: CNPG's default `preferred` is a scheduler hint, so when CNPG recreates an instance
|
||||
# after a node loss it may stack two instances on one node — which is how `instances: 3` decays
|
||||
# back into the single-node failure domain this change removes. `required` makes it a constraint;
|
||||
# the cost is that an instance whose node is gone stays Pending, which is correct (2 of 3 still
|
||||
# serve). Satisfiable on both clusters: prod and dev each have 3 schedulable, untainted nodes.
|
||||
#
|
||||
# APPLYING THIS CHANGES THE POD SPEC, so CNPG rolls the instances: standbys first, then one
|
||||
# switchover. Expect a single brief connection reset for Semaphore.
|
||||
affinity:
|
||||
enablePodAntiAffinity: true
|
||||
podAntiAffinityType: required
|
||||
primaryUpdateMethod: switchover
|
||||
# SYNCHRONOUS REPLICATION — quorum of ONE standby, PREFERRED durability (2026-09-07). Full
|
||||
# rationale in platform/keycloak/postgres-cluster.yaml. Short version: async replication loses the
|
||||
# last few transactions when a primary dies before its standbys catch up, and `method: any` +
|
||||
# `number: 1` makes COMMIT wait for one standby to confirm. dataDurability MUST stay `preferred`,
|
||||
# never `required` — `required` blocks writes outright when fewer than `number` healthy standbys
|
||||
# are available, which on a 3-node cluster sharing nodes with etcd and the CNI turns one degraded
|
||||
# standby into a total write outage. `preferred` relaxes to async instead.
|
||||
#
|
||||
# NON-DISRUPTIVE: compiles to `synchronous_standby_names`, a reload-only PostgreSQL parameter —
|
||||
# no restart, no switchover, no client disconnect. Land it after the anti-affinity roll.
|
||||
postgresql:
|
||||
synchronous:
|
||||
method: any
|
||||
number: 1
|
||||
dataDurability: preferred
|
||||
storage:
|
||||
storageClass: longhorn
|
||||
size: 5Gi
|
||||
@@ -106,38 +69,25 @@ spec:
|
||||
memory: 256Mi
|
||||
limits:
|
||||
memory: 512Mi
|
||||
# --- Off-cluster DR backup (R-STATE-03) — CNPG-native Barman → object storage ---
|
||||
# --- Off-cluster DR backup (R-STATE-03) — CNPG-native Barman → the on-Proxmox MinIO `cnpg` bucket ---
|
||||
# Continuous WAL archiving + periodic base backups = proper Postgres PITR (realises the SD's
|
||||
# backup.policy daily-offcluster). CNPG appends serverName (defaults to the Cluster name) to
|
||||
# destinationPath, so objects land under ${CNPG_BACKUP_PATH}/semaphore-db/ — dev
|
||||
# s3://cnpg/semaphore-db/ on the on-Proxmox MinIO, prod
|
||||
# s3://k3s-prod-backup/barman/semaphore-db/ on Cloudflare R2. Both endpointURL and
|
||||
# destinationPath are cluster-vars knobs (R-ENV-02), substituted by `apps-wave-0`.
|
||||
# Credentials: PER-ENVIRONMENT (2026-09-07). dev reads semaphore-db-backup-credentials (the
|
||||
# scoped MinIO key), prod reads semaphore-db-backup-credentials-prod (the Cloudflare R2
|
||||
# token). BOTH Secrets are applied to BOTH clusters by apps/wave-0/kustomization.yaml (a
|
||||
# generated golden — render.py places the -prod sidecar in this DB's wave); the
|
||||
# cluster-vars knob ${BACKUP_CREDENTIALS_SUFFIX} (dev "" | prod "-prod") chooses which
|
||||
# one this Cluster reads. Both are in-namespace, which is all a pod can read. They were
|
||||
# ONE shared Secret until the split, so giving prod its R2 token would have left dev
|
||||
# signing MinIO requests with an R2 key id — 403 on every archive command, i.e. the
|
||||
# disk-full clock below rather than a merely missing backup.
|
||||
#
|
||||
# archive_mode is ON, so a FAILING archive command is a disk-full clock, not a degraded backup:
|
||||
# Postgres refuses to recycle a WAL segment it has not archived, and the pgdata PVC fills until
|
||||
# the primary PANICs. Prod demonstrated this from bootstrap to 2026-09-07 with a placeholder
|
||||
# endpoint. Treat ContinuousArchiving=False here as an availability incident.
|
||||
# backup.policy daily-offcluster). destinationPath is the bucket ROOT; CNPG appends serverName
|
||||
# (defaults to the Cluster name), so objects land under s3://cnpg/semaphore-db/. Path-style HTTP.
|
||||
# Credentials: the scoped key from the in-namespace SOPS Secret semaphore-db-backup-credentials
|
||||
# (same-ns only), kept in sync with platform/backup/minio-credentials.sops.yaml.
|
||||
# endpointURL is ${S3_BACKUP_ENDPOINT} (cluster-vars, R-ENV-02; dev http://192.168.1.9:9000),
|
||||
# substituted by the `apps-wave-0` Flux Kustomization. Prod supplies its own off-cluster target.
|
||||
backup:
|
||||
retentionPolicy: "30d"
|
||||
barmanObjectStore:
|
||||
destinationPath: ${CNPG_BACKUP_PATH}
|
||||
destinationPath: s3://cnpg
|
||||
endpointURL: ${S3_BACKUP_ENDPOINT}
|
||||
s3Credentials:
|
||||
accessKeyId:
|
||||
name: semaphore-db-backup-credentials${BACKUP_CREDENTIALS_SUFFIX}
|
||||
name: semaphore-db-backup-credentials
|
||||
key: ACCESS_KEY_ID
|
||||
secretAccessKey:
|
||||
name: semaphore-db-backup-credentials${BACKUP_CREDENTIALS_SUFFIX}
|
||||
name: semaphore-db-backup-credentials
|
||||
key: ACCESS_SECRET_KEY
|
||||
wal:
|
||||
compression: gzip
|
||||
|
||||
@@ -1,159 +0,0 @@
|
||||
---
|
||||
# Namespace (trust zone; default-deny + policies apply)
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: vm-demo
|
||||
labels:
|
||||
homelab.keithjackson.dev/app-namespace: 'true'
|
||||
annotations:
|
||||
homelab.keithjackson.dev/backup-policy: daily-offcluster
|
||||
---
|
||||
# Default-deny baseline (R-NET-01): deny all, allow only DNS
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: default-deny
|
||||
namespace: vm-demo
|
||||
labels:
|
||||
homelab.keithjackson.dev/baseline: default-deny
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
- Egress
|
||||
ingress: []
|
||||
egress:
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
ports:
|
||||
- protocol: UDP
|
||||
port: 53
|
||||
- protocol: TCP
|
||||
port: 53
|
||||
---
|
||||
# CDI DataVolume (standalone; survives rebuild, R-STATE-01)
|
||||
apiVersion: cdi.kubevirt.io/v1beta1
|
||||
kind: DataVolume
|
||||
metadata:
|
||||
name: vm-demo-rootdisk
|
||||
namespace: vm-demo
|
||||
spec:
|
||||
source:
|
||||
registry:
|
||||
url: docker://quay.io/containerdisks/fedora:41
|
||||
storage:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: longhorn
|
||||
resources:
|
||||
requests:
|
||||
storage: 6Gi
|
||||
---
|
||||
# KubeVirt VirtualMachine (R-PLAT-03)
|
||||
apiVersion: kubevirt.io/v1
|
||||
kind: VirtualMachine
|
||||
metadata:
|
||||
name: vm-demo
|
||||
namespace: vm-demo
|
||||
labels:
|
||||
app.kubernetes.io/name: vm-demo
|
||||
spec:
|
||||
running: false
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
kubevirt.io/vm: vm-demo
|
||||
homelab.keithjackson.dev/egress-class: internet
|
||||
homelab.keithjackson.dev/auth-mode: none
|
||||
spec:
|
||||
domain:
|
||||
cpu:
|
||||
cores: 1
|
||||
memory:
|
||||
guest: 1536Mi
|
||||
devices:
|
||||
disks:
|
||||
- name: rootdisk
|
||||
disk:
|
||||
bus: virtio
|
||||
- name: cloudinit
|
||||
disk:
|
||||
bus: virtio
|
||||
interfaces:
|
||||
- name: default
|
||||
masquerade: {}
|
||||
resources:
|
||||
requests:
|
||||
memory: 1536Mi
|
||||
networks:
|
||||
- name: default
|
||||
pod: {}
|
||||
volumes:
|
||||
- name: rootdisk
|
||||
dataVolume:
|
||||
name: vm-demo-rootdisk
|
||||
- name: cloudinit
|
||||
cloudInitNoCloud:
|
||||
userData: "#cloud-config\npackage_update: true\npackages:\n - dnf-automatic\nwrite_files:\n - path: /etc/dnf/automatic.conf\n content: |\n [commands]\n upgrade_type = default\n random_sleep = 0\n download_updates = yes\n apply_updates = yes\nruncmd:\n - [ systemctl, enable, --now, dnf-automatic.timer ]\n"
|
||||
---
|
||||
# CiliumNetworkPolicy world egress allow (egress class governs the path; R-NET-05): vm-demo-allow-internet-egress
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
metadata:
|
||||
name: vm-demo-allow-internet-egress
|
||||
namespace: vm-demo
|
||||
labels:
|
||||
app.kubernetes.io/name: vm-demo
|
||||
spec:
|
||||
endpointSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: vm-demo
|
||||
egress:
|
||||
- toEntities:
|
||||
- world
|
||||
---
|
||||
# ServiceMonitor (R-OBS-01)
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: vm-demo
|
||||
namespace: vm-demo
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
app.kubernetes.io/name: vm-demo
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: vm-demo
|
||||
namespaceSelector:
|
||||
matchNames:
|
||||
- vm-demo
|
||||
endpoints:
|
||||
- port: metrics
|
||||
interval: 30s
|
||||
---
|
||||
# PrometheusRule SLI alerts (R-OBS-01)
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: vm-demo-slis
|
||||
namespace: vm-demo
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
app.kubernetes.io/name: vm-demo
|
||||
spec:
|
||||
groups:
|
||||
- name: vm-demo.slis
|
||||
rules:
|
||||
- alert: vm-demo-down
|
||||
expr: up{job="vm-demo"} == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
service: vm-demo
|
||||
annotations:
|
||||
summary: vm-demo is down
|
||||
description: 'vm-demo has been unreachable for 5m (SLI: up).'
|
||||
@@ -1,228 +0,0 @@
|
||||
---
|
||||
# Namespace (trust zone; default-deny + policies apply)
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: keycloak
|
||||
labels:
|
||||
homelab.keithjackson.dev/app-namespace: 'true'
|
||||
gateway-access: shared
|
||||
annotations:
|
||||
homelab.keithjackson.dev/backup-policy: daily-offcluster
|
||||
---
|
||||
# Default-deny baseline (R-NET-01): deny all, allow only DNS
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: default-deny
|
||||
namespace: keycloak
|
||||
labels:
|
||||
homelab.keithjackson.dev/baseline: default-deny
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
- Egress
|
||||
ingress: []
|
||||
egress:
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
ports:
|
||||
- protocol: UDP
|
||||
port: 53
|
||||
- protocol: TCP
|
||||
port: 53
|
||||
---
|
||||
# PVC keycloak-db (R-STATE)
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: keycloak-db
|
||||
namespace: keycloak
|
||||
labels:
|
||||
app.kubernetes.io/name: keycloak
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: longhorn
|
||||
resources:
|
||||
requests:
|
||||
storage: 10Gi
|
||||
---
|
||||
# Deployment (R-PLAT-02)
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: keycloak
|
||||
namespace: keycloak
|
||||
labels:
|
||||
app.kubernetes.io/name: keycloak
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: keycloak
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: keycloak
|
||||
homelab.keithjackson.dev/egress-class: internet
|
||||
homelab.keithjackson.dev/auth-mode: native
|
||||
spec:
|
||||
containers:
|
||||
- name: keycloak
|
||||
image: quay.io/keycloak/keycloak
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
volumeMounts:
|
||||
- name: db
|
||||
mountPath: /data/db
|
||||
volumes:
|
||||
- name: db
|
||||
persistentVolumeClaim:
|
||||
claimName: keycloak-db
|
||||
---
|
||||
# Service
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: keycloak
|
||||
namespace: keycloak
|
||||
labels:
|
||||
app.kubernetes.io/name: keycloak
|
||||
spec:
|
||||
selector:
|
||||
app.kubernetes.io/name: keycloak
|
||||
ports:
|
||||
- name: port-8080
|
||||
port: 8080
|
||||
targetPort: 8080
|
||||
---
|
||||
# CiliumNetworkPolicy egress allow (R-NET-02): keycloak-allow-keycloak-db-5432
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
metadata:
|
||||
name: keycloak-allow-keycloak-db-5432
|
||||
namespace: keycloak
|
||||
labels:
|
||||
app.kubernetes.io/name: keycloak
|
||||
spec:
|
||||
endpointSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: keycloak
|
||||
egress:
|
||||
- toEndpoints:
|
||||
- matchLabels:
|
||||
app.kubernetes.io/name: keycloak-db
|
||||
k8s:io.kubernetes.pod.namespace: keycloak-db
|
||||
toPorts:
|
||||
- ports:
|
||||
- port: '5432'
|
||||
protocol: TCP
|
||||
---
|
||||
# CiliumNetworkPolicy gateway-ingress allow (R-NET-01/02): keycloak-allow-gateway-ingress
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
metadata:
|
||||
name: keycloak-allow-gateway-ingress
|
||||
namespace: keycloak
|
||||
labels:
|
||||
app.kubernetes.io/name: keycloak
|
||||
spec:
|
||||
endpointSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: keycloak
|
||||
ingress:
|
||||
- fromEntities:
|
||||
- ingress
|
||||
toPorts:
|
||||
- ports:
|
||||
- port: '8080'
|
||||
protocol: TCP
|
||||
---
|
||||
# CiliumNetworkPolicy world egress allow (egress class governs the path; R-NET-05): keycloak-allow-internet-egress
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
metadata:
|
||||
name: keycloak-allow-internet-egress
|
||||
namespace: keycloak
|
||||
labels:
|
||||
app.kubernetes.io/name: keycloak
|
||||
spec:
|
||||
endpointSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: keycloak
|
||||
egress:
|
||||
- toEntities:
|
||||
- world
|
||||
---
|
||||
# HTTPRoute keycloak-8080 (R-TLS/R-DNS)
|
||||
apiVersion: gateway.networking.k8s.io/v1
|
||||
kind: HTTPRoute
|
||||
metadata:
|
||||
name: keycloak-8080
|
||||
namespace: keycloak
|
||||
labels:
|
||||
app.kubernetes.io/name: keycloak
|
||||
spec:
|
||||
parentRefs:
|
||||
- name: shared
|
||||
namespace: gateway
|
||||
sectionName: https
|
||||
hostnames:
|
||||
- keycloak.${BASE_DOMAIN}
|
||||
rules:
|
||||
- matches:
|
||||
- path:
|
||||
type: PathPrefix
|
||||
value: /
|
||||
backendRefs:
|
||||
- name: keycloak
|
||||
port: 8080
|
||||
---
|
||||
# ServiceMonitor (R-OBS-01)
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: keycloak
|
||||
namespace: keycloak
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
app.kubernetes.io/name: keycloak
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: keycloak
|
||||
namespaceSelector:
|
||||
matchNames:
|
||||
- keycloak
|
||||
endpoints:
|
||||
- port: metrics
|
||||
interval: 30s
|
||||
---
|
||||
# PrometheusRule SLI alerts (R-OBS-01)
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: keycloak-slis
|
||||
namespace: keycloak
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
app.kubernetes.io/name: keycloak
|
||||
spec:
|
||||
groups:
|
||||
- name: keycloak.slis
|
||||
rules:
|
||||
- alert: keycloak-down
|
||||
expr: up{job="keycloak"} == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
service: keycloak
|
||||
annotations:
|
||||
summary: keycloak is down
|
||||
description: 'keycloak has been unreachable for 5m (SLI: up).'
|
||||
@@ -35,34 +35,6 @@ spec:
|
||||
- protocol: TCP
|
||||
port: 53
|
||||
---
|
||||
# PersistentVolume nextcloud-legacy-archive - static external NFS (bulk tier)
|
||||
apiVersion: v1
|
||||
kind: PersistentVolume
|
||||
metadata:
|
||||
name: nextcloud-legacy-archive
|
||||
labels:
|
||||
app.kubernetes.io/name: nextcloud
|
||||
spec:
|
||||
capacity:
|
||||
storage: 270Gi
|
||||
accessModes:
|
||||
- ReadOnlyMany
|
||||
persistentVolumeReclaimPolicy: Retain
|
||||
storageClassName: ''
|
||||
mountOptions:
|
||||
- nfsvers=4.1
|
||||
- hard
|
||||
- noatime
|
||||
- timeo=600
|
||||
- retrans=2
|
||||
nfs:
|
||||
server: ${BULK_NFS_SERVER}
|
||||
path: /mnt/Storage/nextcloud
|
||||
readOnly: true
|
||||
claimRef:
|
||||
namespace: nextcloud
|
||||
name: nextcloud-legacy-archive
|
||||
---
|
||||
# PVC nextcloud-data (R-STATE)
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
@@ -79,23 +51,6 @@ spec:
|
||||
requests:
|
||||
storage: 20Gi
|
||||
---
|
||||
# PVC nextcloud-legacy-archive (R-STATE)
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: nextcloud-legacy-archive
|
||||
namespace: nextcloud
|
||||
labels:
|
||||
app.kubernetes.io/name: nextcloud
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadOnlyMany
|
||||
storageClassName: ''
|
||||
volumeName: nextcloud-legacy-archive
|
||||
resources:
|
||||
requests:
|
||||
storage: 270Gi
|
||||
---
|
||||
# Deployment (R-PLAT-02)
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
@@ -164,10 +119,6 @@ spec:
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /var/www/html
|
||||
- name: legacy-archive
|
||||
mountPath: /mnt/legacy-nextcloud
|
||||
subPath: nextcloud
|
||||
readOnly: true
|
||||
lifecycle:
|
||||
postStart:
|
||||
exec:
|
||||
@@ -181,9 +132,6 @@ spec:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: nextcloud-data
|
||||
- name: legacy-archive
|
||||
persistentVolumeClaim:
|
||||
claimName: nextcloud-legacy-archive
|
||||
---
|
||||
# Service
|
||||
apiVersion: v1
|
||||
|
||||
+1
-59
@@ -35,33 +35,6 @@ spec:
|
||||
- protocol: TCP
|
||||
port: 53
|
||||
---
|
||||
# PersistentVolume radarr-media - static external NFS (bulk tier)
|
||||
apiVersion: v1
|
||||
kind: PersistentVolume
|
||||
metadata:
|
||||
name: radarr-media
|
||||
labels:
|
||||
app.kubernetes.io/name: radarr
|
||||
spec:
|
||||
capacity:
|
||||
storage: 9Ti
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
persistentVolumeReclaimPolicy: Retain
|
||||
storageClassName: ''
|
||||
mountOptions:
|
||||
- nfsvers=4.1
|
||||
- hard
|
||||
- noatime
|
||||
- timeo=600
|
||||
- retrans=2
|
||||
nfs:
|
||||
server: ${BULK_NFS_SERVER}
|
||||
path: /mnt/Storage/plex
|
||||
claimRef:
|
||||
namespace: radarr
|
||||
name: radarr-media
|
||||
---
|
||||
# PVC radarr-config (R-STATE)
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
@@ -78,23 +51,6 @@ spec:
|
||||
requests:
|
||||
storage: 5Gi
|
||||
---
|
||||
# PVC radarr-media (R-STATE)
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: radarr-media
|
||||
namespace: radarr
|
||||
labels:
|
||||
app.kubernetes.io/name: radarr
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
storageClassName: ''
|
||||
volumeName: radarr-media
|
||||
resources:
|
||||
requests:
|
||||
storage: 9Ti
|
||||
---
|
||||
# Deployment (R-PLAT-02)
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
@@ -125,18 +81,9 @@ spec:
|
||||
image: lscr.io/linuxserver/radarr:5.21.1
|
||||
ports:
|
||||
- containerPort: 7878
|
||||
env:
|
||||
- name: PUID
|
||||
value: '1000'
|
||||
- name: PGID
|
||||
value: '3000'
|
||||
- name: TZ
|
||||
value: Etc/UTC
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /config
|
||||
- name: media
|
||||
mountPath: /media
|
||||
mountPath: /data/config
|
||||
- name: oauth2-proxy
|
||||
image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1
|
||||
imagePullPolicy: IfNotPresent
|
||||
@@ -201,15 +148,10 @@ spec:
|
||||
port: http
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 15
|
||||
securityContext:
|
||||
fsGroup: 3000
|
||||
volumes:
|
||||
- name: config
|
||||
persistentVolumeClaim:
|
||||
claimName: radarr-config
|
||||
- name: media
|
||||
persistentVolumeClaim:
|
||||
claimName: radarr-media
|
||||
---
|
||||
# Service
|
||||
apiVersion: v1
|
||||
|
||||
@@ -5,15 +5,6 @@
|
||||
# credential from its own namespace (a Pod cannot read a Secret in another namespace).
|
||||
# access-key-encryption: FIXED base64 32-byte AES key Semaphore uses to encrypt stored SSH keys /
|
||||
# credentials; MUST stay stable across restarts or stored secrets become undecryptable.
|
||||
# oidc-client-secret: the Keycloak `semaphore` CONFIDENTIAL client secret, and ONLY that. It MIRRORS
|
||||
# `semaphore-client-secret` in platform/keycloak/keycloak-realm-clients.sops.yaml (the realm import
|
||||
# injects that copy into the client), same cross-namespace mirror pattern as db-password above.
|
||||
# It replaced the former `oidc-providers-json` key (2026-09-07): that key held the WHOLE
|
||||
# SEMAPHORE_OIDC_PROVIDERS blob, which buried the environment-specific realm-issuer and redirect
|
||||
# URLs inside ciphertext where Flux's ${BASE_DOMAIN} postBuild substitution could not reach them,
|
||||
# so prod ran with dev's URLs. Those URLs are PUBLIC and now live in plaintext in
|
||||
# catalog/semaphore.yaml (rendered to apps/wave-1/semaphore.yaml); the kubelet splices THIS value
|
||||
# back into the blob via $(SEMAPHORE_OIDC_CLIENT_SECRET). Only the secret half is encrypted.
|
||||
# Recover: infra/devcontainer/dev.sh run sops -d apps/semaphore-bootstrap-secret.sops.yaml
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
@@ -24,11 +15,11 @@ metadata:
|
||||
app.kubernetes.io/name: semaphore
|
||||
type: Opaque
|
||||
stringData:
|
||||
admin-username: ENC[AES256_GCM,data:ju3wVdjNzRUP/w==,iv:d/5ZxgQZ5bympu3gT2ck3DJy9iUPPBAtZYsJvD3To7I=,tag:0UtBaDVqNSK4RCJrQicchw==,type:str]
|
||||
admin-password: ENC[AES256_GCM,data:pxIwpgh3BoND2TI0mPTixOTJoictzT77dqtuRdMKgY/2WaW4wM1Ra6H6n/dxnyR2,iv:eYbFQGFYVJsnYGwn6cOI3BgAdzMcin9YggAhiuL+xqM=,tag:/sPWGv4gue0vjc2nMk7wWA==,type:str]
|
||||
db-password: ENC[AES256_GCM,data:hBFpWXdpk1R5a23Pntup03t56NBRPS+SZU8CrHVb/ypIaIpgf7kjYtVy6nhTF6u0,iv:eBzoQgH5ZhWWpfKZ6AQmwaSal5QaiuN9gfexQ7GsVFs=,tag:j2k8EwhKDlj/Ud6O39ioVQ==,type:str]
|
||||
access-key-encryption: ENC[AES256_GCM,data:SKtAV4HxJ9pNYDqUK/1uOdI1rtMji3pTfprI2rhQxFQyhc6dFlhqI+OUEcY=,iv:rKCP+FLidcOltOwmMMcO5M8tz4qiifIMI+W5soGO2M8=,tag:HaHzkX/JI+9wmcg2SD9GAQ==,type:str]
|
||||
oidc-client-secret: ENC[AES256_GCM,data:uCo7TNzbM2QKw9ie+xzyUG9MFKco1OAlwr5qPn/AGpNvo1YyXSfbZyyM0CLJ1NS821hGg5Zr/m3R/xpy84s3FQ==,iv:O8a4haskufpmorPaxOkJ1Q99Y5GTlzmDFJAsHS+irLk=,tag:VojJpA9jNMbNQmIe1Qr44w==,type:str]
|
||||
admin-username: ENC[AES256_GCM,data:uD/vO9A9MjerFg==,iv:i/NAqVD6rWg9oWWSGBrRkxzwcPp2IW1HLIKY0pdcPPg=,tag:r27tKLAJ+ydE7Ty3ccMNUw==,type:str]
|
||||
admin-password: ENC[AES256_GCM,data:544nWY1UmBnlsdPJAQZJMXRumu4W9X7UXZvZ6vHN/ArjRgLXKTArOe5JkT/nNrNw,iv:AC1DDbZj2HfpU19EjzA7T+WpIaR4nZc3PNN61y8bAzU=,tag:r28LFi5w6ZeCCaRaX2jRdg==,type:str]
|
||||
db-password: ENC[AES256_GCM,data:ySkdvaGRCW9fMQqChs5l8dY4zCG19i4HOEinRC48mgLuh9WOrjd16T6jwFgiwB55,iv:+1gztJ22HbwgQJiolZMMWOcKIsxrBJpnruJdtxdooOU=,tag:8CfJGwyPTjDGyr0CvalXMQ==,type:str]
|
||||
access-key-encryption: ENC[AES256_GCM,data:1YAMLo1nVkpJWZc+KBZ7B4ia+Y8P+xMyOhHMcEGGJOXd3o+WC5FqirLBVh4=,iv:j28ozeaDhGWDjoaxFOhn2t9YbYE7L3ah/Mfi+i2DWwY=,tag:czG9Wxc/7Nsuh1W5iATOww==,type:str]
|
||||
oidc-providers-json: ENC[AES256_GCM,data:90yt5mIVWLVI2KrlJWpxhL/BMDpEgRhv7+BL+mBRt79diRvU7pN9vtzhZprAKzcU1sEywlT4EwFlsvJpYyo5Qbkp+ZVknbyhAlGvHhJOBHukOPWgf4LignLerO2GzhXjOOzFqWfZ4EyUxNMWTbSe/xc0zIVHDDNQiYplGVRK+E/ukiI7sD/Tn2RrbrdvpC5GyK0NXWRsXn9ffhE3fCoiRHXagj9i/q+uUrQI5EKpPobrDUAGqvKoBXTl/wJ4phnWg9IqtCTE/c3yA7fLLz4j5dCtPZNSVPT4TKDx8Mdv8CfCgziVicjG2s+yHuWpsQn8lf6nvdJmpYPvIqTtLWdxFWuD8Vfo6Qbq2mqPbKR+KmDOGUnjl4gbZdL1SkE9JiqBUobz7UXeLXwYzxMIsinOfYZLcCk7alrD5Yndj9JDxDJZQ54sMMBfVbxDl94W1VHy03RxAR46KKAbPaxOP5wH4Q4z3SDceNwmuGCDRGChRXiHhN5X2X0GqYCbc8E6GdkIMcpmD80e26ydEmAjw7F89mCF4F/yMzJNiG82qaZIn8HeqgGA8urous9r9bYJNlitYTYQ,iv:jbfVdrrZ9fGiOxhQeoTJfFg1QwiOMkoQyaadiVfxX/w=,tag:JkepcRs34EuEgNb4buTTDQ==,type:str]
|
||||
sops:
|
||||
kms: []
|
||||
gcp_kms: []
|
||||
@@ -38,14 +29,14 @@ sops:
|
||||
- recipient: age175l8mdpdfu7hq0aml3exs3gykarwjmuvr23sq56s3wjejnyk44wsp4mfzg
|
||||
enc: |
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAyTzF6NTBtcFVWK1Vtc0I2
|
||||
VGdySHBsRGgvc1h0VzgvQ01xYVBJV3g2Q1NRCjJyZHVYZGtuamo1MFJHWlBwWkxt
|
||||
cHA4VkY1TkRpN3M4UTdMOWZTbFVGemMKLS0tIGVNdDBrWVJveU56VjVBU2JWdHF4
|
||||
RGtaYStXYUpONTZjZCtmM0NyZ3Rka1UKIQpmQai8dz+uRAsXxyeuGaRlRwsFqIPQ
|
||||
9AfrZJcslYwlaK3ptm1oqaCdrlxcMar2nI7thSqzbLe+e1nzA2z7vQ==
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBuRDlqRFJ0dk14UHluSGRP
|
||||
S1pESy9PWlpMakU0aTkzNVhqUGdtbWNEdTJjCm5RNG14azJHc28rdEJ4L0RzR0xO
|
||||
MGRocGZ1V1JZTENWQkN3VGZzamp2eTgKLS0tIGt6QkNNaW5hdTBJVHY0SVo5d3JX
|
||||
elp0SWxHcjg0RWF3aW9mWTJHZ0lhcUEKc0rU5VilAVDleCwqq4SOF+Y9sniu1Pka
|
||||
w/MGDA9THSB0b+YgWRWiX/KiimFqp0ZFODOJ9cHVwFGY71+VadPwOg==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
lastmodified: "2026-09-07T07:01:50Z"
|
||||
mac: ENC[AES256_GCM,data:Xutm1rQt57J4RILElD8FkNOsQT1HK5jZlPtqqLyZ/VqOoDwo+IS1wp7hROprlUvQC0bUXy79tRRIl5Q0CCOD+fvAGIdmade+vB7qXlYN6Q1441dGhuSTPvckso6FaB2ddRjKhKGLc/Z4bJoz9VaRNNxJNpWXIrENTg+I+QhNhlo=,iv:klJOrEQ6QN5dTGkQfGnbMbgCWsgWwZPb5YXDMeWeE34=,tag:fuJr4t3Tt3dT9u3/0SFo8Q==,type:str]
|
||||
lastmodified: "2026-08-27T22:12:32Z"
|
||||
mac: ENC[AES256_GCM,data:nHftJTGnTr0O5h8yGXwgz3eflggXJfIXM3m8n3bzDJE/P4nvSpZzFeMb+sc2on+AL+t+SeKrIl8blzm33XsQ1lkaY85Zm+bc5cnED1g04BMDVmmK3brh/8w50Bp2wPEVzb4I01MDzhhyIbOYg7pFvZAH3/VuqvM1t6XmfF1NrOA=,iv:4KBa9PH/dfdGwLC16RenZIVB6BtSoGo6x3qt3h1R43I=,tag:KULiA1klP/RZMNigkloGDA==,type:str]
|
||||
pgp: []
|
||||
encrypted_regex: ^(data|stringData)$
|
||||
version: 3.9.4
|
||||
@@ -118,13 +118,11 @@ spec:
|
||||
key: access-key-encryption
|
||||
- name: SEMAPHORE_PORT
|
||||
value: :3000
|
||||
- name: SEMAPHORE_OIDC_CLIENT_SECRET
|
||||
- name: SEMAPHORE_OIDC_PROVIDERS
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: semaphore-bootstrap
|
||||
key: oidc-client-secret
|
||||
- name: SEMAPHORE_OIDC_PROVIDERS
|
||||
value: '{"keycloak":{"display_name":"Keycloak","provider_url":"https://keycloak.${BASE_DOMAIN}/realms/homelab","client_id":"semaphore","client_secret":"$(SEMAPHORE_OIDC_CLIENT_SECRET)","redirect_url":"https://semaphore.${BASE_DOMAIN}/api/auth/oidc/keycloak/redirect","scopes":["openid","profile","email"],"username_claim":"preferred_username","email_claim":"email","name_claim":"name","order":1}}'
|
||||
key: oidc-providers-json
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /etc/semaphore
|
||||
|
||||
+1
-59
@@ -35,33 +35,6 @@ spec:
|
||||
- protocol: TCP
|
||||
port: 53
|
||||
---
|
||||
# PersistentVolume sonarr-media - static external NFS (bulk tier)
|
||||
apiVersion: v1
|
||||
kind: PersistentVolume
|
||||
metadata:
|
||||
name: sonarr-media
|
||||
labels:
|
||||
app.kubernetes.io/name: sonarr
|
||||
spec:
|
||||
capacity:
|
||||
storage: 9Ti
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
persistentVolumeReclaimPolicy: Retain
|
||||
storageClassName: ''
|
||||
mountOptions:
|
||||
- nfsvers=4.1
|
||||
- hard
|
||||
- noatime
|
||||
- timeo=600
|
||||
- retrans=2
|
||||
nfs:
|
||||
server: ${BULK_NFS_SERVER}
|
||||
path: /mnt/Storage/plex
|
||||
claimRef:
|
||||
namespace: sonarr
|
||||
name: sonarr-media
|
||||
---
|
||||
# PVC sonarr-config (R-STATE)
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
@@ -78,23 +51,6 @@ spec:
|
||||
requests:
|
||||
storage: 5Gi
|
||||
---
|
||||
# PVC sonarr-media (R-STATE)
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: sonarr-media
|
||||
namespace: sonarr
|
||||
labels:
|
||||
app.kubernetes.io/name: sonarr
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
storageClassName: ''
|
||||
volumeName: sonarr-media
|
||||
resources:
|
||||
requests:
|
||||
storage: 9Ti
|
||||
---
|
||||
# Deployment (R-PLAT-02)
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
@@ -125,18 +81,9 @@ spec:
|
||||
image: lscr.io/linuxserver/sonarr:4.0.15
|
||||
ports:
|
||||
- containerPort: 8989
|
||||
env:
|
||||
- name: PUID
|
||||
value: '1000'
|
||||
- name: PGID
|
||||
value: '3000'
|
||||
- name: TZ
|
||||
value: Etc/UTC
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /config
|
||||
- name: media
|
||||
mountPath: /media
|
||||
mountPath: /data/config
|
||||
- name: oauth2-proxy
|
||||
image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1
|
||||
imagePullPolicy: IfNotPresent
|
||||
@@ -201,15 +148,10 @@ spec:
|
||||
port: http
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 15
|
||||
securityContext:
|
||||
fsGroup: 3000
|
||||
volumes:
|
||||
- name: config
|
||||
persistentVolumeClaim:
|
||||
claimName: sonarr-config
|
||||
- name: media
|
||||
persistentVolumeClaim:
|
||||
claimName: sonarr-media
|
||||
---
|
||||
# Service
|
||||
apiVersion: v1
|
||||
|
||||
@@ -23,17 +23,6 @@ storage:
|
||||
- name: config
|
||||
size: 2Gi
|
||||
class: longhorn # indexer definitions/API keys on 3-replica Longhorn (docs/04; R-STATE-02)
|
||||
mountPath: /config # THE APP WRITES /config, NOT the renderer default /data/config.
|
||||
# LSIO images run with `-data=/config` (Jackett/qBittorrent likewise use
|
||||
# /config), so without this the Longhorn PVC mounted at /data/config sat
|
||||
# holding nothing but lost+found while the app wrote its entire
|
||||
# configuration to the container's EPHEMERAL writable layer — destroyed on
|
||||
# every pod recreation: restart, reschedule, image bump. Measured in-cluster
|
||||
# 2026-09-12: /data/config was dev=2080 with [lost+found], /config was
|
||||
# dev=2053 on the node's own disk with [Sentry asp config.xml logs].
|
||||
# This is the SAME failure catalog/plex.yaml:54 already records fixing.
|
||||
# The live /config contents were copied onto each PVC before this landed,
|
||||
# so API keys and credentials other apps reference were preserved.
|
||||
dependencies: [] # default-deny: reaches only DNS; arr reach it north of its own egress (docs/03; R-NET-01/02)
|
||||
ingressRoutes:
|
||||
- host: jackett.${BASE_DOMAIN}
|
||||
|
||||
@@ -223,44 +223,6 @@ storage:
|
||||
mountPath: /var/www/html # 3-replica tier; irreplaceable, DR-critical (docs/04; R-STATE-02).
|
||||
# The whole html dir is persisted so config.php + user data survive a
|
||||
# rebuild (the default /data/<name> would strand them on ephemeral rootfs).
|
||||
# THE LEGACY NEXTCLOUD DATASET (Storage/nextcloud, 270 G) — MOUNTED READ-ONLY, DELIBERATELY, AND
|
||||
# NOT AS THIS INSTANCE'S DATA DIRECTORY. Read this before changing either decision.
|
||||
#
|
||||
# WHAT IS ACTUALLY IN THERE (measured 2026-09-07, not assumed):
|
||||
# /mnt/Storage/nextcloud/nextcloud/config 103 M
|
||||
# /mnt/Storage/nextcloud/nextcloud/config_old 3.4 G
|
||||
# /mnt/Storage/nextcloud/nextcloud/config_backup 267 G <- 99% of the dataset
|
||||
# There is NO `data/` directory. This is the OLD, non-k3s Nextcloud's configuration plus a backup
|
||||
# blob — an ARCHIVE, not a live user-data tree.
|
||||
#
|
||||
# WHY READ-ONLY. Two independent reasons, either sufficient:
|
||||
# 1. The old instance is STILL LIVE on it. The export is currently shared rw to 192.168.1.6 and
|
||||
# 192.168.1.5, and the operator has said the existing non-k3s prod must remain active. Two
|
||||
# Nextcloud instances writing one instance's config directory, each with its own database and
|
||||
# therefore its own oc_filecache, is a corruption scenario with no clean recovery.
|
||||
# 2. Nothing here needs writing. An archive is read.
|
||||
# "Do not grant more than needed" therefore lands on `ro`. If the operator's intent is in fact a
|
||||
# MIGRATION (read this data in, then decommission the old instance), that is a different and
|
||||
# larger change than a mount, and it should be a deliberate one — flip `readOnly` and update the
|
||||
# export at that point, not before. Enforced by the PV's nfs.readOnly, so it is a real `ro` mount.
|
||||
#
|
||||
# WHY NOT /var/www/html/data. That path is the LONGHORN PVC above and is this instance's own
|
||||
# state; mounting a foreign tree into it would have Nextcloud scan and index an archive it does
|
||||
# not own. A neutral path keeps the two completely separate. To expose it to users, add it as a
|
||||
# Nextcloud External Storage (Local) mount pointing at this path — an in-app, per-user decision.
|
||||
#
|
||||
# subPath skips the redundant single-child directory: the dataset root holds exactly one entry,
|
||||
# `nextcloud/`, so mounting that subtree puts config/ config_old/ config_backup/ at the top.
|
||||
#
|
||||
# UID: nothing to do. The tree is 0755 owned 1000:3000 and the mount is read-only, so www-data
|
||||
# (uid 33) can read and traverse it. (fsGroup would be a no-op on an NFS volume in any case.)
|
||||
- name: legacy-archive
|
||||
class: bulk
|
||||
export: /mnt/Storage/nextcloud # the ZFS dataset Storage/nextcloud, on ${BULK_NFS_SERVER}
|
||||
subPath: nextcloud
|
||||
mountPath: /mnt/legacy-nextcloud
|
||||
readOnly: true
|
||||
size: 270Gi # ADVISORY ONLY on a static NFS PV — the dataset's used size.
|
||||
dependencies:
|
||||
- to: nextcloud-db # explicit allow: Nextcloud -> its PostgreSQL (docs/13 graph; R-NET-02).
|
||||
port: 5432 # Renders the egress half; the DB namespace mirrors keycloak-db (no
|
||||
|
||||
+15
-30
@@ -4,11 +4,21 @@
|
||||
# auth: native (Plex accounts, NOT Keycloak — docs/13 + docs/15 documented exception).
|
||||
# Validates against schemas/servicedefinition.schema.json.
|
||||
#
|
||||
# BULK MEDIA — NOW ATTACHED (docs/04 bulk tier; the long-standing OQ-2 prerequisite, resolved for
|
||||
# the STATIC case). The library is 8.33 T of ZFS on the Proxmox host at /mnt/Storage/plex, and it
|
||||
# already exists: nothing is provisioned, an existing dataset is mounted. See
|
||||
# docs/BULK-NFS-STORAGE.md for the layout, the export change and the uid analysis. (The bulk-tier
|
||||
# requirement ID is deliberately still not typed here — see that document's "gate" note.)
|
||||
# BULK MEDIA (spec bulk-tier requirement, docs/04) — OPEN PREREQUISITE, intentionally NOT declared
|
||||
# here yet (the requirement ID is cited in the WS-APPS report, not as a token here: this change does
|
||||
# NOT exercise the bulk tier, so it must not be selected as a "touched" requirement whose in-cluster
|
||||
# conformance test is still a Phase-1 gap).
|
||||
# The Plex library belongs on the `bulk` tier (docs/04 / docs/13 cross-cutting note: metadata DB
|
||||
# on replicated Longhorn, large library on bulk). The `bulk` StorageClass exists
|
||||
# (platform/longhorn/storageclasses.yaml) and the schema+renderer both support `class: bulk`, but
|
||||
# that class is a documented STUB: OQ-2 (external NFS/SMB backing) is UNRESOLVED and the NFS CSI
|
||||
# driver is not installed, so a `bulk` PVC will not bind and would leave the Plex pod Pending.
|
||||
# To onboard the running Plex shell now (compute + config + public exposure), the media volume is
|
||||
# omitted. Once OQ-2 is resolved and `bulk` provisions, attach the library with a one-line add:
|
||||
# storage:
|
||||
# - name: media
|
||||
# class: bulk # external mount; size omitted (sized outside the cluster)
|
||||
# then re-render. See the WS-APPS report for the full bulk-media finding.
|
||||
name: plex
|
||||
kind: container
|
||||
source:
|
||||
@@ -60,31 +70,6 @@ storage:
|
||||
# UI sign-in, the operator adds a PLEX_CLAIM env from a fresh https://plex.tv/claim token
|
||||
# (4-min TTL) — deliberately NOT wired here: the env secretKeyRef pattern has no `optional`,
|
||||
# so a placeholder/absent claim Secret would crashloop the pod. See operator hand-off.
|
||||
# THE LIBRARY (docs/04 bulk tier). One READ-ONLY mount of the export ROOT, giving /media/movies,
|
||||
# /media/tv and /media/photos — the whole tree the layout actually has (`ls /mnt/Storage/plex`:
|
||||
# downloads/ movies/ photos/ tv/).
|
||||
#
|
||||
# WHY THE ROOT AND NOT THREE subPath MOUNTS. Nothing is gained by narrowing: the mount is `ro`,
|
||||
# so no subPath could remove a privilege that matters, while three mounts would triple the NFS
|
||||
# mounts per pod and mean a manifest change every time a top-level library is added.
|
||||
#
|
||||
# WHY READ-ONLY, when the export is `rw` and the other consumers write. Plex is a SERVER: it
|
||||
# indexes and streams. Every write on that tree belongs to Sonarr and Radarr. Marking it here
|
||||
# costs nothing and removes Plex from the set of things that can damage 8.33 T. Enforced by the
|
||||
# PV's nfs.readOnly (a real `ro` mount), not only by the volumeMount flag.
|
||||
#
|
||||
# UID: no change needed. The tree is 0755 owned 1000:3000 throughout, so it is world-readable and
|
||||
# world-traversable, and PLEX_UID/PLEX_GID above already make the process uid 1000 in any case.
|
||||
# (fsGroup would have been a no-op here regardless — the in-tree NFS plugin does no ownership
|
||||
# management. That is the trap the arrs' PUID/PGID exists to avoid; Plex never hits it.)
|
||||
- name: media
|
||||
class: bulk
|
||||
export: /mnt/Storage/plex # the ZFS dataset Storage/plex, on ${BULK_NFS_SERVER}
|
||||
mountPath: /media
|
||||
readOnly: true
|
||||
size: 9Ti # ADVISORY ONLY (nothing enforces a static NFS PV's capacity): the
|
||||
# dataset's 8.33 T used, rounded up, so `kubectl get pv` reports
|
||||
# something true. The real limit is the ZFS pool.
|
||||
dependencies: [] # default-deny: reaches only DNS; clients/Seerr reach IT (docs/03; R-NET-01/02)
|
||||
ingressRoutes:
|
||||
- host: plex.${BASE_DOMAIN}
|
||||
|
||||
@@ -38,17 +38,6 @@ storage:
|
||||
- name: config
|
||||
size: 5Gi
|
||||
class: longhorn # settings/categories/session on 3-replica Longhorn; DR-critical (docs/04; R-STATE-02)
|
||||
mountPath: /config # THE APP WRITES /config, NOT the renderer default /data/config.
|
||||
# LSIO images run with `-data=/config` (Jackett/qBittorrent likewise use
|
||||
# /config), so without this the Longhorn PVC mounted at /data/config sat
|
||||
# holding nothing but lost+found while the app wrote its entire
|
||||
# configuration to the container's EPHEMERAL writable layer — destroyed on
|
||||
# every pod recreation: restart, reschedule, image bump. Measured in-cluster
|
||||
# 2026-09-12: /data/config was dev=2080 with [lost+found], /config was
|
||||
# dev=2053 on the node's own disk with [Sentry asp config.xml logs].
|
||||
# This is the SAME failure catalog/plex.yaml:54 already records fixing.
|
||||
# The live /config contents were copied onto each PVC before this landed,
|
||||
# so API keys and credentials other apps reference were preserved.
|
||||
dependencies: [] # default-deny: reaches only DNS; arr reach it north of its own egress (docs/03; R-NET-01/02)
|
||||
ingressRoutes:
|
||||
- host: qbittorrent.${BASE_DOMAIN}
|
||||
|
||||
+5
-77
@@ -9,23 +9,6 @@ name: radarr
|
||||
kind: container
|
||||
source:
|
||||
image: lscr.io/linuxserver/radarr:5.21.1 # pinned stable v5 tag (R-UPD-01); update.range governs advancement
|
||||
# LinuxServer images run their service as the user `abc`, whose uid/gid these two set. They are
|
||||
# LOAD-BEARING for the bulk mount, not cosmetic: see the storage note below. The default is 911,
|
||||
# which cannot write a 1000:3000-owned 0755 tree.
|
||||
env:
|
||||
- name: PUID
|
||||
value: "1000" # owner uid of every directory under /mnt/Storage/plex
|
||||
- name: PGID
|
||||
value: "3000" # dominant group there; keeps files this writes readable to the
|
||||
# EXISTING non-k3s consumers of the same export, which stay live
|
||||
- name: TZ
|
||||
value: Etc/UTC # same safe default plex uses; the arrs schedule by local time
|
||||
podSecurityContext:
|
||||
# For the LONGHORN config PVC only. fsGroup does nothing on the NFS volume (the in-tree plugin
|
||||
# does no ownership management), but on a Longhorn filesystem the kubelet applies it, which is
|
||||
# what makes /config writable by the now-uid-1000 process WITHOUT relying on the image chowning
|
||||
# /config for itself when PUID changes. Matches PGID so the primary group owns the config too.
|
||||
fsGroup: 3000
|
||||
exposure: internal # LAN-only web UI at radarr.${BASE_DOMAIN} (docs/13 row 3; R-DNS/R-NET)
|
||||
egress: vpn # indexer scrape + downloader handoff steered via the shared VPN egress gateway,
|
||||
# kill-switch enforced OUTSIDE the workload (docs/03; ADR-0015; R-NET-05/06)
|
||||
@@ -33,66 +16,11 @@ storage:
|
||||
- name: config
|
||||
size: 5Gi
|
||||
class: longhorn # app DB / config on the 3-replica tier; DR-critical (docs/04; R-STATE-02)
|
||||
mountPath: /config # THE APP WRITES /config, NOT the renderer default /data/config.
|
||||
# LSIO images run with `-data=/config` (Jackett/qBittorrent likewise use
|
||||
# /config), so without this the Longhorn PVC mounted at /data/config sat
|
||||
# holding nothing but lost+found while the app wrote its entire
|
||||
# configuration to the container's EPHEMERAL writable layer — destroyed on
|
||||
# every pod recreation: restart, reschedule, image bump. Measured in-cluster
|
||||
# 2026-09-12: /data/config was dev=2080 with [lost+found], /config was
|
||||
# dev=2053 on the node's own disk with [Sentry asp config.xml logs].
|
||||
# This is the SAME failure catalog/plex.yaml:54 already records fixing.
|
||||
# The live /config contents were copied onto each PVC before this landed,
|
||||
# so API keys and credentials other apps reference were preserved.
|
||||
# THE MEDIA TREE (docs/04 bulk tier) — ONE read-WRITE mount of the export ROOT at /media, giving
|
||||
# /media/movies (the library) and /media/downloads (the import source) UNDER THE SAME MOUNT.
|
||||
#
|
||||
# *** THE ROOT MOUNT IS THE POINT, NOT A CONVENIENCE. *** Radarr imports by HARDLINKING or
|
||||
# atomically renaming a finished download into the library. Both are same-FILESYSTEM operations,
|
||||
# and to the kernel "same filesystem" means SAME MOUNT. The layout on the host is:
|
||||
# /mnt/Storage/plex/movies/ the library
|
||||
# /mnt/Storage/plex/downloads/complete/radarr/ where finished releases land
|
||||
# — siblings under one dataset, so a hardlink between them is legal on the server. But mounting
|
||||
# them as two subPath volumes (subPath: movies at /media/movies, subPath: downloads elsewhere) creates
|
||||
# TWO mounts, and link() across mounts fails EXDEV no matter what the server thinks. Radarr then
|
||||
# silently degrades to copy-then-delete: every import rewrites the whole file over NFS and the
|
||||
# seeding copy is destroyed. Their smallest common ancestor IS the export root, so the root mount
|
||||
# is the MINIMAL mount that preserves hardlinks — least privilege bounded below by a hard
|
||||
# requirement, not a shortcut. (The cost is stated plainly: Radarr can also write /media/tv,
|
||||
# which is sonarr's. Narrowing that would cost hardlinks, which is a worse trade.)
|
||||
#
|
||||
# *** /media, NOT /data — AND THAT IS NOT A STYLE CHOICE. *** The config PVC above renders at the
|
||||
# renderer default /data/config. A media mount at /data would be mounted OVER it and MASK the
|
||||
# config volume entirely; the pod would come up with its app database shadowed by an 8 T media
|
||||
# tree. /media collides with nothing in a LinuxServer image and matches the path plex uses for
|
||||
# the same export, so when qbittorrent is eventually given this mount too, the arrs and the
|
||||
# download client will already agree on absolute paths — which is the other precondition for
|
||||
# hardlinked imports (otherwise every import needs a Remote Path Mapping).
|
||||
#
|
||||
# SEPARATE LATENT BUG, NOT FIXED HERE (deliberately — it changes live on-disk state): /data/config
|
||||
# is almost certainly the wrong place for this PVC in the first place. LinuxServer's arr images run
|
||||
# with `-data=/config`, so the app writes /config — the container's EPHEMERAL layer — while the
|
||||
# Longhorn PVC at /data/config sits unused. That is the EXACT failure plex's own mountPath comment
|
||||
# documents having already been bitten by. Fixing it means starting from an empty config volume, so
|
||||
# it belongs in its own change with the operator present.
|
||||
#
|
||||
# UID/GID — THE FAILURE THIS AVOIDS IS "MOUNTS FINE, CANNOT WRITE". The export is no_root_squash
|
||||
# + sec=sys, so the client's numeric uid is passed through verbatim and the server's ownership is
|
||||
# the whole story. The tree is 0755 owned 1000:3000 (some dirs 1000:1000), so ONLY uid 1000 can
|
||||
# write it. LinuxServer images default to PUID/PGID 911 — which would mount cleanly and then fail
|
||||
# every import with EACCES. The PUID/PGID set above make the process uid 1000, matching the owner.
|
||||
#
|
||||
# AND NOTE fsGroup CANNOT SUBSTITUTE FOR THAT. The in-tree NFS volume plugin performs no
|
||||
# ownership management, so podSecurityContext.fsGroup is a NO-OP on this volume — it is not the
|
||||
# lever here, and reaching for it is how this failure normally gets misdiagnosed. The fsGroup
|
||||
# that IS set above exists for the LONGHORN config PVC, so that changing PUID does not depend on
|
||||
# the image re-chowning /config for itself.
|
||||
- name: media
|
||||
class: bulk
|
||||
export: /mnt/Storage/plex # the ZFS dataset Storage/plex, on ${BULK_NFS_SERVER}
|
||||
mountPath: /media # => /media/movies (Root Folder) and /media/downloads (import source)
|
||||
size: 9Ti # ADVISORY ONLY on a static NFS PV: the dataset's 8.33 T used,
|
||||
# rounded up, so `kubectl get pv` reports something true.
|
||||
# NOTE (bulk-media PREREQUISITE): docs/13 row 3 places the media library / downloads volume on the
|
||||
# `bulk` tier (docs/04 bulk storage). The `bulk` StorageClass exists (platform/longhorn/storageclasses.yaml)
|
||||
# but is a NON-provisioning STUB pending OQ-2 (NFS/SMB backing = REPLACE-ME). Adding a `class: bulk`
|
||||
# volume now would leave the PVC (and therefore the pod) Pending. The media/downloads volume is
|
||||
# therefore DEFERRED until OQ-2 is resolved; config-only on longhorn keeps Radarr deployable today.
|
||||
dependencies:
|
||||
- to: qbittorrent # arr → download client (docs/13 dependency graph; R-NET-02 explicit allow)
|
||||
port: 8080
|
||||
|
||||
+11
-186
@@ -47,36 +47,13 @@ LBL = "homelab.keithjackson.dev"
|
||||
# The platform hostname suffix, emitted as the Flux post-build TOKEN rather than a literal.
|
||||
# Every rendered manifest lands under a Flux path whose Kustomization declares
|
||||
# postBuild.substituteFrom cluster-vars, so `${BASE_DOMAIN}` is resolved per environment at
|
||||
# reconcile time: dev renders home.keithjackson.dev, prod renders prod.home.keithjackson.dev.
|
||||
# reconcile time: dev renders dev.home.keithjackson.dev, prod renders home.keithjackson.dev (the
|
||||
# APEX — operator decision 2026-09-04; prod IS the apex, so there is no `prod.` label and no CNAMEs).
|
||||
# It is a DECLARED cluster-vars knob (clusters/base/cluster-vars.yaml), so it is written BARE —
|
||||
# the escape contract below reserves `$${...}` for expansions meant for a container's shell.
|
||||
# Hardcoding the literal here is what let prod be authored to serve dev's public hostnames,
|
||||
# and prod's external-dns then deleted dev's records (infra/PROD-PROVISIONING.md §10).
|
||||
ZONE = "${BASE_DOMAIN}"
|
||||
# The bulk (external NFS) tier's SERVER, emitted as a Flux post-build TOKEN for exactly the same
|
||||
# reason as ZONE: dev and prod reach the same Proxmox ZFS host by DIFFERENT addresses (dev's nodes
|
||||
# are on the host's own LAN and dial it directly; prod's nodes are on the far side of pfSense and
|
||||
# must be given whichever address the still-pending routing decision produces). Hardcoding either
|
||||
# one authors a manifest that is silently wrong in the other environment. Declared in
|
||||
# clusters/base/cluster-vars.yaml and patched by BOTH overlays; written BARE (it is a real knob,
|
||||
# not a shell expansion), so StrictPostBuildSubstitutions fails the tier if an overlay forgets it.
|
||||
BULK_NFS_SERVER = "${BULK_NFS_SERVER}"
|
||||
# Mount options for every bulk NFS volume. These are a PLATFORM decision, not a per-service one,
|
||||
# so they live here rather than on the ServiceDefinition surface:
|
||||
# nfsvers=4.1 NFSv4 carries mount+lock IN the protocol, so it needs ONE port (tcp/2049) and no
|
||||
# rpcbind/111, mountd or statd. That is what makes the prod path a single pfSense
|
||||
# rule instead of four, and it is why v4 is chosen over v3. The server advertises
|
||||
# +3 +4 +4.1 +4.2; 4.1 is the conservative floor every node kernel already speaks.
|
||||
# hard A `soft` mount turns a server hiccup into a short read / EIO handed to the
|
||||
# application, which for a media library and an archive is silent corruption.
|
||||
# `hard` blocks instead. The cost is explicit: an NFS outage HANGS I/O rather than
|
||||
# failing it, so an outage shows up as stuck pods, not as damaged files. Chosen
|
||||
# deliberately -- hung is recoverable, truncated is not.
|
||||
# noatime The datasets are relatime, so every read would otherwise generate a metadata
|
||||
# WRITE back to the pool. Plex walking an 8 T library must not do that.
|
||||
# timeo=600/retrans=2 the TCP defaults, stated explicitly so a future reader knows the retry
|
||||
# budget was considered rather than inherited.
|
||||
BULK_NFS_MOUNT_OPTIONS = ["nfsvers=4.1", "hard", "noatime", "timeo=600", "retrans=2"]
|
||||
GATEWAY_NAME = "shared"
|
||||
GATEWAY_NS = "gateway"
|
||||
KP_RELEASE = "kube-prometheus-stack" # ServiceMonitor/PrometheusRule discovery label
|
||||
@@ -135,10 +112,8 @@ def apply_defaults(sd: dict) -> dict:
|
||||
obs.setdefault("slis", [])
|
||||
s["observability"] = obs
|
||||
|
||||
# storage entry class default is longhorn (replicated tier); readOnly default is False
|
||||
# (grant-nothing-extra is expressed by SETTING it, so the default stays back-compatible and
|
||||
# every pre-existing golden renders byte-identical).
|
||||
s["storage"] = [{**{"class": "longhorn", "readOnly": False}, **v} for v in s["storage"]]
|
||||
# storage entry class default is longhorn (replicated tier).
|
||||
s["storage"] = [{**{"class": "longhorn"}, **v} for v in s["storage"]]
|
||||
# dependency protocol default is tcp.
|
||||
s["dependencies"] = [{**{"protocol": "tcp"}, **d} for d in s["dependencies"]]
|
||||
return s
|
||||
@@ -226,139 +201,18 @@ def _service_ports(s: dict) -> list:
|
||||
return [{"name": f"port-{p}", "port": p, "targetPort": p} for p in ports]
|
||||
|
||||
|
||||
def _bulk_access_mode(vol: dict) -> str:
|
||||
"""A bulk volume's access mode follows its read/write posture.
|
||||
|
||||
Every bulk volume is `...Many`: the SAME export is consumed by several namespaces at once
|
||||
(plex, sonarr and radarr all sit on /mnt/Storage/plex) and by several nodes at once, which is
|
||||
precisely what ReadWriteOnce cannot express. WHICH `Many` it is says whether this consumer
|
||||
writes -- so `kubectl get pv` reports the intent, not merely the topology.
|
||||
"""
|
||||
return "ReadOnlyMany" if vol.get("readOnly") else "ReadWriteMany"
|
||||
|
||||
|
||||
def r_bulk_pvs(s: dict) -> list:
|
||||
"""One STATIC PersistentVolume per `class: bulk` storage entry (docs/04 bulk tier).
|
||||
|
||||
WHY A PV AT ALL, RATHER THAN AN INLINE `nfs:` POD VOLUME
|
||||
-------------------------------------------------------
|
||||
An inline volume is fewer objects and it would work. It is rejected for four reasons, in
|
||||
descending order of what they would cost to discover later:
|
||||
|
||||
1. RECLAIM POLICY. `persistentVolumeReclaimPolicy: Retain` is a declared, reviewable
|
||||
property of a PV. There is no equivalent for an inline volume -- and the data behind
|
||||
these mounts is 8.33 TiB of media and a 267 GiB archive that ALSO has live non-k3s
|
||||
consumers. A statement in git that nothing in this cluster may reclaim it is worth an
|
||||
object.
|
||||
2. READ-ONLY LIVES IN ONE PLACE. `spec.nfs.readOnly: true` becomes a real `ro` mount option
|
||||
for every pod that ever binds this volume. The same intent expressed on an inline volume
|
||||
is one Deployment edit away from being dropped silently, with no diff anywhere near the
|
||||
storage. (It is still a CLIENT-side guarantee -- the export cannot distinguish plex from
|
||||
sonarr, since both may run on any node -- but it is one that lives in one reviewable
|
||||
place.)
|
||||
3. VISIBILITY. `kubectl get pv,pvc` is where an operator looks for "what external storage is
|
||||
attached, and who holds it". An inline volume is invisible there, and invisible to
|
||||
anything that reasons over PVCs.
|
||||
4. IT IS THE CONTRACT THE REPO ALREADY HAS. Every other storage entry is a PVC; the
|
||||
renderer, the Kyverno ServiceDefinition policy and the backup wiring all key on that. An
|
||||
inline volume would be a second, parallel storage model inside the same renderer.
|
||||
|
||||
CROSS-NAMESPACE. A PV binds 1:1 to a PVC, so "plex, sonarr and radarr share one export" is NOT
|
||||
one shared PV -- it is three PVs, one per namespace, each naming the same server+path. That is
|
||||
the correct model rather than a workaround: the sharing happens on the NFS server, which is
|
||||
built for concurrent clients, and each namespace keeps its own independently reviewable access
|
||||
mode (plex ReadOnlyMany, the arrs ReadWriteMany). One shared PV would have forced a single
|
||||
posture on all three.
|
||||
|
||||
BINDING IS PINNED FROM BOTH ENDS -- `claimRef` here and `volumeName` on the PVC. Either alone
|
||||
leaves a race in which some other PVC of matching size and mode grabs the volume first;
|
||||
together the binding is deterministic. `storageClassName: ""` (explicitly empty, NOT the `bulk`
|
||||
StorageClass) is what tells the binder this volume is statically provisioned: it suppresses the
|
||||
default-StorageClass annotation and guarantees no provisioner is ever asked to create anything.
|
||||
It also sidesteps a trap -- a StorageClass's `provisioner` is IMMUTABLE, so re-pointing the
|
||||
existing `bulk` class (whose stub names an NFS CSI driver that is not installed) at this data
|
||||
would make Flux fail forever on that one object.
|
||||
"""
|
||||
out = []
|
||||
for vol in s["storage"]:
|
||||
if vol.get("class") != "bulk":
|
||||
continue
|
||||
name = f"{s['name']}-{vol['name']}"
|
||||
# `export` and `size` are both structurally required of a bulk entry, and the JSON
|
||||
# Schema only enforces the first. A PersistentVolume with no `capacity` is rejected by
|
||||
# the API server, so a missing size must fail HERE with a sentence a human can act on
|
||||
# rather than as a bare KeyError three frames down.
|
||||
for field, why in (
|
||||
("export", "the absolute NFS export path this volume attaches"),
|
||||
("size", "the PV capacity (advisory, but the API requires the field)"),
|
||||
):
|
||||
if not vol.get(field):
|
||||
raise ValueError(
|
||||
f"{s['name']}: storage entry {vol['name']!r} has class 'bulk' but no "
|
||||
f"'{field}' — {why}. A bulk volume attaches storage that already exists "
|
||||
f"outside the cluster, so it cannot be rendered without it."
|
||||
)
|
||||
nfs = {"server": BULK_NFS_SERVER, "path": vol["export"]}
|
||||
if vol.get("readOnly"):
|
||||
nfs["readOnly"] = True
|
||||
spec = {
|
||||
"capacity": {"storage": vol["size"]},
|
||||
"accessModes": [_bulk_access_mode(vol)],
|
||||
# Retain: deleting the PVC must NEVER be able to reach the data. There is nothing to
|
||||
# "reclaim" here in any case -- the volume was not provisioned by the cluster.
|
||||
"persistentVolumeReclaimPolicy": "Retain",
|
||||
"storageClassName": "",
|
||||
"mountOptions": list(BULK_NFS_MOUNT_OPTIONS),
|
||||
"nfs": nfs,
|
||||
"claimRef": {"namespace": s["name"], "name": name},
|
||||
}
|
||||
out.append(
|
||||
{
|
||||
"apiVersion": "v1",
|
||||
"kind": "PersistentVolume",
|
||||
"metadata": {
|
||||
"name": name,
|
||||
"labels": {"app.kubernetes.io/name": s["name"]},
|
||||
},
|
||||
"spec": spec,
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def r_pvcs(s: dict) -> list:
|
||||
"""One PVC per storage entry (R-STATE-01/02/04).
|
||||
|
||||
longhorn -> 3-replica replicated tier, dynamically provisioned, ReadWriteOnce.
|
||||
bulk -> the STATIC external NFS PV rendered by r_bulk_pvs, pre-bound by volumeName.
|
||||
longhorn -> 3-replica replicated tier; bulk -> external media tier (not 3x replicated).
|
||||
"""
|
||||
out = []
|
||||
for vol in s["storage"]:
|
||||
cls = vol.get("class", "longhorn")
|
||||
if cls == "bulk":
|
||||
# Mirror image of the PV: the same explicitly-empty class (so nothing dynamically
|
||||
# provisions), the same access mode, and volumeName completing the two-sided binding.
|
||||
out.append(
|
||||
{
|
||||
"apiVersion": "v1",
|
||||
"kind": "PersistentVolumeClaim",
|
||||
"metadata": {
|
||||
"name": f"{s['name']}-{vol['name']}",
|
||||
"namespace": s["name"],
|
||||
"labels": {"app.kubernetes.io/name": s["name"]},
|
||||
},
|
||||
"spec": {
|
||||
"accessModes": [_bulk_access_mode(vol)],
|
||||
"storageClassName": "",
|
||||
"volumeName": f"{s['name']}-{vol['name']}",
|
||||
"resources": {"requests": {"storage": vol["size"]}},
|
||||
},
|
||||
}
|
||||
)
|
||||
continue
|
||||
storage_class = "longhorn" if cls == "longhorn" else "bulk"
|
||||
spec = {
|
||||
"accessModes": ["ReadWriteOnce"],
|
||||
"storageClassName": "longhorn",
|
||||
"storageClassName": storage_class,
|
||||
}
|
||||
if vol.get("size"):
|
||||
spec["resources"] = {"requests": {"storage": vol["size"]}}
|
||||
@@ -404,17 +258,7 @@ def r_deployment(s: dict) -> dict:
|
||||
# it verbatim for apps that need a fixed on-disk layout (e.g. Gitea keeps all state
|
||||
# directly under /data). Omitting it preserves the /data/<name> default (back-compat).
|
||||
mount_path = vol.get("mountPath", f"/data/{vol['name']}")
|
||||
mount = {"name": vol["name"], "mountPath": mount_path}
|
||||
# subPath mounts a subtree of the volume instead of its root. Emitted only when declared,
|
||||
# so every pre-existing golden stays byte-identical.
|
||||
if vol.get("subPath"):
|
||||
mount["subPath"] = vol["subPath"]
|
||||
# readOnly on the mount as well as on the PV's nfs source. Belt and braces on purpose: the
|
||||
# PV field is what makes the kernel mount `ro`; this one is what a reviewer reading the
|
||||
# Deployment sees. An edit that drops one still leaves the other.
|
||||
if vol.get("readOnly"):
|
||||
mount["readOnly"] = True
|
||||
volume_mounts.append(mount)
|
||||
volume_mounts.append({"name": vol["name"], "mountPath": mount_path})
|
||||
volumes.append(
|
||||
{
|
||||
"name": vol["name"],
|
||||
@@ -763,7 +607,7 @@ def r_internet_egress(s: dict) -> dict:
|
||||
def r_httproutes(s: dict) -> list:
|
||||
"""One HTTPRoute per ingressRoute, attaching host -> service port on the shared Gateway.
|
||||
|
||||
TLS is the Gateway's wildcard *.home.keithjackson.dev cert (no per-service cert step);
|
||||
TLS is the Gateway's wildcard *.${BASE_DOMAIN} cert (no per-service cert step);
|
||||
ExternalDNS manages the record from the HTTPRoute (R-TLS/R-DNS, renderer-contract).
|
||||
"""
|
||||
out = []
|
||||
@@ -1252,14 +1096,6 @@ def render(sd: dict, image_automation: bool = True, inbound: list | None = None)
|
||||
if s["kind"] == "container":
|
||||
# Container PVCs. (For a vm, the CDI DataVolume owns/creates the disk PVC, so we
|
||||
# never render a plain PVC for the VM root disk — it would collide with CDI.)
|
||||
# The static bulk PV is rendered BEFORE the PVC that names it. Document order inside one
|
||||
# kustomize build is not an apply-ordering guarantee, but a PVC whose PV does not exist
|
||||
# yet merely stays Pending and binds on the next reconcile, so this is a readability
|
||||
# convention rather than a correctness requirement.
|
||||
for pv in r_bulk_pvs(s):
|
||||
docs.append(
|
||||
(f"PersistentVolume {pv['metadata']['name']} - static external NFS (bulk tier)", pv)
|
||||
)
|
||||
for pvc in r_pvcs(s):
|
||||
docs.append((f"PVC {pvc['metadata']['name']} (R-STATE)", pvc))
|
||||
docs.append(("Deployment (R-PLAT-02)", r_deployment(s)))
|
||||
@@ -1486,11 +1322,6 @@ def plan_apps_waves(sds: list, present_files: set) -> dict:
|
||||
- `<svc>-db.yaml` -> wave(svc-db) (a strictly-lower wave; CNPG DB)
|
||||
- `<svc>-db-app.sops.yaml` -> wave(svc-db) (CNPG initdb owner secret)
|
||||
- `<svc>-db-backup-credentials.sops.yaml` -> wave(svc-db) (CNPG Barman S3 backup secret)
|
||||
- `<svc>-db-backup-credentials-prod.sops.yaml` -> wave(svc-db) (the PROD half of that
|
||||
secret: the two environments carry DIFFERENT object-store credentials, selected in
|
||||
the Cluster CR by `${BACKUP_CREDENTIALS_SUFFIX}`. Both objects ship to both clusters,
|
||||
so both must be placed in the DB wave — the Secret has to exist before the Cluster
|
||||
that names it.)
|
||||
Non-catalog units (EXTRA_UNITS, e.g. gitea-actions) place by their declared anchor+offset.
|
||||
Only files that actually exist in `present_files` are placed, so the plan is robust to a
|
||||
service having no bootstrap secret or no DB. Deterministic (sorted).
|
||||
@@ -1518,9 +1349,6 @@ def plan_apps_waves(sds: list, present_files: set) -> dict:
|
||||
add(wdb, f"{db}.yaml")
|
||||
add(wdb, f"{db}-app.sops.yaml")
|
||||
add(wdb, f"{db}-backup-credentials.sops.yaml") # CNPG Barman S3 creds (R-STATE-03)
|
||||
# PROD half of the same credential (2026-09-07 per-env split). Placed only when
|
||||
# the file exists, so a DB that has not been split is unaffected.
|
||||
add(wdb, f"{db}-backup-credentials-prod.sops.yaml")
|
||||
|
||||
for unit, cfg in EXTRA_UNITS.items():
|
||||
anchor = cfg["anchor"]
|
||||
@@ -2003,7 +1831,6 @@ def _selftest() -> int:
|
||||
present = {
|
||||
"a.yaml", "svc.yaml", "svc-bootstrap-secret.sops.yaml",
|
||||
"svc-db.yaml", "svc-db-app.sops.yaml", "svc-db-backup-credentials.sops.yaml",
|
||||
"svc-db-backup-credentials-prod.sops.yaml",
|
||||
"gitea.yaml",
|
||||
"gitea-actions.yaml", "gitea-actions-runner-secret.sops.yaml",
|
||||
}
|
||||
@@ -2013,10 +1840,8 @@ def _selftest() -> int:
|
||||
flat = {f: wv for wv, fs in plan.items() for f in fs}
|
||||
check(flat.get("a.yaml") == 0, "leaf app must land in wave 0")
|
||||
check(flat.get("svc-db.yaml") == 0 and flat.get("svc-db-app.sops.yaml") == 0
|
||||
and flat.get("svc-db-backup-credentials.sops.yaml") == 0
|
||||
and flat.get("svc-db-backup-credentials-prod.sops.yaml") == 0,
|
||||
"CNPG DB manifest + its owner secret + BOTH halves of its per-env Barman backup secret "
|
||||
"must be in the DB's (lower) wave")
|
||||
and flat.get("svc-db-backup-credentials.sops.yaml") == 0,
|
||||
"CNPG DB manifest + its owner secret + its Barman backup secret must be in the DB's (lower) wave")
|
||||
check(flat.get("svc.yaml") == 1 and flat.get("svc-bootstrap-secret.sops.yaml") == 1,
|
||||
"a DB-backed app + its bootstrap secret must be in the consumer wave, above the DB")
|
||||
check(flat.get("gitea.yaml") == 0, "gitea (leaf) must be wave 0")
|
||||
|
||||
+11
-31
@@ -78,34 +78,16 @@ env:
|
||||
# (util ConfigType `oidc_providers`, env-tagged SEMAPHORE_OIDC_PROVIDERS). The value is a single
|
||||
# JSON blob for the `keycloak` provider — provider_url is the homelab realm issuer (auto-discovery),
|
||||
# client_id=semaphore, redirect_url=/api/auth/oidc/keycloak/redirect, claims map to the realm's
|
||||
# token. The name is NOT a kubelet service-link var (those are SEMAPHORE_SERVICE_HOST /
|
||||
# SEMAPHORE_PORT_*), so no collision. The LOCAL SEMAPHORE_ADMIN above stays break-glass (docs/15).
|
||||
#
|
||||
# PER-ENVIRONMENT URLS (2026-09-07 fix). The blob USED to live whole inside the SOPS-encrypted
|
||||
# semaphore-bootstrap Secret (key `oidc-providers-json`) because it CONTAINS the confidential
|
||||
# client secret. That hid two environment-specific, NON-SECRET URLs — the realm issuer and the
|
||||
# redirect URI — inside ciphertext, where Flux's ${BASE_DOMAIN} postBuild substitution CANNOT
|
||||
# reach them. Both were frozen at the apex `home.keithjackson.dev`, so PROD (BASE_DOMAIN =
|
||||
# prod.home.keithjackson.dev) pointed its OIDC at DEV's Keycloak — unroutable from prod's
|
||||
# 192.168.2.0/24 (the identical root cause as the Headlamp login timeout; see
|
||||
# platform/headlamp/README.md). Dev was only ever "right" because dev IS the apex.
|
||||
#
|
||||
# THE SPLIT: the URLs are public, so the blob is now a PLAINTEXT literal here and gets
|
||||
# ${BASE_DOMAIN}-substituted per environment like every other hostname. Only the confidential
|
||||
# client secret stays SOPS-encrypted, in its own key (`oidc-client-secret`), and is spliced back
|
||||
# in by the KUBELET's dependent-env-var expansion: an env `value` may reference an EARLIER env
|
||||
# var of the same container as $(NAME), and secretKeyRef-sourced vars are eligible. So
|
||||
# SEMAPHORE_OIDC_CLIENT_SECRET MUST STAY IMMEDIATELY ABOVE — the kubelet resolves env in list
|
||||
# order, and a reference to a not-yet-defined name is left as the LITERAL text $(...). It is
|
||||
# fail-loud in the way that matters: if the Secret key is missing the Pod never starts
|
||||
# (CreateContainerConfigError), rather than silently authenticating against nothing. ---
|
||||
- name: SEMAPHORE_OIDC_CLIENT_SECRET
|
||||
# token. It CONTAINS the confidential client secret, so the whole blob is held SOPS-encrypted in the
|
||||
# semaphore-bootstrap Secret (key oidc-providers-json; same secret value SOPS-held in
|
||||
# keycloak-realm-clients) and injected here by reference — never in cleartext. The name is NOT a
|
||||
# kubelet service-link var (those are SEMAPHORE_SERVICE_HOST / SEMAPHORE_PORT_*), so no collision.
|
||||
# The LOCAL SEMAPHORE_ADMIN above stays as the break-glass path (docs/15). ---
|
||||
- name: SEMAPHORE_OIDC_PROVIDERS
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: semaphore-bootstrap
|
||||
key: oidc-client-secret
|
||||
- name: SEMAPHORE_OIDC_PROVIDERS
|
||||
value: '{"keycloak":{"display_name":"Keycloak","provider_url":"https://keycloak.${BASE_DOMAIN}/realms/homelab","client_id":"semaphore","client_secret":"$(SEMAPHORE_OIDC_CLIENT_SECRET)","redirect_url":"https://semaphore.${BASE_DOMAIN}/api/auth/oidc/keycloak/redirect","scopes":["openid","profile","email"],"username_claim":"preferred_username","email_claim":"email","name_claim":"name","order":1}}'
|
||||
key: oidc-providers-json
|
||||
# The semaphoreui/semaphore image runs as the non-root user UID 1001 / GID 0 (tini -> server-wrapper),
|
||||
# and on boot the wrapper writes its generated config.json into the config dir. A mounted Longhorn PVC
|
||||
# is root:root, so UID 1001 cannot write it -> the pod panics ("permission denied"). fsGroup=1001 makes
|
||||
@@ -135,12 +117,10 @@ auth:
|
||||
groups:
|
||||
- platform-admins # access scoped to an authorized group (R-IDP-06)
|
||||
# WIRED (native OIDC): the Keycloak `semaphore` confidential client (platform/keycloak/keycloak-realm-
|
||||
# import.yaml + live kcadm) is consumed via the SEMAPHORE_OIDC_PROVIDERS env — a PLAINTEXT JSON blob
|
||||
# (see `env` above) whose per-environment URLs are ${BASE_DOMAIN}-substituted, with the confidential
|
||||
# client secret spliced in at runtime from the semaphore-bootstrap Secret key `oidc-client-secret`.
|
||||
# The `groups` claim is exposed for authorization (R-IDP-06). Semaphore KEEPS its LOCAL admin as the
|
||||
# break-glass path so an IdP outage never locks the operator out (docs/15 "Availability caveat").
|
||||
# No secret is embedded here.
|
||||
# import.yaml + live kcadm) is consumed via the SEMAPHORE_OIDC_PROVIDERS env (JSON blob held encrypted
|
||||
# in the semaphore-bootstrap Secret, oidc-providers-json). The `groups` claim is exposed for
|
||||
# authorization (R-IDP-06). Semaphore KEEPS its LOCAL admin as the break-glass path so an IdP outage
|
||||
# never locks the operator out (docs/15 "Availability caveat"). No secret is embedded here.
|
||||
update:
|
||||
channel: stable # R-UPD-01
|
||||
range: ">=2.19.0 <3.0.0" # stay on the 2.x stable line
|
||||
|
||||
+5
-77
@@ -9,23 +9,6 @@ name: sonarr
|
||||
kind: container
|
||||
source:
|
||||
image: lscr.io/linuxserver/sonarr:4.0.15 # pinned stable v4 tag (R-UPD-01); update.range governs advancement
|
||||
# LinuxServer images run their service as the user `abc`, whose uid/gid these two set. They are
|
||||
# LOAD-BEARING for the bulk mount, not cosmetic: see the storage note below. The default is 911,
|
||||
# which cannot write a 1000:3000-owned 0755 tree.
|
||||
env:
|
||||
- name: PUID
|
||||
value: "1000" # owner uid of every directory under /mnt/Storage/plex
|
||||
- name: PGID
|
||||
value: "3000" # dominant group there; keeps files this writes readable to the
|
||||
# EXISTING non-k3s consumers of the same export, which stay live
|
||||
- name: TZ
|
||||
value: Etc/UTC # same safe default plex uses; the arrs schedule by local time
|
||||
podSecurityContext:
|
||||
# For the LONGHORN config PVC only. fsGroup does nothing on the NFS volume (the in-tree plugin
|
||||
# does no ownership management), but on a Longhorn filesystem the kubelet applies it, which is
|
||||
# what makes /config writable by the now-uid-1000 process WITHOUT relying on the image chowning
|
||||
# /config for itself when PUID changes. Matches PGID so the primary group owns the config too.
|
||||
fsGroup: 3000
|
||||
exposure: internal # LAN-only web UI at sonarr.${BASE_DOMAIN} (docs/13 row 2; R-DNS/R-NET)
|
||||
egress: vpn # indexer scrape + downloader handoff steered via the shared VPN egress gateway,
|
||||
# kill-switch enforced OUTSIDE the workload (docs/03; ADR-0015; R-NET-05/06)
|
||||
@@ -33,66 +16,11 @@ storage:
|
||||
- name: config
|
||||
size: 5Gi
|
||||
class: longhorn # app DB / config on the 3-replica tier; DR-critical (docs/04; R-STATE-02)
|
||||
mountPath: /config # THE APP WRITES /config, NOT the renderer default /data/config.
|
||||
# LSIO images run with `-data=/config` (Jackett/qBittorrent likewise use
|
||||
# /config), so without this the Longhorn PVC mounted at /data/config sat
|
||||
# holding nothing but lost+found while the app wrote its entire
|
||||
# configuration to the container's EPHEMERAL writable layer — destroyed on
|
||||
# every pod recreation: restart, reschedule, image bump. Measured in-cluster
|
||||
# 2026-09-12: /data/config was dev=2080 with [lost+found], /config was
|
||||
# dev=2053 on the node's own disk with [Sentry asp config.xml logs].
|
||||
# This is the SAME failure catalog/plex.yaml:54 already records fixing.
|
||||
# The live /config contents were copied onto each PVC before this landed,
|
||||
# so API keys and credentials other apps reference were preserved.
|
||||
# THE MEDIA TREE (docs/04 bulk tier) — ONE read-WRITE mount of the export ROOT at /media, giving
|
||||
# /media/tv (the library) and /media/downloads (the import source) UNDER THE SAME MOUNT.
|
||||
#
|
||||
# *** THE ROOT MOUNT IS THE POINT, NOT A CONVENIENCE. *** Sonarr imports by HARDLINKING or
|
||||
# atomically renaming a finished download into the library. Both are same-FILESYSTEM operations,
|
||||
# and to the kernel "same filesystem" means SAME MOUNT. The layout on the host is:
|
||||
# /mnt/Storage/plex/tv/ the library
|
||||
# /mnt/Storage/plex/downloads/complete/tv-sonarr/ where finished releases land
|
||||
# — siblings under one dataset, so a hardlink between them is legal on the server. But mounting
|
||||
# them as two subPath volumes (subPath: tv at /media/tv, subPath: downloads elsewhere) creates
|
||||
# TWO mounts, and link() across mounts fails EXDEV no matter what the server thinks. Sonarr then
|
||||
# silently degrades to copy-then-delete: every import rewrites the whole file over NFS and the
|
||||
# seeding copy is destroyed. Their smallest common ancestor IS the export root, so the root mount
|
||||
# is the MINIMAL mount that preserves hardlinks — least privilege bounded below by a hard
|
||||
# requirement, not a shortcut. (The cost is stated plainly: Sonarr can also write /media/movies,
|
||||
# which is radarr's. Narrowing that would cost hardlinks, which is a worse trade.)
|
||||
#
|
||||
# *** /media, NOT /data — AND THAT IS NOT A STYLE CHOICE. *** The config PVC above renders at the
|
||||
# renderer default /data/config. A media mount at /data would be mounted OVER it and MASK the
|
||||
# config volume entirely; the pod would come up with its app database shadowed by an 8 T media
|
||||
# tree. /media collides with nothing in a LinuxServer image and matches the path plex uses for
|
||||
# the same export, so when qbittorrent is eventually given this mount too, the arrs and the
|
||||
# download client will already agree on absolute paths — which is the other precondition for
|
||||
# hardlinked imports (otherwise every import needs a Remote Path Mapping).
|
||||
#
|
||||
# SEPARATE LATENT BUG, NOT FIXED HERE (deliberately — it changes live on-disk state): /data/config
|
||||
# is almost certainly the wrong place for this PVC in the first place. LinuxServer's arr images run
|
||||
# with `-data=/config`, so the app writes /config — the container's EPHEMERAL layer — while the
|
||||
# Longhorn PVC at /data/config sits unused. That is the EXACT failure plex's own mountPath comment
|
||||
# documents having already been bitten by. Fixing it means starting from an empty config volume, so
|
||||
# it belongs in its own change with the operator present.
|
||||
#
|
||||
# UID/GID — THE FAILURE THIS AVOIDS IS "MOUNTS FINE, CANNOT WRITE". The export is no_root_squash
|
||||
# + sec=sys, so the client's numeric uid is passed through verbatim and the server's ownership is
|
||||
# the whole story. The tree is 0755 owned 1000:3000 (some dirs 1000:1000), so ONLY uid 1000 can
|
||||
# write it. LinuxServer images default to PUID/PGID 911 — which would mount cleanly and then fail
|
||||
# every import with EACCES. The PUID/PGID set above make the process uid 1000, matching the owner.
|
||||
#
|
||||
# AND NOTE fsGroup CANNOT SUBSTITUTE FOR THAT. The in-tree NFS volume plugin performs no
|
||||
# ownership management, so podSecurityContext.fsGroup is a NO-OP on this volume — it is not the
|
||||
# lever here, and reaching for it is how this failure normally gets misdiagnosed. The fsGroup
|
||||
# that IS set above exists for the LONGHORN config PVC, so that changing PUID does not depend on
|
||||
# the image re-chowning /config for itself.
|
||||
- name: media
|
||||
class: bulk
|
||||
export: /mnt/Storage/plex # the ZFS dataset Storage/plex, on ${BULK_NFS_SERVER}
|
||||
mountPath: /media # => /media/tv (Root Folder) and /media/downloads (import source)
|
||||
size: 9Ti # ADVISORY ONLY on a static NFS PV: the dataset's 8.33 T used,
|
||||
# rounded up, so `kubectl get pv` reports something true.
|
||||
# NOTE (bulk-media PREREQUISITE): docs/13 row 2 places the media library / downloads volume on the
|
||||
# `bulk` tier (docs/04 bulk storage). The `bulk` StorageClass exists (platform/longhorn/storageclasses.yaml)
|
||||
# but is a NON-provisioning STUB pending OQ-2 (NFS/SMB backing = REPLACE-ME). Adding a `class: bulk`
|
||||
# volume now would leave the PVC (and therefore the pod) Pending. The media/downloads volume is
|
||||
# therefore DEFERRED until OQ-2 is resolved; config-only on longhorn keeps Sonarr deployable today.
|
||||
dependencies:
|
||||
- to: qbittorrent # arr → download client (docs/13 dependency graph; R-NET-02 explicit allow)
|
||||
port: 8080
|
||||
|
||||
@@ -137,82 +137,6 @@ else
|
||||
esac
|
||||
fi
|
||||
|
||||
# ---- Stage 3d: Keycloak client-secret declaration + reconciler (UNCONDITIONAL) ----
|
||||
# The Keycloak realm import resolves each OIDC client's secret from an `(env:NAME)`
|
||||
# placeholder. When the operator cannot resolve one it stores the LITERAL placeholder as
|
||||
# the client secret AND THE IMPORT JOB STILL REPORTS SUCCESS, then rests at Done=True and
|
||||
# never re-runs. On 2026-09-06 that had all 12 prod clients unusable ("Invalid client or
|
||||
# Invalid client credentials") behind an entirely green cluster; dev had one. The check
|
||||
# asserts the declaration is self-consistent (every placeholder defined, every Secret key
|
||||
# present, names matching the reconciler's contract), that the Flux ordering is a REAL
|
||||
# dependsOn edge rather than a comment, and — by extracting the shipped script from its
|
||||
# ConfigMap and running its --selftest — that the reconciler is compare-then-write,
|
||||
# idempotent, non-logging and blast-radius-bounded.
|
||||
#
|
||||
# WHY ITS OWN UNCONDITIONAL STAGE, same reasoning as 3c: Stage 3b scopes the conformance
|
||||
# run to the TOUCHED requirements (R-TEST-03). This check's invariant spans platform/,
|
||||
# platform-config/ and clusters/, so a change that breaks it need not attribute R-IDP-03/04
|
||||
# and the requirement-scoped selection would filter it out of exactly the change that
|
||||
# introduces the bug. It is a sub-second offline scan plus a loopback selftest, with no
|
||||
# side effects, so running it twice when 3b also selects it is not worth avoiding.
|
||||
KEYCLOAK_SECRET_CHECK="${KEYCLOAK_SECRET_CHECK:-${REPO_ROOT}/conformance/tests/l0-static/keycloak-client-secret-reconciler/check.sh}"
|
||||
log "Stage: Keycloak client-secret reconciler safety (${KEYCLOAK_SECRET_CHECK})"
|
||||
if [ ! -f "$KEYCLOAK_SECRET_CHECK" ]; then
|
||||
skip "keycloak client-secret check not present at '${KEYCLOAK_SECRET_CHECK}' — UNWIRED, not green"
|
||||
summary "- keycloak-client-secrets: **SKIPPED** (check missing)"
|
||||
if [ "$rc" -eq 0 ]; then rc=3; fi
|
||||
else
|
||||
kc_rc=0
|
||||
bash "$KEYCLOAK_SECRET_CHECK" || kc_rc=$?
|
||||
case "$kc_rc" in
|
||||
0) ok "Keycloak client-secret declaration + reconciler passed"
|
||||
summary "- keycloak-client-secrets: **passed**" ;;
|
||||
3) skip "keycloak client-secret check SKIPPED (platform/keycloak absent) — not a pass"
|
||||
summary "- keycloak-client-secrets: **SKIPPED** (nothing to audit)"
|
||||
if [ "$rc" -eq 0 ]; then rc=3; fi ;;
|
||||
*) err "Keycloak client-secret declaration + reconciler FAILED"
|
||||
summary "- keycloak-client-secrets: **FAILED**"; rc=1 ;;
|
||||
esac
|
||||
fi
|
||||
|
||||
# ---- Stage 3e: bulk tier distinct from the replicated tier (UNCONDITIONAL) ----
|
||||
# The `bulk` StorageClass sat as a REPLACE-ME stub for six roadmap phases, and the only
|
||||
# thing keeping a multi-terabyte media library off the 3-replica `longhorn` class was four
|
||||
# hand-written comments in ServiceDefinitions. A bulk volume that lands on the replicated
|
||||
# tier does not fail loudly — Longhorn accepts the PVC and starts replicating 3x of data
|
||||
# three mini-PCs cannot hold. The check asserts the tier separation, that a bulk volume is
|
||||
# statically pre-bound from BOTH ends (so nothing can dynamically provision an EMPTY
|
||||
# stand-in that mounts and Binds perfectly while the real data is never attached), Retain,
|
||||
# hard/NFSv4 mount options, the read-only coupling between PV and volumeMount, that no
|
||||
# workload mounts one export twice (which breaks Sonarr/Radarr hardlinks with EXDEV), and
|
||||
# that the server is a per-environment knob every overlay supplies.
|
||||
#
|
||||
# WHY ITS OWN UNCONDITIONAL STAGE, same reasoning as 3c/3d: Stage 3b scopes the conformance
|
||||
# run to the TOUCHED requirements (R-TEST-03), and ci/touched-requirements.sh's path map
|
||||
# keys NEITHER apps/* nor platform/longhorn/* to the bulk requirement. So the change most
|
||||
# likely to break this — moving a bulk volume onto the replicated tier, or narrowing one
|
||||
# export into two mounts — could attribute no requirement at all and be gated by nothing.
|
||||
# The invariant spans catalog/, apps/ and clusters/, so it runs on EVERY gate invocation.
|
||||
BULK_TIER_CHECK="${BULK_TIER_CHECK:-${REPO_ROOT}/conformance/tests/l0-static/bulk-tier-distinct-from-replicated/check.sh}"
|
||||
log "Stage: bulk tier distinct from replicated tier (${BULK_TIER_CHECK})"
|
||||
if [ ! -f "$BULK_TIER_CHECK" ]; then
|
||||
skip "bulk tier check not present at '${BULK_TIER_CHECK}' — UNWIRED, not green"
|
||||
summary "- bulk-tier: **SKIPPED** (check missing)"
|
||||
if [ "$rc" -eq 0 ]; then rc=3; fi
|
||||
else
|
||||
bulk_rc=0
|
||||
bash "$BULK_TIER_CHECK" || bulk_rc=$?
|
||||
case "$bulk_rc" in
|
||||
0) ok "Bulk tier distinctness + static binding passed"
|
||||
summary "- bulk-tier: **passed**" ;;
|
||||
3) skip "bulk tier check SKIPPED (catalog/ or apps/ absent) — not a pass"
|
||||
summary "- bulk-tier: **SKIPPED** (nothing to audit)"
|
||||
if [ "$rc" -eq 0 ]; then rc=3; fi ;;
|
||||
*) err "Bulk tier distinctness + static binding FAILED"
|
||||
summary "- bulk-tier: **FAILED**"; rc=1 ;;
|
||||
esac
|
||||
fi
|
||||
|
||||
case "$rc" in
|
||||
0) ok "L0 stage complete" ;;
|
||||
3) skip "L0 stage incomplete (unwired seam / missing tool) — not a pass" ;;
|
||||
|
||||
@@ -180,41 +180,9 @@ path_requirements() {
|
||||
platform/externaldns/*) echo "R-DNS-02" ;;
|
||||
platform/velero/*) echo "R-STATE-03" ;;
|
||||
platform/monitoring/*) echo "R-OBS-01 R-OBS-02" ;;
|
||||
# platform/cnpg was UNMAPPED until 2026-09-06. It is the PostgreSQL DR path for keycloak-db /
|
||||
# nextcloud-db / semaphore-db (barmanObjectStore WAL archiving + ScheduledBackup), so a change
|
||||
# there is an R-IDP-05 / R-STATE-03 change; it now also carries the platform-health alert rules
|
||||
# for that path (R-OBS-04). Without this line an edit under platform/cnpg/ contributed ZERO
|
||||
# requirements unless it happened to retype an R-ID on a non-comment line, and escaped the gate.
|
||||
platform/cnpg/*) echo "R-IDP-05 R-STATE-03 R-OBS-04" ;;
|
||||
# vpn-egress was UNMAPPED until 2026-09-06. platform/vpn-egress/egress-steering.yaml IS the
|
||||
# R-NET-04 datapath (the rule that selects between the two egress paths) and the R-NET-05/06/07
|
||||
# steering policy — it even self-declares those IDs in its `homelab.keithjackson.dev/requirements`
|
||||
# annotation. But the content scan only reads ADDED/REMOVED lines, so any edit that does not
|
||||
# happen to retype an R-ID (a selector change, an excludedCIDRs change, a comment) contributed
|
||||
# ZERO requirements and escaped the gate entirely. Mapped explicitly; the annotation still wins
|
||||
# when a diff does carry the tags.
|
||||
platform/vpn-egress/*) echo "R-NET-04 R-NET-05 R-NET-06 R-NET-07" ;;
|
||||
# Keycloak is the whole SSO surface: the realm import (declarative identity config),
|
||||
# the SOPS client secrets, and the reconciler that heals them. Added 2026-09-06 after
|
||||
# the client-secret incident, in which editing platform/keycloak/ attributed NO
|
||||
# requirement, so the L2 keycloak tests were filtered out of the merge gate's touched
|
||||
# subset — the change that could break every OIDC login was gated by nothing.
|
||||
# R-IDP-05 is in scope because platform/keycloak/ also carries the CNPG DB Cluster and
|
||||
# its off-cluster backup credential, not only the realm.
|
||||
platform/keycloak/*) echo "R-IDP-01 R-IDP-02 R-IDP-03 R-IDP-04 R-IDP-05 R-IDP-06" ;;
|
||||
platform/oauth2-proxy/*) echo "R-IDP-02 R-IDP-06" ;;
|
||||
platform/image-automation/*) echo "R-UPD-01 R-UPD-02" ;;
|
||||
conformance/*) echo "R-TEST-01 R-TEST-02 R-TEST-03 R-TEST-04" ;;
|
||||
clusters/*) echo "R-GITOPS-01 R-ENV-02" ;;
|
||||
# bootstrap/ was UNMAPPED until 2026-09-06, and the omission was load-bearing:
|
||||
# bootstrap/standup.sh carries the placeholder guards that decide whether `velero` may be
|
||||
# enabled against a real backup endpoint. A change touching standup.sh ALONGSIDE any
|
||||
# already-mapped path scoped the L0 run to that other path's requirements and silently
|
||||
# excluded conformance/tests/l0-static/placeholder-guard-coupling — the check that exists
|
||||
# precisely to catch a guard/value mismatch. The 2026-09-05 guard bug (a prefix glob that
|
||||
# stopped matching when the value it guards changed, in a different file) is exactly the
|
||||
# class that omission would hide.
|
||||
bootstrap/*) echo "R-STATE-03 R-GITOPS-01" ;;
|
||||
.gitea/*|ci/*) echo "R-GITOPS-01 R-TEST-02 R-TEST-03" ;;
|
||||
*) : ;;
|
||||
esac
|
||||
|
||||
@@ -97,27 +97,6 @@ Kustomization: apps-wave-1 (./apps/wave-1, dependsOn apps-wave-0)
|
||||
Kustomization: apps-wave-N (./apps/wave-N, dependsOn apps-wave-N-1)
|
||||
```
|
||||
|
||||
### Second-order CRDs need a THIRD pass (`platform-config-cdi`, added 2026-09-05)
|
||||
|
||||
The controllers/configs split (`platform` → `platform-config`) assumes a CRD is registered by the
|
||||
controller **install**. CDI breaks that: `cdi-operator.yaml` registers only `cdis.cdi.kubevirt.io`,
|
||||
and cdi-operator creates `storageprofiles` / `datavolumes` / `datasources` **only after it has
|
||||
reconciled the `CDI` CR**. Since kustomize-controller server-side **dry-runs every object in a pass
|
||||
before applying any of them**, having the `CDI` CR and the CDI `StorageProfile`s in one pass was a
|
||||
**deadlock, not a race**: unknown kind → pass aborts → CR never applied → CRD never created →
|
||||
`platform-config` can never converge → every tier below it blocked. Invisible on dev (its CDI CRDs
|
||||
have existed for months); it cost 12h on prod's cold start.
|
||||
|
||||
So there is a third tier, `platform-config-cdi` (`./platform-config-cdi`, `dependsOn:
|
||||
platform-config`), and a rule:
|
||||
|
||||
> A CR belongs in `platform-config-cdi`, not `platform-config`, when the CRD it needs is created by a
|
||||
> **controller at runtime** rather than by a manifest in git.
|
||||
|
||||
`apps-vms` `dependsOn` it as well as `platform-config`, because CDI reads
|
||||
`StorageProfile/longhorn-vm` at DataVolume **provisioning** time — a VM disk provisioned before that
|
||||
profile exists is stamped RWO/Filesystem and is permanently not live-migratable.
|
||||
|
||||
Wave 0 `dependsOn` `platform-config` (transitively `platform`) so CNI/storage/TLS/DNS/gateway
|
||||
exist before any workload reconciles. The app tier is split into dependency-ordered WAVES
|
||||
(`catalog/render.py --emit-waves`): each `apps-wave-<N>` health-gates (wait:true) ONLY its own
|
||||
|
||||
@@ -4,11 +4,8 @@
|
||||
# NON-GATING (wait:false): unlike the app waves, this does NOT health-wait. A VM's first-boot image
|
||||
# import (CDI, minutes) + in-guest software install mean a VM can be intentionally long-not-Ready, so
|
||||
# health-gating it would wedge reconciliation for no benefit. VMs have no in-cluster dependents.
|
||||
# dependsOn platform-config so KubeVirt/CDI + the longhorn-vm StorageClass exist first, AND
|
||||
# platform-config-cdi so the CDI StorageProfiles do (2026-09-05: the StorageProfiles MOVED OUT of
|
||||
# platform-config into their own tier, so `dependsOn: platform-config` alone NO LONGER guarantees
|
||||
# them — see the dependsOn block). prune:true — this Kustomization solely owns the VM manifests (no
|
||||
# adoption concern).
|
||||
# dependsOn platform-config so KubeVirt/CDI + the longhorn-vm StorageClass + CDI StorageProfiles
|
||||
# exist first. prune:true — this Kustomization solely owns the VM manifests (no adoption concern).
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
@@ -22,28 +19,6 @@ spec:
|
||||
wait: false
|
||||
dependsOn:
|
||||
- name: platform-config
|
||||
# LOAD-BEARING, not decoration. apps/vms/unifi/20-datavolume.yaml provisions a DataVolume on
|
||||
# storageClassName `longhorn-vm`, and CDI reads StorageProfile/longhorn-vm AT PROVISIONING TIME to
|
||||
# decide the claim's accessModes/volumeMode. Without that profile's `claimPropertySets`
|
||||
# (ReadWriteMany + Block) the PVC is stamped RWO/Filesystem and the VM disk is PERMANENTLY not
|
||||
# live-migratable — a silent, unrecoverable-without-recreate loss. The profiles used to arrive
|
||||
# with platform-config; since 2026-09-05 they arrive with `platform-config-cdi`, so the edge has
|
||||
# to move with them. platform-config-cdi runs wait:true and StorageProfile has no status
|
||||
# conditions (kstatus -> Current on apply), so this edge cannot stall this tier.
|
||||
#
|
||||
# HONEST LIMIT: CDI provisions from the profile's STATUS, which its storageprofile-controller
|
||||
# derives from our spec. This edge proves the spec is applied, not that CDI has folded it into
|
||||
# status — that is a watch-triggered round-trip of seconds against a ~30s dependency requeue, so
|
||||
# it is a timing argument rather than a guarantee. See clusters/base/platform-config-cdi.yaml.
|
||||
#
|
||||
# This edge also makes the StorageProfile prune/recreate transition safe: platform-config's prune
|
||||
# deletes the two profiles for the ~30-60s before platform-config-cdi re-applies them, and this
|
||||
# dependsOn is what guarantees Flux cannot create a DataVolume inside that window.
|
||||
#
|
||||
# The platform-config edge above is kept although platform-config-cdi already dependsOn it: the
|
||||
# edges are a set, there is no cycle, and stating both means a future edit to platform-config-cdi
|
||||
# cannot silently drop this tier's KubeVirt/StorageClass gate.
|
||||
- name: platform-config-cdi
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
|
||||
@@ -34,9 +34,6 @@ spec:
|
||||
substituteFrom:
|
||||
- kind: ConfigMap
|
||||
name: cluster-vars
|
||||
# SOPS decryption for the `cloud` INI credentials — velero-cloud.sops.yaml (dev's scoped MinIO key)
|
||||
# and velero-cloud-prod.sops.yaml (prod's Cloudflare R2 token) — is layered per environment by the
|
||||
# overlay (clusters/dev and clusters/prod patch THIS Kustomization by name), identical to how
|
||||
# headlamp/oauth2-proxy/crafty decrypt their own SOPS secrets. Both Secrets are applied on both
|
||||
# clusters; ${BACKUP_CREDENTIALS_SUFFIX} (substituted from cluster-vars above) decides which one the
|
||||
# velero Deployment actually mounts. See clusters/dev/kustomization.yaml.
|
||||
# SOPS decryption for velero-cloud.sops.yaml (the scoped MinIO `cloud` credential) is layered per
|
||||
# environment by the overlay (clusters/dev patches THIS Kustomization by name), identical to how
|
||||
# headlamp/oauth2-proxy/crafty decrypt their own SOPS secrets. See clusters/dev/kustomization.yaml.
|
||||
+72
-205
@@ -26,36 +26,49 @@ data:
|
||||
# authored to serve the SAME hostnames out of the SAME Cloudflare zone, so prod's external-dns
|
||||
# (policy: sync, owner `default`) concluded it owned dev's public records and DELETED ALL OF THEM.
|
||||
# See infra/PROD-PROVISIONING.md §10. The fix is architectural: each environment gets its OWN
|
||||
# subdomain, so the two clusters can no longer describe the same name.
|
||||
# suffix, so the two clusters can no longer describe the same name.
|
||||
#
|
||||
# dev -> home.keithjackson.dev (UNCHANGED — see below)
|
||||
# prod -> prod.home.keithjackson.dev
|
||||
# dev -> dev.home.keithjackson.dev (a subdomain; = this base default)
|
||||
# prod -> home.keithjackson.dev (THE APEX — prod IS the apex, no `prod.` label)
|
||||
#
|
||||
# DEV DELIBERATELY STAYS ON THE APEX `home.keithjackson.dev`. The target architecture puts dev on
|
||||
# `dev.home.keithjackson.dev` with apex CNAMEs `<svc>.home` -> `<svc>.prod.home`, but moving dev
|
||||
# relocates its Keycloak issuer, every OIDC client redirect URI, kubectl SSO, its wildcard cert and
|
||||
# the Gitea host that IS the GitOps source — on a live cluster. Deferred to phase 2
|
||||
# (infra/PROD-PROVISIONING.md §10). Base default is therefore the DEV value, the same convention as
|
||||
# GATEWAY_VIP / LONGHORN_DATA_PATH: the base default is the known-good working value, dev restates
|
||||
# it verbatim, and prod overrides it. Dev's rendered output is byte-identical to the pre-
|
||||
# tokenization hardcoded manifests (verified 2026-09-04, all 12 substituted paths).
|
||||
BASE_DOMAIN: "home.keithjackson.dev"
|
||||
# OPERATOR DECISION 2026-09-04 (supersedes the interim split committed earlier the same day, which
|
||||
# had prod on `prod.home.` with apex CNAMEs pointing at it). PROD IS THE APEX. That deletes the
|
||||
# whole CNAME layer — `<svc>.home.keithjackson.dev` is prod's real name, not an alias — and it is
|
||||
# DEV that carries a label. Convention is unchanged from GATEWAY_VIP / LONGHORN_DATA_PATH: the base
|
||||
# default is the DEV value (dev restates it verbatim) and prod overrides it. What changed is which
|
||||
# string dev holds.
|
||||
#
|
||||
# DEV'S VALUE IS A MIGRATION, NOT A VARIABLE CHANGE. The live dev cluster is serving the APEX right
|
||||
# now: its wildcard cert, every public DNS record, its Keycloak issuer, ~12 OIDC clients' redirect
|
||||
# URIs, kubectl SSO, and gitea.home.keithjackson.dev — which IS this repo's GitOps source AND the
|
||||
# push target. Flipping this value on dev without first standing up the new host set STRANDS DEV:
|
||||
# Flux's GitRepository url moves to gitea.dev.home.keithjackson.dev, which has no DNS record and no
|
||||
# cert, so dev stops reconciling and cannot be fixed by pushing. ORDERED CUTOVER, operator present:
|
||||
# infra/PROD-PROVISIONING.md §10.
|
||||
#
|
||||
# DEV MUST VACATE THE APEX BEFORE PROD PUBLISHES THERE. Both clusters describing
|
||||
# `*.home.keithjackson.dev` at once is the 2026-09-04 incident precondition, re-created.
|
||||
BASE_DOMAIN: "dev.home.keithjackson.dev"
|
||||
|
||||
# ExternalDNS `--domain-filter` (platform/externaldns/helmrelease.yaml). SAFETY NET #2 behind
|
||||
# EXTERNALDNS_TXT_OWNER_ID: a record whose name falls outside this filter is not even considered,
|
||||
# so it cannot be created, updated or deleted regardless of TXT ownership.
|
||||
#
|
||||
# DEV KEEPS THE ZONE APEX `keithjackson.dev`, and must: dev serves the `home.keithjackson.dev`
|
||||
# apex itself, and the Cloudflare provider filters the ACCOUNT'S ZONES by this value — a filter
|
||||
# that is a strict subdomain of the zone dropped the parent zone outright ("no hosted zone matching
|
||||
# record DNS Name", DECISIONS-LOG 2026-08) on the version in use then. So dev's value is unchanged
|
||||
# from the retired `DNS_ZONE` knob this replaces, and dev renders byte-identically.
|
||||
# BOTH ENVIRONMENTS USE THE CLOUDFLARE **ZONE**, `keithjackson.dev`. This value is NOT a per-
|
||||
# environment isolation knob and must not be narrowed to either environment's BASE_DOMAIN.
|
||||
#
|
||||
# PROD NARROWS TO ITS OWN SUBTREE `prod.home.keithjackson.dev`. VERIFY ON PROD'S FIRST RECONCILE
|
||||
# (infra/PROD-PROVISIONING.md §10): if the zone-lookup limitation above still applies to the pinned
|
||||
# chart, prod's external-dns will select NO zone and create NO records. That failure is FAIL-CLOSED
|
||||
# — it can never delete another environment's records — and the fallback is to set prod's value
|
||||
# back to "keithjackson.dev", leaving EXTERNALDNS_TXT_OWNER_ID as the sole (and sufficient) guard.
|
||||
# FINDING F-DNS1 (DECISIONS-LOG 2026-08-17) — the Cloudflare provider uses --domain-filter for ZONE
|
||||
# SELECTION as well as name filtering, so a filter that is a STRICT SUBDOMAIN of the hosted zone
|
||||
# matches NO zone: ExternalDNS generates the record and then skips it with "no hosted zone matching
|
||||
# record DNS Name", publishing NOTHING. This has been tried and reverted more than once (an agent
|
||||
# narrowed prod to prod.home.keithjackson.dev on 2026-09-04; overridden in review). DO NOT "harden"
|
||||
# it — the change is not defence in depth, it is a total outage of DNS publication.
|
||||
#
|
||||
# WHERE ISOLATION ACTUALLY COMES FROM, and it is sufficient:
|
||||
# (1) the two environments request DISJOINT hostname sets (dev.home.… vs the apex), and
|
||||
# (2) EXTERNALDNS_TXT_OWNER_ID differs, so neither registry ever claims the other's records.
|
||||
# That pairing is what the 2026-09-04 incident needed; a domain filter cannot provide it, because
|
||||
# both environments live in the one zone either way.
|
||||
EXTERNALDNS_DOMAIN_FILTER: "keithjackson.dev"
|
||||
|
||||
# ExternalDNS TXT registry owner id (`--txt-owner-id`). THE ROOT CAUSE OF THE 2026-09-04 INCIDENT:
|
||||
@@ -64,14 +77,35 @@ data:
|
||||
# `policy: sync`, to DELETE when no local resource claims it. Two clusters sharing an owner id on
|
||||
# one zone therefore garbage-collect each other. Each environment MUST have a distinct value.
|
||||
#
|
||||
# DEV MUST RENDER EXACTLY `default`. Dev's LIVE public records are already registered in Cloudflare
|
||||
# under owner `default`; changing dev's value would make dev's own external-dns stop recognising
|
||||
# them, and it would re-create duplicates alongside the orphaned originals. Dev's overlay pins it.
|
||||
# dev -> `dev` prod -> `prod` base default -> a non-colliding sentinel (below)
|
||||
#
|
||||
# THE BASE DEFAULT IS A NON-COLLIDING SENTINEL, deliberately NOT `default`. A new environment that
|
||||
# forgets this knob must not silently inherit dev's identity — that is precisely the incident. With
|
||||
# this sentinel such a cluster owns nothing, so it can DELETE nothing; it would publish under a
|
||||
# visibly wrong owner id (grep the TXT records) instead of destroying a live environment's DNS.
|
||||
# DEV IS `dev`, NOT `default` (changed 2026-09-04 with the move to dev.home.keithjackson.dev). Until
|
||||
# this migration dev HAD to render `default`, because its LIVE apex records were already registered
|
||||
# in Cloudflare's TXT registry under that owner and renaming would have orphaned every one of them.
|
||||
# THAT CONSTRAINT NO LONGER BINDS: dev is publishing an ENTIRELY NEW record set under
|
||||
# *.dev.home.keithjackson.dev, with no pre-existing registry entries to recognise, so the owner id
|
||||
# is now free to be chosen on its merits. Two reasons to take `dev`:
|
||||
# - `default` is THE VALUE YOU GET BY FORGETTING (it is the external-dns chart default, and both
|
||||
# clusters silently shared it, which IS the incident). An environment whose identity is the
|
||||
# forget-value can never be told apart from a misconfigured newcomer. No environment should
|
||||
# hold it.
|
||||
# - It makes vacating the apex an EXPLICIT, ORDERED, VERIFIABLE step. Dev's old apex records stay
|
||||
# owned by `default`, which no environment now claims, so nothing deletes them implicitly on
|
||||
# some future reconcile — they are retired deliberately, and prod's publish is gated on that
|
||||
# retirement being confirmed in Cloudflare (infra/PROD-PROVISIONING.md §10) rather than on a
|
||||
# race between two reconcile loops.
|
||||
#
|
||||
# PROD STAYS `prod` AND MUST NOT BECOME `default`, even though prod now serves the apex names that
|
||||
# dev's `default`-owned records currently occupy. Giving prod `default` would hand prod ownership of
|
||||
# the LIVE dev cluster's records while dev is still serving them, and under `policy: sync` prod
|
||||
# would delete the ones no prod source claims. That is the 2026-09-04 incident, re-armed, with the
|
||||
# roles swapped. Prod inherits the apex NAMES; it must never inherit the apex OWNER ID.
|
||||
#
|
||||
# THE BASE DEFAULT IS A NON-COLLIDING SENTINEL, deliberately NOT `default` and not any real
|
||||
# environment's id. A new environment that forgets this knob must not silently inherit another's
|
||||
# identity — that is precisely the incident. With this sentinel such a cluster owns nothing, so it
|
||||
# can DELETE nothing; it would publish under a visibly wrong owner id (grep the TXT records)
|
||||
# instead of destroying a live environment's DNS.
|
||||
EXTERNALDNS_TXT_OWNER_ID: "unset-see-cluster-vars"
|
||||
|
||||
ACME_CLUSTER_ISSUER: "letsencrypt-staging" # dev: staging | prod: letsencrypt-prod (ADR-0006)
|
||||
@@ -79,7 +113,7 @@ data:
|
||||
# ingress VIP of the `shared` Gateway. MUST equal the FIRST address of
|
||||
# ${CILIUM_LB_POOL_START}..${CILIUM_LB_POOL_STOP} below — the Gateway
|
||||
# takes the pool's first free IP, and coredns-custom answers
|
||||
# *.home.keithjackson.dev with THIS so pod -> public-hostname traffic
|
||||
# *.${BASE_DOMAIN} with THIS so pod -> public-hostname traffic
|
||||
# hairpins through the Gateway (correct SNI -> wildcard cert).
|
||||
# Base default = the DEV LAN VIP (was 10.73.0.240, an address on the
|
||||
# long-RETIRED libvirt network — corrected 2026-08-31 so an overlay that
|
||||
@@ -96,55 +130,15 @@ data:
|
||||
#
|
||||
# Substitution coverage (verify before adding a knob here): every path that consumes these declares
|
||||
# postBuild.substituteFrom cluster-vars — platform (K3S_API_HOST), platform-config
|
||||
# (CILIUM_LB_POOL_*, and keycloak-db's S3_BACKUP_ENDPOINT + CNPG_BACKUP_PATH: the Cluster lives at
|
||||
# platform/keycloak/postgres-cluster.yaml but is rendered by platform-config, which is the ONLY
|
||||
# tier that lists it), crafty (CRAFTY_L4_VIP), apps-wave-0 (nextcloud-db + semaphore-db:
|
||||
# S3_BACKUP_ENDPOINT, CNPG_BACKUP_PATH), velero (S3_BACKUP_ENDPOINT, S3_BACKUP_REGION,
|
||||
# VELERO_BACKUP_BUCKET, VELERO_BACKUP_PREFIX), apps-vms (UNIFI_L4_VIP). A ${TOKEN} in an
|
||||
# UNSUBSTITUTED path would ship the literal string into the cluster.
|
||||
#
|
||||
# ADDING A KNOB IS SAFE UNDER StrictPostBuildSubstitutions ONLY BECAUSE OF THE ROOT EDGE. An
|
||||
# UNDEFINED variable is a HARD Kustomization failure, so a manifest that references a knob the
|
||||
# live ConfigMap does not yet carry breaks its whole tier. `clusters/base/platform.yaml` gained
|
||||
# `dependsOn: [flux-system]` for exactly this: dependsOn is REVISION-gated, so no substituting
|
||||
# tier can reconcile revision N until the ROOT Kustomization has APPLIED revision N — i.e. until
|
||||
# the new cluster-vars key EXISTS. Every substituting consumer chains transitively through
|
||||
# `platform`, so one edge covers the graph, and a new knob plus its first reference may ship in
|
||||
# ONE commit. (A hand-run `kubectl apply -k clusters/<env>` bypasses that ordering entirely: apply
|
||||
# the overlay BEFORE the tier that references the knob, or the tier fails until it retries.)
|
||||
# (CILIUM_LB_POOL_*, S3_BACKUP_ENDPOINT), crafty (CRAFTY_L4_VIP), apps-wave-0 (S3_BACKUP_ENDPOINT),
|
||||
# velero (S3_BACKUP_ENDPOINT), apps-vms (UNIFI_L4_VIP). A ${TOKEN} in an UNSUBSTITUTED path would
|
||||
# ship the literal string into the cluster.
|
||||
|
||||
# k3s API endpoint Cilium's kube-proxy replacement dials (platform/cilium/helmrelease.yaml).
|
||||
#
|
||||
# *** THIS IS THE CONTROL-PLANE VIP (kube-vip), NOT A NODE ADDRESS. *** Operator decision
|
||||
# 2026-09-06. It was each environment's PRIMARY NODE IP until then, which made the CNI's
|
||||
# control-plane path depend on one machine in a cluster whose entire purpose is HA: verified on
|
||||
# live prod, all three cilium-agents carried KUBERNETES_SERVICE_HOST=192.168.2.10 (k3s-prod-1's
|
||||
# own address) while kube-vip held 192.168.2.9. Losing that node leaves the datapath running off
|
||||
# Cilium's cache while NO service or policy update lands — a silent, unbounded staleness, not an
|
||||
# outage you would notice. A virtual address that follows the control plane is the fix, and it is
|
||||
# what kube-vip was stood up to provide.
|
||||
#
|
||||
# dev -> 192.168.1.49 (= kube_vip_address, group_vars/all.yml; base default, below)
|
||||
# prod -> 192.168.2.9 (= kube_vip_address, group_vars/k3s_prod/vars.yml)
|
||||
#
|
||||
# THE NAME IS NOW SLIGHTLY WRONG AND IS KEPT ANYWAY: renaming the knob would be a second,
|
||||
# unrelated change to every substituting Kustomization at the same time as a live CNI endpoint
|
||||
# move. Read it as "the API host Cilium dials", which is what it always meant.
|
||||
#
|
||||
# [parity] MUST equal the Ansible bootstrap's **k3s_apiserver_endpoint** for the SAME environment
|
||||
# (infra/ansible/inventory/group_vars/all.yml -> roles/cilium/templates/cilium-values.yaml.j2,
|
||||
# already inventory-parameterized). NOT k3s_api_host — that variable still exists and still holds
|
||||
# the FOUNDER's address, because the cold-bootstrap `--server` join needs a machine that is
|
||||
# already up before any VIP exists. The two are deliberately separate; see the TWO API ADDRESSES
|
||||
# block at the top of group_vars/all.yml.
|
||||
#
|
||||
# Wrong value = Cilium cannot reach the API server on adopt and the CNI breaks. Port stays 6443
|
||||
# in both envs, so it is NOT a knob.
|
||||
#
|
||||
# TLS needs nothing: k3s_tls_sans has always carried kube_vip_address alongside every node IP, so
|
||||
# both addresses are already in the serving cert (verified live 2026-09-06 — prod answers through
|
||||
# https://192.168.2.9:6443 under full CA verification). No cert reissue, no k3s restart.
|
||||
K3S_API_HOST: "192.168.1.49"
|
||||
# [parity] MUST equal the Ansible bootstrap's k3s_api_host (infra/ansible/roles/cilium/defaults/
|
||||
# main.yml -> cilium-values.yaml.j2, already inventory-parameterized). Wrong value = Cilium cannot
|
||||
# reach the API server on adopt and the CNI breaks. Port stays 6443 in both envs, so it is NOT a knob.
|
||||
K3S_API_HOST: "192.168.1.50"
|
||||
|
||||
# CiliumLoadBalancerIPPool `default-pool` (platform-config/cilium/l2-ip-pool.yaml): the VIP range
|
||||
# L2-announced on the flat LAN. Must sit OUTSIDE the node IPs and any DHCP scope, and must match the
|
||||
@@ -160,46 +154,12 @@ data:
|
||||
UNIFI_L4_VIP: "192.168.1.54" # UniFi controller VM: inform/STUN/adoption (apps/vms/unifi)
|
||||
CRAFTY_L4_VIP: "192.168.1.55" # Crafty: Minecraft Java (TCP) + Bedrock (UDP) game ports
|
||||
|
||||
# --- Bulk (external NFS) storage host (docs/04 bulk tier) ---------------------------------------
|
||||
# THE address of the Proxmox host whose ZFS datasets back every `class: bulk` volume: the media
|
||||
# library (Storage/plex, 8.33 T — plex, sonarr, radarr) and the legacy Nextcloud archive
|
||||
# (Storage/nextcloud, 270 G). The renderer emits it into each static NFS PersistentVolume's
|
||||
# `spec.nfs.server` (catalog/render.py, BULK_NFS_SERVER); the export PATH is per-dataset and is
|
||||
# declared in the ServiceDefinition, because the path is identical in every environment and only
|
||||
# the ADDRESS is not.
|
||||
#
|
||||
# WHY THIS IS A KNOB AND NOT A LITERAL — the two clusters are on OPPOSITE SIDES of pfSense.
|
||||
# The storage host has exactly one interface, vmbr0 192.168.1.9/24, on pfSense's LAN.
|
||||
# dev nodes are 192.168.1.40/.41/.42 — the SAME L2 segment. They reach it directly, and one
|
||||
# NFS export to them already exists on that host (/mnt/Storage/k8s-volumes in
|
||||
# /etc/exports), so this path is proven, not theoretical. Base default = the dev value,
|
||||
# per the convention of the LAN block above.
|
||||
# prod nodes are 192.168.2.10/.11/.12 — pfSense's WAN. tcp/2049 does NOT traverse that today;
|
||||
# 22 and 8006 answer only because pfSense port-forwards them via 192.168.2.102. Opening
|
||||
# the storage path is a PENDING OPERATOR DECISION with two candidate outcomes that give
|
||||
# DIFFERENT values here: 192.168.2.102 (another port-forward) or 192.168.1.9 (a route).
|
||||
# So prod holds a loud TBD-PROD-* placeholder — see clusters/prod/kustomization.yaml for the
|
||||
# blast radius of shipping it unset, which is NOT the usual "fails to apply".
|
||||
BULK_NFS_SERVER: "192.168.1.9"
|
||||
|
||||
# Off-cluster S3 backup target (R-STATE-03, R-IDP-05) — the on-Proxmox MinIO, plain HTTP, path-style.
|
||||
# Consumed by BOTH backup paths: every CNPG Cluster's barmanObjectStore.endpointURL (keycloak-db,
|
||||
# nextcloud-db, semaphore-db) and Velero's backupStorageLocation s3Url. Full URL incl. scheme+port.
|
||||
# Prod needs its OWN target: a backup that lands on the dev LAN's MinIO is not off-cluster DR —
|
||||
# and, measured 2026-09-05 from a pod INSIDE prod, it CANNOT reach it anyway (prod is on the SOHO
|
||||
# network 192.168.2.0/24, which is pfSense's WAN, while this MinIO sits inside pfSense's LAN:
|
||||
# `curl http://192.168.1.9:9000/minio/health/live` timed out after 15s). Prod uses Cloudflare R2.
|
||||
# Prod needs its OWN target: a backup that lands on the dev LAN's MinIO is not off-cluster DR.
|
||||
S3_BACKUP_ENDPOINT: "http://192.168.1.9:9000"
|
||||
|
||||
# S3 region. Every S3 client REQUIRES one; MinIO ignores whatever it is; Cloudflare R2 demands
|
||||
# exactly `auto` (its `head-bucket` reports the physical hint `WNAM`, which is NOT a usable region
|
||||
# string). This was hardcoded `us-east-1` inside platform/velero/helmrelease.yaml — correct-by-
|
||||
# accident for MinIO and wrong for anything else, which is why it is a knob now. The base default
|
||||
# is dev's LIVE value, deliberately: dev's Velero has been signing requests with `us-east-1`
|
||||
# against that MinIO since bootstrap, and this repo's rule is that base == the known-good dev
|
||||
# value. Prod overrides to `auto`.
|
||||
S3_BACKUP_REGION: "us-east-1"
|
||||
|
||||
# --- storage (ADR-0003) ---
|
||||
LONGHORN_DEFAULT_REPLICA_COUNT: "3" # default StorageClass replica count (3 on 3 nodes)
|
||||
# Longhorn's defaultDataPath (platform/longhorn/helmrelease.yaml): the on-NODE directory every
|
||||
@@ -252,99 +212,6 @@ data:
|
||||
VELERO_BACKUP_SCHEDULE: "0 3 * * *" # cron; overlays may differ
|
||||
VELERO_RETENTION: "168h" # dev short | prod long
|
||||
|
||||
# ==========================================================================================
|
||||
# OBJECT-STORE TENANCY — the three knobs that keep two clusters and two backup tools from
|
||||
# writing into each other's data. THESE ARE SAFETY KNOBS, not preferences.
|
||||
# ==========================================================================================
|
||||
# Nothing about the object layout was per-environment before this: Velero's bucket was the
|
||||
# hardcoded literal `velero` with no prefix, and every CNPG Cluster used the hardcoded
|
||||
# `s3://cnpg` with barman's `serverName` defaulting to the CLUSTER NAME — and BOTH environments
|
||||
# have a keycloak-db, a nextcloud-db and a semaphore-db. Point two clusters at one bucket under
|
||||
# that layout and:
|
||||
# (a) each cluster's Velero ADOPTS the other's backups and TTL-DELETES them on retention; and
|
||||
# (b) two different Postgres servers stream WAL into ONE Barman prefix, which does not merely
|
||||
# mix files — it CORRUPTS THE CHAIN and destroys PITR for both. That is the worse half:
|
||||
# dev's DR restore is PROVEN (R-STATE-03) and this would silently un-prove it.
|
||||
#
|
||||
# WHY THERE IS NO SINGLE `S3_BACKUP_BUCKET`. The two consumers take differently-SHAPED values:
|
||||
# the Velero chart wants `bucket` and `prefix` as separate fields, while CNPG's
|
||||
# barmanObjectStore.destinationPath is ONE full s3:// URL whose first path segment IS the bucket.
|
||||
# CNPG's bucket therefore already lives inside CNPG_BACKUP_PATH; adding a second knob that had to
|
||||
# hold the same bucket name a second time would be a rotation waiting to drift — the exact class
|
||||
# of duplication this repo has been bitten by. Three knobs, each shaped for its own consumer, and
|
||||
# no two of them can contradict each other.
|
||||
#
|
||||
# BASE = DEV'S LIVE LAYOUT, BYTE-FOR-BYTE, AND MUST STAY THAT WAY. Dev's existing backups are at
|
||||
# `s3://velero/` (bucket root, no prefix) and `s3://cnpg/<clusterName>/` on that MinIO. Giving dev
|
||||
# a prefix, or renaming its buckets, would ORPHAN the very backups its proven restore depends on —
|
||||
# the same reason dev's EXTERNALDNS_TXT_OWNER_ID stays `default` and its LONGHORN_DATA_PATH stays
|
||||
# /var/lib/longhorn. Verified: with an EMPTY prefix the Velero chart omits `objectStorage.prefix`
|
||||
# from the rendered BackupStorageLocation entirely (helm template, chart 12.1.0), so dev's live
|
||||
# BSL is unchanged by this parameterization rather than merely equivalent to it.
|
||||
#
|
||||
# PROD, which has no history to preserve, gets ONE bucket with TWO disjoint top-level prefixes:
|
||||
#
|
||||
# s3://k3s-prod-backup/velero/… Velero (kopia repo + backup metadata)
|
||||
# s3://k3s-prod-backup/barman/<clusterName>/… CNPG WAL + base backups
|
||||
#
|
||||
# The operator created exactly one R2 bucket, so velero-vs-barman separation has to come from the
|
||||
# prefix rather than the bucket. The BUCKET ROOT IS DELIBERATELY LEFT EMPTY: if a prefix ever
|
||||
# renders wrong, its objects land somewhere obviously bogus instead of silently colliding with a
|
||||
# real tenant's data. Dev↔prod isolation is doubly independent — different endpoint AND different
|
||||
# bucket — so neither cluster can reach the other's objects even with a mis-substituted prefix.
|
||||
VELERO_BACKUP_BUCKET: "velero" # dev: the `velero` MinIO bucket | prod: k3s-prod-backup
|
||||
VELERO_BACKUP_PREFIX: "" # dev: bucket root (live layout) | prod: "velero"
|
||||
CNPG_BACKUP_PATH: "s3://cnpg" # dev: the `cnpg` MinIO bucket root | prod: s3://k3s-prod-backup/barman
|
||||
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# WHICH CREDENTIAL SECRET THE BACKUP CONSUMERS READ (added 2026-09-07 — the per-env credential split)
|
||||
# ------------------------------------------------------------------------------------------------
|
||||
# The three knobs above say WHERE the backups go. This one says WHICH KEY signs the requests, and
|
||||
# it exists because the endpoint moved per-environment while the credential did not.
|
||||
#
|
||||
# THE BUG IT FIXES. The four backup-credential Secrets — velero-cloud.sops.yaml and the three
|
||||
# <db>-backup-credentials.sops.yaml — are SHARED BASE manifests: platform/ and apps/ are rendered
|
||||
# by the same Flux tiers on both clusters, and cluster-vars substitution is the ONLY thing that
|
||||
# differs (R-ENV-02). So writing prod's R2 token into those four files did not GIVE prod a
|
||||
# credential, it REPLACED DEV'S: dev's Velero and all three dev CNPG Clusters would have started
|
||||
# signing MinIO requests with an R2 key id. For Velero that is a broken BSL; for CNPG it is worse,
|
||||
# because `archive_mode=on` means Postgres REFUSES TO RECYCLE a WAL segment it could not archive,
|
||||
# so a 403 loop is a monotonically filling pgdata PVC ending in a PANIC — the same failure prod
|
||||
# spent its whole bring-up in with a placeholder endpoint. One shared Secret also means rotating
|
||||
# either environment's key silently breaks the other; that is the structural reason for the split,
|
||||
# independent of dev's eventual decommission.
|
||||
#
|
||||
# THE MECHANISM. Each credential is now TWO Secret objects, both applied to BOTH clusters (they are
|
||||
# base resources; nothing here forks a manifest per environment). The consumer names one:
|
||||
#
|
||||
# velero: credentials.existingSecret: velero-cloud-credentials${BACKUP_CREDENTIALS_SUFFIX}
|
||||
# CNPG: s3Credentials.*.name: <db>-backup-credentials${BACKUP_CREDENTIALS_SUFFIX}
|
||||
#
|
||||
# ONE knob, not four, ON PURPOSE. Four independent secret-name knobs could be updated three-
|
||||
# quarters of the way, leaving one consumer pointed at the OTHER environment's key with nothing
|
||||
# rendering wrong — and StrictPostBuildSubstitutions cannot catch that, because it only fails on an
|
||||
# UNDEFINED variable, never on a defined-but-stale one. With a single suffix the four move together
|
||||
# or not at all. The cost is that the rendered Secret name is not greppable from the consumer; the
|
||||
# names are spelled out in full in each kustomization.yaml beside the resources.
|
||||
#
|
||||
# THE BASE DEFAULT IS THE EMPTY STRING, AND THAT IS LOAD-BEARING. It renders the UNSUFFIXED names,
|
||||
# which are the objects that already exist and that the live dev cluster is running. A default
|
||||
# naming a Secret that does not exist would be worse than the shared-Secret state it replaces:
|
||||
# Velero's pod would sit in ContainerCreating forever on a missing `existingSecret` volume, and a
|
||||
# CNPG Cluster whose s3Credentials Secret is absent stops archiving WAL — i.e. the disk-full clock
|
||||
# again — so an unresolvable default breaks the referencing cluster HARDER than a wrong key does.
|
||||
# Empty is also the value dev must carry, so base and dev agree, exactly like VELERO_BACKUP_PREFIX.
|
||||
#
|
||||
# An EMPTY VALUE IS SAFE UNDER StrictPostBuildSubstitutions: strict mode fails on a variable that is
|
||||
# not DEFINED, not on one defined as "". VELERO_BACKUP_PREFIX above has shipped empty since the
|
||||
# gate was enabled and proves it. Both overlays state this knob explicitly — dev "" and prod
|
||||
# "-prod" — so neither inherits it silently.
|
||||
#
|
||||
# ADDING A NEW BACKUP CREDENTIAL: create BOTH halves (`x-credentials.sops.yaml` and
|
||||
# `x-credentials-prod.sops.yaml`), list both in the kustomization, and reference
|
||||
# `x-credentials${BACKUP_CREDENTIALS_SUFFIX}`. Never a single shared Secret again.
|
||||
BACKUP_CREDENTIALS_SUFFIX: "" # dev: "" (MinIO creds) | prod: "-prod" (R2 creds)
|
||||
|
||||
# --- placement ---
|
||||
PLATFORM_NODE_SELECTOR_KEY: "kubernetes.io/os" # base no-op selector; prod may pin roles
|
||||
PLATFORM_NODE_SELECTOR_VALUE: "linux"
|
||||
@@ -14,15 +14,6 @@ resources:
|
||||
- cilium.yaml # Flux Kustomization -> ./platform/cilium (dependsOn flux-system, wait:true)
|
||||
- platform.yaml # Flux Kustomization -> ./platform (CONTROLLERS: HelmReleases / CRDs)
|
||||
- platform-config.yaml # Flux Kustomization -> ./platform-config (CONFIGS: CRs; dependsOn platform)
|
||||
# SECOND-ORDER CDI configs — CRs whose CRD is created by a controller AT RUNTIME, not by any
|
||||
# manifest in git. cdi-operator.yaml registers only `cdis.cdi.kubevirt.io`; `storageprofiles` (and
|
||||
# datavolumes, datasources, …) appear only after cdi-operator reconciles the `CDI` CR that
|
||||
# platform-config applies. Because Flux dry-runs every object in a pass before applying any of them,
|
||||
# keeping the StorageProfiles in platform-config was a TRUE DEADLOCK (unknown kind -> pass aborts ->
|
||||
# CR never applied -> CRD never created -> tier never converges), invisible on dev and fatal on any
|
||||
# cold cluster — observed on prod 2026-09-05. This tier is the durable fix; `standup.sh crds` stays
|
||||
# as the belt-and-braces repair for clusters already wedged. See clusters/base/platform-config-cdi.yaml.
|
||||
- platform-config-cdi.yaml # Flux Kustomization -> ./platform-config-cdi (dependsOn platform-config)
|
||||
# App tier, split into dependency-ordered WAVES (was the single ./apps `apps` Kustomization).
|
||||
# Each apps-wave-<N> dependsOn wave N-1 (wave 0 dependsOn platform-config) and health-gates
|
||||
# (wait:true) ONLY its own wave, so a crashloop in a later wave can't stall an earlier one.
|
||||
|
||||
@@ -1,135 +0,0 @@
|
||||
---
|
||||
# Flux Kustomization: SECOND-ORDER CDI CONFIGS — reconciles ./platform-config-cdi.
|
||||
# Serves R-GITOPS-01 (all desired state in git, reconciled by Flux) and R-ENV-02.
|
||||
#
|
||||
# WHY THIS TIER EXISTS — the durable fix for the 2026-09-05 prod CDI DEADLOCK
|
||||
#
|
||||
# `platform-config` applied the `CDI` CR and the CDI `StorageProfile`s in the SAME pass.
|
||||
# kustomize-controller server-side DRY-RUNS EVERY object before applying ANY of them, and
|
||||
# `storageprofiles.cdi.kubevirt.io` does not exist until cdi-operator has reconciled that `CDI` CR.
|
||||
# So the unknown kind aborted the whole tier → the CR was never applied → the CRD was never created →
|
||||
# platform-config could NEVER converge and every tier below it stayed blocked. Not a race: a
|
||||
# deadlock. cdi-operator was Available for 12h on prod with no CR. Invisible on dev (its CDI CRDs
|
||||
# have existed for months), FATAL on any cold cluster. See DECISIONS-LOG 2026-09-05.
|
||||
#
|
||||
# `bootstrap/standup.sh crds` pre-applies the `CDI` CR to break it by hand. That is a WORKAROUND and
|
||||
# stays as belt-and-braces for clusters already wedged; THIS Kustomization is the fix that makes the
|
||||
# graph order itself, so a cold cluster needs no human in the loop.
|
||||
#
|
||||
# THE ORDERING GUARANTEE, precisely: `dependsOn` is REVISION-gated, not merely Ready-gated (verified
|
||||
# against kustomize-controller v1.4.0 `kustomization_controller.go :: checkDependencies`, our
|
||||
# version): sharing a GitRepository with the dependency, this Kustomization cannot reconcile revision
|
||||
# N until `platform-config` has APPLIED revision N — and, because platform-config runs `wait: true`,
|
||||
# not until the `CDI` CR it applied is HEALTHY. cdi-operator creates the remaining CDI CRDs as part of
|
||||
# reaching that state, so by the time this tier's first dry-run runs, `StorageProfile` is a known
|
||||
# kind. One edge, no pre-apply.
|
||||
#
|
||||
# WHY NOT just move the CDI CR earlier, into `platform`? Because `platform` is the tier the CNI gates
|
||||
# and everything else waits on: putting CDI's health on that critical path is the mistake the
|
||||
# oauth2-proxy/headlamp splits were made to undo. The CR is correctly a platform-config CR; it is the
|
||||
# StorageProfiles that were in the wrong pass.
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: platform-config-cdi
|
||||
namespace: flux-system
|
||||
spec:
|
||||
interval: 10m
|
||||
# 2m (short, deliberately). kustomize-controller caches API discovery, so on a COLD cluster the
|
||||
# very first attempt here can still fail `no matches for kind "StorageProfile"` even though the CRD
|
||||
# now exists — a stale RESTMapper, not a missing CRD (the failure mode `standup.sh crds` warns
|
||||
# about). The controller resets the mapper and the retry succeeds, so the only cost of that stale
|
||||
# cache is one retryInterval. Keep it short.
|
||||
retryInterval: 2m
|
||||
# 2m is ample: this tier applies two tiny cluster-scoped CRs. There is no workload to roll out, no
|
||||
# image to pull and no controller to become Available, so a slow apply here means a real error, not
|
||||
# a slow rollout — failing fast surfaces it instead of burning 5m per attempt.
|
||||
timeout: 2m
|
||||
# TRUE, and this tier is the SOLE declarer of both StorageProfiles (checked repo-wide), so prune is
|
||||
# component-scoped: the stale set can only ever be a StorageProfile this file stopped rendering.
|
||||
# It is NOT the platform/cilium case — nothing here declares a shared Namespace, or any object a
|
||||
# second path also declares. See the transition note at the bottom of this file.
|
||||
prune: true
|
||||
# TRUE, but understand exactly what it buys and what it does NOT.
|
||||
#
|
||||
# CANNOT STALL: `StorageProfile`'s status type (CDI v1beta1 StorageProfileStatus: storageClass,
|
||||
# provisioner, claimPropertySets, cloneStrategy, dataImportCronSourceFormat) carries NO
|
||||
# `conditions` and NO `observedGeneration`, so kstatus's generic path reports it Current the moment
|
||||
# it is applied. `wait: true` therefore adds a Ready condition without adding a way to hang — which
|
||||
# is why it is safe to let `apps-vms` gate on it (contrast the headlamp gating stall).
|
||||
# If a resource that CAN be unhealthy is ever added to ./platform-config-cdi, re-derive this:
|
||||
# `apps-vms` waits on it.
|
||||
#
|
||||
# WHAT IT DOES NOT PROVE (honest gap): CDI reads the profile's **status**, not its spec, when it
|
||||
# provisions a claim (status.claimPropertySets / status.cloneStrategy), and CDI's
|
||||
# storageprofile-controller computes that status from our spec override plus inference. So Ready
|
||||
# here means "the spec override is accepted by the API server", not "CDI has folded it into
|
||||
# status". That reconcile is a watch-triggered round-trip of seconds, and the dependency requeue
|
||||
# below it is ~30s, so a DataVolume cannot realistically beat it — but it is a timing argument, not
|
||||
# a guarantee, and no `wait` setting can turn it into one (there is no condition to wait on).
|
||||
wait: true
|
||||
dependsOn:
|
||||
# platform-config applies the `CDI` CR and (wait:true) waits for it to be healthy — which is what
|
||||
# causes cdi-operator to create `storageprofiles.cdi.kubevirt.io`. Revision-gated, so this is an
|
||||
# ordering guarantee and not a hope. Transitively this also chains platform -> cilium ->
|
||||
# flux-system, so `cluster-vars` at revision N is applied before this renders revision N.
|
||||
- name: platform-config
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
path: ./platform-config-cdi
|
||||
# NO SOPS `decryption` stanza: ../platform/kubevirt/cdi-storageprofiles.yaml is two plain CRs with
|
||||
# no Secret and no *.sops.yaml, so neither overlay needs to patch one in (unlike
|
||||
# oauth2-proxy/headlamp/crafty/velero, which each carry an encrypted Secret). Nothing to add to
|
||||
# clusters/dev or clusters/prod for this tier.
|
||||
#
|
||||
# substituteFrom cluster-vars: the manifest currently contains NO `${TOKEN}`, so this stanza
|
||||
# substitutes nothing today. It is declared anyway for two concrete reasons, not for symmetry:
|
||||
# 1. COVERAGE. conformance/tests/l0-static/flux-postbuild-substitution/check.sh only audits paths
|
||||
# whose Kustomization declares substituteFrom. Omitting it would move
|
||||
# cdi-storageprofiles.yaml OUT of the audited set — a blind spot where a future `${TOKEN}`
|
||||
# could ship as a literal string, precisely the class of bug that check exists to stop.
|
||||
# 2. FAIL-CLOSED. kustomize-controller runs --feature-gates=StrictPostBuildSubstitutions=true, so
|
||||
# with this stanza an undefined token FAILS this Kustomization loudly instead of rendering
|
||||
# empty. Without the stanza there is no substitution at all and the literal ships silently.
|
||||
postBuild:
|
||||
substituteFrom:
|
||||
- kind: ConfigMap
|
||||
name: cluster-vars
|
||||
# ── FIRST RECONCILE AFTER MERGE: a TRANSIENT DELETE-THEN-RECREATE. READ BEFORE MERGING. ──────────
|
||||
# Moving cdi-storageprofiles.yaml out of ./platform-config drops both StorageProfiles from THAT
|
||||
# Kustomization's inventory, and platform-config has `prune: true` — so its next apply DELETES
|
||||
# StorageProfile/longhorn and StorageProfile/longhorn-vm, and this Kustomization recreates them on
|
||||
# the following reconcile (revision-gated: strictly after platform-config's apply, so there is no
|
||||
# dual-ownership fight — the objects are pruned before they are re-applied under this tier's
|
||||
# `kustomize.toolkit.fluxcd.io/name=platform-config-cdi` label).
|
||||
#
|
||||
# Why that is acceptable here — and it is NOT the platform/cilium case:
|
||||
# * Neither object is the sole (or any) declarer of a shared object. No Namespace, no ConfigMap,
|
||||
# nothing else in the repo declares a StorageProfile. The stale set is exactly two
|
||||
# cluster-scoped CDI CRs.
|
||||
# * The objects do not actually go missing: CDI's storageprofile-controller creates a
|
||||
# StorageProfile per StorageClass and recreates a deleted one, with an EMPTY `.spec` and
|
||||
# inferred `.status`. So the transient is "our .spec overrides are absent", not "the object is
|
||||
# absent".
|
||||
# * A StorageProfile is read at DataVolume/PVC PROVISIONING and CLONE time only. Existing PVCs,
|
||||
# existing DataVolumes and running VMs do not consult it again, so nothing already provisioned
|
||||
# is disturbed and no data path is touched.
|
||||
# * The only exposure is a DataVolume/clone STARTED inside the window: it would clone by `copy`
|
||||
# instead of `snapshot`, and on `longhorn-vm` be stamped RWO/Filesystem instead of RWX/Block —
|
||||
# permanently non-live-migratable for that disk. Flux cannot cause that, because `apps-vms` now
|
||||
# dependsOn THIS tier (clusters/base/apps-vms.yaml), so no Flux-managed DataVolume can be
|
||||
# created while the overrides are absent. A human running `virtctl`/`kubectl` in that minute
|
||||
# could.
|
||||
#
|
||||
# PRECONDITION THE OPERATOR MUST CHECK, because it is what bounds the window: the recreate cannot
|
||||
# happen until `platform-config` is Ready, and platform-config runs `wait: true` over the whole
|
||||
# config tier (ClusterIssuers, wildcard Certificate, Gateway, Keycloak, CNPG, KubeVirt, CDI…). On a
|
||||
# healthy cluster that is seconds and the window is ~30-60s (kustomize-controller's
|
||||
# --requeue-dependency default is 30s). If platform-config is NOT Ready when this merges, the
|
||||
# StorageProfiles are pruned and stay at CDI defaults until it recovers. So:
|
||||
# `flux get kustomization platform-config` must be Ready BEFORE merging.
|
||||
# If it is not, do the two-step the cilium move used instead: merge with `platform-config.prune`
|
||||
# temporarily set to `false` (the objects are then dropped from its inventory but NOT deleted, and
|
||||
# this tier ADOPTS them with zero window), verify the ownership label transferred, then restore
|
||||
# `prune: true` in a second commit.
|
||||
+85
-100
@@ -5,6 +5,20 @@
|
||||
# reconciliation graph and patches the shared cluster-vars ConfigMap with dev-scoped values (R-ENV-02).
|
||||
# It also adds a self-managing ROOT Kustomization (sync.yaml) so the whole graph reconciles from git
|
||||
# on every push — no more manual `kubectl apply -k` for cluster-vars/overlay changes (F-OPS1).
|
||||
#
|
||||
# ##############################################################################################
|
||||
# ### *** DO NOT MERGE THIS FILE TO `main` UNATTENDED (as of 2026-09-04) *** ###
|
||||
# ### ###
|
||||
# ### This overlay now carries dev's DNS MIGRATION target: BASE_DOMAIN moves from the apex ###
|
||||
# ### `home.keithjackson.dev` to `dev.home.keithjackson.dev`, and the GitRepository url ###
|
||||
# ### moves with it. sync.yaml reconciles ./clusters/dev from `main` EVERY 10 MINUTES, so a ###
|
||||
# ### merge to main IS the cutover — it executes on the live dev cluster with nobody present. ###
|
||||
# ### Dev's Gitea is both its GitOps source and the push target: if the new host is not up ###
|
||||
# ### with a Ready cert first, dev stops reconciling and pushing cannot repair it. ###
|
||||
# ### ###
|
||||
# ### Run the ordered runbook: infra/PROD-PROVISIONING.md §10 (steps D1-D8), operator ###
|
||||
# ### present. Merging is step D3, not step one. ###
|
||||
# ##############################################################################################
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
@@ -106,11 +120,8 @@ patches:
|
||||
secretRef:
|
||||
name: sops-age
|
||||
# velero (backup controller) has its own post-platform Flux Kustomization (clusters/base/backup.yaml).
|
||||
# It carries BOTH halves of the per-env `cloud` INI credential — velero-cloud.sops.yaml (dev's scoped
|
||||
# MinIO key) and velero-cloud-prod.sops.yaml (prod's R2 token) — so it needs its own SOPS decryption
|
||||
# stanza (same as headlamp/oauth2-proxy/crafty). Which one the Deployment mounts is chosen by
|
||||
# BACKUP_CREDENTIALS_SUFFIX in the cluster-vars patch below, not here. The `sops-age` Secret is
|
||||
# out-of-band (name only).
|
||||
# It carries velero-cloud.sops.yaml (the scoped MinIO `cloud` credential), so it needs its own SOPS
|
||||
# decryption stanza (same as headlamp/oauth2-proxy/crafty). The `sops-age` Secret is out-of-band (name only).
|
||||
- patch: |-
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
@@ -134,7 +145,24 @@ patches:
|
||||
# via coredns -> Gateway VIP, and the LE-prod wildcard cert is publicly trusted so NO
|
||||
# secretRef/caCert is needed. Recovery: restart bootstrap Gitea (192.168.2.103:3000) and
|
||||
# revert this url (docs/05).
|
||||
url: https://gitea.home.keithjackson.dev/Home-Network/home-network-k3s-impl.git
|
||||
#
|
||||
# *** THE STRAND. THIS URL IS THE MOST DANGEROUS LINE IN THE REPOSITORY. ***
|
||||
# It is BOTH dev's GitOps source AND the operator's push target, and it is NOT tokenized —
|
||||
# it cannot be: the `flux-system` root Kustomization that renders ./clusters/dev declares NO
|
||||
# postBuild.substituteFrom, so `gitea.${BASE_DOMAIN}` here would ship the LITERAL token and
|
||||
# break the source outright (the L0 flux-postbuild-substitution check asserts exactly this).
|
||||
# So it is a hand-maintained literal that must be moved in lockstep with BASE_DOMAIN below.
|
||||
#
|
||||
# It has been moved to dev's NEW suffix as part of the 2026-09-04 migration authoring. That
|
||||
# makes this file the TARGET state, NOT a safe unattended merge: dev's root Kustomization
|
||||
# (sync.yaml) reconciles ./clusters/dev from `main` every 10m, so MERGING THIS TO main WHILE
|
||||
# DEV IS LIVE MIGRATES DEV UNATTENDED. If gitea.dev.home.keithjackson.dev does not already
|
||||
# resolve and serve a valid cert at that instant, dev stops reconciling and CANNOT BE FIXED
|
||||
# BY PUSHING — the broken thing is the path Flux uses to read pushes.
|
||||
# ORDERING THAT AVOIDS IT (infra/PROD-PROVISIONING.md §10, steps D1-D3): suspend dev's root
|
||||
# Kustomization or park this url on the bootstrap Gitea (http://192.168.2.132:3000/...) for
|
||||
# the migration window, and only restore the in-cluster url after the new host is proven.
|
||||
url: https://gitea.dev.home.keithjackson.dev/Home-Network/home-network-k3s-impl.git
|
||||
ref:
|
||||
branch: main
|
||||
# --- dev-scoped cluster-vars (R-ENV-02) ---
|
||||
@@ -151,33 +179,60 @@ patches:
|
||||
- op: replace
|
||||
path: /data/ACME_CLUSTER_ISSUER
|
||||
value: "letsencrypt-prod"
|
||||
# --- DNS identity (R-DNS-01/02/03) -----------------------------------------------------
|
||||
# DEV STAYS ON THE APEX. Restated explicitly even though it equals the base default, exactly
|
||||
# like the LAN block and LONGHORN_DATA_PATH below: this is what the LIVE cluster is running.
|
||||
# Every one of dev's public DNS records, its `*.home.keithjackson.dev` wildcard cert, its
|
||||
# Keycloak issuer, every OIDC client redirect URI, kubectl SSO, and gitea.home.keithjackson.dev
|
||||
# (which IS this repo's GitOps source) are all on this exact suffix RIGHT NOW. Prod moved to
|
||||
# prod.home.keithjackson.dev instead (2026-09-04) — that alone ended the collision, so dev did
|
||||
# not have to move. Relocating dev to dev.home.keithjackson.dev is phase 2 and is a migration,
|
||||
# not a variable change: infra/PROD-PROVISIONING.md §10.
|
||||
# ==========================================================================================
|
||||
# DEV DNS IDENTITY (R-DNS-01/02/03) — DEV MOVES OFF THE APEX. **NOT YET APPLIED.**
|
||||
# ==========================================================================================
|
||||
# OPERATOR DECISION 2026-09-04, superseding the interim split committed earlier the same day:
|
||||
#
|
||||
# dev -> dev.home.keithjackson.dev (this knob; = the base default, restated)
|
||||
# prod -> home.keithjackson.dev (THE APEX — prod IS the apex, no `prod.` label,
|
||||
# and therefore no apex CNAME layer at all)
|
||||
#
|
||||
# *** THIS VALUE IS AHEAD OF THE LIVE CLUSTER. *** The dev cluster is serving the APEX right
|
||||
# now. Merging this file does not perform the migration — it DEFINES the target. The cutover is
|
||||
# an ORDERED, OPERATOR-PRESENT runbook: infra/PROD-PROVISIONING.md §10.
|
||||
#
|
||||
# WHY IT CANNOT BE A PLAIN VARIABLE FLIP — THE STRAND. `spec.url` of the GitRepository ~20
|
||||
# lines above is https://gitea.home.keithjackson.dev/... . Dev's Gitea is BOTH dev's GitOps
|
||||
# source AND the operator's push target. The moment BASE_DOMAIN moves, dev's Gitea HTTPRoute,
|
||||
# DNS record and wildcard cert all move to gitea.dev.home.keithjackson.dev. If that host does
|
||||
# not ALREADY resolve with a valid, Ready cert, dev stops reconciling — and it CANNOT BE FIXED
|
||||
# BY PUSHING, because the thing that is broken is the path Flux uses to read pushes. Recovery
|
||||
# is then hands-on (point the source at the bootstrap Gitea on 192.168.2.132:3000, or repair
|
||||
# in-cluster by hand). §10 states the ordering that avoids it: publish and verify the new host
|
||||
# FIRST, or park the source on the bootstrap Gitea for the window.
|
||||
#
|
||||
# ALSO MOVED BY THIS ONE LINE, all on a live cluster: the `*.` wildcard Certificate (a fresh
|
||||
# ACME order), Keycloak's issuer URL and ~12 OIDC clients' redirect URIs, kubectl SSO (a k3s
|
||||
# server flag => a rolling restart + every user re-issuing their kubeconfig), coredns-custom's
|
||||
# split-horizon zone, Homepage's links, and 6 L2 chainsaw tests that hardcode the apex.
|
||||
- op: replace
|
||||
path: /data/BASE_DOMAIN
|
||||
value: "home.keithjackson.dev"
|
||||
# ExternalDNS scope. MUST stay the zone apex: dev serves the home.keithjackson.dev APEX itself,
|
||||
# and the Cloudflare provider selects zones with this value (a strict-subdomain filter dropped
|
||||
# the parent zone — DECISIONS-LOG 2026-08). Same value the retired DNS_ZONE knob held.
|
||||
value: "dev.home.keithjackson.dev"
|
||||
# ExternalDNS scope. MUST stay the Cloudflare ZONE — NOT dev.home.keithjackson.dev. FINDING
|
||||
# F-DNS1 (DECISIONS-LOG 2026-08-17): the Cloudflare provider uses this for ZONE SELECTION, so a
|
||||
# strict-subdomain filter matches no zone and publishes NOTHING ("no hosted zone matching
|
||||
# record DNS Name"). Dev moving to a subdomain does NOT make narrowing safe — it makes it
|
||||
# newly tempting, which is why this comment is here. Same value the retired DNS_ZONE knob held.
|
||||
- op: replace
|
||||
path: /data/EXTERNALDNS_DOMAIN_FILTER
|
||||
value: "keithjackson.dev"
|
||||
# DO NOT CHANGE THIS VALUE. Dev's LIVE public records are registered in Cloudflare's TXT
|
||||
# registry under owner `default` (the external-dns chart default, which is what both clusters
|
||||
# silently shared until the 2026-09-04 incident). Dev's external-dns identifies its own records
|
||||
# BY this string: any other value and it stops recognising every record it manages, then
|
||||
# re-creates duplicates beside the orphaned originals. Prod is `prod` — distinct, which is the
|
||||
# whole point. The base default is a non-colliding sentinel, so this line is load-bearing.
|
||||
# OWNER ID: `dev` (was `default` until 2026-09-04). The old value was FORCED — dev's live apex
|
||||
# records were already in Cloudflare's TXT registry under owner `default`, so renaming would
|
||||
# have orphaned every one of them. THE MIGRATION DISSOLVES THAT CONSTRAINT: on the new suffix
|
||||
# dev publishes a brand-new record set with no prior registry entries to recognise, so the id
|
||||
# is chosen on its merits, and `default` has none — it is the value external-dns uses when you
|
||||
# FORGET the flag, which is exactly how both clusters came to share it. Consequences, both
|
||||
# deliberate:
|
||||
# - dev's OLD apex records remain owned by `default`, which no environment now claims, so
|
||||
# nothing deletes them implicitly. Retiring them is an explicit §10 step, and prod's first
|
||||
# publish is gated on that retirement being CONFIRMED — not on two reconcile loops racing.
|
||||
# - prod stays `prod` and must NEVER be given `default`, or prod would own the live dev
|
||||
# cluster's apex records and sync-delete them. Prod inherits the apex NAMES, not its OWNER.
|
||||
# dev != prod is the load-bearing invariant; the base default stays a non-colliding sentinel.
|
||||
- op: replace
|
||||
path: /data/EXTERNALDNS_TXT_OWNER_ID
|
||||
value: "default"
|
||||
value: "dev"
|
||||
# --- dev LAN addressing (192.168.1.0/24 on the Proxmox LAN) -----------------------------
|
||||
# EVERY knob in cluster-vars' "LAN addressing" block is restated here EXPLICITLY, even where it
|
||||
# equals the base default. These values are what the LIVE dev cluster is running right now, so
|
||||
@@ -186,29 +241,15 @@ patches:
|
||||
#
|
||||
# Split-horizon DNS target: the Cilium-L2 VIP of the `shared` Gateway on the Proxmox LAN
|
||||
# (= CILIUM_LB_POOL_START, the pool's first address). coredns-custom answers
|
||||
# *.home.keithjackson.dev with this so in-cluster OIDC clients reach Keycloak via the Gateway.
|
||||
# *.dev.home.keithjackson.dev with this so in-cluster OIDC clients reach Keycloak via the Gateway.
|
||||
- op: replace
|
||||
path: /data/GATEWAY_VIP
|
||||
value: "192.168.1.53"
|
||||
# k3s API endpoint Cilium's kube-proxy replacement dials — DEV'S CONTROL-PLANE VIP,
|
||||
# the address kube-vip floats, NOT the primary node .50.
|
||||
# [parity] equals the Ansible inventory's k3s_apiserver_endpoint for dev
|
||||
# (= kube_vip_address, group_vars/all.yml). See clusters/base/cluster-vars.yaml for why.
|
||||
#
|
||||
# *** THE ONE EXCEPTION TO "NEVER LET THIS BLOCK DRIFT FROM THE LIVE CLUSTER" ***
|
||||
# Every other value in this block is what dev is running right now. This one is not:
|
||||
# the live dev cilium-agents dial 192.168.1.50 (k3s-1). Changing it here is a
|
||||
# DELIBERATE, OPERATOR-APPROVED move (2026-09-06) and reconciling it ROLLS THE CILIUM
|
||||
# AGENT DAEMONSET on dev — one node at a time (updateStrategy.maxUnavailable: 1, kept
|
||||
# in parity for exactly this reason). Do not push this to dev-live casually, and not
|
||||
# while dev's DNS migration is mid-flight; the L2 datapath survives an agent restart
|
||||
# but a wrong value here is the 2026-09-01 incident. .49 is in dev's serving-cert SANs
|
||||
# already (k3s_tls_sans) and dev's own kubeconfig validates against it via
|
||||
# tls-server-name, so TLS is not the risk — the risk is reachability, and .49 is on the
|
||||
# same LAN as the nodes and ARP-announced by kube-vip on every server.
|
||||
# k3s API endpoint Cilium's kube-proxy replacement dials — the primary control-plane node
|
||||
# (k3s-1). [parity] equals the Ansible inventory's k3s_api_host for dev.
|
||||
- op: replace
|
||||
path: /data/K3S_API_HOST
|
||||
value: "192.168.1.49"
|
||||
value: "192.168.1.50"
|
||||
# Cilium LB IP pool = the tofu reservation cilium_l2_vip_range .53-.60, outside the node IPs
|
||||
# (.50/.51/.52) and outside the DHCP scope. .53 Gateway, .54 unifi, .55 crafty; .56-.60 free.
|
||||
- op: replace
|
||||
@@ -225,67 +266,11 @@ patches:
|
||||
- op: replace
|
||||
path: /data/CRAFTY_L4_VIP
|
||||
value: "192.168.1.55"
|
||||
# Bulk (external NFS) storage host: the Proxmox host itself, on the SAME LAN as dev's nodes
|
||||
# (192.168.1.40/.41/.42), which already hold an NFS export from it (/mnt/Storage/k8s-volumes).
|
||||
# Restated verbatim rather than inherited, per this overlay's "supply every knob" convention.
|
||||
# Dev is therefore the environment in which the bulk manifests can actually be VALIDATED
|
||||
# end-to-end before prod's network path exists — provided the additive `zfs set sharenfs=`
|
||||
# change described in docs/BULK-NFS-STORAGE.md is applied for the dev node addresses.
|
||||
- op: replace
|
||||
path: /data/BULK_NFS_SERVER
|
||||
value: "192.168.1.9"
|
||||
# Off-cluster backup target: the on-Proxmox MinIO (plain HTTP, path-style) that every CNPG
|
||||
# barmanObjectStore and Velero's backupStorageLocation write to.
|
||||
#
|
||||
# DEV IS DELIBERATELY STAYING ON THIS MinIO while prod goes to Cloudflare R2 (2026-09-07).
|
||||
# Prod could not use it at all — it cannot even route to it — but dev's move is a separate,
|
||||
# operator-gated decision, because R2 would start dev on an EMPTY prefix: Velero would write
|
||||
# a fresh kopia repository, CNPG would take new base backups, and the 30-day Barman window
|
||||
# would rebuild from zero. Dev's DR restore is the one PROVEN recovery path in this repo
|
||||
# (R-STATE-03); resetting its PITR window is not a side effect to take while fixing prod.
|
||||
- op: replace
|
||||
path: /data/S3_BACKUP_ENDPOINT
|
||||
value: "http://192.168.1.9:9000"
|
||||
# DO NOT CHANGE THESE FOUR. They are dev's LIVE object layout, restated verbatim for the same
|
||||
# reason as LONGHORN_DATA_PATH and EXTERNALDNS_TXT_OWNER_ID below/above: they equal the base
|
||||
# defaults today, and a silent drift in base would ORPHAN dev's existing backups — the ones
|
||||
# its proven restore depends on — without any manifest looking wrong.
|
||||
#
|
||||
# `us-east-1` is what dev's Velero has signed every request with since bootstrap; MinIO
|
||||
# ignores the value, but changing it is a needless risk against a live, working backup path.
|
||||
# An EMPTY VELERO_BACKUP_PREFIX means the bucket root, which is where dev's backups already
|
||||
# are — and the Velero chart omits `objectStorage.prefix` entirely when it is empty (verified
|
||||
# against chart 12.1.0), so dev's live BackupStorageLocation is byte-unchanged by this.
|
||||
- op: replace
|
||||
path: /data/S3_BACKUP_REGION
|
||||
value: "us-east-1"
|
||||
- op: replace
|
||||
path: /data/VELERO_BACKUP_BUCKET
|
||||
value: "velero"
|
||||
- op: replace
|
||||
path: /data/VELERO_BACKUP_PREFIX
|
||||
value: ""
|
||||
- op: replace
|
||||
path: /data/CNPG_BACKUP_PATH
|
||||
value: "s3://cnpg"
|
||||
# WHICH CREDENTIAL SIGNS THOSE REQUESTS. Empty suffix = the UNSUFFIXED Secrets
|
||||
# (velero-cloud-credentials, <db>-backup-credentials) — the scoped MinIO keys dev's Velero and
|
||||
# its three CNPG Clusters are running RIGHT NOW, byte-identical to what is on the cluster. Prod
|
||||
# renders "-prod" and reads the Cloudflare R2 token from the parallel `-prod` Secrets instead.
|
||||
#
|
||||
# RESTATED EXPLICITLY even though it equals the base default, for the same reason as
|
||||
# EXTERNALDNS_TXT_OWNER_ID and the four backup-layout knobs above: the four Secrets are SHARED
|
||||
# base manifests, so before this split prod's R2 token landed in the objects DEV reads. Dev
|
||||
# signing MinIO with an R2 key id is 403 on every barman-cloud-wal-archive, and archive_mode=on
|
||||
# turns that into a filling pgdata PVC — the disk-full clock, on the one cluster whose DR
|
||||
# restore is PROVEN (R-STATE-03). This line is what keeps dev on its own credential.
|
||||
#
|
||||
# DEV IS NOT BEING BACKED UP GOING FORWARD and is slated for decommission (operator,
|
||||
# 2026-09-07), so this is not protecting a future backup — it is protecting the EXISTING
|
||||
# backups and the WAL chain until dev is retired. Do not "simplify" it away while dev runs.
|
||||
- op: replace
|
||||
path: /data/BACKUP_CREDENTIALS_SUFFIX
|
||||
value: ""
|
||||
# dev mirrors prod's 3-node topology, so the replicated tier keeps 3 replicas.
|
||||
- op: replace
|
||||
path: /data/LONGHORN_DEFAULT_REPLICA_COUNT
|
||||
|
||||
@@ -129,14 +129,23 @@ patches:
|
||||
#
|
||||
# WHY THE IN-CLUSTER URL CANNOT BE USED YET (blocker found 2026-09-01, still true; the
|
||||
# unusable value was `https://gitea.home.keithjackson.dev/Home-Network/home-network-k3s-impl.git`).
|
||||
# Two independent reasons, both consequences of the split-horizon design:
|
||||
# 1. SELF-HOSTING CIRCULARITY. Gitea runs ON THE DEV CLUSTER. Prod syncing from it means prod's
|
||||
# GitOps source lives inside the very cluster slated for decommission. Gitea must move to
|
||||
# prod (or off-cluster) BEFORE dev is retired, or prod loses its source of truth.
|
||||
# 2. SPLIT-HORIZON DNS SENDS IT TO THE WRONG PLACE. platform/coredns/coredns-custom.yaml
|
||||
# answers the ENTIRE home.keithjackson.dev zone with ${GATEWAY_VIP}. On prod that is
|
||||
# 192.168.2.20 — PROD's own gateway, where no Gitea is listening. Prod's Flux would resolve
|
||||
# gitea.home.keithjackson.dev to itself and never reach a repository.
|
||||
# REASONING UPDATED 2026-09-04 for the new suffix direction (prod = the apex, dev = dev.home.),
|
||||
# which changes WHY the blocker holds but not THAT it holds:
|
||||
# 1. SELF-HOSTING CIRCULARITY — still true, and now the only reason. The Gitea this repo is
|
||||
# pushed to runs ON THE DEV CLUSTER. Prod syncing from it would put prod's GitOps source
|
||||
# inside the very cluster slated for decommission. Gitea must move to prod (or off-cluster)
|
||||
# BEFORE dev is retired, or prod loses its source of truth.
|
||||
# NOTE the suffix change makes the two Giteas DIFFERENT HOSTS: dev's is now
|
||||
# gitea.dev.home.keithjackson.dev and prod's is gitea.home.keithjackson.dev. Prod pointing at
|
||||
# the apex name therefore means "prod's OWN Gitea", never dev's — which is what makes the
|
||||
# cutover below a one-line change once prod actually runs Gitea.
|
||||
# 2. SPLIT-HORIZON DNS — NO LONGER A BLOCKER, and the note is kept because the old reasoning is
|
||||
# easy to re-derive incorrectly. platform/coredns/coredns-custom.yaml answers the whole
|
||||
# ${BASE_DOMAIN} zone with ${GATEWAY_VIP}; when prod's BASE_DOMAIN was `prod.home.` that meant
|
||||
# prod resolved gitea.home.keithjackson.dev to PROD's gateway (192.168.2.20) where no Gitea
|
||||
# listened — a misdirection. Now that prod IS the apex, resolving gitea.home.keithjackson.dev
|
||||
# to prod's own Gateway is CORRECT: that is where prod's Gitea will be served from. What
|
||||
# remains is purely reason 1 — the Gitea is not there YET.
|
||||
#
|
||||
# WHAT THE TEMPORARY SOURCE IS (stood up 2026-09-04 on explicit operator direction):
|
||||
# bootstrap/gitea/bootstrap-gitea.sh with HOST_NAT_IP=192.168.2.132 — a podman container on the
|
||||
@@ -185,25 +194,9 @@ patches:
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
spec:
|
||||
# CUTOVER DONE 2026-09-06 (operator direction: "move the repo to prod"). Prod now reconciles
|
||||
# from PROD'S OWN in-cluster Gitea. The workstation bootstrap container
|
||||
# (http://192.168.2.132:3000) is retired as prod's source — it was a podman container on the
|
||||
# operator's workstation holding a DHCP lease on WIFI, i.e. production's source of truth was a
|
||||
# laptop. It is kept only as a recovery path (bootstrap/gitea/bootstrap-gitea.sh).
|
||||
#
|
||||
# THIS LINE IS SELF-REFERENTIAL AND THAT IS THE WHOLE DIFFICULTY. The root Kustomization
|
||||
# renders ./clusters/prod, which contains this patch — so a live `kubectl patch` of
|
||||
# GitRepository.spec.url is REVERTED on the next reconcile unless the authored value moves
|
||||
# too. The cutover therefore had to be: (1) push the repo to prod's Gitea, (2) fast-forward
|
||||
# its `main` to the exact commit prod was already running so the source swap changed nothing
|
||||
# else, (3) live-patch the url to open a window, (4) land THIS commit through that window so
|
||||
# the authored value agrees. Dev hit the identical trap on 2026-09-05
|
||||
# (infra/PROD-PROVISIONING.md §10a "D0/D1 as executed").
|
||||
#
|
||||
# Verified before the switch: a throwaway pod INSIDE prod resolved this host through
|
||||
# in-cluster CoreDNS to the Gateway VIP and cloned over the public LE cert with no CA
|
||||
# supplied. The workstation being able to reach it proves nothing about pod egress.
|
||||
url: https://gitea.prod.home.keithjackson.dev/Home-Network/home-network-k3s-impl.git
|
||||
# TEMPORARY bootstrap source — workstation container, NOT in any cluster. See the block
|
||||
# above for the cutover to https://gitea.home.keithjackson.dev/... . Do not treat as final.
|
||||
url: http://192.168.2.132:3000/Home-Network/home-network-k3s-impl.git
|
||||
ref:
|
||||
branch: main
|
||||
- target:
|
||||
@@ -218,28 +211,41 @@ patches:
|
||||
path: /data/ACME_CLUSTER_ISSUER
|
||||
value: "letsencrypt-prod"
|
||||
# ==========================================================================================
|
||||
# PROD DNS IDENTITY (R-DNS-01/02/03) — the fix for the 2026-09-04 incident.
|
||||
# PROD DNS IDENTITY (R-DNS-01/02/03) — PROD **IS** THE APEX.
|
||||
# ==========================================================================================
|
||||
# PROD SERVES ITS OWN SUBDOMAIN. Until 2026-09-04 every hostname was HARDCODED to
|
||||
# home.keithjackson.dev in the base manifests, so prod was authored to serve the EXACT SAME
|
||||
# public names as the live dev cluster out of the SAME Cloudflare zone. Two consequences, one
|
||||
# of which already happened:
|
||||
# OPERATOR DECISION 2026-09-04, superseding the interim split committed earlier the same day
|
||||
# (prod on `prod.home.` behind apex CNAMEs). The simpler target, and the one authored here:
|
||||
#
|
||||
# dev -> dev.home.keithjackson.dev (dev carries the label; clusters/dev)
|
||||
# prod -> home.keithjackson.dev (THE APEX — this knob)
|
||||
#
|
||||
# There is NO `prod.` label and NO CNAME layer. `<svc>.home.keithjackson.dev` is prod's REAL
|
||||
# name, so nothing has to alias anything: prod's Gitea is gitea.home.keithjackson.dev, its
|
||||
# Keycloak keycloak.home.keithjackson.dev, and the wildcard cert is `*.home.keithjackson.dev`.
|
||||
#
|
||||
# BACKGROUND — WHY THE SUFFIXES HAD TO SPLIT AT ALL. Until 2026-09-04 every hostname was
|
||||
# HARDCODED to home.keithjackson.dev in the base manifests, so prod was authored to serve the
|
||||
# EXACT SAME public names as the live dev cluster out of the SAME Cloudflare zone. Two
|
||||
# consequences, one of which already happened:
|
||||
# 1. HAPPENED: prod's external-dns (policy: sync, owner id `default` — the same implicit
|
||||
# default dev had) concluded it owned dev's records and DELETED EVERY ONE of them.
|
||||
# 2. QUEUED BEHIND IT: platform-config would have requested a `*.home.keithjackson.dev`
|
||||
# wildcard from letsencrypt-prod for a cert dev already holds, and both clusters would
|
||||
# have raced each other's `_acme-challenge` TXT records on that one zone.
|
||||
# Moving prod to its own suffix dissolves both: the two clusters can no longer name the same
|
||||
# record. This is what makes ACME_CLUSTER_ISSUER: letsencrypt-prod above safe here.
|
||||
#
|
||||
# dev -> home.keithjackson.dev (UNCHANGED — dev is live; phase 2 moves it)
|
||||
# prod -> prod.home.keithjackson.dev (this knob)
|
||||
#
|
||||
# CONSEQUENCE — prod's hostnames are NOT the apex names. Prod's Gitea is
|
||||
# gitea.prod.home.keithjackson.dev, its Keycloak keycloak.prod.home.keithjackson.dev, and so
|
||||
# on; the wildcard cert becomes `*.prod.home.keithjackson.dev`. The apex CNAMEs that make
|
||||
# `<svc>.home.keithjackson.dev` point at prod are PHASE 2, deliberately not done here — they
|
||||
# would take traffic off the live dev cluster. infra/PROD-PROVISIONING.md §10.
|
||||
# *** THE ORDERING CONSTRAINT THIS VALUE CREATES — READ BEFORE UNSUSPENDING ANYTHING. ***
|
||||
# Prod now claims the very names the LIVE dev cluster is serving today. Both hazards above are
|
||||
# therefore still armed until DEV HAS VACATED THE APEX. Prod's `platform` and its 10 downstream
|
||||
# tiers are SUSPENDED for exactly this reason. The gate is not "review is done", it is:
|
||||
# (a) dev is reconciling on *.dev.home.keithjackson.dev with a Ready wildcard cert, AND
|
||||
# (b) dev's old apex A/CNAME records and their TXT registry entries are GONE from Cloudflare
|
||||
# (verified against Cloudflare directly — a pfSense override is NOT evidence about public
|
||||
# DNS; that mistake was made on 2026-09-04), AND
|
||||
# (c) dev's old `*.home.keithjackson.dev` cert/Certificate is released, because prod ordering
|
||||
# that same name set while dev's still exists counts against LE's duplicate-certificate
|
||||
# limit (5 per week per identical name set).
|
||||
# Full ordered runbook: infra/PROD-PROVISIONING.md §10. Unsuspending before (a)-(c) re-creates
|
||||
# the 2026-09-04 incident with the roles reversed.
|
||||
#
|
||||
# [parity] MUST equal k3s_base_domain in infra/ansible/inventory/group_vars/k3s_prod/vars.yml.
|
||||
# The kube-apiserver's oidc-issuer-url is an INSTALL-TIME k3s flag, not a manifest, so it is
|
||||
@@ -247,30 +253,33 @@ patches:
|
||||
# trusts a different Keycloak than the one prod actually runs.
|
||||
- op: replace
|
||||
path: /data/BASE_DOMAIN
|
||||
value: "prod.home.keithjackson.dev"
|
||||
# SAFETY NET #2 (independent of the owner id below): narrow ExternalDNS's name scope to prod's
|
||||
# OWN subtree, so prod cannot create, update or delete any record outside it no matter what the
|
||||
# TXT registry says. VERIFY ON FIRST RECONCILE — the Cloudflare provider uses this value for
|
||||
# ZONE selection too, and on the version in use in 2026-08 a strict-subdomain filter dropped
|
||||
# the parent zone ("no hosted zone matching record DNS Name"). If prod's external-dns logs no
|
||||
# matching zone, it will create NOTHING — fail-closed, and it can still never delete dev's
|
||||
# records. Fallback in that case: set this to "keithjackson.dev" (dev's value) and rely on
|
||||
# EXTERNALDNS_TXT_OWNER_ID below, which is sufficient on its own.
|
||||
value: "home.keithjackson.dev"
|
||||
- op: replace
|
||||
path: /data/EXTERNALDNS_DOMAIN_FILTER
|
||||
# MUST be the Cloudflare ZONE, not prod's subdomain. FINDING F-DNS1 (DECISIONS-LOG,
|
||||
# 2026-08-17) proved a subdomain filter DROPS THE PARENT ZONE: ExternalDNS generated the
|
||||
# record then skipped it with "no hosted zone matching record DNS Name", so prod would
|
||||
# publish NOTHING. Isolation from dev does NOT come from this filter — it comes from
|
||||
# (1) prod only requesting *.prod.home.keithjackson.dev hostnames, and
|
||||
# (2) txtOwnerId: prod, so prod's registry never claims dev's `default`-owned records.
|
||||
# MUST be the Cloudflare ZONE. Prod now serves the apex, so `home.keithjackson.dev` looks
|
||||
# like a tighter, more honest filter than the zone — IT IS NOT, IT IS AN OUTAGE. FINDING
|
||||
# F-DNS1 (DECISIONS-LOG 2026-08-17) proved a STRICT SUBDOMAIN of the hosted zone drops the
|
||||
# zone entirely: ExternalDNS generates the record then skips it with "no hosted zone matching
|
||||
# record DNS Name", so prod publishes NOTHING. An agent narrowed this to
|
||||
# prod.home.keithjackson.dev on 2026-09-04 and it was reverted in review; do not retry it.
|
||||
# Isolation from dev does NOT come from this filter — it comes from
|
||||
# (1) the two clusters requesting DISJOINT hostname sets (apex vs dev.home.), and
|
||||
# (2) txtOwnerId: prod vs dev, so neither registry ever claims the other's records.
|
||||
# That pairing is what the 2026-09-04 incident needed and this filter cannot provide.
|
||||
value: "keithjackson.dev"
|
||||
# SAFETY NET #1 — THE ACTUAL ROOT CAUSE OF THE INCIDENT. Both clusters left this unset and so
|
||||
# both inherited the chart default `default`; prod then treated dev's TXT-registry entries as
|
||||
# its own and, under policy: sync, deleted the records behind them. This value MUST differ from
|
||||
# dev's (`default`, pinned in clusters/dev — dev's live records are registered under it and
|
||||
# cannot be renamed without orphaning them). `prod` is distinct, which is the entire fix.
|
||||
# its own and, under policy: sync, deleted the records behind them.
|
||||
#
|
||||
# PROD MUST NOT BE GIVEN `default`, however tempting it now looks. Prod is taking over the apex
|
||||
# NAMES, and dev's existing apex records are registered under owner `default` — so `default` is
|
||||
# the one value that would let prod adopt them. It would also let prod SYNC-DELETE the ones no
|
||||
# prod source claims, on a cluster dev is still serving. Prod inherits the apex names; it must
|
||||
# never inherit the apex OWNER ID. `prod` keeps prod fail-safe: records owned by `default` are
|
||||
# invisible to it, so an early unsuspend cannot destroy dev's DNS the way 2026-09-04 did — it
|
||||
# simply fails to publish until those records are retired, which is the correct behaviour.
|
||||
# dev is `dev` (clusters/dev, changed with dev's migration); the base default is a
|
||||
# non-colliding sentinel. All three differ.
|
||||
- op: replace
|
||||
path: /data/EXTERNALDNS_TXT_OWNER_ID
|
||||
value: "prod"
|
||||
@@ -356,32 +365,14 @@ patches:
|
||||
- op: replace
|
||||
path: /data/CILIUM_LB_POOL_STOP
|
||||
value: "192.168.2.30"
|
||||
# --- still UNDECIDED -> loud placeholders (must be real before `flux bootstrap`) ------------
|
||||
# k3s API endpoint Cilium's kube-proxy replacement dials. This is the prod CONTROL-PLANE VIP
|
||||
# (kube-vip), NOT .20 (the ingress VIP) and NOT the single node address .10.
|
||||
#
|
||||
# *** DECIDED 2026-09-06 by the operator: the VIP, as intended. *** (This block previously
|
||||
# read "Not yet decided" while shipping .10 — the founder's own address — and that is what
|
||||
# the LIVE prod cluster is running: all three cilium-agents carry
|
||||
# KUBERNETES_SERVICE_HOST=192.168.2.10 while kube-vip holds .9, verified 2026-09-06. On a
|
||||
# 3-node HA cluster that makes the CNI's control-plane path depend on k3s-prod-1: lose it and
|
||||
# the datapath keeps running off Cilium's cache while no service or policy update lands, with
|
||||
# nothing looking broken. The VIP is a virtual address that reaches the cluster wherever the
|
||||
# control plane is, which is the whole reason kube-vip exists here.)
|
||||
#
|
||||
# It is owned Ansible-side by **k3s_apiserver_endpoint** (group_vars/all.yml, derived from
|
||||
# this environment's kube_vip_address = 192.168.2.9 in group_vars/k3s_prod/vars.yml) and MUST
|
||||
# equal that value — [parity] contract in platform/cilium/helmrelease.yaml. NOT k3s_api_host:
|
||||
# that variable still exists and still holds the FOUNDER's .10, because the cold-bootstrap
|
||||
# `--server` join needs a host that is up before any VIP is floated. Two addresses, two
|
||||
# variables, deliberately.
|
||||
#
|
||||
# .9 sits OUTSIDE the Cilium LB pool (.20-.30) and is not a node address, so kube-vip and
|
||||
# LB-IPAM can never be handed the same IP — asserted by
|
||||
# conformance/tests/l0-static/ha-control-plane-as-code.
|
||||
# (kube-vip), NOT .20 (the ingress VIP) and NOT the single node address .10. It is owned
|
||||
# Ansible-side (prod inventory k3s_api_host) and MUST equal that value — [parity] contract in
|
||||
# platform/cilium/helmrelease.yaml. Not yet decided.
|
||||
- op: replace
|
||||
path: /data/K3S_API_HOST
|
||||
value: "192.168.2.9"
|
||||
# --- still UNDECIDED -> loud placeholders (must be real before `flux bootstrap`) ------------
|
||||
value: "192.168.2.10"
|
||||
# Pinned L4 VIPs, both carved out of the pool by lbipam.cilium.io/ips. Not yet assigned for
|
||||
# prod. Leaving the dev literals in place would hand a prod LoadBalancer a dev-LAN address, so
|
||||
# they stay placeholders; whatever is chosen MUST fall inside [POOL_START, POOL_STOP] and not
|
||||
@@ -392,146 +383,10 @@ patches:
|
||||
- op: replace
|
||||
path: /data/CRAFTY_L4_VIP
|
||||
value: "192.168.2.22"
|
||||
# ------------------------------------------------------------------------------------------
|
||||
# BULK (EXTERNAL NFS) STORAGE HOST — *** NOT YET DECIDED. THIS PLACEHOLDER IS LOAD-BEARING. ***
|
||||
# ------------------------------------------------------------------------------------------
|
||||
# Backs every `class: bulk` volume: the 8.33 T media library (Storage/plex — plex, sonarr,
|
||||
# radarr) and the 270 G legacy Nextcloud archive (Storage/nextcloud).
|
||||
#
|
||||
# THE STORAGE HOST IS NOT REACHABLE FROM PROD AT ALL RIGHT NOW. It has one interface,
|
||||
# vmbr0 192.168.1.9/24, on pfSense's LAN; prod's nodes are on 192.168.2.0/24, pfSense's WAN.
|
||||
# 22 and 8006 answer from prod only because pfSense port-forwards them via 192.168.2.102;
|
||||
# 2049 and 111 do not. Opening that path is an OPERATOR decision, and the two candidate
|
||||
# outcomes give DIFFERENT values here:
|
||||
# port-forward 192.168.2.102:2049 -> 192.168.1.9:2049 => "192.168.2.102"
|
||||
# a WAN->LAN route to 192.168.1.0/24 => "192.168.1.9"
|
||||
# Guessing either one authors a prod cluster that mounts nothing, or — worse, if the guess
|
||||
# ever became reachable — mounts something unintended. So it stays a placeholder.
|
||||
#
|
||||
# WHICH PORTS THE DECISION MUST OPEN: exactly ONE, tcp/2049. The PVs mount nfsvers=4.1, and
|
||||
# NFSv4 carries mount and lock in-protocol — no rpcbind/111, no mountd, no statd. Choosing v4
|
||||
# over v3 was in part this decision's cost: v3 would have needed four ports and three of them
|
||||
# on ephemeral numbers.
|
||||
#
|
||||
# WHAT THE EXPORT MUST ALLOW IS ALREADY SETTLED, and it is the same under BOTH outcomes.
|
||||
# Measured 2026-09-07 from this workstation (192.168.2.132) through the existing port-forward:
|
||||
# the storage host logged the connection as coming FROM 192.168.2.132, not from a pfSense LAN
|
||||
# address (`ss -tn` peer 192.168.2.132, and the host's only route off-LAN is its default via
|
||||
# 192.168.1.1). pfSense DNATs WITHOUT source NAT, so the NFS server will see prod's REAL node
|
||||
# addresses either way. The additive `zfs set sharenfs=` values are in docs/BULK-NFS-STORAGE.md.
|
||||
#
|
||||
# *** BLAST RADIUS — READ BEFORE MERGING THIS TO A BRANCH PROD RECONCILES. ***
|
||||
# A PersistentVolume's spec.nfs.server is an unvalidated string: the API server ACCEPTS this
|
||||
# placeholder, the PV is created and the PVC BINDS to it (binding is pure control-plane). The
|
||||
# failure surfaces only when a pod mounts — MountVolume.SetUp "Failed to resolve server
|
||||
# TBD-PROD-BULK-NFS-SERVER" — so plex, sonarr, radarr and nextcloud would sit in
|
||||
# ContainerCreating on prod. Unlike the LAN knobs above, this placeholder does NOT fail at
|
||||
# apply time and there is no CRD validation to lean on. Set this value BEFORE prod reconciles
|
||||
# a revision containing the bulk volumes; the dev overlay renders a working value, which is
|
||||
# where these manifests are meant to be proven first.
|
||||
- op: replace
|
||||
path: /data/BULK_NFS_SERVER
|
||||
# 192.168.2.102 is pfSense's WAN address, which DNATs tcp/2049 through to the Proxmox
|
||||
# host's REAL address 192.168.1.9 on pfSense's LAN. Prod sits on the WAN side and cannot
|
||||
# reach 192.168.1.9 directly, so the cluster mounts the NAT address. Dev is on the LAN and
|
||||
# keeps the base value 192.168.1.9.
|
||||
#
|
||||
# PROVEN from k3s-prod-1 on 2026-09-12, not inferred:
|
||||
# - mount of 192.168.2.102:/mnt/Storage/plex negotiated nfs4 vers=4.1 proto=tcp
|
||||
# - a write as uid 1000 succeeded. Tested as 1000 and NOT root deliberately: the export
|
||||
# carries no_root_squash, so a root write would have passed and proved nothing, while
|
||||
# LSIO images default to PUID 911 and would have gone Ready then failed every import
|
||||
# with EACCES. The tree is 0755 1000:3000, so uid 1000 is the only writer.
|
||||
# - ln() across downloads/ -> tv/ returned link count 2 (same inode), so the arrs get
|
||||
# real atomic moves instead of silently degrading to copy-then-delete on EXDEV.
|
||||
# pfSense DNATs WITHOUT source NAT (the host logs prod's own node addresses as the peer),
|
||||
# so the ZFS export ACL names 192.168.2.10/.11/.12 directly rather than one translated IP.
|
||||
value: "192.168.2.102"
|
||||
# ==========================================================================================
|
||||
# OFF-CLUSTER DR TARGET — Cloudflare R2 (operator, 2026-09-05; bucket + token 2026-09-07).
|
||||
# ==========================================================================================
|
||||
# Dev's versitygw/MinIO (192.168.1.9:9000) sits on the DEV side of the network and is NOT an
|
||||
# option for prod, for two independent reasons:
|
||||
# 1. UNREACHABLE. Prod is on the SOHO network 192.168.2.0/24 — pfSense's WAN — while that
|
||||
# MinIO is inside pfSense's LAN. Measured from a pod INSIDE prod (the workstation proves
|
||||
# nothing about pod egress): `curl http://192.168.1.9:9000/minio/health/live` → connection
|
||||
# timed out after 15s. Reaching it would need a WAN→LAN rule that does not exist.
|
||||
# 2. NOT OFF-CLUSTER DR. It runs on the same Proxmox host as dev's VMs, so it already shares
|
||||
# a physical failure domain with dev. R2 is genuinely off-site, and its free egress means
|
||||
# restore REHEARSALS cost nothing — R-STATE-03 has no financial disincentive to stay proven.
|
||||
#
|
||||
# THIS UNBLOCKS AN ACTIVE INCIDENT, not just a roadmap item. All three prod CNPG Clusters ship
|
||||
# `archive_mode=on` with `endpointURL: TBD-PROD-S3-BACKUP-ENDPOINT`, so barman-cloud-wal-archive
|
||||
# exits 4 on every segment and Postgres — correctly — REFUSES TO RECYCLE WAL. Verified live
|
||||
# 2026-09-07: keycloak-db / nextcloud-db / semaphore-db all report
|
||||
# ContinuousArchiving=False/ContinuousArchivingFailing, and keycloak-db's 5 GiB PVC is at
|
||||
# 579 MiB used / 4360 MiB free and climbing monotonically. A disk-full PANIC there takes the
|
||||
# cluster's SSO database down, and with it kubectl OIDC, Grafana, Headlamp, Gitea and
|
||||
# forward-auth. Fixing the endpoint lets Postgres drain the whole backlog.
|
||||
#
|
||||
# ENDPOINT IS ACCOUNT-SCOPED, so it stays a prod-overlay value even though it is not a LAN
|
||||
# address: this is the operator's Cloudflare account id, confirmed from inside a prod pod
|
||||
# (`head-bucket` on k3s-prod-backup → BucketRegion WNAM; PUT and DELETE both succeed).
|
||||
# It is NOT put in base, because base is dev's live MinIO and dev is deliberately STAYING
|
||||
# there — moving dev to R2 would reset its PITR window and orphan the backups its PROVEN
|
||||
# restore depends on. Two environments, two targets, isolated by URL *and* by bucket.
|
||||
# Off-cluster DR target. Dev's versitygw/MinIO (192.168.1.9:9000) sits on the DEV side of the
|
||||
# network — a prod backup written there is not off-cluster DR and would mix two environments'
|
||||
# PITR streams in one bucket. Operator supplies the prod endpoint (full URL incl. scheme+port).
|
||||
- op: replace
|
||||
path: /data/S3_BACKUP_ENDPOINT
|
||||
value: "https://7ce452a1b3d6c86daf2c020edcb82d37.r2.cloudflarestorage.com"
|
||||
# R2 requires the region string to be exactly `auto`. (`head-bucket` answers
|
||||
# `{"BucketRegion":"WNAM"}` — that is a PHYSICAL LOCATION HINT, not a signing region; using it
|
||||
# here produces AuthorizationHeaderMalformed.) Dev keeps the base `us-east-1` its live MinIO
|
||||
# has always been signed with.
|
||||
- op: replace
|
||||
path: /data/S3_BACKUP_REGION
|
||||
value: "auto"
|
||||
# ------------------------------------------------------------------------------------------
|
||||
# OBJECT TENANCY INSIDE THE ONE BUCKET. The operator created a SINGLE R2 bucket, so velero and
|
||||
# barman cannot be separated by bucket the way they are on dev's MinIO (`velero` and `cnpg`).
|
||||
# They are separated by disjoint TOP-LEVEL PREFIXES instead:
|
||||
#
|
||||
# s3://k3s-prod-backup/velero/… Velero: kopia repo + backup metadata
|
||||
# s3://k3s-prod-backup/barman/<clusterName>/… CNPG: WAL + base backups
|
||||
#
|
||||
# The bucket ROOT is left empty on purpose — a misrendered prefix then lands somewhere
|
||||
# obviously wrong instead of silently colliding with a live tenant. Velero's retention TTL
|
||||
# deletes objects under its own prefix only, so it can never reap a WAL segment; and barman's
|
||||
# per-cluster serverName subdirectory keeps the three Postgres servers' chains apart, which
|
||||
# matters because interleaving two servers' WAL into one prefix corrupts the chain outright.
|
||||
# Dev cannot be reached from any of these: different endpoint AND different bucket names.
|
||||
# ------------------------------------------------------------------------------------------
|
||||
- op: replace
|
||||
path: /data/VELERO_BACKUP_BUCKET
|
||||
value: "k3s-prod-backup"
|
||||
- op: replace
|
||||
path: /data/VELERO_BACKUP_PREFIX
|
||||
value: "velero"
|
||||
- op: replace
|
||||
path: /data/CNPG_BACKUP_PATH
|
||||
value: "s3://k3s-prod-backup/barman"
|
||||
# ------------------------------------------------------------------------------------------
|
||||
# WHICH CREDENTIAL SIGNS THE REQUESTS TO THAT BUCKET (per-env credential split, 2026-09-07).
|
||||
# ------------------------------------------------------------------------------------------
|
||||
# The knobs above moved the ENDPOINT and the OBJECT LAYOUT per environment. The credential did
|
||||
# not move with them, and it could not: the four backup-credential Secrets are SHARED BASE
|
||||
# manifests, applied to both clusters by the same Flux tiers. Writing the R2 token into them
|
||||
# gave prod a working key by TAKING DEV'S AWAY — dev would have signed MinIO requests with an
|
||||
# R2 key id, and under archive_mode=on that is a filling pgdata PVC, not a missing backup.
|
||||
#
|
||||
# So each credential is now two objects, and this suffix selects prod's:
|
||||
# velero-cloud-credentials-prod the R2 token in the AWS INI `cloud` shape
|
||||
# <db>-backup-credentials-prod the same token as ACCESS_KEY_ID / ACCESS_SECRET_KEY
|
||||
# for keycloak-db, nextcloud-db and semaphore-db. Both halves ship to both clusters (they are
|
||||
# base resources); ONLY this value decides which one each consumer reads. It is a SUFFIX rather
|
||||
# than four secret-name knobs deliberately — four could be updated three-quarters of the way,
|
||||
# leaving one consumer on dev's MinIO key with nothing rendering wrong, and strict-mode
|
||||
# substitution cannot catch a defined-but-stale variable.
|
||||
#
|
||||
# THE R2 TOKEN IS THE ONE PROVEN FROM INSIDE PROD (chore/r2-credentials 0a6a862): head-bucket on
|
||||
# k3s-prod-backup returns BucketRegion WNAM, and PUT and DELETE both succeed. Rotating it now
|
||||
# touches ONLY the four `-prod` files; dev's MinIO keys are untouched objects. That isolation is
|
||||
# the structural reason for the split, independent of dev's eventual decommission.
|
||||
- op: replace
|
||||
path: /data/BACKUP_CREDENTIALS_SUFFIX
|
||||
value: "-prod"
|
||||
value: "TBD-PROD-S3-BACKUP-ENDPOINT"
|
||||
# TODO(WS-INFRA, Phase 7): pin prod node placement once the bare-metal role labels exist.
|
||||
@@ -57,13 +57,6 @@ rules:
|
||||
verbs: [create, delete, get, list, watch, patch]
|
||||
- apiGroups: [""]
|
||||
resources: [pods, services, configmaps, secrets, persistentvolumeclaims]
|
||||
# LOAD-BEARING `get` ON CONFIGMAPS, BEYOND FIXTURE CREATION (R-ENV-02): the environment-
|
||||
# agnostic L2 tests (dns-exposure, dns-internal-resolution, gateway-ingress-reachable,
|
||||
# oidc-sso-grafana, trusted-tls) resolve the cluster's own BASE_DOMAIN / GATEWAY_VIP at run
|
||||
# time from `flux-system/cluster-vars` — the same ConfigMap Flux substitutes from — via
|
||||
# chainsaw `x_k8s_get($client, 'v1', 'ConfigMap', 'flux-system', 'cluster-vars')` bindings.
|
||||
# If this rule is ever narrowed to the test's own namespace, those bindings fail and the
|
||||
# tests error out. Narrow it only alongside a namespace-scoped read of that ConfigMap.
|
||||
verbs: [create, delete, get, list, watch, patch]
|
||||
- apiGroups: [""]
|
||||
resources: [pods/exec]
|
||||
|
||||
+19
-213
@@ -297,109 +297,6 @@ _EXECUTABLE_RE = re.compile(
|
||||
_BLOCKED_RE = re.compile(
|
||||
r"^\s*" + re.escape(LABEL_PREFIX) + r"/blocked-by:\s*(.+)$", re.M)
|
||||
|
||||
# ---------------------------------------------------------- scaffold-marker detection
|
||||
# WHY THIS IS NOT A WHOLE-FILE `"TODO" in text`
|
||||
# ---------------------------------------------------
|
||||
# It used to be, and that mis-reported a SAFETY-CRITICAL requirement. A test whose SUBJECT
|
||||
# is placeholder detection has to name the idioms it hunts for:
|
||||
# `l0-static/placeholder-guard-coupling/check.sh` carries
|
||||
# `SEED = ["TBD", "TODO", "FIXME", "PLACEHOLDER", ...]` — its detection vocabulary, as data.
|
||||
# A whole-file substring test read that as the test declaring itself unfinished, classified
|
||||
# a 32-assertion executable check as a scaffold, raised a permanent `--strict` annotation
|
||||
# conflict, and (because the same file is tagged R-STATE-03) reported R-STATE-03 as
|
||||
# `exec=1 scaffold=1`. A previous engineer worked around it by rewording prose.
|
||||
#
|
||||
# The rule below is derived from the corpus (43 tests), not invented. Every genuine
|
||||
# scaffold in this tree — `l2-behavioral/onboarding-completeness`, `l3-fault-injection/
|
||||
# alerting`, `l3-fault-injection/gitops-drift`, and no others — declares itself in BOTH of
|
||||
# two structural positions, and no executable test does either:
|
||||
#
|
||||
# 1. a `# STATUS: ... scaffold` line in the file's documentation header. This is the
|
||||
# repo's DEDICATED declaration slot (tests/README.md): 3 of 43 tests say
|
||||
# `STATUS: scaffold` and they are exactly the three scaffolds; 19 others declare
|
||||
# `STATUS: EXECUTABLE` / `IMPLEMENTED` / `RUNNABLE`; the rest declare nothing.
|
||||
# 2. a TODO/FIXME marker in the BODY that CARRIES A MESSAGE — `echo "TODO: assert PVC
|
||||
# bound ..."`, `TODO(T-4.1): hermetic SMTP capture sink`. A marker that carries a
|
||||
# message is a note about work that is missing. A marker that is the ENTIRE content of
|
||||
# a quoted literal (`"TODO",` in a vocabulary list) carries no message: it is the
|
||||
# marker as a *datum*, which is what a placeholder DETECTOR necessarily contains.
|
||||
#
|
||||
# Both are still evaluated over the body of `chainsaw-test.yaml` AND of any
|
||||
# `render-check.sh` / `check.sh`, and either one still OUTRANKS an
|
||||
# `executable: "true"` annotation (see classify_test) — the invariant that an annotation
|
||||
# must never outrank the body is unchanged. Ambiguity still only ever DOWNGRADES: an
|
||||
# unrecognised marker shape, a marker in body code outside a quoted literal, or a marker
|
||||
# carrying any trailing text all classify as scaffold.
|
||||
#
|
||||
# KNOWN, ACCEPTED IMPRECISION (stated rather than hidden): a test that embeds a marker with
|
||||
# text attached but as data — e.g. `re.compile(r"TODO|FIXME")`, or a single string holding
|
||||
# several idioms — is still called a scaffold. That is the conservative direction, and the
|
||||
# author's remedy is the same structural discipline the corpus already uses: keep each
|
||||
# idiom a standalone literal in a list.
|
||||
|
||||
# Vocabulary: only the idioms that DECLARE unfinished work. Deliberately excludes `TBD`,
|
||||
# `PLACEHOLDER`, `XXX`, `REPLACE-ME` — tests in this tree legitimately carry those in
|
||||
# executable code as data (`l0-static/egress-paths-selected-by-rules/check.sh` compares
|
||||
# authored values against `"TBD"`; `l0-static/no-plaintext-secrets/check.sh` matches
|
||||
# `placeholder.*`), and none of them is used here to mark a placeholder body.
|
||||
# Case-insensitive: no English word contains either token, and the corpus carries no
|
||||
# lower-case occurrence, so folding case only ever ADDS detection (`echo "todo: ..."`).
|
||||
_MARKER_RE = re.compile(r"\b(?:TODO|FIXME)\b", re.I)
|
||||
|
||||
# The dedicated declaration slot, on a comment line, case-insensitive. The declared state
|
||||
# must be the FIRST token after `STATUS:` — a `STATUS:` line declares exactly one state,
|
||||
# and all three real scaffolds read `# STATUS: scaffold — <what is missing>`. Anchoring it
|
||||
# there is what keeps `# STATUS: EXECUTABLE ... this is not a scaffold` from matching; a
|
||||
# loose `.*scaffold` would turn any prose that merely uses the word into a declaration.
|
||||
_STATUS_SCAFFOLD_RE = re.compile(
|
||||
r"^[ \t]*#[ \t]*STATUS:[ \t]*(?:scaffold|placeholder)\b", re.M | re.I)
|
||||
|
||||
# A quoted literal whose entire content is one marker word: the marker as data.
|
||||
_MARKER_AS_DATUM_RE = re.compile(r"(['\"])[ \t]*(?:TODO|FIXME)[ \t]*\1", re.I)
|
||||
|
||||
|
||||
def _split_doc_header(text: str) -> tuple[str, str]:
|
||||
"""Split a test file into (documentation header, body).
|
||||
|
||||
Both file kinds in this tree open with a banner comment block that is PROSE about the
|
||||
test: the Given/When/Then, the incident being closed, the mandatory "what this does NOT
|
||||
prove" list (tests/README.md, L0 authoring rule 1). Prose necessarily discusses
|
||||
placeholder idioms when the test hunts them, so an incidental mention there is
|
||||
documentation, not a placeholder body.
|
||||
|
||||
The body starts at the first line that is neither blank nor a `#` comment:
|
||||
*.sh -> `set -euo pipefail` (the shebang is a comment)
|
||||
*.yaml -> `apiVersion: chainsaw.kyverno.io/v1alpha1`
|
||||
Comments BELOW that point are body, deliberately: a scaffold marks a missing fixture
|
||||
with an inline comment (`l3-fault-injection/alerting`: `file: fixtures/smtp-sink.yaml
|
||||
# TODO(T-4.1): hermetic SMTP capture sink`), and that must keep counting.
|
||||
"""
|
||||
lines = text.splitlines(keepends=True)
|
||||
for i, ln in enumerate(lines):
|
||||
s = ln.strip()
|
||||
if s and not s.startswith("#"):
|
||||
return "".join(lines[:i]), "".join(lines[i:])
|
||||
return text, "" # no executable line at all: it is all header, body is empty
|
||||
|
||||
|
||||
def scaffold_marker(text: str) -> str:
|
||||
"""Why `text` declares itself a placeholder body, or "" if it does not.
|
||||
|
||||
Returns a short human-readable reason (used verbatim in the hygiene report) so the
|
||||
defect is actionable rather than a bare path.
|
||||
"""
|
||||
head, body = _split_doc_header(text)
|
||||
m = _STATUS_SCAFFOLD_RE.search(head) or _STATUS_SCAFFOLD_RE.search(body)
|
||||
if m:
|
||||
return m.group(0).strip().lstrip("#").strip()[:80]
|
||||
# Blank out markers held as pure data before looking for a marker that has a message.
|
||||
m2 = _MARKER_RE.search(_MARKER_AS_DATUM_RE.sub('""', body))
|
||||
if m2:
|
||||
line = body[:m2.start()].count("\n")
|
||||
snippet = body.splitlines()[line].strip()
|
||||
return f"placeholder marker in body: {snippet[:80]}"
|
||||
return ""
|
||||
|
||||
|
||||
def classify_test(t: dict) -> dict:
|
||||
"""Decide EXECUTABLE vs SCAFFOLD for one discovered test, conservatively.
|
||||
@@ -407,18 +304,14 @@ def classify_test(t: dict) -> dict:
|
||||
A tagged test is only worth counting if it can actually assert something. Rules, in
|
||||
order, chosen so ambiguity NEVER upgrades a test:
|
||||
|
||||
scaffold the test DECLARES itself a placeholder — a `# STATUS: scaffold`
|
||||
header line, or a TODO/FIXME marker carrying a message in the
|
||||
body (see `scaffold_marker` for the rule and the corpus it was
|
||||
derived from) — *even if* it is annotated executable. An
|
||||
annotation must never outrank the body. That mismatch is also
|
||||
reported as a hygiene defect.
|
||||
executable annotated `conformance.home.k3s/executable: "true"` and no
|
||||
scaffold declaration.
|
||||
executable-untagged no scaffold declaration and a real runnable body, but the
|
||||
annotation is missing (true of the L1 render tests, which
|
||||
execute a render-check.sh + golden diff, of the L0 static
|
||||
checks, and of the META self-check).
|
||||
scaffold the body contains TODO markers (a placeholder), *even if* it
|
||||
is annotated executable — an annotation must never outrank the
|
||||
body. That mismatch is also reported as a hygiene defect.
|
||||
executable annotated `conformance.home.k3s/executable: "true"` and no TODOs.
|
||||
executable-untagged no TODOs and a real runnable body, but the annotation is
|
||||
missing (true of the L1 render tests, which execute a
|
||||
render-check.sh + golden diff, of the L0 static checks, and
|
||||
of the META self-check).
|
||||
Counted as executable, and flagged as an annotation gap.
|
||||
scaffold anything else.
|
||||
"""
|
||||
@@ -429,26 +322,16 @@ def classify_test(t: dict) -> dict:
|
||||
# downgrades the test to a scaffold — an annotation must never outrank the body,
|
||||
# whichever file the body lives in.
|
||||
extra = ""
|
||||
scripts: list[tuple[str, str]] = []
|
||||
for script in ("render-check.sh", "check.sh"):
|
||||
sp = p.parent / script
|
||||
if sp.is_file():
|
||||
body = sp.read_text()
|
||||
extra += body
|
||||
scripts.append((script, body))
|
||||
extra += sp.read_text()
|
||||
declared = bool(_EXECUTABLE_RE.search(text))
|
||||
# Each file is classified on ITS OWN structure — the yaml and the script have different
|
||||
# header blocks, so they must not be concatenated before the header/body split.
|
||||
reason = scaffold_marker(text)
|
||||
for script, body in scripts:
|
||||
if reason:
|
||||
break
|
||||
r = scaffold_marker(body)
|
||||
reason = f"{script}: {r}" if r else ""
|
||||
has_todo = "TODO" in text or "TODO" in extra
|
||||
bm = _BLOCKED_RE.search(text)
|
||||
has_body = bool(extra) or bool(re.search(r"^\s+- (script|assert|apply):", text, re.M))
|
||||
|
||||
if reason:
|
||||
if has_todo:
|
||||
state = "scaffold"
|
||||
elif declared:
|
||||
state = "executable"
|
||||
@@ -465,8 +348,7 @@ def classify_test(t: dict) -> dict:
|
||||
"name": t["name"], "path": rel, "layer": t["layer"],
|
||||
"state": state,
|
||||
"declared_executable": declared,
|
||||
"annotation_conflict": bool(declared and reason),
|
||||
"scaffold_reason": reason,
|
||||
"annotation_conflict": bool(declared and has_todo),
|
||||
"blocked_by": bm.group(1).strip().strip('"') if bm else "",
|
||||
}
|
||||
|
||||
@@ -758,10 +640,6 @@ def build(repo: Path, registry: list[dict], tests: list[dict], impl: dict[str, l
|
||||
# --- hygiene defects this tool owns -------------------------------------------
|
||||
hygiene = {
|
||||
"annotation_conflicts": [t["path"] for t in classified if t["annotation_conflict"]],
|
||||
# Same set, with WHY each one was called a placeholder body, so the fix is obvious
|
||||
# from the report instead of requiring the reader to re-derive the rule.
|
||||
"annotation_conflict_reasons": {t["path"]: t["scaffold_reason"]
|
||||
for t in classified if t["annotation_conflict"]},
|
||||
"executable_untagged": [t["path"] for t in classified
|
||||
if t["state"] == "executable-untagged"],
|
||||
"registry_not_in_spec": [],
|
||||
@@ -1032,10 +910,8 @@ def render_md(rep: dict, nodes: list[dict], advisory: dict | None) -> str:
|
||||
else:
|
||||
o.append("- registry ↔ spec parity: **NOT CHECKED** (spec tree not available; "
|
||||
"this is a skip, not a pass)")
|
||||
o.append("- tests annotated executable but declaring a placeholder body: "
|
||||
+ (", ".join(f"`{k}` ({v})"
|
||||
for k, v in sorted(h["annotation_conflict_reasons"].items()))
|
||||
if h["annotation_conflicts"] else "none"))
|
||||
o.append(f"- tests annotated executable but containing TODO bodies: "
|
||||
f"{h['annotation_conflicts'] or 'none'}")
|
||||
o.append(f"- executable tests missing the `{LABEL_PREFIX}/executable` annotation: "
|
||||
f"{h['executable_untagged'] or 'none'}")
|
||||
|
||||
@@ -1054,12 +930,9 @@ def render_md(rep: dict, nodes: list[dict], advisory: dict | None) -> str:
|
||||
"reconciled |")
|
||||
o.append("| verifying test | `conformance.home.k3s/R-*` label on a Chainsaw test "
|
||||
"(the authoritative tag, same source as `coverage.md`) | exact |")
|
||||
o.append("| executable vs scaffold | a `# STATUS: scaffold` header line, or a "
|
||||
"TODO/FIXME marker CARRYING A MESSAGE in the body (a marker held as a bare "
|
||||
"quoted datum — a placeholder detector's own vocabulary — is not one); "
|
||||
"either outranks the `executable` annotation, and a real body without the "
|
||||
"annotation still counts and is flagged | conservative — ambiguity never "
|
||||
"upgrades |")
|
||||
o.append("| executable vs scaffold | TODO markers in the test body outrank the "
|
||||
"`executable` annotation; a real body without the annotation still counts "
|
||||
"and is flagged | conservative — ambiguity never upgrades |")
|
||||
o.append("| **test actually passed** | the committed `runs/ledger.jsonl` row for "
|
||||
"every verifying test: JUnit result **plus** that test's own pass sentinel "
|
||||
"token grepped from the run log | exact where a run was recorded, refused "
|
||||
@@ -1254,70 +1127,6 @@ def _selftest() -> int:
|
||||
classify_test({"name": "s", "path": str(f), "layer": "L2"})["state"]
|
||||
== "scaffold")
|
||||
|
||||
# === THE BUG THIS DETECTOR WAS REWRITTEN FOR ==============================
|
||||
# A whole-file `"TODO" in text` mis-classified `l0-static/placeholder-guard-
|
||||
# coupling` — a 32-assertion executable check whose SUBJECT is placeholder
|
||||
# detection, so it must name the idioms it hunts — as a scaffold, and through it
|
||||
# reported the SAFETY-CRITICAL R-STATE-03 as `exec=1 scaffold=1`. These
|
||||
# assertions pin BOTH directions: a marker held as DATA must not downgrade, and a
|
||||
# real placeholder body must still be caught. Both halves are load-bearing; the
|
||||
# second is what stops this fix from degrading into "always executable".
|
||||
yml = ("metadata:\n annotations:\n"
|
||||
" conformance.home.k3s/executable: \"true\"\n"
|
||||
"spec:\n steps:\n - script:\n"
|
||||
" content: bash ./check.sh\n")
|
||||
f.write_text(yml)
|
||||
detector = td2 / "check.sh"
|
||||
# Shaped exactly like the real check: a prose header that discusses the idioms,
|
||||
# then executable code holding them as a vocabulary list.
|
||||
detector.write_text(
|
||||
"#!/usr/bin/env bash\n"
|
||||
"# L0 static check — placeholder vocabulary\n"
|
||||
"# * SEED — loud-placeholder idioms (TBD, TODO, FIXME, PLACEHOLDER)\n"
|
||||
"# STATUS: EXECUTABLE. A pure offline assertion script.\n"
|
||||
"set -euo pipefail\n"
|
||||
"python3 - <<'EOF'\n"
|
||||
"SEED = [\"TBD\", \"TODO\", \"FIXME\", \"PLACEHOLDER\"]\n"
|
||||
"assert any(w in open('/etc/hostname').read().upper() for w in SEED) is False\n"
|
||||
"EOF\n")
|
||||
c = classify_test({"name": "s", "path": str(f), "layer": "L0"})
|
||||
check("a marker in HEADER PROSE does not downgrade an executable check",
|
||||
c["state"] == "executable" and not c["annotation_conflict"])
|
||||
check("a marker held as a bare quoted DATUM does not downgrade (the R-STATE-03 bug)",
|
||||
c["scaffold_reason"] == "")
|
||||
|
||||
# ... and the protection is intact: a real placeholder body is still caught, in
|
||||
# each of the three shapes the corpus actually uses.
|
||||
detector.write_text("#!/usr/bin/env bash\nset -euo pipefail\n"
|
||||
"echo \"TODO: assert the PVC is bound\"\n")
|
||||
c = classify_test({"name": "s", "path": str(f), "layer": "L0"})
|
||||
check("a TODO carrying a message in check.sh IS still a scaffold",
|
||||
c["state"] == "scaffold" and c["annotation_conflict"])
|
||||
check("the scaffold reason names the file and the offending line",
|
||||
c["scaffold_reason"].startswith("check.sh:")
|
||||
and "TODO: assert the PVC is bound" in c["scaffold_reason"])
|
||||
detector.write_text("#!/usr/bin/env bash\nset -euo pipefail\n"
|
||||
"# TODO(T-4.1) write the hermetic sink fixture\ntrue\n")
|
||||
check("a TODO(T-x.y) roadmap marker in an inline body comment IS still a scaffold",
|
||||
classify_test({"name": "s", "path": str(f), "layer": "L0"})["state"]
|
||||
== "scaffold")
|
||||
detector.write_text("#!/usr/bin/env bash\nset -euo pipefail\n"
|
||||
"echo \"FIXME the extraction before trusting this\"\n")
|
||||
check("an unpunctuated FIXME carrying trailing prose IS still a scaffold",
|
||||
classify_test({"name": "s", "path": str(f), "layer": "L0"})["state"]
|
||||
== "scaffold")
|
||||
# The dedicated declaration slot outranks a body that would otherwise look real —
|
||||
# this is how all three genuine scaffolds in the tree declare themselves.
|
||||
detector.unlink()
|
||||
f.write_text("# L3 FAULT-INJECTION SCAFFOLD — x\n"
|
||||
"# STATUS: scaffold — needs T-1.4 (Flux bootstrapped).\n" + yml)
|
||||
c = classify_test({"name": "s", "path": str(f), "layer": "L3"})
|
||||
check("a `# STATUS: scaffold` header declaration IS a scaffold, body notwithstanding",
|
||||
c["state"] == "scaffold" and c["annotation_conflict"])
|
||||
check("`# STATUS: EXECUTABLE` is not read as a scaffold declaration",
|
||||
scaffold_marker("# STATUS: EXECUTABLE. `check.sh` is a pure offline "
|
||||
"assertion script; covers no scaffold.\nset -e\n") == "")
|
||||
|
||||
# ---------------------------------------------------------------- PROOF / ledger
|
||||
# These are the assertions that stand between this report and a fake green.
|
||||
check("empty ledger parses to no rows (pre-ledger behaviour is preserved)",
|
||||
@@ -1532,11 +1341,8 @@ def main() -> int:
|
||||
f"{h['registry_not_in_spec']} {h['spec_not_in_registry']}\n")
|
||||
rc = 1
|
||||
if h["annotation_conflicts"]:
|
||||
sys.stderr.write(
|
||||
"[status] STRICT: tests annotated executable that declare a placeholder "
|
||||
"body: " + "; ".join(f"{k} ({v})" for k, v in
|
||||
sorted(h["annotation_conflict_reasons"].items()))
|
||||
+ "\n")
|
||||
sys.stderr.write(f"[status] STRICT: tests annotated executable with TODO "
|
||||
f"bodies: {h['annotation_conflicts']}\n")
|
||||
rc = 1
|
||||
if rep["safety_critical_without_executable_l3"]:
|
||||
sys.stderr.write(f"[status] STRICT: safety-critical requirement(s) whose L3 "
|
||||
|
||||
@@ -93,52 +93,9 @@ and coverage extraction exact and robust.
|
||||
break the thing and assert fail-closed / no-data-loss, with `finally` restoring state.
|
||||
- **Never weaken a test to go green (R-TEST-06).** If the spec can't be met, escalate to
|
||||
the PM — do not relax the assertion or the requirement.
|
||||
- **Never write a per-environment value into a test (R-ENV-02).** No hostname suffix, no
|
||||
Gateway VIP, no LB-pool range. One authored suite must run green against dev *and* prod,
|
||||
so a test **asks the cluster it is pointed at** what it is. The mechanism is a chainsaw
|
||||
binding over the same ConfigMap Flux substitutes from:
|
||||
|
||||
```yaml
|
||||
spec:
|
||||
bindings:
|
||||
- name: baseDomain
|
||||
value: (x_k8s_get($client, 'v1', 'ConfigMap', 'flux-system', 'cluster-vars').data.BASE_DOMAIN)
|
||||
```
|
||||
|
||||
Declare it at **test level** (step `outputs`/`bindings` are step-scoped), then consume it:
|
||||
`($baseDomain)` for a pod `env:` value, `(join('.', ['myhost', $baseDomain]))` where a
|
||||
full hostname is needed — including inside a `fixtures/*.yaml` applied with `apply: file:`,
|
||||
which chainsaw templates with the same bindings. Prefer reading the *actual* observed value
|
||||
over restating a range (e.g. assert the DNS answer equals `GATEWAY_VIP`, or equals the LB IP
|
||||
the Service was assigned) — it is both stricter and environment-independent. Never take the
|
||||
environment from a CLI flag or an exported variable: a flag someone forgets is how a suite
|
||||
ends up silently asserting the other cluster's expectations. L1 goldens are the one
|
||||
exception: they are rendering fixtures and carry the literal `${BASE_DOMAIN}` token.
|
||||
|
||||
### How a scaffold declares itself (and how `status.py` tells one from a verifier)
|
||||
|
||||
Scaffolds in this tree carry `TODO(T-x.y)` markers where a fixture/assertion lands as a
|
||||
later roadmap phase delivers the capability under test. `lib/status.py` classifies each
|
||||
test EXECUTABLE or SCAFFOLD from the test's own files — `chainsaw-test.yaml`,
|
||||
`render-check.sh`, `check.sh` — and either signal below **outranks**
|
||||
`conformance.home.k3s/executable: "true"`, so a placeholder body can never be reported as
|
||||
an executable test. Declare a scaffold in **both** positions:
|
||||
|
||||
1. **`# STATUS: scaffold — <what is missing>`** in the file's header comment block. The
|
||||
declared state must be the first token after `STATUS:` (executable tests say
|
||||
`# STATUS: EXECUTABLE` / `IMPLEMENTED` / `RUNNABLE`).
|
||||
2. **A `TODO`/`FIXME` marker carrying a message, in the body** — the file from its first
|
||||
non-comment line onward. `echo "TODO(T-4.2): inject a fault to fire a rule"`,
|
||||
or an inline comment on a missing fixture:
|
||||
`file: fixtures/smtp-sink.yaml # TODO(T-4.1): hermetic SMTP capture sink`.
|
||||
|
||||
What is **not** a scaffold marker, and why the distinction is load-bearing: a marker
|
||||
mentioned in the **header prose** (the Given/When/Then and the mandatory "what this does
|
||||
NOT prove" list), and a marker that is the **entire content of a quoted literal** — a
|
||||
placeholder *detector*'s own vocabulary, e.g. `SEED = ["TBD", "TODO", "FIXME", ...]` in
|
||||
`l0-static/placeholder-guard-coupling/check.sh`. A whole-file `"TODO" in text` test used to
|
||||
classify that 32-assertion check as a scaffold and, through it, report the safety-critical
|
||||
R-STATE-03 as `exec=1 scaffold=1`. If your check must hold a marker as data, keep each
|
||||
idiom a **standalone quoted literal in a list**; a marker with text attached (a regex
|
||||
alternation, several idioms in one string) still classifies as a scaffold, because
|
||||
ambiguity here only ever downgrades.
|
||||
later roadmap phase delivers the capability under test. `lib/status.py` treats such a
|
||||
marker — in the `chainsaw-test.yaml`, in `render-check.sh`, or in `check.sh` — as
|
||||
outranking any `conformance.home.k3s/executable: "true"` annotation, so a placeholder body
|
||||
can never be reported as an executable test.
|
||||
@@ -1,106 +0,0 @@
|
||||
# =============================================================================
|
||||
# L0 STATIC — the five auditability proofs can be produced on demand (R-SEC-03) [EXECUTABLE]
|
||||
# Catalog: docs/14 L0 row ("Pre-apply, in CI") — runs WITHOUT a cluster.
|
||||
# Spec: docs/00 R-SEC-03 ("All the above properties MUST be demonstrable/auditable (an
|
||||
# operator can prove isolation, egress path, cert validity, and backup integrity)"),
|
||||
# made concrete by docs/07 §"Auditability requirements (R-SEC-03) — what an operator
|
||||
# must be able to prove": Isolation (denied-probe test), Egress path (egress IP
|
||||
# check), Kill-switch (fault-injection test), TLS (cert status + external check),
|
||||
# Backups (DR drill) — "These proofs are part of the platform acceptance suite, not
|
||||
# one-time checks", with the acceptance gate "The five auditability proofs above can
|
||||
# be produced on demand."
|
||||
#
|
||||
# HOW THIS AVOIDS BEING A TAUTOLOGY (the whole design question for a meta-requirement)
|
||||
# -----------------------------------------------------------------------------------
|
||||
# "A test exists for R-NET-01, therefore isolation is auditable" would pass in any suite that
|
||||
# compiles and would keep passing while every proof rotted into a scaffold. Four distinct
|
||||
# failure modes are asserted instead, each by DRIVING the real machinery:
|
||||
# (a) each of the five proofs has an EXECUTABLE test — not a placeholder scaffold — at the layer
|
||||
# the spec's own words require (fault injection / a DR drill => L3), taken from
|
||||
# lib/status.py's own verdict so this check cannot disagree with the status report;
|
||||
# (b) the proofs are PRODUCIBLE by the documented command: `run.sh --requirements <the five>`
|
||||
# must resolve to real test dirs AND pass the merge gate's coverage assertion;
|
||||
# (c) each Chainsaw-layer proof declares a PASS SENTINEL that lib/record_run.py can
|
||||
# attribute — a proof that passes but records nothing leaves the operator with a green
|
||||
# terminal and no evidence;
|
||||
# (d) the evidence chain REFUSES FORGERIES: this check mounts two of its own against the
|
||||
# real tools — a green JUnit for a real proof test whose log holds only that test's SKIP
|
||||
# token, and a ledger row claiming `proven` at the repository's ROOT commit — and
|
||||
# requires record_run.py to report not-proven and status.py to report `stale-proof`. It
|
||||
# also drives `record_run.py --selftest` and `status.py --selftest`, which NOTHING in
|
||||
# ci/ or run.sh ran before this test existed.
|
||||
# The five proofs are not hand-listed where the spec can be read: when the read-only spec repo
|
||||
# is mounted, docs/07's bullet labels are parsed and must match this check's mapping exactly,
|
||||
# so a sixth proof added to the spec fails here instead of being silently unverified.
|
||||
#
|
||||
# WHY L0: every clause is a property of the repository and its own tooling, needs no cluster,
|
||||
# and belongs in the merge gate — an audit chain is worth having only if it is intact BEFORE
|
||||
# anyone needs it.
|
||||
#
|
||||
# GIVEN the repository as git tracks it (plus the read-only spec repo when mounted),
|
||||
# WHEN lib/status.py, lib/common.sh's requirement resolver, run.sh's coverage gate,
|
||||
# lib/record_run.py's sentinel parser and both evidence tools' self-tests are DRIVEN,
|
||||
# and two synthetic forgeries are presented to them,
|
||||
# THEN each of docs/07's five proofs maps to an executable test at the required layer, the
|
||||
# documented one-command invocation selects and asserts them, each is capturable as a
|
||||
# ledger row, and neither a self-skip nor a stale row can be recorded as proof.
|
||||
#
|
||||
# STATUS: EXECUTABLE. `check.sh` is a pure offline assertion script (git tree + the harness's
|
||||
# own stdlib tooling; no cluster, no network); conformance/run.sh --layer l0 and ci/l0.sh run
|
||||
# it directly, and this Chainsaw Test wraps the SAME script.
|
||||
#
|
||||
# WHAT THIS TEST FIXED WHILE BEING WRITTEN (recorded, not hidden): three of the tests that ARE
|
||||
# the automated half of docs/07's isolation and TLS proofs — l2-behavioral/default-deny,
|
||||
# l2-behavioral/trusted-tls and l2-behavioral/gateway-ingress-reachable — declared NO pass
|
||||
# sentinel, so lib/record_run.py could never record them as proven and those two audit proofs
|
||||
# were structurally unrecordable. Each now ends with a proof-of-pass sentinel step. The
|
||||
# assertion here was NOT relaxed to accommodate them (R-TEST-06).
|
||||
#
|
||||
# KNOWN, DELIBERATE GAPS (enumerated in check.sh's header; this test claims none of them):
|
||||
# * That the five proofs PASS. This asserts they are present, strong, runnable by one
|
||||
# command, capturable, and that the chain refuses forgeries — not that isolation holds
|
||||
# today (R-NET-01's test) or that the restore works (R-STATE-03's).
|
||||
# * HUBBLE. docs/07 names Hubble flow logs as part of the isolation proof; no test in this
|
||||
# suite queries Hubble. Only the denied-probe half is automated — a real, named gap.
|
||||
# * LEDGER COMPLETENESS OVER TIME. Nothing offline can prove a row was never removed; that
|
||||
# would need signed history.
|
||||
# * R-SEC-03's wording is broader than docs/07's five bullets. Suite-wide coverage of every
|
||||
# requirement is R-TEST-01/04's claim (tests/meta/harness-selfcheck) and is deliberately
|
||||
# not re-asserted here.
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: auditability-proofs-producible
|
||||
labels:
|
||||
conformance.home.k3s/R-SEC-03: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-SEC-03"
|
||||
conformance.home.k3s/layer: "L0"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
Given the tracked tree and the harness's own tooling, when status.py, the requirement
|
||||
resolver, run.sh's coverage gate, record_run.py's sentinel parser and both evidence
|
||||
tools' self-tests are driven and two synthetic forgeries are presented to them, then each
|
||||
of docs/07's five auditability proofs has an executable test at the layer the spec
|
||||
requires, is selectable and asserted by the documented one-command invocation, is
|
||||
capturable as a ledger row, and neither a self-skip nor a stale proof can be recorded as
|
||||
evidence.
|
||||
spec:
|
||||
steps:
|
||||
- name: then-each-of-the-five-proofs-is-producible-and-the-evidence-chain-refuses-forgeries
|
||||
description: >-
|
||||
Run the offline assertion script over the tracked tree (same script ci/l0.sh and
|
||||
conformance/run.sh --layer l0 execute).
|
||||
try:
|
||||
- script:
|
||||
content: bash ./check.sh
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
# NOTE ON WORDING (deliberate, do not "improve"): this file and check.sh say "placeholder
|
||||
# scaffold" rather than the four-letter marker lib/status.py looks for, because that detector is
|
||||
# a bare substring test over the whole file. Writing the marker in PROSE — even to say a proof
|
||||
# must NOT be one — makes status.py classify THIS test as a scaffold, so the R-SEC-03 test would
|
||||
# report itself as unexecutable. Observed and worked around here; the detector's crudeness is
|
||||
# reported to the PM rather than loosened, since loosening a fake-green guard to suit one file is
|
||||
# the wrong trade (R-TEST-06).
|
||||
@@ -1,431 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# L0 static check (offline, no cluster) — auditability-proofs-producible
|
||||
# Serves: R-SEC-03 ("All the above properties MUST be demonstrable/auditable (an operator can
|
||||
# prove isolation, egress path, cert validity, and backup integrity)."
|
||||
# — spec docs/00-vision-and-scope.md).
|
||||
#
|
||||
# The spec makes this meta-requirement concrete. docs/07-security-and-secrets.md
|
||||
# §"Auditability requirements (R-SEC-03) — what an operator must be able to prove":
|
||||
# - **Isolation**: "service X cannot reach service Y" — via Hubble flow logs + a
|
||||
# denied-probe test.
|
||||
# - **Egress path**: "service X's public IP is the VPN's" — via an egress IP check.
|
||||
# - **Kill-switch**: "with VPN down, X leaks nothing" — via a fault-injection test.
|
||||
# - **TLS**: "every endpoint serves a valid, non-self-signed, unexpired cert" — via cert
|
||||
# status + external check.
|
||||
# - **Backups**: "irreplaceable data is restorable" — via a DR drill.
|
||||
# "These proofs are part of the platform acceptance suite, not one-time checks."
|
||||
# and its acceptance gate: "The five auditability proofs above can be produced on demand."
|
||||
#
|
||||
# HOW A META-REQUIREMENT IS TESTED WITHOUT BECOMING A TAUTOLOGY
|
||||
# ------------------------------------------------------------
|
||||
# The trap is obvious and this check is built to avoid it: "a test exists for R-NET-01,
|
||||
# therefore isolation is auditable" is true of any suite that compiles, proves nothing, and
|
||||
# would keep passing while every proof rotted into a scaffold. Four separate failure modes
|
||||
# have to be caught for "an operator can PROVE it" to mean anything, and each one is asserted
|
||||
# by DRIVING the real machinery rather than by reading files:
|
||||
#
|
||||
# (a) THE PROOF EXISTS, AT THE RIGHT STRENGTH. Each of the five named proofs maps to a
|
||||
# requirement that must have an EXECUTABLE verifying test — not a placeholder scaffold — and,
|
||||
# where the spec's own words name fault injection ("with VPN down"; "a DR drill"), an
|
||||
# executable L3 test specifically. Executable-vs-scaffold is not re-derived here: this
|
||||
# check RUNS lib/status.py and reads its verdict, so it cannot disagree with the
|
||||
# repository's own status report.
|
||||
# (b) THE PROOF IS PRODUCIBLE ON DEMAND, BY THE DOCUMENTED COMMAND. `run.sh
|
||||
# --requirements <the five IDs>` must resolve to a non-empty test set and must ASSERT
|
||||
# their coverage (that flag is the merge gate's own path: it passes --require to
|
||||
# coverage.py). This check drives run.sh itself, so if the resolver, the labels or the
|
||||
# gate wiring break, R-SEC-03 fails.
|
||||
# (c) THE PROOF IS CAPTURABLE. A test that passes but records nothing leaves an operator
|
||||
# with a green terminal and no evidence. For each Chainsaw-layer proof, the test must
|
||||
# declare a PASS SENTINEL that lib/record_run.py can actually attribute — computed with
|
||||
# record_run.py's OWN parser, so deleting a test's sentinel makes this check fail
|
||||
# instead of silently making the requirement unprovable.
|
||||
# (d) THE EVIDENCE CANNOT BE FORGED, AND IS NOT HONOURED WHEN STALE. This is the clause
|
||||
# that makes the check bite. Two INDEPENDENT forgery attempts are mounted against the
|
||||
# real tools, in a throwaway directory:
|
||||
# * a Chainsaw run in which the JUnit says a real proof test PASSED while the run log
|
||||
# carries only that test's SKIP token — the exact shape of a self-skipping L2/L3
|
||||
# test. record_run.py must refuse to call it `proven`.
|
||||
# * a ledger row claiming `proven` at a commit that PREDATES the requirement's
|
||||
# implementing code — status.py must degrade it to `stale-proof`, never green.
|
||||
# Both tools' own built-in self-tests (`--selftest`) are also driven, because nothing
|
||||
# in ci/ or run.sh runs them today: the guards that keep a skip from becoming a proof
|
||||
# were themselves unexercised in the merge gate until this check.
|
||||
#
|
||||
# The FIVE PROOFS ARE NOT HAND-LISTED where the spec can be read: when the read-only spec
|
||||
# repo is mounted, the bullet labels are parsed out of docs/07 and must match this check's
|
||||
# mapping EXACTLY. Add a sixth proof to the spec and this check fails as unmapped rather
|
||||
# than quietly ignoring it.
|
||||
#
|
||||
# WHY L0: every clause above is a property of the repository and its own tooling, provable
|
||||
# offline, and it belongs in the merge gate — the point of an audit chain is that it is intact
|
||||
# BEFORE anyone needs it. Nothing here needs a cluster, and nothing here would be more true
|
||||
# with one.
|
||||
#
|
||||
# WHAT THIS DOES *NOT* PROVE (honest gaps — do not read this check as claiming them):
|
||||
# - THAT THE FIVE PROOFS PASS. This asserts the proofs are present, strong, runnable by one
|
||||
# documented command, capturable as evidence, and that the evidence chain refuses
|
||||
# forgeries. Whether isolation actually holds today is R-NET-01's test; whether the DR
|
||||
# restore works is R-STATE-03's. R-SEC-03 is about the operator's ABILITY to produce the
|
||||
# proof, which is what is asserted.
|
||||
# - HUBBLE. docs/07 names "Hubble flow logs" as part of the isolation proof. No test in this
|
||||
# suite queries Hubble, and this check does not pretend otherwise: it asserts the
|
||||
# denied-probe half, which is the half that is automated. The Hubble half remains an
|
||||
# operator-interactive capability — a real, named gap in R-SEC-03's coverage.
|
||||
# - THE OPERATOR-FACING NARRATIVE. That a human can find and read the evidence (the status
|
||||
# report's prose, the ledger's provenance fields) is a usability property, not a testable
|
||||
# one.
|
||||
# - LEDGER COMPLETENESS OVER TIME. The ledger is append-only by convention and by review;
|
||||
# nothing here can prove a row was never removed from it (that would need signed history).
|
||||
# - THAT THE PROOFS COVER "ALL THE ABOVE PROPERTIES". R-SEC-03's own wording is broader than
|
||||
# the five bullets docs/07 enumerates. The five are what the spec made concrete and are
|
||||
# what is asserted; suite-wide coverage of every requirement is R-TEST-01/04's claim,
|
||||
# tested by tests/meta/harness-selfcheck, and is deliberately NOT re-asserted here.
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT="$(cd "${DIR}/../../../.." && pwd)" # repo root
|
||||
PYTHON="${PYTHON:-python3}"
|
||||
: "${SPEC_REPO:=${ROOT}/../home-network-k3s}"
|
||||
export SPEC_REPO
|
||||
|
||||
cd "${ROOT}"
|
||||
"${PYTHON}" - "${ROOT}" <<'PY'
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(sys.argv[1])
|
||||
CONF = ROOT / "conformance"
|
||||
LIB = CONF / "lib"
|
||||
FAILURES: list[str] = []
|
||||
NOTES: list[str] = []
|
||||
CHECKS = 0
|
||||
|
||||
|
||||
def check(cond, msg):
|
||||
global CHECKS
|
||||
CHECKS += 1
|
||||
if not cond:
|
||||
FAILURES.append(msg)
|
||||
|
||||
|
||||
def note(msg):
|
||||
NOTES.append(msg)
|
||||
|
||||
|
||||
def die():
|
||||
print(f"CHECK FAIL (auditability-proofs-producible): {len(FAILURES)} of {CHECKS} "
|
||||
"assertions failed", file=sys.stderr)
|
||||
for f in FAILURES:
|
||||
print(f" - {f}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
def run(cmd, **kw) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(cmd, capture_output=True, text=True, cwd=kw.pop("cwd", CONF), **kw)
|
||||
|
||||
|
||||
# ================================================= THE FIVE PROOFS docs/07 ENUMERATES
|
||||
# key = the bullet label docs/07 uses (verified against the spec below when it is mounted)
|
||||
# reqs = the requirement(s) whose verifying test IS that proof
|
||||
# layer= the WEAKEST layer the spec's own words allow. Where docs/07 says "a fault-injection
|
||||
# test" / "a DR drill", an L2 happy-path test does not discharge the proof.
|
||||
PROOFS = {
|
||||
"Isolation": {"reqs": ["R-NET-01"], "layers": {"L2", "L3"},
|
||||
"why": '"service X cannot reach service Y" — via Hubble flow logs + a '
|
||||
"denied-probe test"},
|
||||
"Egress path": {"reqs": ["R-NET-05"], "layers": {"L2", "L3"},
|
||||
"why": '"service X\'s public IP is the VPN\'s" — via an egress IP check'},
|
||||
"Kill-switch": {"reqs": ["R-NET-06"], "layers": {"L3"},
|
||||
"why": '"with VPN down, X leaks nothing" — via a fault-injection test'},
|
||||
"TLS": {"reqs": ["R-TLS-01"], "layers": {"L2", "L3"},
|
||||
"why": '"every endpoint serves a valid, non-self-signed, unexpired cert" '
|
||||
"— via cert status + external check"},
|
||||
"Backups": {"reqs": ["R-STATE-03"], "layers": {"L3"},
|
||||
"why": '"irreplaceable data is restorable" — via a DR drill'},
|
||||
}
|
||||
PROOF_REQS = sorted({r for p in PROOFS.values() for r in p["reqs"]})
|
||||
|
||||
# ---- SELF-POLICING: the proof list must still be the SPEC's proof list ---------------
|
||||
spec = Path(os.environ.get("SPEC_REPO", ""))
|
||||
sec_doc = spec / "docs" / "07-security-and-secrets.md"
|
||||
if sec_doc.is_file():
|
||||
text = sec_doc.read_text()
|
||||
m = re.search(r"^##\s+Auditability requirements \(R-SEC-03\).*?$(.*?)^##\s", text,
|
||||
re.M | re.S)
|
||||
check(m is not None,
|
||||
"docs/07 no longer carries an '## Auditability requirements (R-SEC-03)' section — "
|
||||
"the proof list this check is coupled to has moved or been removed, so the mapping "
|
||||
"below can no longer be trusted (R-TEST-06: follow the spec, never the other way)")
|
||||
if m:
|
||||
labels = re.findall(r"^\s*-\s+\*\*([^*]+)\*\*:", m.group(1), re.M)
|
||||
check(set(labels) == set(PROOFS),
|
||||
f"docs/07 enumerates the auditability proofs {sorted(labels)} but this check "
|
||||
f"maps {sorted(PROOFS)} — every proof the spec names must map to a verifying "
|
||||
"test, and a proof this check does not know about would be silently unverified")
|
||||
check(len(labels) == 5,
|
||||
f"docs/07 enumerates {len(labels)} auditability proof(s); its own acceptance "
|
||||
"gate says 'The five auditability proofs above can be produced on demand' — the "
|
||||
"prose and the list have drifted apart")
|
||||
else:
|
||||
note(f"the read-only spec repo is not mounted at {spec} — the proof list could not be "
|
||||
"re-derived from docs/07, so the mapping below was used as written. Every other "
|
||||
"assertion still ran. (ci/l0.sh mounts the spec; a bare local run may not.)")
|
||||
|
||||
# ============================================ (a) EACH PROOF EXISTS, AT THE RIGHT STRENGTH
|
||||
# Driven through lib/status.py so this check and the repository's own status report cannot
|
||||
# disagree about what is executable and what is a scaffold.
|
||||
tmp = Path(tempfile.mkdtemp(prefix="r-sec-03-"))
|
||||
try:
|
||||
out = tmp / "status"
|
||||
st = run([sys.executable, str(LIB / "status.py"), "--repo", str(ROOT),
|
||||
"--registry", str(CONF / "requirements" / "registry.yaml"),
|
||||
"--tests", str(CONF / "tests"), "--out", str(out)])
|
||||
check(st.returncode == 0,
|
||||
f"lib/status.py exited {st.returncode} — the report an operator would use to find "
|
||||
f"the audit evidence does not generate: {(st.stderr or st.stdout).strip()[:400]}")
|
||||
status = {}
|
||||
sj = out / "status.json"
|
||||
if sj.is_file():
|
||||
status = json.loads(sj.read_text())
|
||||
check(bool(status.get("by_requirement")),
|
||||
"lib/status.py produced no per-requirement report — the audit posture of each "
|
||||
"requirement is not derivable, so R-SEC-03 cannot be assessed at all")
|
||||
by_req = status.get("by_requirement", {})
|
||||
|
||||
for label, spec_proof in sorted(PROOFS.items()):
|
||||
for rid in spec_proof["reqs"]:
|
||||
rec = by_req.get(rid)
|
||||
check(rec is not None,
|
||||
f"proof '{label}' maps to {rid}, which the status report does not know — "
|
||||
"either the registry lost the requirement or this mapping is stale")
|
||||
if not rec:
|
||||
continue
|
||||
tests = (rec.get("verification") or {}).get("tests") or []
|
||||
executable = [t for t in tests if t.get("state", "").startswith("executable")]
|
||||
check(bool(executable),
|
||||
f"proof '{label}' ({rid}) has no EXECUTABLE verifying test — its tests are "
|
||||
f"{[t.get('name') + ':' + t.get('state', '?') for t in tests]}. docs/07: "
|
||||
f"{spec_proof['why']}. A placeholder scaffold cannot be produced on demand, so the "
|
||||
"operator cannot prove this property")
|
||||
at_layer = [t for t in executable if t.get("layer") in spec_proof["layers"]]
|
||||
check(bool(at_layer),
|
||||
f"proof '{label}' ({rid}) has no executable test at layer(s) "
|
||||
f"{sorted(spec_proof['layers'])} — it has "
|
||||
f"{[t.get('name') + '@' + str(t.get('layer')) for t in executable]}. "
|
||||
f"docs/07 requires this proof to be produced {spec_proof['why']}, and a "
|
||||
"weaker layer does not discharge it (a rendering check is not a drill)")
|
||||
|
||||
# ==================================== (b) PRODUCIBLE ON DEMAND, BY THE DOCUMENTED COMMAND
|
||||
# The resolver: --requirements must translate the five R-IDs into real test dirs.
|
||||
resolver = run(["bash", "-c",
|
||||
f'export CONF_ROOT="{CONF}"; . "{LIB}/common.sh"; '
|
||||
f'conf_dirs_for_requirements "{" ".join(PROOF_REQS)}"'])
|
||||
dirs = [l for l in resolver.stdout.splitlines() if l.strip()]
|
||||
check(resolver.returncode == 0 and len(dirs) >= len(PROOF_REQS),
|
||||
f"`run.sh --requirements {','.join(PROOF_REQS)}` resolves to {len(dirs)} test dir(s) "
|
||||
f"for {len(PROOF_REQS)} proof requirement(s) — the single command docs/07's "
|
||||
"acceptance gate depends on ('produced on demand') cannot select the proofs")
|
||||
|
||||
# The gate: the same flag ASSERTS coverage (run.sh passes --require to coverage.py), so a
|
||||
# zero exit here is the repository's own statement that all five proofs are covered.
|
||||
gate = run(["./run.sh", "--coverage-only", "--requirements", ",".join(PROOF_REQS)])
|
||||
check(gate.returncode == 0,
|
||||
f"`./run.sh --coverage-only --requirements {','.join(PROOF_REQS)}` exited "
|
||||
f"{gate.returncode} — the merge gate's own coverage assertion rejects the audit "
|
||||
f"proof set: {(gate.stderr or gate.stdout).strip()[:400]}")
|
||||
|
||||
# ================================================ (c) THE PROOF IS CAPTURABLE AS EVIDENCE
|
||||
# Use record_run.py's OWN sentinel parser: a Chainsaw-layer proof with no declared pass
|
||||
# token can pass forever and never be recorded as proof of anything.
|
||||
sys.path.insert(0, str(LIB))
|
||||
os.environ.setdefault("PYTHONDONTWRITEBYTECODE", "1")
|
||||
sys.dont_write_bytecode = True
|
||||
import record_run as rr # noqa: E402
|
||||
|
||||
smap = rr.sentinel_map(CONF / "tests")
|
||||
check(len(smap) >= 10,
|
||||
f"record_run.py's sentinel map contains {len(smap)} test(s) — the parser lost its "
|
||||
"grip on the test tree, so the assertions below would assert nothing")
|
||||
for label, spec_proof in sorted(PROOFS.items()):
|
||||
for rid in spec_proof["reqs"]:
|
||||
rec = by_req.get(rid) or {}
|
||||
tests = (rec.get("verification") or {}).get("tests") or []
|
||||
names = [t.get("name") for t in tests
|
||||
if t.get("state", "").startswith("executable")
|
||||
and t.get("layer") in spec_proof["layers"]]
|
||||
for name in names:
|
||||
ent = smap.get(name)
|
||||
check(ent is not None,
|
||||
f"proof '{label}': record_run.py does not know the test {name!r} — its "
|
||||
"result could never be attributed to a ledger row")
|
||||
if not ent:
|
||||
continue
|
||||
if ent.get("layer") in rr.SENTINEL_EXEMPT_LAYERS:
|
||||
continue # offline layers are truthful by JUnit alone
|
||||
check(bool(ent.get("pass")),
|
||||
f"proof '{label}': the Chainsaw test {name!r} declares NO pass sentinel, "
|
||||
"so lib/record_run.py can never record it as proven. It would go green "
|
||||
"in the terminal and leave the operator with no evidence — exactly the "
|
||||
"gap the ledger exists to close (docs/07: the proofs are 'part of the "
|
||||
"platform acceptance suite, not one-time checks')")
|
||||
check(bool(ent.get("terminal_pass")),
|
||||
f"proof '{label}': the test {name!r} has no TERMINAL pass token, so "
|
||||
"record_run.py cannot decide its verdict")
|
||||
|
||||
# =============================== (d) THE EVIDENCE CANNOT BE FORGED, NOR HONOURED WHEN STALE
|
||||
# Both tools' own guards, which NOTHING in ci/ or run.sh drives today.
|
||||
for tool in ("record_run.py", "status.py"):
|
||||
r = run([sys.executable, str(LIB / tool), "--selftest"])
|
||||
check(r.returncode == 0,
|
||||
f"`lib/{tool} --selftest` exited {r.returncode} — the guards that keep a "
|
||||
"self-skipped or stale run from being recorded as proof are broken, so nothing "
|
||||
f"in the evidence chain can be trusted: {(r.stderr or r.stdout).strip()[:500]}")
|
||||
|
||||
# ---- FORGERY 1: a green JUnit for a REAL proof test whose log holds only a SKIP token.
|
||||
# This is the precise shape of a self-skipping L2/L3 test, and it must NOT become proof.
|
||||
forge_target, forge_skip = None, None
|
||||
for label, spec_proof in sorted(PROOFS.items()):
|
||||
for rid in spec_proof["reqs"]:
|
||||
for name, ent in smap.items():
|
||||
if rid in (ent.get("requirements") or []) and ent.get("skip") \
|
||||
and ent.get("layer") not in rr.SENTINEL_EXEMPT_LAYERS:
|
||||
forge_target, forge_skip = name, ent["skip"][0]
|
||||
break
|
||||
if forge_target:
|
||||
break
|
||||
if forge_target:
|
||||
break
|
||||
check(forge_target is not None,
|
||||
"no Chainsaw-layer proof test declares a skip token, so the forgery attempt below "
|
||||
"could not be mounted. That is not a pass: it means this check could not verify "
|
||||
"that a self-skip is refused as proof, which is the whole point of the ledger")
|
||||
if forge_target:
|
||||
run_dir = tmp / "20260101T000000Z-forged"
|
||||
run_dir.mkdir(parents=True)
|
||||
(run_dir / "report.junit.xml").write_text(
|
||||
'<?xml version="1.0" encoding="UTF-8"?>\n'
|
||||
'<testsuites><testsuite name="chainsaw" tests="1" failures="0" errors="0">'
|
||||
f'<testcase name="{forge_target}" classname="forged"/>'
|
||||
"</testsuite></testsuites>\n")
|
||||
(run_dir / "run.log").write_text(
|
||||
f"=== RUN chainsaw/{forge_target}\n"
|
||||
f" === STDOUT\n {forge_skip}\n")
|
||||
forged = run([sys.executable, str(LIB / "record_run.py"), "--run-dir", str(run_dir),
|
||||
"--dry-run", "--ledger", str(tmp / "never-written.jsonl"),
|
||||
"--tests", str(CONF / "tests"), "--repo", str(ROOT)])
|
||||
verdicts = re.findall(rf"^\[record\]\s+\S+\s+(\S+)\s+{re.escape(forge_target)}\b",
|
||||
forged.stdout, re.M)
|
||||
check(verdicts and all(v != "proven" for v in verdicts),
|
||||
f"lib/record_run.py recorded a FORGED run as {verdicts!r} for {forge_target!r}: "
|
||||
"a JUnit saying `passed` with nothing in the log but that test's own SKIP token "
|
||||
f"({forge_skip}). A self-skip must never become proof — with this hole every "
|
||||
"audit proof in docs/07 could be produced by a cluster that skipped it")
|
||||
check(not (tmp / "never-written.jsonl").exists(),
|
||||
"record_run.py --dry-run WROTE to the ledger — a tool that records while "
|
||||
"claiming not to cannot be trusted to record only real runs")
|
||||
|
||||
# ---- FORGERY 2: a ledger row claiming `proven` at a commit that predates the code.
|
||||
# status.py must degrade it to stale-proof (not green), or an old green run would keep
|
||||
# vouching for code it never executed.
|
||||
root_sha = run(["git", "-C", str(ROOT), "rev-list", "--max-parents=0", "HEAD"])
|
||||
old_sha = (root_sha.stdout.strip().splitlines() or [""])[0]
|
||||
check(len(old_sha) == 40,
|
||||
"the repository's root commit could not be resolved, so the stale-proof guard could "
|
||||
"not be exercised (that is a gap in this check, not a pass)")
|
||||
if len(old_sha) == 40 and forge_target:
|
||||
stale_ledger = tmp / "stale.jsonl"
|
||||
rid = sorted(smap[forge_target].get("requirements") or ["R-NET-06"])[0]
|
||||
stale_ledger.write_text(json.dumps({
|
||||
"ts_utc": "2026-01-01T00:00:00Z", "git_sha": old_sha, "env": "dev",
|
||||
"chainsaw_version": "v0.0.0", "kube_version": "unknown",
|
||||
"layer": smap[forge_target].get("layer", "L3"), "test": forge_target,
|
||||
"requirements": [rid], "junit_result": "passed",
|
||||
"sentinel": (smap[forge_target].get("terminal_pass") or "X"),
|
||||
"proof": "proven", "run_id": "20260101T000000Z-forged",
|
||||
"note": "synthetic row written by the R-SEC-03 conformance check",
|
||||
}) + "\n")
|
||||
out2 = tmp / "status-stale"
|
||||
st2 = run([sys.executable, str(LIB / "status.py"), "--repo", str(ROOT),
|
||||
"--registry", str(CONF / "requirements" / "registry.yaml"),
|
||||
"--tests", str(CONF / "tests"), "--out", str(out2),
|
||||
"--ledger", str(stale_ledger)])
|
||||
check(st2.returncode == 0,
|
||||
f"lib/status.py failed against a synthetic ledger: "
|
||||
f"{(st2.stderr or st2.stdout).strip()[:300]}")
|
||||
s2 = json.loads((out2 / "status.json").read_text()) if (out2 / "status.json").is_file() else {}
|
||||
rec2 = (s2.get("by_requirement") or {}).get(rid) or {}
|
||||
per = {p.get("test"): p.get("state") for p in ((rec2.get("proof") or {}).get("per_test") or [])}
|
||||
check(per.get(forge_target) == "stale-proof",
|
||||
f"a ledger row claiming `proven` for {forge_target!r} at the repository's ROOT "
|
||||
f"commit was reported as {per.get(forge_target)!r} instead of 'stale-proof'. A "
|
||||
"proof from a run that predates the code it vouches for is not evidence, and "
|
||||
"honouring it would let one old green run certify every later change")
|
||||
check((rec2.get("proof") or {}).get("state") != "proven",
|
||||
f"{rid} is reported PROVEN on the strength of a row from the root commit — the "
|
||||
"freshness guard is not holding")
|
||||
|
||||
# ---- the ledger itself is machine-readable, and no row proves a test that is gone ----
|
||||
ledger = CONF / "runs" / "ledger.jsonl"
|
||||
if ledger.is_file():
|
||||
rows, bad = [], []
|
||||
for i, line in enumerate(ledger.read_text().splitlines(), 1):
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
rows.append(json.loads(line))
|
||||
except json.JSONDecodeError:
|
||||
bad.append(i)
|
||||
check(not bad,
|
||||
f"conformance/runs/ledger.jsonl has unparsable line(s) {bad} — the audit record "
|
||||
"is not machine-readable, so no proof can be derived from it")
|
||||
for r in rows:
|
||||
for key in ("ts_utc", "git_sha", "test", "requirements", "junit_result", "proof",
|
||||
"run_id"):
|
||||
check(key in r,
|
||||
f"a ledger row for {r.get('test', '?')!r} lacks {key!r} — a row without "
|
||||
"provenance cannot be audited")
|
||||
if r.get("proof") == "proven":
|
||||
check(bool(r.get("sentinel")),
|
||||
f"a ledger row claims `proven` for {r.get('test')!r} with NO sentinel — "
|
||||
"the sentinel is the only thing separating a real pass from a self-skip")
|
||||
check(r.get("test") in smap,
|
||||
f"a ledger row claims `proven` for {r.get('test')!r}, which is not a "
|
||||
"test in the suite any more — a proof of something that no longer exists")
|
||||
check("-dirty" not in str(r.get("git_sha", "")),
|
||||
f"a ledger row claims `proven` for {r.get('test')!r} from a DIRTY tree "
|
||||
f"({r.get('git_sha')}) — what ran is not what is committed")
|
||||
else:
|
||||
note("conformance/runs/ledger.jsonl does not exist yet — the ledger-shape assertions "
|
||||
"were skipped; every other assertion ran, and the forgery probes above use "
|
||||
"synthetic ledgers, not this file")
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
# --------------------------------------------------------------------------- verdict
|
||||
# NO-OP GUARD: a check that quietly asserts nothing is worse than no check.
|
||||
check(CHECKS >= 30,
|
||||
f"only {CHECKS} assertions ran — the check degenerated to a near-no-op and its pass "
|
||||
"would be meaningless")
|
||||
|
||||
if FAILURES:
|
||||
die()
|
||||
for n in NOTES:
|
||||
print(f" note: {n}")
|
||||
print(f"check OK: auditability-proofs-producible — {CHECKS} assertions; the five docs/07 "
|
||||
f"proofs ({', '.join(sorted(PROOFS))}) each map to an executable test at the layer the "
|
||||
f"spec's own wording requires, `run.sh --requirements {','.join(PROOF_REQS)}` resolves "
|
||||
"and asserts them, each Chainsaw-layer proof declares a sentinel record_run.py can "
|
||||
"attribute, both evidence tools' self-tests pass, and two independent forgeries — a "
|
||||
"self-skip presented as a pass, and a `proven` row from the root commit — were REFUSED. "
|
||||
"R-SEC-03's producibility half only: whether each proof PASSES is its own "
|
||||
"requirement's test, and the Hubble half of the isolation proof is automated nowhere "
|
||||
"(see the header's gap list).")
|
||||
PY
|
||||
@@ -1,99 +0,0 @@
|
||||
# =============================================================================
|
||||
# L0 STATIC — the bulk tier's capacity strategy is DISTINCT from the replicated tier's
|
||||
# Catalog: docs/14 L0 row ("Pre-apply, in CI") — runs WITHOUT a cluster.
|
||||
#
|
||||
# THE BUG CLASS THIS CLOSES. The `bulk` StorageClass existed from Phase 0 as a stub:
|
||||
# `provisioner: nfs.csi.k8s.io`, `server: REPLACE-ME`, `share: REPLACE-ME`, TODO(OQ-2).
|
||||
# Four ServiceDefinitions carried a comment promising to attach their media once OQ-2 was
|
||||
# resolved, and the ONLY thing keeping a multi-terabyte library off the 3-replica
|
||||
# `longhorn` class was those four comments. A bulk volume that lands on the replicated
|
||||
# tier does not fail loudly — Longhorn accepts the PVC and starts replicating 3x of data
|
||||
# three mini-PCs cannot hold. R-STATE-04 is precisely that the two tiers are distinct, so
|
||||
# the distinctness is asserted rather than reviewed.
|
||||
#
|
||||
# It also pins the four decisions that make the STATIC form of this tier safe, each with a
|
||||
# plausible "simplification" that breaks something silently: `Retain` (bulk data pre-exists
|
||||
# the cluster and has consumers outside it, so anything else makes `delete pvc` a data-loss
|
||||
# operation); NO dynamic provisioning (a dynamic bulk class hands out a FRESH EMPTY
|
||||
# directory that mounts and Binds perfectly while the real 8 TiB is never attached — the
|
||||
# most deceptive failure available here); `hard` not `soft` (a soft mount hands a server
|
||||
# hiccup to the app as a short read, which for a media library is silent corruption); and
|
||||
# NFSv4 (mount and lock in-protocol ⇒ ONE reachable port, tcp/2049, which is what the
|
||||
# operator must open in pfSense for prod to reach this server at all).
|
||||
#
|
||||
# AND the EXDEV trap, which is not about storage. Sonarr/Radarr import by hardlinking or
|
||||
# atomically renaming, both same-filesystem operations, and to the kernel "same filesystem"
|
||||
# means SAME MOUNT. Splitting one export into two subPath mounts looks TIGHTER and breaks
|
||||
# link() with EXDEV; the arrs then degrade to copy-then-delete, rewriting every import over
|
||||
# NFS and destroying the seeding copy. It surfaces weeks later as "imports are slow and my
|
||||
# ratio is gone". A reviewer cannot see that; this check can.
|
||||
#
|
||||
# GIVEN the repository as authored,
|
||||
# WHEN every `class: bulk` storage entry in catalog/ is compared against the objects the
|
||||
# COMMITTED apps/ goldens carry for it (what Flux reconciles — deliberately not the
|
||||
# renderer, which would only prove it agrees with itself),
|
||||
# THEN (1) no bulk volume is on a Longhorn-provisioned class, and its class is explicitly
|
||||
# EMPTY so nothing can dynamically provision an empty stand-in;
|
||||
# (2) its capacity is declared and is at least an order of magnitude above the same
|
||||
# service's replicated volume — the requirement stated as arithmetic;
|
||||
# (3) PV and PVC are pre-bound from BOTH ends (claimRef + volumeName) and agree;
|
||||
# (4) the PV reclaim policy is Retain;
|
||||
# (5) mountOptions carry `hard` and exactly one `nfsvers=4.x`, and never `soft`;
|
||||
# (6) no workload mounts one export more than once (the EXDEV trap);
|
||||
# (7) a readOnly entry renders nfs.readOnly AND a readOnly volumeMount AND
|
||||
# ReadOnlyMany; a writable one renders neither and is ReadWriteMany; every bulk
|
||||
# access mode is a `...Many`, since one export serves several namespaces and
|
||||
# several nodes;
|
||||
# (8) the PV's nfs.server is a BARE cluster-vars token (never a literal — dev and
|
||||
# prod reach this host by different addresses), that knob has a base default, and
|
||||
# EVERY overlay supplies it;
|
||||
# AND (9) the check is LOAD-BEARING (ci/l0.sh runs it unconditionally, ci/gate.sh blocks
|
||||
# on the l0 stage) and NON-VACUOUS (an empty bulk corpus FAILS rather than
|
||||
# passing trivially, and a floor on the assertion count keeps a pass from
|
||||
# degenerating into a no-op).
|
||||
#
|
||||
# STATUS: EXECUTABLE. `check.sh` is a pure offline assertion script (standard library +
|
||||
# PyYAML); conformance/run.sh --layer l0 runs it directly and this Chainsaw Test wraps the
|
||||
# SAME script.
|
||||
#
|
||||
# NOT proven — the runtime half, which needs a cluster: that the server is reachable, that
|
||||
# the export admits these clients, that the PVC binds, that a write succeeds, or that a
|
||||
# hardlink between the library and the download path actually works. The uid analysis is
|
||||
# reasoning (docs/BULK-NFS-STORAGE.md §6) and the in-cluster probes that would settle it
|
||||
# are written down in §9 of that document, unrun. Nor can this prove prod's address is
|
||||
# CORRECT — only that it is a supplied knob; while it is a placeholder, prod's pods will
|
||||
# not mount, and catching that statically needs the placeholder guard, which is absent from
|
||||
# main (§7). Read-only on the media tree is a CLIENT-side guarantee only: the export serves
|
||||
# one reader and two writers from the same node addresses, so the server cannot tell them
|
||||
# apart (§5).
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: bulk-tier-distinct-from-replicated
|
||||
labels:
|
||||
conformance.home.k3s/R-STATE-04: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-STATE-04"
|
||||
conformance.home.k3s/layer: "L0"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
Bulk media storage is provably NOT on the 3-replica Longhorn tier, is sized for a
|
||||
different order of magnitude, cannot be silently replaced by a dynamically
|
||||
provisioned empty volume, is never reclaimable by the cluster, mounts hard over
|
||||
NFSv4, and is never split into two mounts of one export — so hardlinked imports keep
|
||||
working. Its server is a per-environment knob, not a literal, because the two
|
||||
clusters reach the same storage host by different addresses.
|
||||
spec:
|
||||
steps:
|
||||
- name: assert-bulk-tier-is-distinct-and-statically-bound
|
||||
description: >-
|
||||
Read every `class: bulk` storage entry from catalog/, pair it with the
|
||||
PersistentVolume, PersistentVolumeClaim and volumeMount the committed apps/ goldens
|
||||
carry for it, and assert the tier separation, the static pre-binding, the reclaim
|
||||
and mount-option decisions, the read-only coupling, the one-export-one-mount
|
||||
invariant, and that the server address is supplied by every environment overlay.
|
||||
try:
|
||||
- script:
|
||||
content: bash ./check.sh
|
||||
@@ -1,480 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# L0 static check (offline, no cluster) — bulk-tier-distinct-from-replicated
|
||||
# Serves: R-STATE-04 ("Bulk media storage has a capacity strategy distinct from the
|
||||
# replicated tier." — spec docs/04-storage-and-state.md)
|
||||
#
|
||||
# WHY THIS EXISTS — THE BULK TIER WAS A STUB FOR SIX ROADMAP PHASES, AND STUBS ROT QUIETLY
|
||||
# ----------------------------------------------------------------------------------------
|
||||
# `platform/longhorn/storageclasses.yaml` declared a `bulk` StorageClass from Phase 0 with
|
||||
# `provisioner: nfs.csi.k8s.io`, `server: REPLACE-ME`, `share: REPLACE-ME` and a TODO(OQ-2).
|
||||
# Four ServiceDefinitions carried a comment promising to attach their media "once OQ-2 is
|
||||
# resolved", and each of them said, correctly, that adding `class: bulk` before then would
|
||||
# leave the PVC unbound and the pod Pending. So the tier existed as a NAME with nothing
|
||||
# behind it, and the only thing preventing a multi-terabyte media library from being
|
||||
# authored onto the 3-replica `longhorn` class was four hand-written comments.
|
||||
#
|
||||
# That is the failure this check makes impossible to reintroduce. R-STATE-04 is not "bulk
|
||||
# storage exists" — it is that bulk storage's capacity strategy is DISTINCT from the
|
||||
# replicated tier's. Three mini-PCs cannot hold 3x of 8.33 TiB. A bulk volume that lands on
|
||||
# `longhorn` does not fail loudly: Longhorn accepts the PVC and starts replicating, and the
|
||||
# tier fills up. Distinctness is therefore a property worth asserting statically, on every
|
||||
# gate run, rather than trusting to review.
|
||||
#
|
||||
# It also pins the four decisions that make the STATIC form of this tier safe, each of
|
||||
# which has a plausible-looking "simplification" that silently breaks something:
|
||||
#
|
||||
# * RECLAIM POLICY. Bulk data pre-exists the cluster, is larger than the cluster, and has
|
||||
# consumers OUTSIDE it. `Retain` is the whole reason a PV is used here instead of an
|
||||
# inline volume. `Delete` on a dynamically-provisioned bulk class would make `kubectl
|
||||
# delete pvc` a data-loss operation.
|
||||
# * NO DYNAMIC PROVISIONING. A dynamic bulk class hands out FRESH EMPTY directories. The
|
||||
# pod mounts successfully, the PVC is Bound, everything is green — and the 8 TiB of
|
||||
# existing data was never attached. This is the single most deceptive failure available
|
||||
# in this tier, so `storageClassName` must be explicitly empty and binding must be
|
||||
# pinned from BOTH ends (claimRef + volumeName).
|
||||
# * `hard`, NOT `soft`. A soft NFS mount converts a server hiccup into a short read or
|
||||
# EIO handed to the application. For a media library that is silent corruption. `hard`
|
||||
# hangs instead — recoverable, where truncated is not.
|
||||
# * NFSv4. v4 carries mount and lock in-protocol, so the tier needs ONE reachable port
|
||||
# (tcp/2049) instead of v3's rpcbind/111 + mountd + statd on ephemeral numbers. That is
|
||||
# a FIREWALL fact as much as a storage one: prod reaches this server through pfSense,
|
||||
# and the number of ports the operator must open is a consequence of this field.
|
||||
#
|
||||
# AND ONE INVARIANT THAT IS NOT ABOUT STORAGE AT ALL — the EXDEV trap (assertion 6).
|
||||
# Sonarr and Radarr import by hardlinking or atomically renaming a finished download into
|
||||
# the library. Both are same-filesystem operations, and to the kernel "same filesystem"
|
||||
# means SAME MOUNT. Mounting one export twice (say `subPath: tv` and `subPath: downloads`)
|
||||
# looks TIGHTER — less of the tree exposed per mount — and breaks `link()` with EXDEV no
|
||||
# matter that the server knows they are one dataset. The arrs do not fail loudly on that:
|
||||
# they degrade to copy-then-delete, rewriting every imported file over NFS and destroying
|
||||
# the seeding copy. The symptom appears weeks later as "imports are slow and my ratio is
|
||||
# gone". A reviewer cannot see it, so this check does: one export, at most one mount, per
|
||||
# workload.
|
||||
#
|
||||
# GIVEN the repository as authored,
|
||||
# WHEN every `class: bulk` storage entry in catalog/ is compared against the manifests
|
||||
# render.py produces for it (read from the committed apps/ goldens, i.e. what Flux
|
||||
# actually reconciles — not from the renderer, which would only agree with itself),
|
||||
# THEN (1) DISTINCTNESS: no bulk volume is on the replicated tier. Its PV/PVC
|
||||
# storageClassName is explicitly empty — never `longhorn`, never any class the
|
||||
# repo declares with the Longhorn provisioner — and its PV's source is an
|
||||
# EXTERNAL one (nfs), not a Longhorn volume;
|
||||
# (2) CAPACITY STRATEGY: every bulk PV declares a capacity, and for each service the
|
||||
# bulk volume is at least an ORDER OF MAGNITUDE larger than that service's own
|
||||
# replicated volume. This is the requirement stated as arithmetic: the two tiers
|
||||
# exist because the data has two magnitudes, so a "bulk" volume sized like a
|
||||
# config PVC means the strategy was named and not actually applied;
|
||||
# (3) NOT DYNAMICALLY PROVISIONED, AND PRE-BOUND BOTH WAYS: every bulk PVC names its
|
||||
# PV by volumeName, every bulk PV names its PVC by claimRef (namespace+name), and
|
||||
# the two agree — so no other claim can win a race, and nothing can provision an
|
||||
# empty replacement;
|
||||
# (4) RECLAIM: every bulk PV is `Retain`;
|
||||
# (5) MOUNT SAFETY: every bulk PV declares mountOptions carrying `hard` and an
|
||||
# `nfsvers=4.x`, and carrying NEITHER `soft` NOR an `nfsvers=3`;
|
||||
# (6) ONE EXPORT, ONE MOUNT PER WORKLOAD (the EXDEV trap above);
|
||||
# (7) READ-ONLY IS REAL AND CONSISTENT: a bulk entry declared readOnly renders BOTH
|
||||
# `spec.nfs.readOnly: true` on the PV AND `readOnly: true` on the volumeMount,
|
||||
# and its access mode is ReadOnlyMany; a writable one renders NEITHER and is
|
||||
# ReadWriteMany. Every bulk access mode is a `...Many` — one export is consumed
|
||||
# by several namespaces and several nodes, which ReadWriteOnce cannot express;
|
||||
# (8) AUTHORED ONCE, PROMOTED (the R-ENV-02 seam this tier depends on): every bulk
|
||||
# PV's `nfs.server` is the BARE cluster-vars token, never a literal address; that
|
||||
# knob has a base default; and EVERY environment overlay supplies it — so the two
|
||||
# clusters, which reach this storage by different addresses, cannot be served by
|
||||
# one hardcoded value;
|
||||
# AND (9) LOAD-BEARING + NON-VACUOUS: ci/l0.sh runs this check unconditionally and
|
||||
# ci/gate.sh blocks the merge when the l0 stage fails; and the corpus is NOT
|
||||
# empty — a repo with zero bulk volumes fails here rather than passing trivially,
|
||||
# because a check that is satisfiable by deleting the feature proves nothing.
|
||||
#
|
||||
# WHAT THIS DOES *NOT* PROVE (honest gaps)
|
||||
# - Nothing about RUNTIME. Not that the server is reachable, not that the export permits
|
||||
# these clients, not that the PVC binds, not that a write succeeds. The uid analysis in
|
||||
# docs/BULK-NFS-STORAGE.md §6 is reasoning, and the write/hardlink probes in §9 are the
|
||||
# in-cluster proof — they need a cluster and are not run here.
|
||||
# - It cannot prove the prod address is CORRECT, only that it is a supplied knob. While
|
||||
# prod's value is a placeholder, prod's pods will not mount (docs/BULK-NFS-STORAGE.md
|
||||
# §7); catching that needs the placeholder guard, which is absent from main.
|
||||
# - Read-only for the media tree is a CLIENT-side guarantee, and this check can only
|
||||
# assert the client side. The export serves a reader and two writers from the same node
|
||||
# addresses, so the server cannot distinguish them (docs/BULK-NFS-STORAGE.md §5).
|
||||
# - Assertion (2)'s magnitude ratio is a heuristic with a deliberately generous floor. It
|
||||
# catches "bulk volume sized like a config volume", not a subtly wrong capacity — and a
|
||||
# static NFS PV's capacity is advisory anyway, since the ZFS pool is the real limit.
|
||||
# - Assertion (6) reasons over ONE workload's mounts. It cannot see two different
|
||||
# services hardlinking between each other's mounts, which is not a thing that happens.
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT="$(cd "${DIR}/../../../.." && pwd)" # repo root
|
||||
PYTHON="${PYTHON:-python3}"
|
||||
|
||||
cd "${ROOT}"
|
||||
|
||||
if [ ! -d "${ROOT}/catalog" ] || [ ! -d "${ROOT}/apps" ]; then
|
||||
echo "check SKIPPED (bulk-tier-distinct-from-replicated): catalog/ or apps/ absent — nothing to audit" >&2
|
||||
exit 3
|
||||
fi
|
||||
|
||||
"${PYTHON}" - "${ROOT}" <<'PY'
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError: # pragma: no cover
|
||||
print("check FAIL: PyYAML is required (same dependency as catalog/render.py)",
|
||||
file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
ROOT = Path(sys.argv[1])
|
||||
FAILURES: list[str] = []
|
||||
CHECKS = 0
|
||||
|
||||
|
||||
def check(cond, msg):
|
||||
global CHECKS
|
||||
CHECKS += 1
|
||||
if not cond:
|
||||
FAILURES.append(msg)
|
||||
|
||||
|
||||
def bail(msg):
|
||||
"""A failure that makes the rest of the check meaningless: report and stop."""
|
||||
print(f"CHECK FAIL (bulk-tier-distinct-from-replicated): {msg}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
def qty_to_bytes(q):
|
||||
"""Kubernetes quantity -> bytes. Only the binary suffixes the schema permits."""
|
||||
m = re.fullmatch(r"([0-9]+)(Mi|Gi|Ti)", str(q).strip())
|
||||
if not m:
|
||||
return None
|
||||
return int(m.group(1)) * {"Mi": 2 ** 20, "Gi": 2 ** 30, "Ti": 2 ** 40}[m.group(2)]
|
||||
|
||||
|
||||
# =====================================================================================
|
||||
# The replicated tier, named by the repo rather than by this file. Any StorageClass the
|
||||
# repo declares with the Longhorn provisioner is "the replicated tier" for assertion (1);
|
||||
# deriving it means adding a new Longhorn class cannot create a blind spot here.
|
||||
# =====================================================================================
|
||||
REPLICATED_CLASSES: set[str] = set()
|
||||
LONGHORN_DIR = ROOT / "platform" / "longhorn"
|
||||
if LONGHORN_DIR.is_dir():
|
||||
for f in sorted(LONGHORN_DIR.rglob("*.yaml")):
|
||||
for doc in yaml.safe_load_all(f.read_text()):
|
||||
if isinstance(doc, dict) and doc.get("kind") == "StorageClass":
|
||||
if "longhorn" in str(doc.get("provisioner", "")):
|
||||
REPLICATED_CLASSES.add(doc.get("metadata", {}).get("name", ""))
|
||||
# `longhorn` is the name every ServiceDefinition uses for the replicated tier, so it is in
|
||||
# the set even if platform/longhorn/ is not present in this checkout.
|
||||
REPLICATED_CLASSES.add("longhorn")
|
||||
REPLICATED_CLASSES.discard("")
|
||||
|
||||
# =====================================================================================
|
||||
# The bulk corpus: every `class: bulk` storage entry in every ServiceDefinition, paired
|
||||
# with the objects the COMMITTED goldens carry for it. Reading the goldens (not calling
|
||||
# the renderer) is deliberate: the goldens are what Flux reconciles, and render==golden is
|
||||
# a separate gate stage. Asking the renderer would only prove it agrees with itself.
|
||||
# =====================================================================================
|
||||
sds: dict[str, dict] = {}
|
||||
for f in sorted((ROOT / "catalog").glob("*.yaml")):
|
||||
doc = yaml.safe_load(f.read_text())
|
||||
if isinstance(doc, dict) and doc.get("name"):
|
||||
sds[doc["name"]] = doc
|
||||
|
||||
if not sds:
|
||||
bail("no ServiceDefinitions under catalog/ — cannot audit the bulk tier")
|
||||
|
||||
# Every rendered app-tier document, indexed by (kind, name).
|
||||
rendered: dict[tuple, dict] = {}
|
||||
for f in sorted((ROOT / "apps").rglob("*.yaml")):
|
||||
if f.name == "kustomization.yaml" or ".sops." in f.name:
|
||||
continue
|
||||
try:
|
||||
docs = list(yaml.safe_load_all(f.read_text()))
|
||||
except yaml.YAMLError:
|
||||
continue
|
||||
for doc in docs:
|
||||
if isinstance(doc, dict) and doc.get("kind") and doc.get("metadata", {}).get("name"):
|
||||
rendered[(doc["kind"], doc["metadata"]["name"])] = doc
|
||||
|
||||
# (svc, vol_entry) for every bulk entry, and the replicated sizes alongside for (2).
|
||||
corpus: list[tuple[str, dict]] = []
|
||||
replicated_bytes: dict[str, list[int]] = {}
|
||||
for svc, sd in sds.items():
|
||||
for vol in sd.get("storage") or []:
|
||||
cls = vol.get("class", "longhorn")
|
||||
if cls == "bulk":
|
||||
corpus.append((svc, vol))
|
||||
else:
|
||||
b = qty_to_bytes(vol.get("size"))
|
||||
if b:
|
||||
replicated_bytes.setdefault(svc, []).append(b)
|
||||
|
||||
# ------------------------------------------------------------------- (9) NON-VACUOUS
|
||||
# A corpus of zero would make every loop below trivially satisfied. The bulk tier is
|
||||
# DELIVERED; a change that removes the last bulk volume must fail here and be argued for,
|
||||
# not sail through on an empty iteration.
|
||||
check(len(corpus) > 0,
|
||||
"NO `class: bulk` storage entry exists in catalog/ — this check would then pass "
|
||||
"vacuously, which is exactly the posture that let the bulk tier sit as a REPLACE-ME "
|
||||
"stub for six roadmap phases. If the tier is genuinely being retired, retire this "
|
||||
"check in the same change and say so; do not let it pass on an empty corpus")
|
||||
if not corpus:
|
||||
bail("empty bulk corpus — refusing to report a vacuous pass (see assertion 9)")
|
||||
|
||||
inventory: list[str] = []
|
||||
|
||||
for svc, vol in corpus:
|
||||
vname = vol.get("name", "<unnamed>")
|
||||
label = f"{svc}/{vname}"
|
||||
obj = f"{svc}-{vname}"
|
||||
ro = bool(vol.get("readOnly"))
|
||||
|
||||
# `export` is what the schema requires of a bulk entry; without it there is nothing to
|
||||
# attach and the rest of the assertions have no subject.
|
||||
export = vol.get("export")
|
||||
check(bool(export) and str(export).startswith("/"),
|
||||
f"{label}: bulk entry has no absolute `export` path — a bulk volume attaches an "
|
||||
f"EXISTING dataset, so it must say which one (schema requires it when class is bulk)")
|
||||
|
||||
pv = rendered.get(("PersistentVolume", obj))
|
||||
pvc = rendered.get(("PersistentVolumeClaim", obj))
|
||||
check(pv is not None,
|
||||
f"{label}: no PersistentVolume '{obj}' in the committed apps/ goldens. A bulk "
|
||||
f"entry that renders no PV has NO external storage behind it — the pod would get "
|
||||
f"an unbound PVC, or (worse, with a dynamic class) a fresh EMPTY directory")
|
||||
check(pvc is not None,
|
||||
f"{label}: no PersistentVolumeClaim '{obj}' in the committed apps/ goldens")
|
||||
if pv is None or pvc is None:
|
||||
continue
|
||||
|
||||
pvs, pvcs = pv.get("spec", {}) or {}, pvc.get("spec", {}) or {}
|
||||
|
||||
# ------------------------------------------------------ (1) DISTINCT FROM REPLICATED
|
||||
for what, spec in (("PV", pvs), ("PVC", pvcs)):
|
||||
sc = spec.get("storageClassName")
|
||||
check(sc not in REPLICATED_CLASSES,
|
||||
f"{label}: bulk {what} storageClassName is '{sc}', which the repo declares "
|
||||
f"with the Longhorn provisioner. That puts BULK data on the REPLICATED tier "
|
||||
f"— 3x of a multi-terabyte library across three mini-PCs. Longhorn will not "
|
||||
f"refuse it; it will accept the PVC and fill the tier (R-STATE-04)")
|
||||
check(sc == "",
|
||||
f"{label}: bulk {what} storageClassName is {sc!r}, not the explicitly EMPTY "
|
||||
f"string. Empty is what disables dynamic provisioning AND the default-class "
|
||||
f"annotation. Any named class risks a provisioner handing out a fresh EMPTY "
|
||||
f"directory that mounts and binds cleanly while the real data is never attached")
|
||||
check("nfs" in pvs,
|
||||
f"{label}: bulk PV declares no external `nfs` source — the bulk tier's whole "
|
||||
f"premise is that durability lives OUTSIDE the cluster (docs/04)")
|
||||
|
||||
# ------------------------------------------------------------ (4) RECLAIM POLICY
|
||||
check(pvs.get("persistentVolumeReclaimPolicy") == "Retain",
|
||||
f"{label}: bulk PV reclaim policy is "
|
||||
f"{pvs.get('persistentVolumeReclaimPolicy')!r}, not Retain. This data pre-exists "
|
||||
f"the cluster and has consumers outside it; anything but Retain makes `kubectl "
|
||||
f"delete pvc` a data-loss operation")
|
||||
|
||||
# ------------------------------------------------- (3) PRE-BOUND FROM BOTH ENDS
|
||||
check(pvcs.get("volumeName") == obj,
|
||||
f"{label}: bulk PVC does not name its PV via volumeName (got "
|
||||
f"{pvcs.get('volumeName')!r}, expected {obj!r}) — without it the claim can be "
|
||||
f"satisfied by some OTHER volume of matching size and mode")
|
||||
cref = pvs.get("claimRef") or {}
|
||||
check(cref.get("name") == obj and cref.get("namespace") == svc,
|
||||
f"{label}: bulk PV claimRef is {cref!r}, expected namespace={svc!r} name={obj!r} "
|
||||
f"— pre-binding must be pinned from BOTH ends or another PVC can win the race")
|
||||
|
||||
# ------------------------------------------------------------- (5) MOUNT SAFETY
|
||||
mopts = [str(o) for o in (pvs.get("mountOptions") or [])]
|
||||
check(bool(mopts),
|
||||
f"{label}: bulk PV declares no mountOptions — then `hard` vs `soft` and the NFS "
|
||||
f"version are whatever the client happens to default to, and both are decisions "
|
||||
f"with consequences (silent short reads; how many firewall ports must open)")
|
||||
check("hard" in mopts,
|
||||
f"{label}: bulk PV mountOptions do not include `hard` ({mopts}). A soft mount "
|
||||
f"hands a server hiccup to the application as a short read or EIO — silent "
|
||||
f"corruption for a media library. `hard` hangs instead, which is recoverable")
|
||||
check("soft" not in mopts,
|
||||
f"{label}: bulk PV mountOptions include `soft` ({mopts}) — see above; this is the "
|
||||
f"one option that turns an outage into damaged data")
|
||||
vers = [o for o in mopts if o.startswith("nfsvers=")]
|
||||
check(len(vers) == 1 and re.fullmatch(r"nfsvers=4(\.[0-9]+)?", vers[0]),
|
||||
f"{label}: bulk PV must pin exactly one NFSv4 `nfsvers=` option (got {vers}). "
|
||||
f"NFSv4 carries mount and lock in-protocol, so the tier needs ONE reachable port "
|
||||
f"(tcp/2049); v3 also needs rpcbind/111, mountd and statd, which changes what the "
|
||||
f"operator has to open in pfSense for prod to reach this server at all")
|
||||
|
||||
# ---------------------------------------------- (7) READ-ONLY IS REAL + CONSISTENT
|
||||
want_mode = "ReadOnlyMany" if ro else "ReadWriteMany"
|
||||
for what, spec in (("PV", pvs), ("PVC", pvcs)):
|
||||
modes = spec.get("accessModes") or []
|
||||
check(modes == [want_mode],
|
||||
f"{label}: bulk {what} accessModes {modes} != ['{want_mode}']. Every bulk "
|
||||
f"volume is a `...Many` (one export, several namespaces, several nodes — "
|
||||
f"ReadWriteOnce cannot express that), and WHICH `Many` is how the read/write "
|
||||
f"posture stays legible in `kubectl get pv`")
|
||||
nfs_ro = bool((pvs.get("nfs") or {}).get("readOnly"))
|
||||
check(nfs_ro == ro,
|
||||
f"{label}: ServiceDefinition readOnly={ro} but the PV's nfs.readOnly={nfs_ro}. "
|
||||
f"The PV field is what makes the kernel mount `ro`; a mismatch means the declared "
|
||||
f"posture is not the one the kernel enforces")
|
||||
|
||||
# The volumeMount half, read out of the rendered Deployment.
|
||||
dep = rendered.get(("Deployment", svc))
|
||||
check(dep is not None, f"{label}: no rendered Deployment '{svc}' to check the mount against")
|
||||
if dep is not None:
|
||||
containers = (((dep.get("spec") or {}).get("template") or {}).get("spec") or {}).get("containers") or []
|
||||
# The app container is containers[0] by the renderer's contract (a forward-auth
|
||||
# sidecar is appended AFTER it).
|
||||
mounts = [m for m in (containers[0].get("volumeMounts") or []) if m.get("name") == vname] if containers else []
|
||||
check(len(mounts) == 1,
|
||||
f"{label}: the app container has {len(mounts)} volumeMount(s) named {vname!r}, "
|
||||
f"expected exactly 1 — a bulk volume that is declared and never mounted is a "
|
||||
f"PV/PVC pair doing nothing, and two mounts of one volume is assertion (6)")
|
||||
if len(mounts) == 1:
|
||||
m = mounts[0]
|
||||
check(bool(m.get("readOnly")) == ro,
|
||||
f"{label}: volumeMount readOnly={bool(m.get('readOnly'))} but the "
|
||||
f"ServiceDefinition says readOnly={ro} — the PV and the mount must agree, "
|
||||
f"so that dropping one still leaves the other")
|
||||
check(m.get("subPath", vol.get("subPath")) == vol.get("subPath"),
|
||||
f"{label}: volumeMount subPath {m.get('subPath')!r} does not match the "
|
||||
f"declared subPath {vol.get('subPath')!r}")
|
||||
|
||||
# ------------------------------------------ (8) AUTHORED ONCE, SUPPLIED PER ENV
|
||||
server = (pvs.get("nfs") or {}).get("server")
|
||||
m = re.fullmatch(r"\$\{([A-Z0-9_]+)\}", str(server or ""))
|
||||
check(m is not None,
|
||||
f"{label}: bulk PV nfs.server is {server!r} — it must be a BARE cluster-vars "
|
||||
f"token, not a literal address. dev and prod reach this storage host by DIFFERENT "
|
||||
f"addresses (dev is on its LAN; prod is across pfSense), so a literal is correct "
|
||||
f"in at most one environment and silently wrong in the other (R-ENV-02)")
|
||||
|
||||
cap = (pvs.get("capacity") or {}).get("storage")
|
||||
req = ((pvcs.get("resources") or {}).get("requests") or {}).get("storage")
|
||||
check(cap is not None and qty_to_bytes(cap) is not None,
|
||||
f"{label}: bulk PV declares no parseable capacity ({cap!r}) — the API requires it, "
|
||||
f"and it is what makes `kubectl get pv` report something true about this tier")
|
||||
check(cap == req,
|
||||
f"{label}: PV capacity {cap!r} != PVC request {req!r}; a request above the "
|
||||
f"capacity never binds")
|
||||
|
||||
inventory.append(
|
||||
f" {label:<28} export={export} {'ro' if ro else 'rw'} {want_mode:<14} "
|
||||
f"cap={cap} server={server} opts={','.join(mopts)}"
|
||||
)
|
||||
|
||||
# ---------------------------------- (2) CAPACITY STRATEGY: two tiers, two magnitudes
|
||||
cap_b = qty_to_bytes(cap)
|
||||
reps = replicated_bytes.get(svc) or []
|
||||
if cap_b and reps:
|
||||
biggest = max(reps)
|
||||
# ONE ORDER OF MAGNITUDE. Calibrated against the real corpus rather than guessed:
|
||||
# the media library is ~460x its service's config volume, and the Nextcloud archive
|
||||
# ~13x its 20Gi Longhorn volume — so a floor above ~13x would fire on a legitimate
|
||||
# bulk volume, and this check would be teaching people to weaken it. 10x cannot be
|
||||
# met by accident and still catches the regression that matters: a "bulk" volume
|
||||
# sized like a config PVC, i.e. the strategy named but never applied.
|
||||
check(cap_b >= 10 * biggest,
|
||||
f"{label}: bulk capacity {cap} is less than 10x this service's largest "
|
||||
f"replicated volume ({biggest // 2 ** 30}Gi). R-STATE-04 exists because the "
|
||||
f"two tiers hold data of two different MAGNITUDES — the replicated tier keeps "
|
||||
f"the small config/DB and the bulk tier keeps what cannot be replicated 3x. A "
|
||||
f"bulk volume sized like a config volume means the tier was named, not applied")
|
||||
|
||||
# ------------------------------------------------- (6) ONE EXPORT, AT MOST ONE MOUNT
|
||||
# The EXDEV trap. Grouped per SERVICE, because the hazard is two mounts inside ONE pod.
|
||||
by_service: dict[str, list[dict]] = {}
|
||||
for svc, vol in corpus:
|
||||
by_service.setdefault(svc, []).append(vol)
|
||||
for svc, vols in sorted(by_service.items()):
|
||||
seen: dict[str, str] = {}
|
||||
for vol in vols:
|
||||
exp = vol.get("export")
|
||||
prev = seen.get(exp)
|
||||
check(prev is None,
|
||||
f"{svc}: mounts the export {exp} TWICE (volumes {prev!r} and "
|
||||
f"{vol.get('name')!r}). Two mounts of one export are two FILESYSTEMS to the "
|
||||
f"kernel, so link() between them fails EXDEV even though the server knows "
|
||||
f"they are one dataset. Sonarr/Radarr do not fail loudly on that — they "
|
||||
f"degrade to copy-then-delete, rewriting every imported file over NFS and "
|
||||
f"destroying the seeding copy. Mount the common ancestor ONCE instead "
|
||||
f"(docs/BULK-NFS-STORAGE.md §4)")
|
||||
seen[exp] = vol.get("name")
|
||||
|
||||
# ------------------------------------- (8b) the knob has a base default AND every overlay
|
||||
tokens = set()
|
||||
for _svc, vol in corpus:
|
||||
pv = rendered.get(("PersistentVolume", f"{_svc}-{vol.get('name')}"))
|
||||
if pv:
|
||||
mm = re.fullmatch(r"\$\{([A-Z0-9_]+)\}", str(((pv.get("spec") or {}).get("nfs") or {}).get("server") or ""))
|
||||
if mm:
|
||||
tokens.add(mm.group(1))
|
||||
|
||||
cv = ROOT / "clusters" / "base" / "cluster-vars.yaml"
|
||||
base_keys: set[str] = set()
|
||||
if cv.is_file():
|
||||
doc = yaml.safe_load(cv.read_text()) or {}
|
||||
base_keys = set((doc.get("data") or {}).keys())
|
||||
|
||||
overlays = sorted(
|
||||
d for d in (ROOT / "clusters").iterdir()
|
||||
if cv.is_file() and d.is_dir() and d.name != "base" and (d / "kustomization.yaml").is_file()
|
||||
) if (ROOT / "clusters").is_dir() else []
|
||||
|
||||
for tok in sorted(tokens):
|
||||
check(tok in base_keys,
|
||||
f"bulk PVs reference ${{{tok}}} but clusters/base/cluster-vars.yaml declares no "
|
||||
f"such knob. Under StrictPostBuildSubstitutions that fails the whole tier; without "
|
||||
f"it the token ships into the cluster as a literal server name")
|
||||
for ov in overlays:
|
||||
text = (ov / "kustomization.yaml").read_text()
|
||||
check(f"/data/{tok}" in text,
|
||||
f"clusters/{ov.name} does not supply ${{{tok}}}. The base default is the DEV "
|
||||
f"value by this repo's convention, so an overlay that omits it silently "
|
||||
f"inherits an address on the WRONG SIDE of pfSense — a mount that either "
|
||||
f"never connects or connects to something unintended (R-ENV-02)")
|
||||
|
||||
# --------------------------------------------------------------- (9) LOAD-BEARING
|
||||
L0 = (ROOT / "ci" / "l0.sh").read_text() if (ROOT / "ci" / "l0.sh").is_file() else ""
|
||||
GATE = (ROOT / "ci" / "gate.sh").read_text() if (ROOT / "ci" / "gate.sh").is_file() else ""
|
||||
check("bulk-tier-distinct-from-replicated/check.sh" in L0,
|
||||
"ci/l0.sh does not run bulk-tier-distinct-from-replicated/check.sh. Stage 3b scopes "
|
||||
"the conformance run to TOUCHED requirements, and ci/touched-requirements.sh maps "
|
||||
"neither apps/* nor platform/longhorn/* to the bulk requirement — so a change that "
|
||||
"moves a bulk volume onto the replicated tier could attribute no requirement at all "
|
||||
"and be gated by nothing. It must be invoked directly and unconditionally")
|
||||
check(re.search(r"^for s in .*\bl0\b.*; do", GATE, re.M) and re.search(r"fail\).*BLOCK=1", GATE),
|
||||
"ci/gate.sh does not set the merge block when the l0 stage fails")
|
||||
|
||||
# ------------------------------------------------------------------------- report
|
||||
print("Bulk tier inventory (from catalog/ + the committed apps/ goldens):")
|
||||
for line in inventory:
|
||||
print(line)
|
||||
print(f"\nreplicated tier StorageClasses (derived from platform/longhorn/): "
|
||||
f"{', '.join(sorted(REPLICATED_CLASSES))}")
|
||||
print(f"bulk volumes: {len(corpus)} distinct exports: "
|
||||
f"{len({v.get('export') for _s, v in corpus})} "
|
||||
f"server knob(s): {', '.join(sorted(tokens)) or 'NONE'}")
|
||||
|
||||
# A floor on the assertion count, for the same reason the corpus may not be empty: a pass
|
||||
# must not be able to degenerate into "nothing was actually asked".
|
||||
MIN_CHECKS = 40
|
||||
check(CHECKS >= MIN_CHECKS,
|
||||
f"only {CHECKS} assertions ran (floor {MIN_CHECKS}) — a pass this thin is not "
|
||||
f"evidence; the corpus or the extraction has degraded")
|
||||
|
||||
print()
|
||||
if FAILURES:
|
||||
print(f"CHECK FAIL (bulk-tier-distinct-from-replicated): {len(FAILURES)} finding(s)",
|
||||
file=sys.stderr)
|
||||
for f in FAILURES:
|
||||
print(f" - {f}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
print(f"check OK: bulk-tier-distinct-from-replicated — {CHECKS} assertions (R-STATE-04)")
|
||||
PY
|
||||
@@ -1,75 +0,0 @@
|
||||
# =============================================================================
|
||||
# L0 STATIC — dev/test nodes are provisioned as code, end to end (R-ENV-01) [EXECUTABLE]
|
||||
# Catalog: docs/14 L0 row ("Pre-apply, in CI") — runs WITHOUT a cluster.
|
||||
# Spec: docs/00 R-ENV-01 ("dev/test MUST be provisioned as local KVMs via IaC:
|
||||
# created, baselined, cluster-joined, and managed as code"); the four stages
|
||||
# are enumerated in docs/02 §"Node lifecycle stages".
|
||||
#
|
||||
# WHY L0: R-ENV-01 is a requirement about HOW the dev nodes came to exist, and a live
|
||||
# Node object cannot answer it — a node built by OpenTofu and one built by hand in a
|
||||
# hypervisor UI are indistinguishable through the Kubernetes API. What is provable, with
|
||||
# no cluster, is that a COMPLETE and EXECUTABLE provisioning path exists for all four
|
||||
# stages and covers EVERY dev node. For the Ansible half that is proven by DRIVING the
|
||||
# real toolchain offline (`--syntax-check`, `--list-hosts` — neither opens a connection
|
||||
# to any node), not by reading files: a chain that does not resolve fails the check.
|
||||
#
|
||||
# ENVIRONMENT-AGNOSTIC: no hostname, address, prefix or node count appears in the test.
|
||||
# The dev group names are read from the playbooks' own default host patterns, the node
|
||||
# names are generated from the IaC module's prefix/count variables, and the two sets are
|
||||
# compared against each other — so the assertion survives a rename or a resize.
|
||||
#
|
||||
# GIVEN the repository as git tracks it,
|
||||
# WHEN the provisioning chain is parsed and (where offline execution is possible) driven,
|
||||
# THEN the IaC module creates the dev nodes on a LOCAL KVM hypervisor (allowlisted
|
||||
# provider — a cloud provider fails), from a for_each map rather than a count index,
|
||||
# each with CPU/RAM/OS disk/dedicated Longhorn data disk/NIC/cloud-init seed; the
|
||||
# node set it creates is EXACTLY the set Ansible manages; the baseline stage applies
|
||||
# the package floor, kernel modules, sysctls, data-disk mount, swap-off and iscsid
|
||||
# declaratively; one k3s role founds the cluster with --cluster-init and joins the
|
||||
# rest with --server while disabling flannel/kube-proxy/traefik/servicelb at
|
||||
# install; and site.yml's chain (baseline -> k3s -> Cilium -> verify -> Flux)
|
||||
# resolves and targets every dev node at every stage.
|
||||
#
|
||||
# STATUS: EXECUTABLE. `check.sh` is a pure offline assertion script (git, PyYAML, and —
|
||||
# where present — ansible-playbook in no-connection modes; no cluster, no network);
|
||||
# conformance/run.sh --layer l0 and ci/l0.sh run it directly, and this Chainsaw Test
|
||||
# wraps the SAME script.
|
||||
#
|
||||
# KNOWN, DELIBERATE GAPS (enumerated in check.sh's header; this test claims none of them):
|
||||
# * That the RUNNING dev VMs were created by this module — needs the OpenTofu state or
|
||||
# hypervisor access, neither available offline.
|
||||
# * `tofu validate` — needs `tofu init` (provider registry), i.e. network. The module is
|
||||
# parsed, not evaluated; a network-allowed CI stage would close this.
|
||||
# * "Never hand-edited on the box" (docs/02 stage 4's second half) — a live drift check.
|
||||
# * REBUILD/re-converge (docs/02's own proof line) — exercised at L3 by
|
||||
# tests/l3-fault-injection/node-loss (R-STATE-02).
|
||||
# * docs/02 prose puts the baseline in cloud-init; the implementation moved it into the
|
||||
# Ansible node_baseline role. R-ENV-01 requires the node to be "baselined ... as code",
|
||||
# not that cloud-init be the tool, so the check asserts the STAGE and RECORDS the
|
||||
# divergence rather than hiding it.
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: dev-nodes-provisioned-as-code
|
||||
labels:
|
||||
conformance.home.k3s/R-ENV-01: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-ENV-01"
|
||||
conformance.home.k3s/layer: "L0"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
Given the tracked tree, when the dev provisioning chain is parsed and driven
|
||||
offline, then every dev node is created by local-KVM IaC, baselined, cluster-joined
|
||||
and handed to Flux by one code path that covers all of them — with no node outside
|
||||
it and no stage missing.
|
||||
spec:
|
||||
steps:
|
||||
- name: then-every-dev-node-is-created-baselined-joined-and-managed-as-code
|
||||
description: >-
|
||||
Run the offline assertion script over the tracked tree (same script ci/l0.sh and
|
||||
conformance/run.sh --layer l0 execute).
|
||||
try:
|
||||
- script:
|
||||
content: bash ./check.sh
|
||||
@@ -1,442 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# L0 static check (offline, no cluster) — dev-nodes-provisioned-as-code
|
||||
# Serves: R-ENV-01 ("dev/test MUST be provisioned as local KVMs via IaC: created,
|
||||
# baselined, cluster-joined, and managed as code." — spec
|
||||
# docs/00-vision-and-scope.md; the four stages are enumerated in
|
||||
# docs/02-environments-and-provisioning.md §"Node lifecycle stages
|
||||
# (R-ENV-01 — 'created, baselined, cluster-joined, managed as code')":
|
||||
# 1. Create — OpenTofu instantiates the VM
|
||||
# 2. Baseline — OS baseline and prerequisites
|
||||
# 3. Cluster-join — Ansible installs/joins k3s with Cilium as CNI; disables
|
||||
# k3s bundled flannel/traefik/servicelb/kube-proxy
|
||||
# 4. Manage — GitOps-managed by Flux; node config changes are made via IaC
|
||||
# and re-applied, never hand-edited on the box.)
|
||||
#
|
||||
# WHY L0 IS THE RIGHT LAYER, AND WHAT IT MEANS TO TEST THIS BEHAVIOURALLY
|
||||
# ----------------------------------------------------------------------
|
||||
# R-ENV-01 is a requirement about HOW the dev nodes come to exist. The live cluster
|
||||
# cannot answer it: a Kubernetes node object looks identical whether it was built by
|
||||
# OpenTofu or by hand in a hypervisor UI (there is no API field for "was I created by
|
||||
# IaC"). What CAN be proven, with no cluster, is that a COMPLETE, EXECUTABLE
|
||||
# provisioning path exists for all four stages and covers every dev node — and, for the
|
||||
# Ansible half, that is proven by RUNNING the real toolchain in a no-connection mode
|
||||
# (`--syntax-check`, `--list-hosts`) rather than by reading the files, so a chain that
|
||||
# does not actually resolve fails here. The IaC half is asserted by parsing the module:
|
||||
# `tofu init/validate` needs the provider registry, and L0 is offline by contract.
|
||||
#
|
||||
# GIVEN the repository as git tracks it,
|
||||
# WHEN the provisioning chain is parsed and (where offline execution is possible)
|
||||
# actually driven,
|
||||
# THEN CREATE: the IaC module targets a LOCAL KVM hypervisor (every declared provider
|
||||
# is a KVM-family hypervisor provider — no cloud provider may appear),
|
||||
# instantiates its nodes from a for_each map (never a count index, so one
|
||||
# node can be replaced without disturbing its siblings), and gives each
|
||||
# node CPU, RAM, an OS disk, a DEDICATED Longhorn data disk, a NIC and a
|
||||
# cloud-init seed carrying the login user + SSH key;
|
||||
# AND the node set it creates is EXACTLY the dev node set Ansible manages —
|
||||
# names derived from the module's own prefix/count, so no node can be
|
||||
# hand-added to the inventory or hand-created in the hypervisor;
|
||||
# BASELINE: a baseline stage exists as declarative Ansible (package floor, kernel
|
||||
# modules persisted AND loaded, sysctls, the Longhorn data disk formatted
|
||||
# and mounted, swap off, iscsid enabled), driven by variables with
|
||||
# defaults rather than per-host special cases;
|
||||
# JOIN: one role installs k3s for both roles of every node — `--cluster-init`
|
||||
# on the primary, `--server` join on the secondaries — with flannel,
|
||||
# kube-proxy, traefik and servicelb disabled AT INSTALL, and Cilium
|
||||
# installed as the CNI afterwards;
|
||||
# MANAGE: site.yml chains baseline -> k3s -> Cilium -> verify -> Flux bootstrap;
|
||||
# `ansible-playbook --syntax-check` on that chain SUCCEEDS; and
|
||||
# `--list-hosts` shows every one of those stages targeting EVERY dev
|
||||
# node, so no node is outside the code path.
|
||||
#
|
||||
# ENVIRONMENT-AGNOSTIC: no hostname, address, prefix or node count is written down here.
|
||||
# The dev group names come from the playbooks' own default host patterns, the node names
|
||||
# from the IaC module's prefix/count variables, and the two sets are compared to each
|
||||
# other. This check therefore keeps working when the estate is renamed or resized.
|
||||
#
|
||||
# WHAT THIS DOES *NOT* PROVE (honest gaps — do not read this test as claiming them):
|
||||
# - That the RUNNING dev VMs were in fact created by this module. Nothing offline can:
|
||||
# it would need the OpenTofu state (a remote R2 backend + credentials) or hypervisor
|
||||
# access. `tofu plan` against the live estate is the operator's own check.
|
||||
# - That the module APPLIES cleanly. `tofu validate` needs `tofu init`, which downloads
|
||||
# the provider from the registry; L0 is offline by contract, so the module is parsed,
|
||||
# not evaluated. ci/lint.sh does not cover .tf either — a gap worth closing in CI
|
||||
# with a network-allowed stage, not here.
|
||||
# - "Never hand-edited on the box" (stage 4's second half) — a drift check against the
|
||||
# live nodes, which is an operator/Ansible-check-mode exercise, not a repo property.
|
||||
# - REBUILD/re-converge ("destroy a dev node VM and rebuild it", docs/02's own proof
|
||||
# line) — that is exercised at L3 by tests/l3-fault-injection/node-loss (R-STATE-02).
|
||||
# - That the baseline actually ran on a node (cluster/host state), and that the
|
||||
# hypervisor is on the operator's own LAN rather than remote — "local" is asserted as
|
||||
# "a KVM hypervisor provider driving a self-hosted endpoint variable", not as a
|
||||
# network topology fact.
|
||||
# - DEVIATION FROM docs/02 PROSE, ASSERTED DELIBERATELY: docs/02 describes stage 2 as
|
||||
# cloud-init applying the OS baseline. The implementation moved the rich baseline into
|
||||
# the Ansible `node_baseline` role (cloud-init keeps hostname/user/SSH/network),
|
||||
# because Proxmox provisioning is API-only and a custom cloud-init snippet needs
|
||||
# SSH/SFTP to the hypervisor. R-ENV-01's own wording requires the node to be
|
||||
# "baselined ... as code", not that cloud-init be the tool, so this check asserts the
|
||||
# STAGE, wherever it lives, and records the divergence rather than hiding it.
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT="$(cd "${DIR}/../../../.." && pwd)" # repo root
|
||||
PYTHON="${PYTHON:-python3}"
|
||||
|
||||
cd "${ROOT}"
|
||||
"${PYTHON}" - "${ROOT}" <<'PY'
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError: # pragma: no cover
|
||||
print("check FAIL: PyYAML is required (same dependency as catalog/render.py)",
|
||||
file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
ROOT = Path(sys.argv[1])
|
||||
FAILURES: list[str] = []
|
||||
NOTES: list[str] = []
|
||||
CHECKS = 0
|
||||
|
||||
|
||||
def check(cond, msg):
|
||||
global CHECKS
|
||||
CHECKS += 1
|
||||
if not cond:
|
||||
FAILURES.append(msg)
|
||||
|
||||
|
||||
def note(msg):
|
||||
NOTES.append(msg)
|
||||
|
||||
|
||||
# A LOCAL DEPENDENCY the repo deliberately does not carry — the vault passphrase (kept
|
||||
# outside the repo, R-SEC-01) or the Galaxy collections (gitignored; installed from
|
||||
# collections/requirements.yml). Their absence is an ENVIRONMENT gap, not a repo defect,
|
||||
# and is the ONLY tolerated reason the driven sub-assertions may not run. Deliberately
|
||||
# narrow: every other non-zero exit is a real failure. The parse assertions always run.
|
||||
ENV_ONLY_FAILURE = re.compile(
|
||||
r"vault password|vault-id|Attempting to decrypt"
|
||||
r"|No module named 'ansible_collections|couldn't resolve module/action", re.I)
|
||||
|
||||
TOFU = ROOT / "infra" / "tofu"
|
||||
ANS = ROOT / "infra" / "ansible"
|
||||
check(TOFU.is_dir(), "infra/tofu/ is missing — the CREATE stage has no IaC at all (R-ENV-01)")
|
||||
check(ANS.is_dir(), "infra/ansible/ is missing — the BASELINE/JOIN/MANAGE stages have no code")
|
||||
if not TOFU.is_dir() or not ANS.is_dir():
|
||||
print(f"CHECK FAIL (dev-nodes-provisioned-as-code): {len(FAILURES)} of {CHECKS} failed",
|
||||
file=sys.stderr)
|
||||
for f in FAILURES:
|
||||
print(f" - {f}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
TF = "\n".join((p.read_text(errors="replace") for p in sorted(TOFU.glob("*.tf"))))
|
||||
check(len(TF) > 500, "infra/tofu/*.tf is empty or a stub — nothing creates the dev nodes")
|
||||
|
||||
# ============================================================ STAGE 1 — CREATE (IaC)
|
||||
# A LOCAL KVM hypervisor, by allowlist: any provider outside this set fails, so moving
|
||||
# dev onto a cloud (or anything that is not a KVM hypervisor the operator runs) is caught
|
||||
# here rather than being noticed later. Each entry is justified, not just listed:
|
||||
# bpg/proxmox Proxmox VE — a KVM/QEMU hypervisor the operator hosts.
|
||||
# dmacvicar/libvirt libvirt/KVM directly (the pre-Proxmox substrate).
|
||||
KVM_PROVIDERS = {"bpg/proxmox", "dmacvicar/libvirt"}
|
||||
providers = dict(re.findall(r'source\s*=\s*"([^"]+)"', TF) and
|
||||
[(m.group(1), m.group(2)) for m in
|
||||
re.finditer(r'(\w[\w-]*)\s*=\s*\{[^}]*?source\s*=\s*"([^"]+)"', TF, re.S)])
|
||||
check(len(providers) > 0,
|
||||
"infra/tofu declares no required_providers — the substrate is unpinned, so 'via IaC' "
|
||||
"cannot be checked at all")
|
||||
for alias, source in sorted(providers.items()):
|
||||
check(source.lower() in KVM_PROVIDERS,
|
||||
f"infra/tofu declares provider {alias}={source!r}, which is not a local KVM hypervisor "
|
||||
f"provider (allowed: {sorted(KVM_PROVIDERS)}) — R-ENV-01 requires dev/test to be LOCAL KVMs")
|
||||
|
||||
# The VM resource: for_each-keyed, and carrying the hardware + seed docs/02 enumerates.
|
||||
vm_blocks = re.findall(r'resource\s+"([\w]*(?:_vm|_domain))"\s+"(\w+)"\s*\{(.*?)\n\}',
|
||||
TF, re.S)
|
||||
check(len(vm_blocks) >= 1,
|
||||
"no VM/domain resource found in infra/tofu — nothing in IaC creates a dev node")
|
||||
for rtype, rname, body in vm_blocks:
|
||||
check("for_each" in body,
|
||||
f'resource "{rtype}" "{rname}" is not for_each-keyed. A count index makes single-node '
|
||||
"replace unsafe (a -target'ed destroy cascaded to sibling nodes — DECISIONS-LOG F1), so "
|
||||
"'created ... as code' would not survive a rebuild")
|
||||
check(not re.search(r"^\s*count\s*=", body, re.M),
|
||||
f'resource "{rtype}" "{rname}" uses a positional count index')
|
||||
for want, why in (
|
||||
(r"\bcpu\b", "CPU"),
|
||||
(r"\bmemory\b", "RAM"),
|
||||
(r"network_device|network_interface", "a NIC"),
|
||||
(r"initialization|cloud_init|cloudinit", "a cloud-init seed"),
|
||||
):
|
||||
check(re.search(want, body) is not None,
|
||||
f'resource "{rtype}" "{rname}" declares no {why} — docs/02 stage 1 requires CPU, RAM, '
|
||||
"disks, NICs and a cloud-init seed to come from IaC")
|
||||
disks = len(re.findall(r"^\s*disk\s*\{", body, re.M))
|
||||
check(disks >= 2,
|
||||
f'resource "{rtype}" "{rname}" declares {disks} disk block(s); docs/02 requires the OS '
|
||||
"disk PLUS a dedicated data disk for Longhorn")
|
||||
check(re.search(r"user_account|ssh_authorized_key|ssh_keys", body) is not None,
|
||||
f'resource "{rtype}" "{rname}" seeds no login user / SSH key — the node could not be '
|
||||
"reached by the BASELINE stage without a hand step")
|
||||
|
||||
# Node identity comes from variables, and those variables define the whole node set.
|
||||
def tf_default(var: str):
|
||||
m = re.search(r'variable\s+"' + re.escape(var) + r'"\s*\{(.*?)\n\}', TF, re.S)
|
||||
if not m:
|
||||
return None
|
||||
d = re.search(r'default\s*=\s*(.+)', m.group(1))
|
||||
return d.group(1).strip() if d else None
|
||||
|
||||
|
||||
prefix_raw = tf_default("node_name_prefix")
|
||||
count_raw = tf_default("node_count")
|
||||
ips_raw = tf_default("node_ips")
|
||||
check(prefix_raw is not None and count_raw is not None,
|
||||
"infra/tofu has no node_name_prefix/node_count variables — the node set is not derived "
|
||||
"from code, so the created nodes cannot be compared with the managed ones")
|
||||
prefix = (prefix_raw or "").strip('"')
|
||||
try:
|
||||
node_count = int((count_raw or "0").strip())
|
||||
except ValueError:
|
||||
node_count = 0
|
||||
check(node_count >= 1, f"node_count default is {count_raw!r} — no node would be created")
|
||||
if ips_raw is not None:
|
||||
n_ips = len(re.findall(r'"[^"]+"', ips_raw))
|
||||
check(n_ips >= node_count,
|
||||
f"node_ips default has {n_ips} address(es) for node_count={node_count} — the module "
|
||||
"cannot address every node it creates")
|
||||
check((TOFU / "outputs.tf").is_file() and
|
||||
re.search(r'output\s+"(ansible_inventory|nodes)"', (TOFU / "outputs.tf").read_text()) is not None,
|
||||
"infra/tofu/outputs.tf surfaces no node inventory output — the JOIN stage would have to be "
|
||||
"told the node facts by hand instead of consuming them from IaC")
|
||||
|
||||
# ------------------------------------------------- the dev node set, derived twice over
|
||||
# Group names are taken from the PLAYBOOKS' own default host patterns, so nothing here
|
||||
# restates the estate's naming.
|
||||
PLAYBOOKS = ANS / "playbooks"
|
||||
|
||||
|
||||
def default_patterns(playbook: str) -> list[str]:
|
||||
p = PLAYBOOKS / playbook
|
||||
if not p.is_file():
|
||||
return []
|
||||
return re.findall(r"default\('([^']+)'\)", p.read_text())
|
||||
|
||||
|
||||
dev_groups = set(default_patterns("node-baseline.yml") + default_patterns("k3s.yml"))
|
||||
check(len(dev_groups) > 0,
|
||||
"no playbook declares a default host pattern — the dev target is not expressed in code")
|
||||
|
||||
INV = ANS / "inventory" / "hosts.ini"
|
||||
check(INV.is_file(), "infra/ansible/inventory/hosts.ini is missing — no node is managed as code")
|
||||
ini_groups: dict[str, list[str]] = {}
|
||||
children: dict[str, list[str]] = {}
|
||||
cur = None
|
||||
for line in (INV.read_text().splitlines() if INV.is_file() else []):
|
||||
s = line.strip()
|
||||
if not s or s.startswith("#"):
|
||||
continue
|
||||
m = re.match(r"^\[([^\]]+)\]$", s)
|
||||
if m:
|
||||
cur = m.group(1)
|
||||
(children if cur.endswith(":children") else ini_groups).setdefault(
|
||||
cur.split(":")[0] if cur.endswith(":children") else cur, [])
|
||||
continue
|
||||
if cur is None or cur.endswith(":vars"):
|
||||
continue
|
||||
host = s.split()[0]
|
||||
if cur.endswith(":children"):
|
||||
children.setdefault(cur.split(":")[0], []).append(host)
|
||||
else:
|
||||
ini_groups.setdefault(cur, []).append(host)
|
||||
|
||||
|
||||
def hosts_in(group: str, seen=None) -> set[str]:
|
||||
seen = seen or set()
|
||||
if group in seen:
|
||||
return set()
|
||||
seen.add(group)
|
||||
out = set(ini_groups.get(group, []))
|
||||
for child in children.get(group, []):
|
||||
out |= hosts_in(child, seen)
|
||||
return out
|
||||
|
||||
|
||||
dev_hosts: set[str] = set()
|
||||
for g in sorted(dev_groups):
|
||||
dev_hosts |= hosts_in(g)
|
||||
check(len(dev_hosts) > 0,
|
||||
f"the dev groups {sorted(dev_groups)} resolve to NO hosts in hosts.ini — nothing is managed")
|
||||
expected = {f"{prefix}{i + 1}" for i in range(node_count)}
|
||||
check(dev_hosts == expected,
|
||||
f"the dev nodes Ansible manages ({sorted(dev_hosts)}) are not exactly the nodes the IaC "
|
||||
f"module creates ({sorted(expected)}) — a node outside IaC is a hand-built node (R-ENV-01)")
|
||||
|
||||
# ============================================================ STAGE 2 — BASELINE (code)
|
||||
BASE_ROLE = ANS / "roles" / "node_baseline"
|
||||
check(BASE_ROLE.is_dir(),
|
||||
"no node_baseline role — the BASELINE stage has no code (docs/02 stage 2)")
|
||||
base_tasks = "\n".join(p.read_text(errors="replace")
|
||||
for p in sorted((BASE_ROLE / "tasks").glob("*.yml"))) if BASE_ROLE.is_dir() else ""
|
||||
base_defaults = "\n".join(p.read_text(errors="replace")
|
||||
for p in sorted((BASE_ROLE / "defaults").glob("*.yml"))) if BASE_ROLE.is_dir() else ""
|
||||
# Each item docs/02 names for stage 2, matched on the ANSIBLE MODULE that would do it —
|
||||
# a declarative, idempotent task, not a shell one-liner.
|
||||
for pattern, what in (
|
||||
(r"ansible\.builtin\.apt:", "a package baseline (apt module)"),
|
||||
(r"modprobe:", "kernel modules loaded (modprobe module)"),
|
||||
(r"modules-load|/etc/modules", "kernel modules PERSISTED across reboot"),
|
||||
(r"sysctl", "sysctl tuning"),
|
||||
(r"filesystem:|mkfs", "a filesystem on the Longhorn data disk"),
|
||||
(r"mount:|ansible\.posix\.mount", "the Longhorn data disk MOUNTED"),
|
||||
(r"swap", "swap disabled"),
|
||||
(r"iscsid", "iscsid enabled (Longhorn prerequisite)"),
|
||||
):
|
||||
check(re.search(pattern, base_tasks) is not None,
|
||||
f"node_baseline does not declare {what} — docs/02 stage 2 requires the OS baseline and "
|
||||
"k3s/Longhorn/KubeVirt prerequisites to be applied as code")
|
||||
check(re.search(r"^node_baseline_packages:", base_defaults, re.M) is not None and
|
||||
re.search(r"^node_baseline_kernel_modules:", base_defaults, re.M) is not None,
|
||||
"node_baseline's package/module sets are not variables with defaults — a per-host special "
|
||||
"case is not 'managed as code' (and could not be scoped per environment, R-ENV-02)")
|
||||
|
||||
# ========================================================== STAGE 3 — CLUSTER-JOIN (k3s)
|
||||
K3S_ROLE = ANS / "roles" / "k3s_server"
|
||||
k3s_text = "\n".join(p.read_text(errors="replace")
|
||||
for p in sorted((K3S_ROLE).rglob("*.yml"))) if K3S_ROLE.is_dir() else ""
|
||||
check(K3S_ROLE.is_dir() and len(k3s_text) > 200,
|
||||
"no k3s_server role — nothing joins the nodes into a cluster (docs/02 stage 3)")
|
||||
check("--cluster-init" in k3s_text,
|
||||
"the k3s role never passes --cluster-init — the HA embedded-etcd cluster is never founded")
|
||||
check(re.search(r"--server\s+https", k3s_text) is not None,
|
||||
"the k3s role never joins secondaries with --server — nodes would not form one cluster")
|
||||
check(re.search(r"K3S_TOKEN|--token", k3s_text) is not None,
|
||||
"the k3s role never supplies a join token — the join could only be completed by hand")
|
||||
# The bundled components docs/02 requires to be disabled AT INSTALL, each checked by the
|
||||
# flag that actually disables it.
|
||||
for pattern, what in (
|
||||
(r"--flannel-backend=none", "flannel (Cilium is the CNI)"),
|
||||
(r"--disable-kube-proxy", "kube-proxy (Cilium kube-proxy replacement)"),
|
||||
(r"traefik", "traefik (Gateway API via Cilium)"),
|
||||
(r"servicelb", "servicelb"),
|
||||
):
|
||||
check(re.search(pattern, k3s_text) is not None,
|
||||
f"the k3s install does not disable {what} — docs/02 stage 3 requires it disabled at "
|
||||
"install, or the bundled component fights the platform's own")
|
||||
check((ANS / "roles" / "cilium").is_dir(),
|
||||
"no cilium role — stage 3 requires Cilium to be installed as the CNI at bootstrap")
|
||||
check((ANS / "roles" / "flux_bootstrap").is_dir(),
|
||||
"no flux_bootstrap role — stage 4's hand-off to GitOps has no code")
|
||||
|
||||
# ============================================== STAGE 4 — MANAGE + the chain is REAL
|
||||
SITE = PLAYBOOKS / "site.yml"
|
||||
check(SITE.is_file(), "no playbooks/site.yml — the provisioning chain is not expressed as one path")
|
||||
site_txt = SITE.read_text() if SITE.is_file() else ""
|
||||
order = [m for m in re.findall(r"import_playbook:\s*(\S+)", site_txt)]
|
||||
check(order[:1] == ["node-baseline.yml"],
|
||||
f"site.yml imports {order} — the BASELINE stage must come first (docs/02 sequencing)")
|
||||
for want in ("k3s.yml", "cilium.yml", "flux-bootstrap.yml"):
|
||||
check(want in order, f"site.yml does not import {want} — a lifecycle stage is missing from the chain")
|
||||
if "k3s.yml" in order and "cilium.yml" in order:
|
||||
check(order.index("k3s.yml") < order.index("cilium.yml"),
|
||||
"site.yml installs Cilium before k3s — the CNI cannot precede the cluster")
|
||||
|
||||
# --- DRIVE THE REAL TOOLCHAIN, OFFLINE ------------------------------------------------
|
||||
# --syntax-check and --list-hosts open NO connection to any node: they resolve the
|
||||
# playbook, roles, collections and inventory only. Running them is the difference between
|
||||
# "the chain is written down" and "the chain resolves".
|
||||
if shutil.which("ansible-playbook") is None:
|
||||
note("ansible-playbook is not installed here (e.g. the in-cluster chainsaw image), so the "
|
||||
"chain was PARSED but not DRIVEN; the merge gate's container has it and does drive it")
|
||||
else:
|
||||
# cwd=infra/ansible so the tracked ansible.cfg (roles_path, collections_path, inventory)
|
||||
# is the one in force — the same resolution a real run gets.
|
||||
def ansible(*args) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(["ansible-playbook", *args], cwd=ANS,
|
||||
capture_output=True, text=True)
|
||||
|
||||
syn = ansible("--syntax-check", "playbooks/site.yml")
|
||||
syn_out = (syn.stderr or "") + (syn.stdout or "")
|
||||
lh = None
|
||||
if syn.returncode != 0 and ENV_ONLY_FAILURE.search(syn_out):
|
||||
# NARROW, SELF-POLICING SKIP — see ENV_ONLY_FAILURE above.
|
||||
note("ansible-playbook lacks a LOCAL dependency kept outside the repo by design (the vault "
|
||||
"passphrase and/or the gitignored Galaxy collections), so --syntax-check/--list-hosts "
|
||||
"were not driven; every parse assertion above still ran")
|
||||
else:
|
||||
check(syn.returncode == 0,
|
||||
"`ansible-playbook --syntax-check playbooks/site.yml` FAILED — the provisioning chain "
|
||||
f"does not even load: {syn_out.strip()[:400]}")
|
||||
lh = ansible("--list-hosts", "playbooks/site.yml")
|
||||
check(lh.returncode == 0,
|
||||
"`ansible-playbook --list-hosts playbooks/site.yml` FAILED — the chain does not "
|
||||
f"resolve against the tracked inventory: "
|
||||
f"{((lh.stderr or '') + (lh.stdout or '')).strip()[:400]}")
|
||||
if lh is not None and lh.returncode == 0:
|
||||
# Parse plays -> hosts. NEVER `ansible-inventory --list`: it prints DECRYPTED vault values.
|
||||
plays: list[tuple[str, set[str]]] = []
|
||||
name, hosts, in_hosts = None, set(), False
|
||||
for line in lh.stdout.splitlines():
|
||||
m = re.match(r"\s*play #\d+ \([^)]*\):\s*(.*?)\s*TAGS:", line)
|
||||
if m:
|
||||
if name is not None:
|
||||
plays.append((name, hosts))
|
||||
name, hosts, in_hosts = m.group(1), set(), False
|
||||
continue
|
||||
if re.match(r"\s*hosts \(\d+\):", line):
|
||||
in_hosts = True
|
||||
continue
|
||||
if in_hosts and line.strip() and not line.strip().startswith("pattern:"):
|
||||
hosts.add(line.strip())
|
||||
if name is not None:
|
||||
plays.append((name, hosts))
|
||||
check(len(plays) >= 4,
|
||||
f"--list-hosts resolved {len(plays)} play(s) — the four-stage chain did not expand")
|
||||
# Every lifecycle stage must cover EVERY dev node. A stage that reaches two of three
|
||||
# nodes means the third was finished by hand.
|
||||
STAGES = {
|
||||
"baseline": r"baseline",
|
||||
"cluster-join": r"^k3s (primary|secondar)",
|
||||
"cni": r"cilium",
|
||||
"manage (flux hand-off)": r"flux",
|
||||
}
|
||||
for stage, rx in STAGES.items():
|
||||
covered: set[str] = set()
|
||||
matched = 0
|
||||
for pname, phosts in plays:
|
||||
if re.search(rx, pname, re.I):
|
||||
matched += 1
|
||||
covered |= phosts
|
||||
check(matched >= 1, f"no play in site.yml implements the {stage} stage")
|
||||
check(covered == dev_hosts,
|
||||
f"the {stage} stage targets {sorted(covered)}, not every dev node "
|
||||
f"{sorted(dev_hosts)} — a node outside the code path is a hand-provisioned "
|
||||
"node (R-ENV-01)")
|
||||
|
||||
# --------------------------------------------------------------------------- verdict
|
||||
# NO-OP GUARD: a check that quietly asserts nothing is worse than no check.
|
||||
check(CHECKS >= 30,
|
||||
f"only {CHECKS} assertions ran — the check degenerated to a near-no-op and its pass "
|
||||
"would be meaningless")
|
||||
|
||||
if FAILURES:
|
||||
print(f"CHECK FAIL (dev-nodes-provisioned-as-code): {len(FAILURES)} of {CHECKS} assertions failed",
|
||||
file=sys.stderr)
|
||||
for f in FAILURES:
|
||||
print(f" - {f}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
for n in NOTES:
|
||||
print(f" note: {n}")
|
||||
print(f"check OK: dev-nodes-provisioned-as-code — {CHECKS} assertions; "
|
||||
f"{len(dev_hosts)} dev node(s) {sorted(dev_hosts)} created by IaC ({sorted(providers.values())}), "
|
||||
f"baselined, cluster-joined and Flux-managed by the same chain. R-ENV-01's authored-path "
|
||||
"clause only — the live VMs' provenance, `tofu validate`, and rebuild/re-converge are NOT "
|
||||
"proven here (see the header's gap list; rebuild is L3 node-loss).")
|
||||
PY
|
||||
@@ -1,94 +0,0 @@
|
||||
# =============================================================================
|
||||
# L0 STATIC — two egress paths, and the rule set that selects between them (R-NET-04) [EXECUTABLE]
|
||||
# Catalog: docs/14 L0 row ("Pre-apply, in CI") — runs WITHOUT a cluster.
|
||||
# Spec: docs/00 R-NET-04 ("The platform MUST support **multiple egress paths** selected by
|
||||
# routing rules ... realized as two paths — the local `internet` path and the
|
||||
# in-cluster `vpn` path — not multi-ISP routing"), elaborated by docs/03 §"Egress
|
||||
# routing: two paths, one physical WAN": "the `internet` path is simply the default
|
||||
# route out"; "**Default is the WAN; the VPN is strictly opt-in.** Every pod — and
|
||||
# the nodes themselves — egress the local/private WAN by default and **never** the
|
||||
# VPN. Only services labeled `egress-class=vpn` are steered to the tunnel, and the
|
||||
# kill-switch blackholes **only** that class."
|
||||
#
|
||||
# WHAT THIS ADDS OVER R-NET-05/07 (already covered, and NOT re-tagged here)
|
||||
# ------------------------------------------------------------------------
|
||||
# R-NET-05/07 are about a SERVICE — it declares a class, enforcement is outside it, the class
|
||||
# flips via GitOps — and are proven by tests/l1-rendering/egress-class-render and
|
||||
# tests/l2-behavioral/egress-class. R-NET-04 is about the PLATFORM's PATH SET and the RULES
|
||||
# that select among them, which nothing else asserts:
|
||||
# * exactly TWO paths, the same closed set in the schema, the admission policy and the
|
||||
# renderer (a third value would be a path nothing routes; "multi-ISP" is out of scope);
|
||||
# * selection is by a ROUTING RULE and the rule set is TOTAL AND DISJOINT — exactly one
|
||||
# Cilium steering rule, selecting only `egress-class=vpn`, and on the gateway node one
|
||||
# `ip rule` scoped to the tunnel address. NOTHING selects the `internet` class, because
|
||||
# absence of a matching rule IS the default path;
|
||||
# * the two paths COEXIST per DESTINATION: private space is excluded from the tunnel, so a
|
||||
# vpn-class pod's cluster/LAN traffic still takes the local path;
|
||||
# * the vpn path never becomes the DEFAULT — its default route lives in a dedicated routing
|
||||
# table, `route-noexec` is set and a pushed `redirect-gateway` is ignored;
|
||||
# * the kill-switch blackholes ONLY that class — the nft chain's policy is accept and every
|
||||
# DROP is scoped to the vpn class's own source set/fwmark, so `internet` fails OPEN.
|
||||
# (R-NET-06's opposite claim, that `vpn` fails CLOSED, is L3's vpn-killswitch.)
|
||||
#
|
||||
# WHY L0: every clause above is a statement about ALL rules — including rules that must be
|
||||
# ABSENT (nothing selects the default path; no unscoped rule; no unqualified drop; no default
|
||||
# route in the main table). A cluster can show which path two pods took; it cannot show that
|
||||
# no third rule exists or that the vocabulary is closed.
|
||||
#
|
||||
# ENVIRONMENT-AGNOSTIC: no address appears. The private ranges the tunnel must exclude are
|
||||
# checked as CONTAINMENT of the addresses each environment's own Flux overlay declares (pod
|
||||
# CIDR, Gateway VIP, LB pool, API host), so a new environment on a different LAN is checked
|
||||
# against its own values.
|
||||
#
|
||||
# GIVEN the repository as git tracks it,
|
||||
# WHEN the egress vocabulary, every CiliumEgressGatewayPolicy, the gateway node's policy
|
||||
# routing / OpenVPN / nftables templates, and every environment's overlay addresses are
|
||||
# parsed and cross-checked,
|
||||
# THEN there are exactly two paths with `internet` as the default; exactly one steering rule
|
||||
# selects exactly the vpn class by label; the gateway is selected by node label with a
|
||||
# named egress interface; all RFC1918 space and every address any environment assigns
|
||||
# itself is excluded from the tunnel; the tunnel's default route and its fail-closed
|
||||
# blackhole live only in a dedicated routing table behind a scoped `ip rule`; OpenVPN
|
||||
# cannot take the main table; and no nftables DROP is unscoped.
|
||||
#
|
||||
# STATUS: EXECUTABLE. `check.sh` is a pure offline assertion script (git tree + PyYAML; no
|
||||
# cluster, no network); conformance/run.sh --layer l0 and ci/l0.sh run it directly, and this
|
||||
# Chainsaw Test wraps the SAME script.
|
||||
#
|
||||
# KNOWN, DELIBERATE GAPS (enumerated in check.sh's header; this test claims none of them):
|
||||
# * BEHAVIOUR. That an unlabelled pod really takes the WAN path is asserted at L2 by
|
||||
# tests/l2-behavioral/egress-default-path (same requirement label); the two labelled
|
||||
# classes are R-NET-05's L2 test; the kill-switch firing is R-NET-06's L3 test.
|
||||
# * The RUNNING gateway node's routing tables / nftables ruleset. This reads the templates
|
||||
# that generate them; a hand-added rule on the box is invisible here.
|
||||
# * PHYSICAL MULTI-WAN's absence is NOT asserted — it is out of scope per OQ-1 and no
|
||||
# honest repository signal distinguishes "one WAN, correctly" from "one WAN, by default".
|
||||
# * IPv6 egress.
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: egress-paths-selected-by-rules
|
||||
labels:
|
||||
conformance.home.k3s/R-NET-04: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-NET-04"
|
||||
conformance.home.k3s/layer: "L0"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
Given the tracked tree, when the egress vocabulary, every steering rule and the gateway
|
||||
node's policy-routing/OpenVPN/nftables templates are parsed, then the platform offers
|
||||
exactly two egress paths with the WAN as the default, exactly one rule selects exactly
|
||||
the vpn class, local destinations stay on the local path in every environment, the
|
||||
tunnel never becomes the default route, and nothing unscoped can blackhole the default
|
||||
path.
|
||||
spec:
|
||||
steps:
|
||||
- name: then-two-paths-exist-and-the-rules-select-between-them-totally-and-disjointly
|
||||
description: >-
|
||||
Run the offline assertion script over the tracked tree (same script ci/l0.sh and
|
||||
conformance/run.sh --layer l0 execute).
|
||||
try:
|
||||
- script:
|
||||
content: bash ./check.sh
|
||||
@@ -1,521 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# L0 static check (offline, no cluster) — egress-paths-selected-by-rules
|
||||
# Serves: R-NET-04 ("The platform MUST support **multiple egress paths** selected by routing
|
||||
# rules. Per OQ-1 the environment has **one physical commercial WAN and no
|
||||
# multi-ISP plans**, so this is realized as two paths — the local `internet` path and
|
||||
# the in-cluster `vpn` path — not multi-ISP routing. Physical multi-WAN (multiple ISP
|
||||
# uplinks) is **out of scope** unless a future ADR reopens it."
|
||||
# — spec docs/00-vision-and-scope.md).
|
||||
#
|
||||
# The elaborating spec text this check quotes and asserts against — docs/03
|
||||
# §"Egress routing: two paths, one physical WAN (R-NET-04/05 — resolved by OQ-1)":
|
||||
# | `internet` | Out the single commercial WAN | Normal pod egress via node -> SOHO edge
|
||||
# router -> internet |
|
||||
# | `vpn` | Routed through the in-cluster VPN tunnel | Steered to the shared VPN
|
||||
# egress gateway (ADR-0013) |
|
||||
# "**Physical multi-WAN (multi-ISP) is out of scope.** With a single WAN there is nothing
|
||||
# to policy-route across; the `internet` path is simply the default route out."
|
||||
# "**Default is the WAN; the VPN is strictly opt-in.** Every pod — and the nodes
|
||||
# themselves — egress the local/private WAN by default and **never** the VPN. Only
|
||||
# services labeled `egress-class=vpn` are steered to the tunnel, and the kill-switch
|
||||
# blackholes **only** that class."
|
||||
# "It is **split-tunnel**: it creates `tun0` but does **not** set the node's default
|
||||
# route (`route-noexec` + policy routing), so the node's own traffic stays on the WAN —
|
||||
# only what Cilium points at `tun0` uses the VPN."
|
||||
#
|
||||
# WHAT R-NET-04 ADDS OVER R-NET-05/07 (which are ALREADY covered) — READ THIS FIRST
|
||||
# ---------------------------------------------------------------------------------
|
||||
# R-NET-05/07 are about a SERVICE: it can declare its class, enforcement happens outside it,
|
||||
# and the class can be flipped via GitOps. Those are proven by
|
||||
# tests/l1-rendering/egress-class-render (the render) and tests/l2-behavioral/egress-class
|
||||
# (two labelled pods observe two different public IPs, and a relabel re-routes the same pod).
|
||||
# Re-tagging either of those with R-NET-04 would be a duplicate wearing a new label.
|
||||
#
|
||||
# R-NET-04 is about the PLATFORM's PATH SET and the RULES THAT SELECT AMONG THEM. Its
|
||||
# distinctive content — none of which any existing test asserts — is:
|
||||
#
|
||||
# (1) THERE ARE EXACTLY TWO PATHS, and the same two everywhere. The class vocabulary is
|
||||
# one closed set of two values agreed by the schema, the admission policy and the
|
||||
# renderer. A third value appearing in one of them but not the others would be a path
|
||||
# nothing routes (or a path nothing validates); "multi-ISP" reappearing as a third
|
||||
# class would contradict OQ-1's resolution.
|
||||
# (2) SELECTION IS BY A ROUTING RULE, and the rule set is TOTAL AND DISJOINT. The `vpn`
|
||||
# path is entered only via rules that select the vpn class — a Cilium
|
||||
# CiliumEgressGatewayPolicy pod-selector, and on the gateway node an `ip rule` scoped
|
||||
# to the tunnel source address. Nothing selects the `internet` class, because
|
||||
# "the `internet` path is simply the default route out": ABSENCE of a matching rule IS
|
||||
# the default path. So an unlabelled pod, and an `internet`-class pod, take the WAN.
|
||||
# (3) THE PATHS COEXIST — the choice is per-destination, not per-pod-lifetime. The vpn
|
||||
# rule tunnels public destinations while EXCLUDING all private space, so a vpn-class
|
||||
# pod's cluster/LAN traffic still takes the local path at the same time as its
|
||||
# internet traffic takes the tunnel.
|
||||
# (4) THE VPN PATH NEVER BECOMES THE DEFAULT. Split tunnel: the tunnel's default route
|
||||
# lives in a DEDICATED routing table reached only through the scoped rule, the
|
||||
# provider's pushed `redirect-gateway` is ignored, and `route-noexec` keeps OpenVPN
|
||||
# from touching the main table. Otherwise there would be one path (the VPN), not two.
|
||||
# (5) THE KILL-SWITCH BLACKHOLES ONLY THAT CLASS. The blackhole default is inside the
|
||||
# dedicated table; the nftables chain's policy is ACCEPT and every DROP is qualified
|
||||
# by the vpn class's own source set / fwmark. The `internet` path therefore fails
|
||||
# OPEN and can never be blackholed. (R-NET-06's opposite claim — that `vpn` fails
|
||||
# CLOSED — is proven at L3 by tests/l3-fault-injection/vpn-killswitch. This check
|
||||
# asserts the OTHER direction, which that test does not.)
|
||||
#
|
||||
# WHY L0. Every clause above is a property of the RULE SET, i.e. of authored configuration:
|
||||
# totality ("nothing selects the default path"), disjointness ("exactly one rule, selecting
|
||||
# exactly one class"), and scoping ("no unqualified drop, no default route in the main
|
||||
# table") are statements about ALL rules, including the ones that would have to be absent.
|
||||
# A running cluster can show you which path two pods took; it cannot show you that no third
|
||||
# rule exists, that no rule is unscoped, or that the vocabulary is closed. The behavioural
|
||||
# complement — that an UNLABELLED pod really does take the WAN path — is at L2 in
|
||||
# tests/l2-behavioral/egress-default-path (same requirement label).
|
||||
#
|
||||
# ENVIRONMENT-AGNOSTIC: no address is written down. The private ranges the vpn rule must
|
||||
# exclude are checked as CONTAINMENT of the addresses each environment's own Flux overlay
|
||||
# declares (pod CIDR, Gateway VIP, LB pool, API host) — so a new environment on a different
|
||||
# LAN is checked against its own addresses, and the check keeps working when they change.
|
||||
#
|
||||
# WHAT THIS DOES *NOT* PROVE (honest gaps — do not read this check as claiming them):
|
||||
# - Behaviour. That a vpn-class pod's packets really leave via tun0, that an unlabelled
|
||||
# pod's really do not, and that the tunnel is up at all. L2 covers the two labelled
|
||||
# classes (tests/l2-behavioral/egress-class, R-NET-05) and the unlabelled default
|
||||
# (tests/l2-behavioral/egress-default-path, R-NET-04); the kill-switch firing is L3.
|
||||
# - THE RUNNING NODE's routing tables and nftables ruleset. This reads the templates that
|
||||
# generate them, driven by no node. A hand-added `ip rule` on the gateway node is
|
||||
# invisible here (and to every other test); drift on the box is an Ansible check-mode
|
||||
# exercise for the operator.
|
||||
# - PHYSICAL MULTI-WAN. Out of scope per OQ-1, so its absence is not asserted: there is no
|
||||
# honest repository signal that would distinguish "one WAN, correctly" from "one WAN,
|
||||
# because nobody tried" — and inventing one would be a hollow assertion.
|
||||
# - IPv6. The policy's CIDRs and the routing rules are IPv4-only, matching the platform.
|
||||
# If IPv6 egress is ever enabled, the `internet` path would need the same analysis and
|
||||
# this check would NOT catch its absence.
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT="$(cd "${DIR}/../../../.." && pwd)" # repo root
|
||||
PYTHON="${PYTHON:-python3}"
|
||||
|
||||
cd "${ROOT}"
|
||||
"${PYTHON}" - "${ROOT}" <<'PY'
|
||||
import ipaddress
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError: # pragma: no cover
|
||||
print("check FAIL: PyYAML is required (same dependency as catalog/render.py)",
|
||||
file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
ROOT = Path(sys.argv[1])
|
||||
FAILURES: list[str] = []
|
||||
CHECKS = 0
|
||||
LBL = "homelab.keithjackson.dev"
|
||||
CLASS_LABEL = f"{LBL}/egress-class"
|
||||
|
||||
|
||||
def check(cond, msg):
|
||||
global CHECKS
|
||||
CHECKS += 1
|
||||
if not cond:
|
||||
FAILURES.append(msg)
|
||||
|
||||
|
||||
def die():
|
||||
print(f"CHECK FAIL (egress-paths-selected-by-rules): {len(FAILURES)} of {CHECKS} "
|
||||
"assertions failed", file=sys.stderr)
|
||||
for f in FAILURES:
|
||||
print(f" - {f}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
def docs(path: Path):
|
||||
"""Every YAML document in a file, skipping empties."""
|
||||
try:
|
||||
return [d for d in yaml.safe_load_all(path.read_text()) if isinstance(d, dict)]
|
||||
except yaml.YAMLError:
|
||||
return []
|
||||
|
||||
|
||||
# Skip dot-DIRECTORIES (.git, .claude, .cache) while still matching dot-FILES such as
|
||||
# .sops.yaml. A bare rglob from ROOT also descends into NESTED WORKTREES —
|
||||
# .claude/worktrees/agent-*/ created for isolated agent runs — which made this check see
|
||||
# 5 CiliumEgressGatewayPolicy objects instead of the 1 authored, failing R-NET-04 for a
|
||||
# reason that has nothing to do with egress. .claude/ is gitignored (.gitignore:43), so
|
||||
# nothing under it is part of the authored surface by definition.
|
||||
TRACKED_YAML = sorted(p for p in ROOT.rglob("*.yaml")
|
||||
if "_artifacts" not in p.parts
|
||||
and not any(part.startswith(".")
|
||||
for part in p.relative_to(ROOT).parent.parts))
|
||||
SCHEMA = ROOT / "schemas" / "servicedefinition.schema.json"
|
||||
RENDER = ROOT / "catalog" / "render.py"
|
||||
VPN_ROLE = ROOT / "infra" / "ansible" / "roles" / "vpn_egress_gateway"
|
||||
CLUSTERS = ROOT / "clusters"
|
||||
|
||||
for p in (SCHEMA, RENDER, VPN_ROLE, CLUSTERS):
|
||||
check(p.exists(), f"{p.relative_to(ROOT)} is missing — the egress path model cannot be "
|
||||
"read at all (R-NET-04)")
|
||||
if FAILURES:
|
||||
die()
|
||||
|
||||
|
||||
# ======================================================= (1) EXACTLY TWO PATHS, AGREED
|
||||
import json # noqa: E402 — only needed for the schema
|
||||
|
||||
schema = json.loads(SCHEMA.read_text())
|
||||
egress_prop = ((schema.get("properties") or {}).get("egress") or {})
|
||||
enum = egress_prop.get("enum")
|
||||
check(isinstance(enum, list) and len(enum) == 2,
|
||||
f"schemas/servicedefinition.schema.json declares egress enum {enum!r} — R-NET-04 is "
|
||||
"realized as EXACTLY TWO paths (OQ-1: one WAN, no multi-ISP). A third value is a path "
|
||||
"nothing routes; one value is not 'multiple egress paths'")
|
||||
PATHS = set(enum or [])
|
||||
check(PATHS == {"internet", "vpn"},
|
||||
f"the declared egress paths are {sorted(PATHS)} — docs/03's table fixes them as "
|
||||
"`internet` (out the single commercial WAN) and `vpn` (the in-cluster tunnel)")
|
||||
check(egress_prop.get("default") == "internet",
|
||||
f"the schema's default egress class is {egress_prop.get('default')!r} — docs/03: "
|
||||
"'Default is the WAN; the VPN is strictly opt-in', and 'the `internet` path is simply "
|
||||
"the default route out'. A service that says nothing must get the WAN")
|
||||
|
||||
# The admission policy must constrain the pod label to the SAME closed set, or a workload
|
||||
# could carry a class the schema never allowed (and that no rule routes).
|
||||
pol_files = sorted((ROOT / "platform" / "kyverno").rglob("*egress*.yaml"))
|
||||
check(len(pol_files) >= 1,
|
||||
"no Kyverno policy file mentions egress under platform/kyverno — nothing stops a pod "
|
||||
"carrying an egress class outside the declared path set")
|
||||
pol_text = "\n".join(p.read_text(errors="replace") for p in pol_files)
|
||||
for cls in sorted(PATHS):
|
||||
check(re.search(rf"\b{re.escape(cls)}\b", pol_text) is not None,
|
||||
f"the egress admission policy never names the {cls!r} path — a class the policy "
|
||||
"does not know cannot be validated against the path set")
|
||||
# No tracked object may carry a class outside the closed set — with ONE self-policing
|
||||
# exemption. A Kyverno policy-test fixture deliberately carries an INVALID class so the
|
||||
# admission test can assert the policy REJECTS it; that pod is exempt only while a
|
||||
# kyverno-test.yaml in the same tests tree still expects `result: fail` for it. Delete the
|
||||
# expectation (or flip it to pass) and the exemption evaporates — never a bare path allowlist.
|
||||
def rejected_by_a_policy_test(resource_name: str, fixture: Path) -> bool:
|
||||
tests_root = fixture.parent
|
||||
while tests_root != ROOT and tests_root.name != "tests":
|
||||
tests_root = tests_root.parent
|
||||
if tests_root == ROOT:
|
||||
return False
|
||||
for tf in sorted(tests_root.rglob("kyverno-test.yaml")):
|
||||
for d in docs(tf):
|
||||
for case in (d.get("results") or []):
|
||||
if case.get("result") == "fail" and resource_name in (case.get("resources") or []):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
stray: list[str] = []
|
||||
_STRAY_RE = re.compile(rf"{re.escape(CLASS_LABEL)}:\s*[\"']?([A-Za-z][A-Za-z0-9-]*)")
|
||||
for f in TRACKED_YAML:
|
||||
for d in docs(f):
|
||||
labels = ((d.get("metadata") or {}).get("labels") or {})
|
||||
cls = labels.get(CLASS_LABEL)
|
||||
# Also catch pod-template labels (Deployments/DaemonSets/Jobs/VMs).
|
||||
tmpl = (((d.get("spec") or {}).get("template") or {}).get("metadata") or {}).get("labels") or {}
|
||||
for value, owner in ((cls, d), (tmpl.get(CLASS_LABEL), d)):
|
||||
if value is None or value in PATHS or value == "CLASS":
|
||||
continue
|
||||
name = ((owner.get("metadata") or {}).get("name") or "?")
|
||||
if rejected_by_a_policy_test(name, f):
|
||||
continue
|
||||
stray.append(f"{f.relative_to(ROOT)}:{name}={value}")
|
||||
# Values written as raw text (templates, docs-as-code) are caught too.
|
||||
for value in set(_STRAY_RE.findall(f.read_text(errors="replace"))):
|
||||
if value in PATHS or value == "CLASS":
|
||||
continue
|
||||
if any(value in s for s in stray):
|
||||
continue
|
||||
# Raw-text hit with no parsed owner: exempt only if EVERY object in the file that
|
||||
# carries this value is a policy-test fixture proven to be rejected.
|
||||
owners = [((d.get("metadata") or {}).get("name") or "?") for d in docs(f)
|
||||
if ((d.get("metadata") or {}).get("labels") or {}).get(CLASS_LABEL) == value]
|
||||
if owners and all(rejected_by_a_policy_test(o, f) for o in owners):
|
||||
continue
|
||||
stray.append(f"{f.relative_to(ROOT)}:<text>={value}")
|
||||
check(not stray,
|
||||
f"tracked YAML assigns egress-class value(s) outside the declared path set "
|
||||
f"{sorted(PATHS)}: {sorted(stray)} — a value no routing rule selects and no admission "
|
||||
"policy allows is a workload on no path at all (the only exemption is a Kyverno "
|
||||
"policy-test fixture that a kyverno-test.yaml still expects to FAIL)")
|
||||
|
||||
# The renderer must project the DECLARED class onto the pod label, unmodified: that label is
|
||||
# what the routing rule selects on, so a renderer that rewrote it would break the coupling
|
||||
# between the declaration and the path.
|
||||
rtext = RENDER.read_text()
|
||||
check(re.search(rf'f?"\{{LBL\}}/egress-class":\s*s\["egress"\]', rtext) is not None
|
||||
or re.search(r'egress-class\"\]?\s*:\s*s\[\"egress\"\]', rtext) is not None,
|
||||
"catalog/render.py does not project the ServiceDefinition's own `egress` value onto the "
|
||||
f"{CLASS_LABEL} pod label — the declaration and the routing rule's selector would be "
|
||||
"two independent facts that could disagree")
|
||||
|
||||
|
||||
# ============================== (2) SELECTION IS BY RULE; THE RULE SET IS TOTAL & DISJOINT
|
||||
policies: list[tuple[Path, dict]] = []
|
||||
for p in TRACKED_YAML:
|
||||
for d in docs(p):
|
||||
if d.get("kind") == "CiliumEgressGatewayPolicy":
|
||||
policies.append((p, d))
|
||||
check(len(policies) == 1,
|
||||
f"{len(policies)} CiliumEgressGatewayPolicy object(s) are authored "
|
||||
f"({[str(p.relative_to(ROOT)) for p, _ in policies]}) — R-NET-04's steering must be ONE "
|
||||
"rule. Two rules can overlap, and which one wins for an overlapping pod is not "
|
||||
"something the repository states")
|
||||
|
||||
for path, pol in policies:
|
||||
rel = str(path.relative_to(ROOT))
|
||||
spec = pol.get("spec") or {}
|
||||
selectors = spec.get("selectors") or []
|
||||
check(len(selectors) == 1,
|
||||
f"{rel}: the steering rule has {len(selectors)} selector(s) — every additional "
|
||||
"selector widens the set of pods steered off the default path, and R-NET-04's "
|
||||
"selection must be exactly 'the vpn class'")
|
||||
for sel in selectors:
|
||||
pod_sel = sel.get("podSelector")
|
||||
check(isinstance(pod_sel, dict) and pod_sel,
|
||||
f"{rel}: a selector has no podSelector — an empty selector in Cilium matches "
|
||||
"EVERY pod, which would put the whole cluster on the VPN path and leave no "
|
||||
"default path at all")
|
||||
ml = (pod_sel or {}).get("matchLabels") or {}
|
||||
me = (pod_sel or {}).get("matchExpressions") or []
|
||||
check(not me,
|
||||
f"{rel}: the podSelector uses matchExpressions {me!r}. The class contract is a "
|
||||
"single label equality; expressions (In/NotIn/Exists) can select pods that "
|
||||
"never declared a class, so the rule set would no longer be disjoint from the "
|
||||
"default path")
|
||||
check(set(ml) == {CLASS_LABEL},
|
||||
f"{rel}: the podSelector matches on {sorted(ml)} — it must select on exactly "
|
||||
f"{CLASS_LABEL}. Any other key makes path selection depend on something other "
|
||||
"than the declared class (and a namespace/app key would silently exempt pods)")
|
||||
check(ml.get(CLASS_LABEL) == "vpn",
|
||||
f"{rel}: the steering rule selects {CLASS_LABEL}={ml.get(CLASS_LABEL)!r}. Only "
|
||||
"the `vpn` class may be steered: docs/03 — 'Only services labeled "
|
||||
"egress-class=vpn are steered to the tunnel'. A rule selecting `internet` would "
|
||||
"route the default path through the tunnel, i.e. leave one path, not two")
|
||||
|
||||
# The gateway is chosen by a NODE LABEL, not a hostname: the path's endpoint is itself a
|
||||
# rule, so the gateway can move (or gain a standby) with no rule rewrite.
|
||||
gw = spec.get("egressGateway") or {}
|
||||
node_sel = (gw.get("nodeSelector") or {}).get("matchLabels") or {}
|
||||
check(bool(node_sel),
|
||||
f"{rel}: the egressGateway selects no node by label — pinning the path's exit to a "
|
||||
"named host means the vpn path cannot fail over (docs/03 requires two gateway "
|
||||
"nodes + Egress Gateway HA)")
|
||||
check("nodeName" not in gw,
|
||||
f"{rel}: the egressGateway pins a nodeName — the exit node must be selected by "
|
||||
"label so the path survives losing that node")
|
||||
check(isinstance(gw.get("interface"), str) and gw.get("interface"),
|
||||
f"{rel}: the egressGateway names no egress interface — without it the steered class "
|
||||
"is not actually sent down a different path from the default")
|
||||
|
||||
# ------------------- (3) THE PATHS COEXIST: the choice is per DESTINATION -------------
|
||||
dests = spec.get("destinationCIDRs") or []
|
||||
excl = spec.get("excludedCIDRs") or []
|
||||
check([str(c) for c in dests] == ["0.0.0.0/0"],
|
||||
f"{rel}: destinationCIDRs is {dests!r}. The vpn path is for PUBLIC destinations "
|
||||
"('Routed through the in-cluster VPN tunnel'), expressed as the whole internet "
|
||||
"minus the exclusions below — narrowing it here would silently leave public "
|
||||
"destinations on the WAN for a vpn-class service")
|
||||
try:
|
||||
excluded = [ipaddress.ip_network(str(c)) for c in excl]
|
||||
except ValueError as e:
|
||||
excluded = []
|
||||
check(False, f"{rel}: excludedCIDRs contains an unparsable network ({e})")
|
||||
check(len(excluded) >= 3,
|
||||
f"{rel}: only {len(excluded)} excluded CIDR(s) — docs/03 requires intra-cluster, "
|
||||
"pod->service and LAN traffic to stay on the LOCAL path, which is all of RFC1918")
|
||||
|
||||
def covered(net: ipaddress.IPv4Network | ipaddress.IPv6Network) -> bool:
|
||||
return any(net.subnet_of(e) for e in excluded if e.version == net.version)
|
||||
|
||||
# RFC1918 in full — computed, not string-matched, so a typo'd mask is caught.
|
||||
for rfc in ("10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"):
|
||||
net = ipaddress.ip_network(rfc)
|
||||
check(covered(net),
|
||||
f"{rel}: the private range {rfc} is NOT excluded from the tunnel — a vpn-class "
|
||||
"pod's cluster/LAN traffic would be SNAT'd out tun0, so the two paths would not "
|
||||
"coexist for the same pod and east-west traffic would break")
|
||||
|
||||
# Every address each ENVIRONMENT declares for itself must fall in the excluded set: that
|
||||
# is what makes 'local stays local' true per environment rather than by coincidence.
|
||||
def overlay_addresses() -> list[tuple[str, str]]:
|
||||
found: list[tuple[str, str]] = []
|
||||
base = CLUSTERS / "base" / "cluster-vars.yaml"
|
||||
data = {}
|
||||
if base.is_file():
|
||||
for d in docs(base):
|
||||
if d.get("kind") == "ConfigMap":
|
||||
data = {k: str(v) for k, v in (d.get("data") or {}).items()}
|
||||
for f in sorted(CLUSTERS.glob("*/kustomization.yaml")):
|
||||
env = {**data}
|
||||
for m in re.finditer(
|
||||
r"-\s+op:\s*replace\s*\n\s*path:\s*/data/(?P<k>[A-Z0-9_]+)\s*\n"
|
||||
r"\s*value:\s*(?P<v>\S.*?)\s*$", f.read_text(), re.M):
|
||||
env[m.group("k")] = m.group("v").strip().strip('"').strip("'")
|
||||
label = f.parent.name
|
||||
for key in ("CLUSTER_POD_CIDR", "GATEWAY_VIP", "CILIUM_LB_POOL_START",
|
||||
"CILIUM_LB_POOL_STOP", "K3S_API_HOST"):
|
||||
v = env.get(key, "")
|
||||
if v and not v.startswith("TBD"):
|
||||
found.append((f"{label}/{key}", v))
|
||||
return found
|
||||
|
||||
addrs = overlay_addresses()
|
||||
check(len(addrs) >= 5,
|
||||
f"only {len(addrs)} environment address(es) could be read from the Flux overlays — "
|
||||
"the 'local traffic stays local' assertion would be checking almost nothing")
|
||||
for label, value in addrs:
|
||||
try:
|
||||
net = ipaddress.ip_network(value, strict=False)
|
||||
except ValueError:
|
||||
continue
|
||||
check(covered(net),
|
||||
f"{rel}: {label}={value}, an address THIS repository assigns to a cluster, is "
|
||||
"NOT inside the tunnel's excludedCIDRs — a vpn-class pod in that environment "
|
||||
"would try to reach its own cluster/LAN through the VPN")
|
||||
|
||||
|
||||
# ============ (4) THE VPN PATH IS A SECOND TABLE, NEVER THE DEFAULT (split tunnel) ========
|
||||
TPL = VPN_ROLE / "templates"
|
||||
up = (TPL / "vpn-egress-up.sh.j2")
|
||||
down = (TPL / "vpn-egress-down.sh.j2")
|
||||
conf = (TPL / "vpn-egress.conf.j2")
|
||||
nft = (TPL / "vpn-egress-killswitch.nft.j2")
|
||||
for f in (up, down, conf, nft):
|
||||
check(f.is_file(), f"{f.relative_to(ROOT)} is missing — the second egress path's routing "
|
||||
"rules are not authored")
|
||||
if FAILURES:
|
||||
die()
|
||||
|
||||
defaults = yaml.safe_load((VPN_ROLE / "defaults" / "main.yml").read_text()) or {}
|
||||
|
||||
|
||||
def render(text: str) -> str:
|
||||
"""Substitute `{{ var }}` with the role's own default value, for every default.
|
||||
|
||||
The templates reference their knobs by NAME, so an analysis that looked for the VALUE
|
||||
(e.g. the nft set name) would silently match nothing and pass vacuously — which is
|
||||
exactly what an earlier draft of this check did. Rendering first means the assertions
|
||||
below examine the text the node actually gets. Only simple `{{ name }}` expressions are
|
||||
substituted; anything else is left as-is (and the assertions are written so an
|
||||
unsubstituted expression fails rather than passes).
|
||||
"""
|
||||
out = text
|
||||
for k, v in defaults.items():
|
||||
if isinstance(v, (str, int)):
|
||||
out = re.sub(r"\{\{\s*" + re.escape(str(k)) + r"\s*\}\}", str(v), out)
|
||||
return out
|
||||
|
||||
|
||||
up_t, conf_t, nft_t = render(up.read_text()), render(conf.read_text()), render(nft.read_text())
|
||||
table_id = str(defaults.get("vpn_egress_gateway_route_table_id", ""))
|
||||
check(table_id.isdigit() and table_id not in ("0", "253", "254", "255"),
|
||||
f"the tunnel's routing table id is {table_id!r} — the second path must live in a "
|
||||
"DEDICATED table. 254 (main), 255 (local) and 253 (default) are the kernel's own, and "
|
||||
"putting the tunnel's default route in any of them makes the VPN the node's default "
|
||||
"route instead of a selectable second path")
|
||||
check(str(defaults.get("vpn_egress_gateway_rule_priority", "")).isdigit(),
|
||||
"no ip-rule priority is declared for the tunnel table — 'selected by routing rules' "
|
||||
"needs a rule, at a defined precedence")
|
||||
|
||||
# Every `ip route ... default` the up-script installs must name the dedicated table.
|
||||
route_lines = [l.strip() for l in up_t.splitlines()
|
||||
if re.search(r"^\s*ip\s+route\s+(replace|add)\b", l)]
|
||||
check(len(route_lines) >= 2,
|
||||
f"the up-script installs {len(route_lines)} route(s) — the vpn path needs its own "
|
||||
"default route via the tunnel AND the fail-closed blackhole behind it")
|
||||
for l in route_lines:
|
||||
check(re.search(r'table\s+"?\$\{?TABLE_ID', l) is not None,
|
||||
f"the up-script installs a route WITHOUT naming the dedicated table: {l!r}. A route "
|
||||
"landing in the main table changes the path every other pod and the node itself "
|
||||
"uses — the VPN would stop being opt-in")
|
||||
check("blackhole" not in l or re.search(r'table\s+"?\$\{?TABLE_ID', l) is not None,
|
||||
f"a blackhole route is installed outside the tunnel table: {l!r}. docs/03: the "
|
||||
"kill-switch 'blackholes **only** that class'; `internet` fails OPEN and must never "
|
||||
"be blackholed")
|
||||
|
||||
# Every `ip rule` must be SCOPED. An unscoped rule (`ip rule add lookup <table>`) sends
|
||||
# everything into the tunnel table, which would leave one path instead of two.
|
||||
rule_lines = [l.strip() for l in up_t.splitlines() if re.search(r"^\s*ip\s+rule\s+add\b", l)]
|
||||
check(len(rule_lines) >= 1,
|
||||
"the up-script adds no `ip rule` — nothing SELECTS the tunnel path, so the second "
|
||||
"routing table would never be consulted (R-NET-04's 'selected by routing rules')")
|
||||
for l in rule_lines:
|
||||
check(re.search(r"\bfrom\s+\S|\bfwmark\s+\S|\biif\s+\S", l) is not None,
|
||||
f"an unscoped ip rule is installed: {l!r}. Without a `from`/`fwmark`/`iif` selector "
|
||||
"EVERY packet is looked up in the tunnel table, so the WAN path disappears and the "
|
||||
"VPN is no longer opt-in")
|
||||
check(re.search(r'lookup\s+"?\$\{?TABLE_ID', l) is not None,
|
||||
f"an ip rule looks up a table other than the tunnel's: {l!r}")
|
||||
check(re.search(r'ip rule add from\s+"\$\{SRC\}"', up_t) is not None,
|
||||
"the steering rule is not scoped to ${SRC}, the tunnel's own local address — that "
|
||||
"address is what Cilium SNATs vpn-class pods to, and it is the ONLY thing that "
|
||||
"distinguishes the steered class from every other flow on the gateway node")
|
||||
check(re.search(r"^\s*if \[ -z \"\$\{SRC\}\" \]", up_t, re.M) is not None,
|
||||
"the up-script does not refuse to run when the tunnel address is unknown — installing "
|
||||
"a partial rule set is how a path ends up half-selected")
|
||||
|
||||
# The provider must not be able to make the tunnel the default route either.
|
||||
check(re.search(r"^\s*route-noexec\s*$", conf_t, re.M) is not None,
|
||||
"the OpenVPN wrapper config lacks `route-noexec` — OpenVPN would install the pushed "
|
||||
"routes itself, in the MAIN table, and the VPN would become the node's default route "
|
||||
"(docs/03: 'does **not** set the node's default route (`route-noexec` + policy "
|
||||
"routing)')")
|
||||
check(re.search(r'pull-filter ignore "redirect-gateway"', conf_t) is not None,
|
||||
"the OpenVPN wrapper config does not ignore a pushed `redirect-gateway` — the VPN "
|
||||
"provider could redirect the node's whole default route into the tunnel, collapsing "
|
||||
"the two paths into one")
|
||||
check(re.search(r"^\s*redirect-gateway", conf_t, re.M) is None,
|
||||
"the OpenVPN wrapper config itself sets `redirect-gateway` — that IS making the VPN the "
|
||||
"default route")
|
||||
|
||||
|
||||
# ================= (5) THE KILL-SWITCH BLACKHOLES ONLY THE VPN CLASS (internet fails OPEN)
|
||||
src_set = str(defaults.get("vpn_egress_gateway_nft_src_set", ""))
|
||||
fwmark = str(defaults.get("vpn_egress_gateway_fwmark", ""))
|
||||
check(bool(src_set),
|
||||
"the role declares no nftables source set — the kill-switch would have no way to name "
|
||||
"'the steered class' and could only match broadly")
|
||||
policies_lines = [l.strip() for l in nft_t.splitlines() if re.search(r"^\s*type filter hook", l)]
|
||||
check(len(policies_lines) >= 1,
|
||||
"the kill-switch nft template declares no base chain — nothing would be enforced")
|
||||
for l in policies_lines:
|
||||
check(re.search(r"policy\s+accept\s*;", l) is not None,
|
||||
f"an nft base chain is not `policy accept`: {l!r}. A default-drop chain here "
|
||||
"blackholes the `internet` path too, and docs/03 requires the two classes to fail "
|
||||
"in OPPOSITE directions — 'internet fails **open** (WAN, never blackholed)'")
|
||||
drop_lines = [l.strip() for l in nft_t.splitlines()
|
||||
if re.search(r"\bdrop\b", l) and not l.lstrip().startswith("#")]
|
||||
check(len(drop_lines) >= 1,
|
||||
"the kill-switch nft template contains no drop rule at all — there is no kill-switch, "
|
||||
"and R-NET-04's 'the kill-switch blackholes only that class' has nothing to scope")
|
||||
for l in drop_lines:
|
||||
scoped = (src_set and src_set in l) or (fwmark and "mark" in l)
|
||||
check(scoped,
|
||||
f"an nft DROP rule is not scoped to the vpn class: {l!r}. It must match the tunnel "
|
||||
f"source set (@{src_set}) or the class's fwmark; an unqualified drop would blackhole "
|
||||
"the default `internet` path, which must never be blackholed")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- verdict
|
||||
# NO-OP GUARD: a check that quietly asserts nothing is worse than no check.
|
||||
check(CHECKS >= 35,
|
||||
f"only {CHECKS} assertions ran — the check degenerated to a near-no-op and its pass "
|
||||
"would be meaningless")
|
||||
|
||||
if FAILURES:
|
||||
die()
|
||||
print(f"check OK: egress-paths-selected-by-rules — {CHECKS} assertions; "
|
||||
f"paths={sorted(PATHS)} (default={egress_prop.get('default')}), "
|
||||
f"{len(policies)} steering rule selecting {CLASS_LABEL}=vpn only, tunnel default route "
|
||||
f"in dedicated table {table_id} behind a scoped ip rule, blackhole and every nft DROP "
|
||||
f"scoped to the vpn class. R-NET-04's RULE-SET half only: the paths' actual BEHAVIOUR "
|
||||
"is L2 (egress-class for the two labelled classes, egress-default-path for the "
|
||||
"unlabelled default) and the kill-switch firing is L3 (vpn-killswitch).")
|
||||
PY
|
||||
@@ -1,88 +0,0 @@
|
||||
# =============================================================================
|
||||
# L0 STATIC — the HA control plane is authored, for every environment (R-PLAT-01) [EXECUTABLE]
|
||||
# Catalog: docs/14 L0 row ("Pre-apply, in CI") — runs WITHOUT a cluster.
|
||||
# Spec: docs/00 R-PLAT-01 ("The platform MUST run on k3s across 3 nodes with
|
||||
# production-grade HA control plane in prod; dev/test MAY run fewer nodes but MUST
|
||||
# exercise the same multi-node topology"), elaborated by docs/01 ("k3s (HA,
|
||||
# embedded etcd)") and docs/02 ("HA embedded etcd (k3s server on all 3)",
|
||||
# "3 physical nodes", "no external datastore", "the *same* Ansible bootstrap runs").
|
||||
#
|
||||
# WHY L0: R-PLAT-01 has an authored half and an observed half. The authored half — that the
|
||||
# configuration in git BUILDS a 3-server embedded-etcd control plane in BOTH environments
|
||||
# from one code path, with exactly one --cluster-init founder, no external datastore, and a
|
||||
# floating API endpoint that is SAN'd into the serving cert and cannot collide with the
|
||||
# Service-LB allocator — is provable with no cluster and is ONLY provable here: a live
|
||||
# cluster cannot tell you what the NEXT environment brought up from this repo would look
|
||||
# like. The observed half is asserted separately at L2 by
|
||||
# tests/l2-behavioral/ha-control-plane, which carries the same requirement label.
|
||||
#
|
||||
# ENVIRONMENT-AGNOSTIC: no hostname, address, node count or group name is written down.
|
||||
# Environments are enumerated from the R-ENV-02 seam itself (k3s_primary_group /
|
||||
# k3s_servers_group in group_vars), and WHICH one is prod is derived from the Flux overlay
|
||||
# that environment bootstraps (flux_bootstrap_env_path) — never from a group's name.
|
||||
#
|
||||
# GIVEN the repository as git tracks it,
|
||||
# WHEN the inventory, the k3s_server and kube_vip roles, their plays and each
|
||||
# environment's Flux overlay are parsed and cross-checked against each other,
|
||||
# THEN k3s is the pinned distro and nothing is installed as an agent; every environment
|
||||
# resolves to an ODD, quorum-capable server count (prod exactly three) with EXACTLY
|
||||
# ONE --cluster-init founder and every other server joining it with --server;
|
||||
# the datastore is k3s's embedded etcd (no --datastore-endpoint, no --disable-etcd);
|
||||
# the API's floating endpoint is a leader-elected, control-plane-ONLY kube-vip
|
||||
# DaemonSet on every server whose address is the environment-scoped VIP, that VIP is
|
||||
# in the apiserver serving cert's SANs together with every server's own address, and
|
||||
# it is neither a node address nor the ingress Gateway VIP nor inside that
|
||||
# environment's Cilium LoadBalancer pool; and dev/test and prod are built by the SAME
|
||||
# roles and plays retargeted by variables only.
|
||||
#
|
||||
# STATUS: EXECUTABLE. `check.sh` is a pure offline assertion script (git tree + PyYAML; no
|
||||
# cluster, no network); conformance/run.sh --layer l0 and ci/l0.sh run it directly, and this
|
||||
# Chainsaw Test wraps the SAME script.
|
||||
#
|
||||
# KNOWN, DELIBERATE GAPS (enumerated in check.sh's header; this test claims none of them):
|
||||
# * The LIVE clusters' shape (three Ready servers, all etcd members, VIP actually floating,
|
||||
# API answering through it) — asserted at L2 by tests/l2-behavioral/ha-control-plane.
|
||||
# * FAILOVER. That the control plane survives losing a server (quorum holds, the VIP moves)
|
||||
# is a fault-injection claim and is proven NOWHERE yet — tests/l3-fault-injection/node-loss
|
||||
# drains a node for R-STATE-02's data claim and asserts nothing about the control plane.
|
||||
# * "Production-grade" as a hardware/performance judgement (operator decision, §2 of
|
||||
# infra/PROD-PROVISIONING.md).
|
||||
#
|
||||
# TWO GAPS CLOSED 2026-09-06 (they used to be listed above; recorded so the change is legible):
|
||||
# * `K3S_API_HOST` (Cilium's k8sServiceHost) was the PRIMARY NODE's address in both
|
||||
# environments rather than the floating VIP, contradicting the prod overlay's own comment.
|
||||
# It was ESCALATED, not asserted, because the workstream could not decide it (R-TEST-06).
|
||||
# The operator decided the VIP, so it is now ASSERTED — on both sides of the [parity]
|
||||
# contract and per environment.
|
||||
# * kube-vip was NOT imported by playbooks/site.yml. It is now, BETWEEN k3s and cilium, and
|
||||
# the order is asserted — not as a tidiness preference but because the CNI's endpoint is
|
||||
# the VIP, so an unfloated VIP means a cold build brings up a CNI that cannot reach its
|
||||
# API server.
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: ha-control-plane-as-code
|
||||
labels:
|
||||
conformance.home.k3s/R-PLAT-01: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-PLAT-01"
|
||||
conformance.home.k3s/layer: "L0"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
Given the tracked tree, when the inventory, the k3s/kube-vip roles and plays and each
|
||||
environment's Flux overlay are parsed and cross-checked, then every environment is
|
||||
authored as an odd, quorum-capable set of k3s servers (prod exactly three) with one
|
||||
embedded-etcd founder, no external datastore, and a leader-elected control-plane-only
|
||||
VIP that is in the serving cert's SANs and collides with no other allocator — built by
|
||||
one code path for both environments.
|
||||
spec:
|
||||
steps:
|
||||
- name: then-every-environment-is-authored-as-an-ha-embedded-etcd-control-plane
|
||||
description: >-
|
||||
Run the offline assertion script over the tracked tree (same script ci/l0.sh and
|
||||
conformance/run.sh --layer l0 execute).
|
||||
try:
|
||||
- script:
|
||||
content: bash ./check.sh
|
||||
@@ -1,729 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# L0 static check (offline, no cluster) — ha-control-plane-as-code
|
||||
# Serves: R-PLAT-01 ("The platform MUST run on k3s across 3 nodes with production-grade
|
||||
# HA control plane in prod; dev/test MAY run fewer nodes but MUST exercise the
|
||||
# same multi-node topology." — spec docs/00-vision-and-scope.md).
|
||||
#
|
||||
# The elaborating spec text this check quotes and asserts against:
|
||||
# docs/01-architecture-overview.md "Kubernetes distro | **k3s** (HA, embedded etcd)"
|
||||
# docs/02-environments-and-provisioning.md, table:
|
||||
# "Control plane | HA embedded etcd (k3s server on all 3) | HA embedded etcd
|
||||
# (k3s server on all 3)" (the SAME cell for dev/test and prod)
|
||||
# "Node count | 3 VMs (mirror prod topology) | 3 physical nodes"
|
||||
# docs/02 §"k3s topology decisions":
|
||||
# "HA control plane with **embedded etcd** across all 3 nodes (no external datastore)."
|
||||
# "Nodes are both control-plane and workload (3-node converged cluster); with only 3
|
||||
# nodes we do not isolate control-plane nodes."
|
||||
# "For prod bare-metal, the *same* Ansible bootstrap runs; only the machine-creation
|
||||
# step differs"
|
||||
#
|
||||
# WHY L0 IS THE RIGHT LAYER FOR THESE CLAUSES (and where the rest lives)
|
||||
# ---------------------------------------------------------------------
|
||||
# R-PLAT-01 splits cleanly in two, and the halves are provable at different layers:
|
||||
#
|
||||
# AUTHORED half — "the configuration in git builds a 3-server HA embedded-etcd control
|
||||
# plane in BOTH environments, from ONE code path": provable with no cluster, and
|
||||
# ONLY provable here. A live cluster cannot tell you whether the *next* environment
|
||||
# brought up from this repo will also be HA, whether a second `--cluster-init`
|
||||
# founder could exist, or whether an external datastore is configured anywhere. That
|
||||
# is this check.
|
||||
# OBSERVED half — "the cluster this suite is pointed at really is 3 Ready servers, all
|
||||
# etcd members, with the API answering through the floating VIP": that is cluster
|
||||
# state and lives in tests/l2-behavioral/ha-control-plane (same requirement label).
|
||||
#
|
||||
# Neither half is sufficient alone, and this file claims only the first.
|
||||
#
|
||||
# ENVIRONMENTS ARE DERIVED, NOT LISTED. The check does not know the words "dev" or "prod"
|
||||
# as inventory names. It enumerates environments from the R-ENV-02 seam itself — the
|
||||
# `k3s_primary_group` / `k3s_servers_group` pair in group_vars/all.yml plus every
|
||||
# group_vars/<group>/vars.yml that overrides them — and identifies WHICH environment is
|
||||
# prod by the Flux overlay that environment bootstraps (`flux_bootstrap_env_path`), not by
|
||||
# its group name. Add a fourth prod node, rename a group, or add a third environment and
|
||||
# the assertions follow; nothing here restates an address, a hostname or a node count.
|
||||
#
|
||||
# GIVEN the repository as git tracks it,
|
||||
# WHEN the Ansible inventory, the k3s/kube-vip roles and plays, and each environment's
|
||||
# Flux overlay are parsed and cross-checked against each other,
|
||||
# THEN DISTRO: k3s is the distro, pinned, installed by the one role, and no node is
|
||||
# installed as an `agent` (an agent is not a control-plane member);
|
||||
# SERVERS: every environment's server group resolves to a quorum-capable, ODD
|
||||
# number of hosts (>= 3, tolerating >= 1 loss), the PROD environment to
|
||||
# EXACTLY three, and each has EXACTLY ONE `--cluster-init` founder while
|
||||
# every other server joins it with `--server` (so a second founder — the
|
||||
# split-brain shape — cannot be authored);
|
||||
# ETCD: the datastore is k3s's EMBEDDED etcd: no `--datastore-endpoint` and no
|
||||
# `--disable-etcd` anywhere in the provisioning code, so every server is
|
||||
# an etcd member (docs/02's "no external datastore", "on all 3");
|
||||
# ENDPOINT: the API has a floating endpoint that survives losing the founder — the
|
||||
# kube-vip VIP is control-plane-only (never Service LB, or it would fight
|
||||
# Cilium LB-IPAM for ARP), leader-elected, deployed to EVERY server, and
|
||||
# baked into the apiserver serving cert's SAN list ALONG WITH every
|
||||
# server's own address (so TLS validates whichever node answers);
|
||||
# ADDRESSES: each environment's control-plane VIP is not one of its node addresses,
|
||||
# is not that environment's ingress Gateway VIP, and lies OUTSIDE that
|
||||
# environment's Cilium LoadBalancer pool — the two allocators can never be
|
||||
# handed the same address;
|
||||
# CNI PATH: the CNI dials the FLOATING endpoint, not a machine (added 2026-09-06). The
|
||||
# two API addresses stay SPLIT — k3s_api_host is the founder and feeds only
|
||||
# the cold `--server` join, k3s_apiserver_endpoint derives from
|
||||
# kube_vip_address and feeds Cilium's k8sServiceHost — the Flux overlay's
|
||||
# K3S_API_HOST equals that environment's VIP and is none of its node
|
||||
# addresses ([parity], both halves), the HelmRelease keeps a per-environment
|
||||
# token rather than a literal, and site.yml floats the VIP BETWEEN k3s and
|
||||
# Cilium with roles/kube_vip blocking until it answers;
|
||||
# ONE PATH: dev/test and prod are brought up by the SAME role and the SAME plays,
|
||||
# retargeted by variables only (docs/02 "the *same* Ansible bootstrap
|
||||
# runs"), so "exercise the same multi-node topology" is a property of the
|
||||
# code rather than a coincidence of two hand-written inventories.
|
||||
#
|
||||
# WHAT THIS DOES *NOT* PROVE (honest gaps — do not read this check as claiming them):
|
||||
# - THE LIVE CLUSTERS. That three nodes are Ready, that all three are etcd members with
|
||||
# a healthy quorum, that kube-vip is actually floating the VIP, and that the API
|
||||
# answers through it. All of that is asserted at L2 by
|
||||
# tests/l2-behavioral/ha-control-plane, which reads it from the cluster under test.
|
||||
# - FAILOVER. That the control plane SURVIVES the loss of a server (etcd quorum holds,
|
||||
# the VIP moves, the API stays up) is a fault-injection claim. The nearest existing
|
||||
# coverage is tests/l3-fault-injection/node-loss, which drains a node for R-STATE-02's
|
||||
# data-availability claim and does NOT assert control-plane survival. A dedicated
|
||||
# "kill the VIP leader / kill the founder" L3 test is the remaining R-PLAT-01 gap and
|
||||
# is NOT claimed here or at L2.
|
||||
# - "PRODUCTION-GRADE" AS A HARDWARE/OPERATIONS JUDGEMENT. Whether three converged
|
||||
# mini-PCs with etcd sharing a spindle with Longhorn is production-grade is an operator
|
||||
# decision recorded in infra/PROD-PROVISIONING.md §2, not a repository property. This
|
||||
# check asserts the control-plane TOPOLOGY the spec fixes, not a performance verdict.
|
||||
# - IPv6 / dual-stack, and any claim about the edge network.
|
||||
#
|
||||
# TWO FORMER GAPS ARE NOW ASSERTED, and the reasoning is worth keeping (2026-09-06):
|
||||
# * CILIUM'S OWN API ENDPOINT used to be listed here as an ESCALATION — `K3S_API_HOST`
|
||||
# (clusters/*/kustomization.yaml), which becomes Cilium's `k8sServiceHost`, was each
|
||||
# environment's PRIMARY NODE address rather than the floating VIP, contradicting the prod
|
||||
# overlay's own comment. Making the check fail over it would have turned the merge gate
|
||||
# red on a design question this workstream may not decide (R-TEST-06), so it was reported
|
||||
# instead. THE OPERATOR DECIDED THE VIP on 2026-09-06, which turns the question into a
|
||||
# repository invariant, and invariants belong in assertions — see the "THE CNI DIALS THE
|
||||
# VIP" section. Note what made it assertable: the fix SPLIT one variable in two, so there
|
||||
# is finally an HA-valued name (`k3s_apiserver_endpoint`) to point the contract at.
|
||||
# * kube-vip WAS NOT PART OF THE ONE-COMMAND BRING-UP (site.yml chained node-baseline ->
|
||||
# k3s -> cilium -> verify -> flux-bootstrap), which was recorded here as an observation
|
||||
# and not asserted, because R-PLAT-01 does not fix the bring-up order. It is asserted now
|
||||
# for a different reason that IS in scope: with the CNI's endpoint being the VIP, an
|
||||
# unfloated VIP means a cold build installs a CNI that cannot reach its API server. The
|
||||
# ordering is no longer a preference about tidiness, it is a dependency of the control
|
||||
# plane coming up at all.
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT="$(cd "${DIR}/../../../.." && pwd)" # repo root
|
||||
PYTHON="${PYTHON:-python3}"
|
||||
|
||||
cd "${ROOT}"
|
||||
"${PYTHON}" - "${ROOT}" <<'PY'
|
||||
import ipaddress
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError: # pragma: no cover
|
||||
print("check FAIL: PyYAML is required (same dependency as catalog/render.py)",
|
||||
file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
ROOT = Path(sys.argv[1])
|
||||
FAILURES: list[str] = []
|
||||
NOTES: list[str] = []
|
||||
CHECKS = 0
|
||||
|
||||
|
||||
def check(cond, msg):
|
||||
global CHECKS
|
||||
CHECKS += 1
|
||||
if not cond:
|
||||
FAILURES.append(msg)
|
||||
|
||||
|
||||
def note(msg):
|
||||
NOTES.append(msg)
|
||||
|
||||
|
||||
def die():
|
||||
print(f"CHECK FAIL (ha-control-plane-as-code): {len(FAILURES)} of {CHECKS} assertions failed",
|
||||
file=sys.stderr)
|
||||
for f in FAILURES:
|
||||
print(f" - {f}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
ANS = ROOT / "infra" / "ansible"
|
||||
INV = ANS / "inventory" / "hosts.ini"
|
||||
GV = ANS / "inventory" / "group_vars"
|
||||
K3S_ROLE = ANS / "roles" / "k3s_server"
|
||||
VIP_ROLE = ANS / "roles" / "kube_vip"
|
||||
PLAYBOOKS = ANS / "playbooks"
|
||||
CLUSTERS = ROOT / "clusters"
|
||||
|
||||
for p, why in ((INV, "the inventory that defines the node set"),
|
||||
(K3S_ROLE, "the role that installs the k3s control plane"),
|
||||
(VIP_ROLE, "the role that floats the control-plane VIP"),
|
||||
(CLUSTERS, "the Flux overlays that identify each environment")):
|
||||
check(p.exists(), f"{p.relative_to(ROOT)} is missing — {why} does not exist, so no HA "
|
||||
"control plane is authored at all (R-PLAT-01)")
|
||||
if FAILURES:
|
||||
die()
|
||||
|
||||
K3S_TEXT = "\n".join(p.read_text(errors="replace") for p in sorted(K3S_ROLE.rglob("*.yml")))
|
||||
VIP_TEXT = "\n".join(p.read_text(errors="replace")
|
||||
for p in sorted(list(VIP_ROLE.rglob("*.yml")) + list(VIP_ROLE.rglob("*.j2"))))
|
||||
|
||||
|
||||
# ===================================================================== inventory parse
|
||||
# Parser contract (same shape lib/coverage.py-style parsers use elsewhere): hosts.ini is a
|
||||
# plain INI inventory — `[group]` sections of `host key=value ...` lines, `[group:children]`
|
||||
# sections naming child groups, `[group:vars]` sections ignored. Comments start with '#'.
|
||||
ini_hosts: dict[str, list[str]] = {}
|
||||
ini_children: dict[str, list[str]] = {}
|
||||
host_vars: dict[str, dict[str, str]] = {}
|
||||
cur = None
|
||||
for line in INV.read_text().splitlines():
|
||||
s = line.strip()
|
||||
if not s or s.startswith("#"):
|
||||
continue
|
||||
m = re.match(r"^\[([^\]]+)\]$", s)
|
||||
if m:
|
||||
cur = m.group(1)
|
||||
continue
|
||||
if cur is None or cur.endswith(":vars"):
|
||||
continue
|
||||
host = s.split()[0]
|
||||
if cur.endswith(":children"):
|
||||
ini_children.setdefault(cur.split(":")[0], []).append(host)
|
||||
else:
|
||||
ini_hosts.setdefault(cur, []).append(host)
|
||||
hv = host_vars.setdefault(host, {})
|
||||
for kv in s.split()[1:]:
|
||||
if "=" in kv:
|
||||
k, v = kv.split("=", 1)
|
||||
hv[k] = v
|
||||
|
||||
|
||||
def hosts_in(group: str, seen=None) -> set[str]:
|
||||
seen = seen or set()
|
||||
if group in seen:
|
||||
return set()
|
||||
seen.add(group)
|
||||
out = set(ini_hosts.get(group, []))
|
||||
for child in ini_children.get(group, []):
|
||||
out |= hosts_in(child, seen)
|
||||
return out
|
||||
|
||||
|
||||
def load_yaml(p: Path) -> dict:
|
||||
try:
|
||||
d = yaml.safe_load(p.read_text()) or {}
|
||||
except yaml.YAMLError:
|
||||
return {}
|
||||
return d if isinstance(d, dict) else {}
|
||||
|
||||
|
||||
ALL_VARS = load_yaml(GV / "all.yml")
|
||||
|
||||
|
||||
# =============================================== ENVIRONMENTS, DERIVED FROM THE SEAM
|
||||
# An "environment" here is a (primary group, servers group) pair. group_vars/all.yml holds
|
||||
# the default pair; every group_vars/<g>/vars.yml that overrides `k3s_primary_group` is
|
||||
# another environment. Nothing below knows the words "dev" or "prod" as group names.
|
||||
class Env:
|
||||
def __init__(self, label, vars_, path):
|
||||
self.label = label
|
||||
self.vars = vars_
|
||||
self.path = path
|
||||
|
||||
def var(self, name, default=None):
|
||||
# group-scoped value wins, else the cluster-wide default from all.yml.
|
||||
if name in self.vars:
|
||||
return self.vars[name]
|
||||
return ALL_VARS.get(name, default)
|
||||
|
||||
|
||||
envs: list[Env] = [Env("group_vars/all.yml (default environment)", ALL_VARS, GV / "all.yml")]
|
||||
for vf in sorted(GV.glob("*/vars.yml")):
|
||||
v = load_yaml(vf)
|
||||
if "k3s_primary_group" in v or "k3s_servers_group" in v:
|
||||
envs.append(Env(str(vf.relative_to(ROOT)), v, vf))
|
||||
|
||||
check(len(envs) >= 2,
|
||||
"only ONE environment is derivable from the inventory's R-ENV-02 seam "
|
||||
"(k3s_primary_group / k3s_servers_group). R-PLAT-01 constrains prod AND requires "
|
||||
"dev/test to exercise the same multi-node topology, so two must be authored")
|
||||
|
||||
# WHICH environment is prod: the one that bootstraps the clusters/prod Flux overlay.
|
||||
# roles/flux_bootstrap's own default supplies the value for an environment that does not
|
||||
# override it, so this is read from code, never guessed from a group name.
|
||||
fb_default = None
|
||||
fb_defaults_file = ANS / "roles" / "flux_bootstrap" / "defaults" / "main.yml"
|
||||
if fb_defaults_file.is_file():
|
||||
fb_default = load_yaml(fb_defaults_file).get("flux_bootstrap_env_path")
|
||||
check(isinstance(fb_default, str) and fb_default.startswith("clusters/"),
|
||||
"roles/flux_bootstrap declares no flux_bootstrap_env_path default — the mapping from "
|
||||
"an inventory environment to its Flux overlay is not expressed in code, so this check "
|
||||
"cannot tell which environment is prod")
|
||||
|
||||
prod_envs = []
|
||||
for e in envs:
|
||||
overlay = e.var("flux_bootstrap_env_path", fb_default)
|
||||
e.overlay = overlay if isinstance(overlay, str) else ""
|
||||
if e.overlay.rstrip("/").endswith("/prod"):
|
||||
prod_envs.append(e)
|
||||
check(len(prod_envs) == 1,
|
||||
f"{len(prod_envs)} environment(s) bootstrap the clusters/prod overlay "
|
||||
f"({[e.label for e in prod_envs]}) — exactly one must be prod, or 'in prod' in "
|
||||
"R-PLAT-01 has no unambiguous referent")
|
||||
|
||||
|
||||
# ============================================== overlay knob values (per environment)
|
||||
# Parser contract: clusters/base/cluster-vars.yaml holds the base ConfigMap `data:` map;
|
||||
# each clusters/<env>/kustomization.yaml overrides values with JSON6902 patches written as
|
||||
# - op: replace
|
||||
# path: /data/<KEY>
|
||||
# value: "<v>"
|
||||
# (verified shape across both overlays). A key not patched keeps the base value.
|
||||
BASE_VARS: dict[str, str] = {}
|
||||
base_cm = CLUSTERS / "base" / "cluster-vars.yaml"
|
||||
if base_cm.is_file():
|
||||
d = load_yaml(base_cm)
|
||||
BASE_VARS = {k: str(v) for k, v in (d.get("data") or {}).items()}
|
||||
check(len(BASE_VARS) > 5,
|
||||
"clusters/base/cluster-vars.yaml exposes no data map — the overlay knob surface "
|
||||
"cannot be read, so the VIP/pool separation cannot be checked")
|
||||
|
||||
_PATCH_RE = re.compile(
|
||||
r"-\s+op:\s*replace\s*\n\s*path:\s*/data/(?P<k>[A-Z0-9_]+)\s*\n\s*value:\s*(?P<v>\S.*?)\s*$",
|
||||
re.M)
|
||||
|
||||
|
||||
def overlay_vars(overlay: str) -> dict[str, str]:
|
||||
out = dict(BASE_VARS)
|
||||
f = ROOT / overlay / "kustomization.yaml"
|
||||
if not f.is_file():
|
||||
return out
|
||||
for m in _PATCH_RE.finditer(f.read_text()):
|
||||
out[m.group("k")] = m.group("v").strip().strip('"').strip("'")
|
||||
return out
|
||||
|
||||
|
||||
# ============================================================ PER-ENVIRONMENT TOPOLOGY
|
||||
def as_ip(s):
|
||||
try:
|
||||
return ipaddress.ip_address(str(s).strip())
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
for e in envs:
|
||||
L = e.label
|
||||
pg = e.var("k3s_primary_group")
|
||||
sg = e.var("k3s_servers_group")
|
||||
check(isinstance(pg, str) and isinstance(sg, str) and pg and sg,
|
||||
f"{L}: k3s_primary_group / k3s_servers_group are not both set — the roles cannot "
|
||||
"resolve 'my environment's primary/servers' and would only ever build dev")
|
||||
if not (isinstance(pg, str) and isinstance(sg, str)):
|
||||
continue
|
||||
primaries = hosts_in(pg)
|
||||
servers = hosts_in(sg)
|
||||
|
||||
# --- SERVERS: quorum-capable, one founder, prod exactly three ---------------------
|
||||
check(len(primaries) == 1,
|
||||
f"{L}: group {pg!r} resolves to {sorted(primaries)} — EXACTLY ONE host may run "
|
||||
"`--cluster-init`. Two founders is two etcd clusters (split brain), zero is no "
|
||||
"cluster at all")
|
||||
check(primaries and primaries <= servers,
|
||||
f"{L}: the primary {sorted(primaries)} is not a member of the servers group "
|
||||
f"{sg!r} ({sorted(servers)}) — the founder must itself be one of the servers, or "
|
||||
"it is not part of the HA control plane it founds")
|
||||
n = len(servers)
|
||||
check(n >= 3,
|
||||
f"{L}: group {sg!r} resolves to {n} server(s) {sorted(servers)}. docs/02 fixes "
|
||||
"'HA embedded etcd (k3s server on all 3)' for dev/test AND prod, and an etcd "
|
||||
"cluster of fewer than 3 members tolerates ZERO losses — it is not HA")
|
||||
check(n % 2 == 1,
|
||||
f"{L}: group {sg!r} has an EVEN number of servers ({n}). An even embedded-etcd "
|
||||
f"member count adds a member without adding fault tolerance and introduces a tie: "
|
||||
f"quorum for {n} is {n // 2 + 1}, the same as for {n - 1}")
|
||||
if n:
|
||||
tolerance = n - (n // 2 + 1)
|
||||
check(tolerance >= 1,
|
||||
f"{L}: {n} server(s) give a quorum of {n // 2 + 1} and therefore tolerate "
|
||||
f"{tolerance} loss(es) — 'production-grade HA' requires surviving at least one")
|
||||
if e in prod_envs:
|
||||
check(n == 3,
|
||||
f"{L} is the PROD environment and resolves to {n} server(s). R-PLAT-01: "
|
||||
"'k3s across 3 nodes ... in prod'; docs/02: 'Node count | ... | 3 physical nodes'")
|
||||
else:
|
||||
check(n >= 2,
|
||||
f"{L}: a non-prod environment resolves to {n} host(s). R-PLAT-01 allows fewer "
|
||||
"nodes than prod but requires dev/test to 'exercise the same multi-node "
|
||||
"topology' — one node exercises no topology")
|
||||
|
||||
# --- ADDRESSES: the VIP can collide with nothing ----------------------------------
|
||||
vip_raw = e.var("kube_vip_address")
|
||||
vip = as_ip(vip_raw)
|
||||
check(vip is not None,
|
||||
f"{L}: kube_vip_address is {vip_raw!r}, not an IP address — the control plane has "
|
||||
"no floating endpoint, so losing the founder takes the API endpoint with it")
|
||||
node_ips = {}
|
||||
for h in sorted(servers):
|
||||
ip = as_ip(host_vars.get(h, {}).get("k3s_node_ip", ""))
|
||||
check(ip is not None,
|
||||
f"{L}: host {h} declares no usable k3s_node_ip — k3s --node-ip, the join "
|
||||
"target and the serving-cert SAN list all derive from it")
|
||||
if ip is not None:
|
||||
node_ips[h] = ip
|
||||
if vip is not None:
|
||||
check(vip not in set(node_ips.values()),
|
||||
f"{L}: kube_vip_address {vip} is ALSO a node address "
|
||||
f"({[h for h, i in node_ips.items() if i == vip]}) — a floating VIP that is a "
|
||||
"node's own address cannot float off that node")
|
||||
ov = overlay_vars(getattr(e, "overlay", ""))
|
||||
gw = as_ip(ov.get("GATEWAY_VIP", ""))
|
||||
lo = as_ip(ov.get("CILIUM_LB_POOL_START", ""))
|
||||
hi = as_ip(ov.get("CILIUM_LB_POOL_STOP", ""))
|
||||
check(gw is not None and lo is not None and hi is not None,
|
||||
f"{L}: overlay {getattr(e, 'overlay', '')!r} does not resolve GATEWAY_VIP + "
|
||||
"CILIUM_LB_POOL_START/STOP — the separation between the control-plane VIP and "
|
||||
"the Service-LB allocator cannot be checked")
|
||||
if gw is not None:
|
||||
check(vip != gw,
|
||||
f"{L}: the control-plane VIP and the ingress Gateway VIP are the SAME "
|
||||
f"address ({vip}) — kube-vip and Cilium would both ARP for it")
|
||||
if lo is not None and hi is not None:
|
||||
check(not (lo <= vip <= hi),
|
||||
f"{L}: the control-plane VIP {vip} lies INSIDE that environment's Cilium "
|
||||
f"LoadBalancer pool {lo}-{hi} — LB-IPAM could hand the kube-API's VIP to a "
|
||||
"Service, and the two allocators would fight over ARP for it")
|
||||
check(lo <= hi,
|
||||
f"{L}: Cilium LB pool {lo}-{hi} is inverted — the pool bound cannot be "
|
||||
"used to prove the VIP is outside it")
|
||||
|
||||
# ==================================================================== DISTRO: k3s only
|
||||
check(re.search(r"^k3s_version:\s*\"?v\d+\.\d+", (K3S_ROLE / "defaults" / "main.yml").read_text(), re.M)
|
||||
is not None,
|
||||
"roles/k3s_server pins no concrete k3s_version (vX.Y...) — an unpinned control-plane "
|
||||
"version is not a reproducible platform (docs/01: the distro is k3s, ADR-0001)")
|
||||
check("get.k3s.io" in K3S_TEXT,
|
||||
"roles/k3s_server does not install from the k3s installer — R-PLAT-01 names k3s as the "
|
||||
"distro (ADR-0001) and no other distro may stand in for it")
|
||||
check(re.search(r"^\s*server\s*$", K3S_TEXT, re.M) is not None,
|
||||
"roles/k3s_server never composes the `server` subcommand — without it k3s installs an "
|
||||
"AGENT, which is not a control-plane member (docs/02: 'k3s server on all 3')")
|
||||
for bad, why in (
|
||||
(r"--datastore-endpoint", "an EXTERNAL datastore (docs/02: 'embedded etcd across all 3 "
|
||||
"nodes (no external datastore)')"),
|
||||
(r"datastore-endpoint\s*:", "an external datastore via config.yaml"),
|
||||
(r"--disable-etcd", "a server with etcd DISABLED, i.e. a control-plane node that is not "
|
||||
"an etcd member (docs/02: etcd 'across all 3 nodes')"),
|
||||
(r"INSTALL_K3S_EXEC\s*[:=]\s*[\"']?agent", "an AGENT-only node"),
|
||||
):
|
||||
hits = [str(p.relative_to(ROOT)) for p in sorted(ANS.rglob("*"))
|
||||
if p.is_file() and p.suffix in (".yml", ".yaml", ".j2", ".sh", ".ini")
|
||||
and re.search(bad, p.read_text(errors="replace"))]
|
||||
check(not hits,
|
||||
f"{hits} configure {why} — R-PLAT-01's control plane is k3s's own embedded-etcd HA "
|
||||
f"mode, and {bad!r} contradicts it")
|
||||
|
||||
# ============================================== FOUNDER / JOINER, EXACTLY ONE OF EACH
|
||||
check(re.search(r"--cluster-init", K3S_TEXT) is not None,
|
||||
"roles/k3s_server never passes --cluster-init — the embedded-etcd cluster is never "
|
||||
"founded, so there is no HA datastore to join")
|
||||
init_when = re.search(r"--cluster-init.*?\n(?:.*\n){0,8}?\s*when:\s*k3s_server_mode\s*==\s*'primary'",
|
||||
K3S_TEXT)
|
||||
check(init_when is not None,
|
||||
"the --cluster-init install task is not gated on k3s_server_mode == 'primary' — an "
|
||||
"ungated founder flag would re-found the cluster on every server (split brain)")
|
||||
join = re.search(r"--server https://\{\{\s*k3s_api_host", K3S_TEXT)
|
||||
check(join is not None,
|
||||
"roles/k3s_server does not join secondaries with `--server https://{{ k3s_api_host }}` "
|
||||
"— without a join target the servers are separate single-node clusters")
|
||||
join_when = re.search(r"--server https://.*?\n(?:.*\n){0,10}?\s*when:\s*k3s_server_mode\s*==\s*'secondary'",
|
||||
K3S_TEXT, re.S)
|
||||
check(join_when is not None,
|
||||
"the join install task is not gated on k3s_server_mode == 'secondary' — the founder "
|
||||
"would also try to join itself")
|
||||
k3s_play = PLAYBOOKS / "k3s.yml"
|
||||
check(k3s_play.is_file(), "playbooks/k3s.yml is missing — the control-plane install has no entrypoint")
|
||||
if k3s_play.is_file():
|
||||
play_txt = k3s_play.read_text()
|
||||
modes = re.findall(r"k3s_server_mode:\s*(\w+)", play_txt)
|
||||
check(sorted(modes) == ["primary", "secondary"],
|
||||
f"playbooks/k3s.yml declares server modes {modes} — exactly one primary play and "
|
||||
"one secondary play must exist, or the founder/joiner split is not authored")
|
||||
check(re.search(r"serial:\s*1", play_txt) is not None,
|
||||
"playbooks/k3s.yml does not restart/join the secondaries with `serial: 1` — "
|
||||
"converging two of three servers at once drops embedded etcd below quorum, which "
|
||||
"is exactly the outage an HA control plane exists to avoid")
|
||||
hosts_patterns = re.findall(r"^\s*hosts:\s*(.+)$", play_txt, re.M)
|
||||
check(len(hosts_patterns) >= 2 and all("{{" in h for h in hosts_patterns),
|
||||
f"playbooks/k3s.yml targets literal host patterns {hosts_patterns} — a literal "
|
||||
"group name means this play can only ever build ONE environment, so prod cannot "
|
||||
"be brought up by the same code (docs/02: 'the *same* Ansible bootstrap runs')")
|
||||
|
||||
# =========================================== SERVING-CERT SANs: VIP + every server
|
||||
sans = str(load_yaml(K3S_ROLE / "defaults" / "main.yml").get("k3s_tls_sans", ""))
|
||||
check("kube_vip_address" in sans,
|
||||
"roles/k3s_server's k3s_tls_sans does not include kube_vip_address — the apiserver's "
|
||||
"serving cert would carry no SAN for the floating VIP, so every client reaching the "
|
||||
"API through it fails TLS validation and the HA endpoint is unusable")
|
||||
check("k3s_servers_group" in sans and "k3s_node_ip" in sans,
|
||||
"k3s_tls_sans does not extract k3s_node_ip for every host of k3s_servers_group — the "
|
||||
"cert must be valid for WHICHEVER server answers behind the VIP, and per-environment "
|
||||
"resolution of that set is what keeps one role serving both environments (R-ENV-02)")
|
||||
check(re.search(r"tls-san:", K3S_TEXT) is not None,
|
||||
"the SAN list is passed only as an install flag and never persisted to "
|
||||
"/etc/rancher/k3s/config.yaml — on an ALREADY-installed cluster the install task is a "
|
||||
"no-op, so a changed SAN set (e.g. a new server) would never reach the serving cert")
|
||||
|
||||
# ============================ THE CNI DIALS THE VIP, NOT THE FOUNDER (added 2026-09-06)
|
||||
# This section used to be an ESCALATION in the header's gap list, not an assertion: Cilium's
|
||||
# k8sServiceHost was each environment's PRIMARY NODE address, so on a 3-server HA control
|
||||
# plane the CNI's control-plane path depended on one machine. Measured on live prod
|
||||
# 2026-09-06: all three cilium-agents carried KUBERNETES_SERVICE_HOST=192.168.2.10
|
||||
# (k3s-prod-1's own address) while kube-vip held 192.168.2.9. That failure is SILENT — the
|
||||
# datapath keeps serving from Cilium's cache, so nothing looks broken while no service or
|
||||
# policy update lands at all.
|
||||
#
|
||||
# The operator decided the VIP (2026-09-06), so this is now a repository property and belongs
|
||||
# here rather than in the gap list. THIS IS THE RIGHT LAYER because the claim is about what
|
||||
# the configuration in git WOULD build, in BOTH environments, including the next one — a live
|
||||
# cluster can only tell you about itself.
|
||||
#
|
||||
# WHAT MAKES THIS CHECKABLE AT ALL is that the fix split one variable into two. Asserting
|
||||
# "Cilium uses the HA address" requires an HA address to name; before the split there was
|
||||
# only k3s_api_host, which had to be the founder for the cold join. So the assertions below
|
||||
# check the SPLIT as much as the value: the join keeps the founder, the CNI gets the VIP, and
|
||||
# the Flux half agrees with the Ansible half per environment.
|
||||
CIL_ROLE = ANS / "roles" / "cilium"
|
||||
cil_values_tpl = CIL_ROLE / "templates" / "cilium-values.yaml.j2"
|
||||
check(cil_values_tpl.is_file(),
|
||||
"roles/cilium ships no cilium-values.yaml.j2 — the bootstrap's CNI values, including the "
|
||||
"API endpoint its kube-proxy replacement dials, are not authored")
|
||||
if cil_values_tpl.is_file():
|
||||
# Parse the rendered-shape template, never its prose: strip comment lines first, exactly
|
||||
# as the kube-vip parser above does, so a reassuring comment cannot satisfy the check.
|
||||
cil_body = "\n".join(l for l in cil_values_tpl.read_text(errors="replace").splitlines()
|
||||
if not l.lstrip().startswith("#"))
|
||||
m = re.search(r"^k8sServiceHost:\s*\"?([^\"\n]*)\"?\s*$", cil_body, re.M)
|
||||
check(m is not None,
|
||||
"roles/cilium's values template sets no k8sServiceHost — with kube-proxy disabled "
|
||||
"(ADR-0001) the agent has no API endpoint to dial and the CNI cannot come up")
|
||||
if m is not None:
|
||||
host_expr = m.group(1).strip()
|
||||
check("k3s_apiserver_endpoint" in host_expr,
|
||||
f"roles/cilium renders k8sServiceHost={host_expr!r}. It MUST be the HA endpoint "
|
||||
"variable {{ k3s_apiserver_endpoint }} (= kube_vip_address). Any node-valued "
|
||||
"expression — k3s_api_host, a primary hostvar, a literal IP — makes the CNI's "
|
||||
"control-plane path depend on ONE server of an HA control plane, and the failure "
|
||||
"is silent: the datapath keeps running from Cilium's cache while no service or "
|
||||
"policy update ever lands (measured on prod 2026-09-06)")
|
||||
check("k3s_api_host" not in host_expr,
|
||||
f"roles/cilium renders k8sServiceHost={host_expr!r}, which references "
|
||||
"k3s_api_host — the FOUNDER's own address. That variable exists only for the "
|
||||
"cold-bootstrap `--server` join and one tls-san entry; it is a single point of "
|
||||
"failure by construction and must never be a steady-state endpoint")
|
||||
|
||||
# The HA endpoint variable must BE the VIP, and must be DERIVED from kube_vip_address rather
|
||||
# than restated: an environment overrides one line (kube_vip_address) and both the floating
|
||||
# address and the CNI's endpoint move together. A literal here could disagree with the VIP the
|
||||
# same environment actually floats, which is the two-places-one-fact bug this repo keeps
|
||||
# hitting.
|
||||
endpoint_def = str(ALL_VARS.get("k3s_apiserver_endpoint", ""))
|
||||
check("kube_vip_address" in endpoint_def,
|
||||
f"group_vars/all.yml defines k3s_apiserver_endpoint as {endpoint_def!r}. It MUST derive "
|
||||
"from kube_vip_address, so that overriding the VIP for an environment also moves what "
|
||||
"that environment's CNI dials. A restated literal can silently disagree with the VIP "
|
||||
"kube-vip is actually floating")
|
||||
check("kube_vip_address" not in str(ALL_VARS.get("k3s_api_host", "")),
|
||||
"group_vars/all.yml's k3s_api_host resolves from kube_vip_address — the two API "
|
||||
"addresses have been re-merged. k3s_api_host is the secondary join target and on a COLD "
|
||||
"build no VIP exists yet, so a VIP-valued join target makes a cold bootstrap unjoinable")
|
||||
|
||||
# THE [parity] CONTRACT, now machine-checked instead of asserted in a comment: the Flux
|
||||
# overlay's K3S_API_HOST becomes Cilium's k8sServiceHost under the HelmRelease, and the
|
||||
# Ansible bootstrap renders k3s_apiserver_endpoint for the same field. If the two disagree,
|
||||
# Flux and the bootstrap describe different control-plane paths and fight over the CNI.
|
||||
for e in envs:
|
||||
L = e.label
|
||||
ov = overlay_vars(getattr(e, "overlay", ""))
|
||||
k3s_api_host_knob = as_ip(ov.get("K3S_API_HOST", ""))
|
||||
vip = as_ip(e.var("kube_vip_address"))
|
||||
check(k3s_api_host_knob is not None,
|
||||
f"{L}: overlay {getattr(e, 'overlay', '')!r} resolves no usable K3S_API_HOST — that "
|
||||
"token becomes Cilium's k8sServiceHost via platform/cilium/helmrelease.yaml, and "
|
||||
"kustomize-controller runs with StrictPostBuildSubstitutions so an unresolvable one "
|
||||
"fails the whole platform Kustomization")
|
||||
if k3s_api_host_knob is not None and vip is not None:
|
||||
check(k3s_api_host_knob == vip,
|
||||
f"{L}: [parity] BROKEN — the Flux overlay sets K3S_API_HOST={k3s_api_host_knob} "
|
||||
f"but this environment's control-plane VIP (kube_vip_address, which is what "
|
||||
f"k3s_apiserver_endpoint and therefore the Ansible-side k8sServiceHost resolve "
|
||||
f"to) is {vip}. Flux and the Ansible bootstrap would render DIFFERENT API "
|
||||
"endpoints into the same CNI and fight over it. If they differ because the "
|
||||
"overlay still holds a NODE address, that is the 2026-09-06 defect: the CNI of "
|
||||
"an HA control plane must not dial one server")
|
||||
check(k3s_api_host_knob not in set(
|
||||
as_ip(host_vars.get(h, {}).get("k3s_node_ip", ""))
|
||||
for h in hosts_in(str(e.var("k3s_servers_group")))),
|
||||
f"{L}: the overlay's K3S_API_HOST {k3s_api_host_knob} is one of this "
|
||||
"environment's own NODE addresses. Cilium's kube-proxy replacement would dial a "
|
||||
"single machine on a control plane built specifically to survive losing one")
|
||||
|
||||
# The Flux half must stay a per-environment TOKEN. A literal IP in the shared HelmRelease
|
||||
# would ship one environment's control-plane address to every environment (R-ENV-02), and an
|
||||
# empty value is the 2026-09-01 incident verbatim (k8sServiceHost: "" made the agent's config
|
||||
# init container fall back to the in-cluster ClusterIP 10.43.0.1:443, which only the agent
|
||||
# being replaced could translate: deadlocked on 2 of 3 nodes).
|
||||
hr = ROOT / "platform" / "cilium" / "helmrelease.yaml"
|
||||
check(hr.is_file(), "platform/cilium/helmrelease.yaml is missing — nothing adopts the CNI "
|
||||
"config after the bootstrap, so the parity contract has only one side")
|
||||
if hr.is_file():
|
||||
hr_body = "\n".join(l for l in hr.read_text(errors="replace").splitlines()
|
||||
if not l.lstrip().startswith("#"))
|
||||
hm = re.search(r"^\s*k8sServiceHost:\s*\"?([^\"\n]*)\"?\s*$", hr_body, re.M)
|
||||
check(hm is not None and hm.group(1).strip() == "${K3S_API_HOST}",
|
||||
f"platform/cilium/helmrelease.yaml sets k8sServiceHost="
|
||||
f"{(hm.group(1) if hm else None)!r} instead of the cluster-vars token "
|
||||
"\"${K3S_API_HOST}\" — a literal there would ship ONE environment's control-plane "
|
||||
"address to every environment (R-ENV-02), and an empty value is the 2026-09-01 "
|
||||
"incident verbatim")
|
||||
|
||||
# ORDERING: the VIP must be floating BEFORE the CNI is installed against it. site.yml is the
|
||||
# one-command bring-up, so this is a property of the authored sequence, not of an operator's
|
||||
# memory. (kube-vip can precede the CNI because its DaemonSet is hostNetwork with
|
||||
# KUBERNETES_SERVICE_HOST=127.0.0.1 — asserted separately below/above via the manifest parse.)
|
||||
site_play = PLAYBOOKS / "site.yml"
|
||||
check(site_play.is_file(), "playbooks/site.yml is missing — there is no one-command bring-up "
|
||||
"whose ORDER can be checked")
|
||||
if site_play.is_file():
|
||||
imports = re.findall(r"import_playbook:\s*(\S+)", site_play.read_text())
|
||||
for needed in ("k3s.yml", "kube-vip.yml", "cilium.yml"):
|
||||
check(needed in imports,
|
||||
f"playbooks/site.yml does not import {needed} (imports: {imports}). Cilium's "
|
||||
"k8sServiceHost is the control-plane VIP, so kube-vip is a PREREQUISITE of the "
|
||||
"CNI, not an optional extra step — leaving it out of the bring-up means a cold "
|
||||
"build installs a CNI pointed at an address nothing announces")
|
||||
if all(n in imports for n in ("k3s.yml", "kube-vip.yml", "cilium.yml")):
|
||||
check(imports.index("k3s.yml") < imports.index("kube-vip.yml") < imports.index("cilium.yml"),
|
||||
f"playbooks/site.yml imports {imports} — the order MUST be k3s.yml -> "
|
||||
"kube-vip.yml -> cilium.yml. kube-vip after cilium installs the CNI against an "
|
||||
"unannounced VIP (the 2026-09-01 shape: agents that cannot reach the API server "
|
||||
"on a cold cluster); kube-vip before k3s cannot work at all, because the VIP "
|
||||
"ships as a k3s auto-deploy manifest and needs a cluster to apply it")
|
||||
check(re.search(r"wait_for", VIP_TEXT) is not None
|
||||
and re.search(r"kube_vip_address", VIP_TEXT) is not None,
|
||||
"roles/kube_vip never WAITS for the VIP to answer — templating the manifest is not the "
|
||||
"same as having a floating endpoint (k3s still has to apply it, the image still has to "
|
||||
"pull, leader election still has to settle). Since the CNI is installed immediately "
|
||||
"afterward against this exact address, an unverified VIP turns a missing endpoint into "
|
||||
"unreachable cilium-agents diagnosed from the far side")
|
||||
|
||||
# ============================================ FLOATING ENDPOINT: kube-vip on every server
|
||||
# Read the DaemonSet's ACTUAL container env out of the template, never the role's prose.
|
||||
# (An earlier draft matched `svc_enable=false` anywhere in the role and passed happily with
|
||||
# the template flipped to true — the comment in defaults/main.yml satisfied it. Parse the
|
||||
# thing that ships.) Parser contract: the template writes each env var as an adjacent
|
||||
# `- name: X` / `value: "Y"` pair; comment lines are dropped first.
|
||||
VIP_TEMPLATES = sorted(VIP_ROLE.rglob("*.j2"))
|
||||
check(len(VIP_TEMPLATES) >= 1,
|
||||
"roles/kube_vip ships no manifest template — nothing deploys the API VIP")
|
||||
vip_manifest = "\n".join(
|
||||
"\n".join(l for l in p.read_text(errors="replace").splitlines()
|
||||
if not l.lstrip().startswith("#"))
|
||||
for p in VIP_TEMPLATES)
|
||||
vip_env = dict(re.findall(r"-\s*name:\s*([A-Za-z_][A-Za-z0-9_]*)\s*\n\s*value:\s*\"?([^\"\n]*)\"?",
|
||||
vip_manifest))
|
||||
check(len(vip_env) >= 6,
|
||||
f"only {len(vip_env)} env var(s) parsed out of the kube-vip manifest template — the "
|
||||
"parser lost its grip on the file, so the assertions below would assert nothing")
|
||||
check(vip_env.get("cp_enable") == "true",
|
||||
f"the kube-vip manifest sets cp_enable={vip_env.get('cp_enable')!r} — control-plane "
|
||||
"mode must be ON or it floats no API VIP at all")
|
||||
check(vip_env.get("svc_enable") == "false",
|
||||
f"the kube-vip manifest sets svc_enable={vip_env.get('svc_enable')!r} — it must be "
|
||||
"false, or kube-vip ALSO serves Service LoadBalancers and contends with Cilium LB-IPAM "
|
||||
"for the same addresses (ADR-0002 gives Service LB to Cilium)")
|
||||
check(vip_env.get("vip_leaderelection") == "true",
|
||||
f"the kube-vip manifest sets vip_leaderelection={vip_env.get('vip_leaderelection')!r} "
|
||||
"— without leader election the VIP has no defined owner to fail over FROM, which is "
|
||||
"the whole HA mechanism")
|
||||
check(vip_env.get("address", "").strip() == "{{ kube_vip_address }}",
|
||||
f"the kube-vip manifest announces address={vip_env.get('address')!r} instead of the "
|
||||
"environment-scoped {{ kube_vip_address }} — a literal here would float DEV's VIP on "
|
||||
"prod (R-ENV-02) and would not match the serving-cert SAN list")
|
||||
check(re.search(r"^kind:\s*DaemonSet\s*$", vip_manifest, re.M) is not None,
|
||||
"the kube-vip manifest is not a DaemonSet — a single-replica VIP holder dies with its "
|
||||
"node, which is the opposite of a floating endpoint")
|
||||
vip_play = PLAYBOOKS / "kube-vip.yml"
|
||||
check(vip_play.is_file(), "playbooks/kube-vip.yml is missing — nothing deploys the API VIP")
|
||||
if vip_play.is_file():
|
||||
vp = vip_play.read_text()
|
||||
vip_hosts = re.findall(r"^\s*hosts:\s*(.+)$", vp, re.M)
|
||||
check(len(vip_hosts) == 1 and "k3s_hosts" in vip_hosts[0],
|
||||
f"playbooks/kube-vip.yml targets {vip_hosts} — it must target the whole servers "
|
||||
"group through the same environment-scoped variable the other plays use, so the "
|
||||
"manifest exists on EVERY server and the VIP survives losing any one of them")
|
||||
check(re.search(r"kube_vip_manifest_path", VIP_TEXT) is not None
|
||||
and re.search(r"k3s_data_dir", VIP_TEXT) is not None,
|
||||
"roles/kube_vip's manifest path is not derived from k3s_data_dir — k3s only auto-applies "
|
||||
"<data-dir>/server/manifests, so an environment that moves --data-dir would write the "
|
||||
"manifest somewhere k3s never reads and come up with no VIP at all")
|
||||
|
||||
# ================================================== ONE CODE PATH FOR BOTH ENVIRONMENTS
|
||||
# docs/02: "For prod bare-metal, the *same* Ansible bootstrap runs; only the machine-creation
|
||||
# step differs". A prod-only role or playbook would be a fork of the control-plane path.
|
||||
role_names = {p.name for p in (ANS / "roles").iterdir() if p.is_dir()}
|
||||
play_names = {p.name for p in PLAYBOOKS.glob("*.yml")}
|
||||
env_words = re.compile(r"prod|dev", re.I)
|
||||
forked = sorted(n for n in (role_names | play_names)
|
||||
if env_words.search(n) and n not in ("memattr_survey", "memattr-survey.yml"))
|
||||
check(not forked,
|
||||
f"environment-named roles/playbooks exist ({forked}) — dev and prod must be built by "
|
||||
"the SAME code retargeted by variables (docs/02), not by a per-environment fork")
|
||||
for e in envs:
|
||||
if e.path.name != "vars.yml":
|
||||
continue
|
||||
# Every key a non-default environment sets must be a variable something already reads,
|
||||
# or it is an invented knob nothing consumes (a silent no-op that looks like config).
|
||||
consumers = "\n".join(p.read_text(errors="replace") for p in sorted(ANS.rglob("*"))
|
||||
if p.is_file() and p.suffix in (".yml", ".yaml", ".j2", ".sh"))
|
||||
for k in sorted(e.vars):
|
||||
if k.startswith("ansible_"):
|
||||
continue
|
||||
check(consumers.count(k) > 1,
|
||||
f"{e.label} sets {k!r}, which appears nowhere else under infra/ansible — an "
|
||||
"environment override that nothing reads is not a scoped override, it is dead "
|
||||
"configuration (and would make prod silently inherit a dev default)")
|
||||
|
||||
# --------------------------------------------------------------------------- verdict
|
||||
# NO-OP GUARD: a check that quietly asserts nothing is worse than no check.
|
||||
check(CHECKS >= 40,
|
||||
f"only {CHECKS} assertions ran — the check degenerated to a near-no-op and its pass "
|
||||
"would be meaningless")
|
||||
|
||||
if FAILURES:
|
||||
die()
|
||||
for n in NOTES:
|
||||
print(f" note: {n}")
|
||||
summary = "; ".join(
|
||||
f"{e.label.split('/')[-1]}[{'PROD' if e in prod_envs else 'non-prod'}]="
|
||||
f"{len(hosts_in(str(e.var('k3s_servers_group'))))} servers"
|
||||
for e in envs)
|
||||
print(f"check OK: ha-control-plane-as-code — {CHECKS} assertions; {summary}. "
|
||||
"R-PLAT-01's AUTHORED half only: k3s, one --cluster-init founder per environment, "
|
||||
"embedded etcd on every server with no external datastore, a leader-elected "
|
||||
"control-plane-only VIP on every server that is SAN'd into the serving cert and "
|
||||
"collides with no other allocator, floated BEFORE the CNI that dials it (and the CNI "
|
||||
"dials that VIP, not the founder, in both environments and on both sides of the "
|
||||
"[parity] contract), from one code path for both environments. The LIVE "
|
||||
"cluster shape is proven at L2 (tests/l2-behavioral/ha-control-plane); control-plane "
|
||||
"SURVIVAL of a node loss is proven NOWHERE yet (see the header's gap list).")
|
||||
PY
|
||||
@@ -1,63 +0,0 @@
|
||||
# =============================================================================
|
||||
# L0 STATIC — Keycloak client secrets are declared consistently, ordered by a REAL
|
||||
# Flux edge, and healed by a reconciler whose behaviour is proven offline
|
||||
# Catalog: docs/14 L0 row ("Pre-apply, in CI") — runs WITHOUT a cluster.
|
||||
#
|
||||
# R-IDP-03 (identity config is declarative) and R-IDP-04 (IdP secrets SOPS-encrypted)
|
||||
# are both implemented through ONE mechanism that FAILS OPEN: the KeycloakRealmImport
|
||||
# resolves each client's secret from a `(env:NAME)` placeholder mapped to a key of the
|
||||
# SOPS-encrypted keycloak-realm-clients Secret. When the operator cannot resolve a
|
||||
# placeholder it stores the LITERAL placeholder text as the client secret AND THE IMPORT
|
||||
# JOB STILL REPORTS SUCCESS — then rests at Done=True / Started=False and never re-runs.
|
||||
#
|
||||
# Measured 2026-09-06: prod had all 12 clients holding the literal placeholder (every
|
||||
# OIDC login failed with "Invalid client or Invalid client credentials"); dev had exactly
|
||||
# one (oauth2-proxy), because that Secret key was added AFTER dev's import completed.
|
||||
# Declarative config that silently does not apply is not declarative config.
|
||||
#
|
||||
# GIVEN the repository as authored,
|
||||
# THEN every confidential client declares a placeholder, every placeholder is defined,
|
||||
# resolves to the keycloak-realm-clients Secret, and names a key that EXISTS in the
|
||||
# SOPS file under the `<clientId>-client-secret` contract the reconciler derives
|
||||
# from; public clients declare no secret; the Flux Kustomization applying the Secret
|
||||
# is a transitive `dependsOn` ancestor (wait: true) of the ones applying the import
|
||||
# and the reconciler — a real edge, not a comment; the reconciler is reconciled from
|
||||
# BOTH cluster entrypoints, mounts only what it needs, carries no ServiceAccount
|
||||
# token and contains no dollar sign; and its own `--selftest` (extracted from the
|
||||
# shipped ConfigMap and executed) proves compare-then-write, idempotence,
|
||||
# no-secret-logging, the endpoint allow-list and fail-closed behaviour.
|
||||
#
|
||||
# STATUS: EXECUTABLE. `check.sh` is a pure offline assertion script (git + PyYAML +
|
||||
# stdlib); conformance/run.sh --layer l0 and ci/l0.sh run it directly, and this Chainsaw
|
||||
# Test wraps the SAME script.
|
||||
#
|
||||
# NOT proven here: that the LIVE realm's stored secrets match git (that is
|
||||
# tests/l2-behavioral/keycloak-client-secrets-match-git), that the SOPS ciphertext
|
||||
# decrypts to a usable value (key PRESENCE is offline-checkable; plaintext is not), or
|
||||
# that the CronJob's schedule fires.
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: keycloak-client-secret-reconciler
|
||||
labels:
|
||||
conformance.home.k3s/R-IDP-03: "true"
|
||||
conformance.home.k3s/R-IDP-04: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-IDP-03, R-IDP-04"
|
||||
conformance.home.k3s/layer: "L0"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
Every OIDC client's secret is declared once, resolvable, ordered behind a real Flux
|
||||
dependency, and continuously reconciled — so an unresolved import placeholder can
|
||||
neither be merged nor silently persist in the realm.
|
||||
spec:
|
||||
steps:
|
||||
- name: audit-declaration-ordering-and-reconciler
|
||||
description: >-
|
||||
Assert the realm import, the SOPS Secret and the reconciler agree; that the Flux
|
||||
ordering is a real dependsOn edge; and run the reconciler's own selftest.
|
||||
try:
|
||||
- script:
|
||||
content: bash ./check.sh
|
||||
@@ -1,529 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# L0 static check (offline, no cluster) — keycloak-client-secret-reconciler
|
||||
# Serves: R-IDP-03 ("Identity config (realms, clients, groups, roles) is declarative."
|
||||
# — spec docs/15-identity-and-access.md)
|
||||
# R-IDP-04 ("IdP secrets SOPS-encrypted; MFA available."
|
||||
# — spec docs/15-identity-and-access.md)
|
||||
#
|
||||
# WHY THIS EXISTS — THE REALM IMPORT IS A ONE-SHOT THAT FAILS OPEN
|
||||
# ---------------------------------------------------------------
|
||||
# platform/keycloak/keycloak-realm-import.yaml declares 12 confidential OIDC clients
|
||||
# whose secrets are `(env:NAME)` import placeholders, mapped by spec.placeholders to
|
||||
# keys of the SOPS-encrypted keycloak-realm-clients Secret. Measured on both clusters
|
||||
# on 2026-09-06:
|
||||
#
|
||||
# * when the Keycloak Operator cannot resolve a placeholder it stores the LITERAL
|
||||
# placeholder text as the client secret AND THE IMPORT JOB STILL REPORTS SUCCESS;
|
||||
# * the KeycloakRealmImport then rests at Done=True / Started=False and NEVER
|
||||
# re-runs, so it never self-heals.
|
||||
#
|
||||
# Consequence: on prod all 12 clients held the literal placeholder and every OIDC
|
||||
# login on the cluster failed ("Invalid client or Invalid client credentials"); on dev
|
||||
# exactly one (oauth2-proxy) was broken, because that Secret key was added AFTER dev's
|
||||
# import had already completed and a completed import is never re-run.
|
||||
#
|
||||
# Nothing in the repo asserted the declaration was self-consistent, and nothing
|
||||
# corrected the realm afterwards. This check is the static half of the fix (the
|
||||
# reconciler CronJob is the runtime half, and this check proves that too).
|
||||
#
|
||||
# GIVEN the repository as authored,
|
||||
# THEN (1) DECLARATION IS COMPLETE: every confidential client in the realm import
|
||||
# declares a secret placeholder; every placeholder it uses is defined in
|
||||
# spec.placeholders; every placeholder resolves to the keycloak-realm-clients
|
||||
# Secret; and every referenced key EXISTS in the SOPS file. A client whose
|
||||
# key is missing is exactly the dev failure, and it is now unmergeable.
|
||||
# (2) THE NAMING CONTRACT HOLDS: each key is `<clientId>-client-secret`. That is
|
||||
# what lets the reconciler derive the client set from the mounted Secret with
|
||||
# no second declaration — so it must be enforced, not assumed. No orphan
|
||||
# placeholder, and no orphan Secret key (an orphan key would make every
|
||||
# reconciler run fail with NOT FOUND).
|
||||
# (3) PUBLIC CLIENTS DECLARE NO SECRET (the `kubernetes` client is PKCE-public;
|
||||
# giving it a secret would be a silent downgrade).
|
||||
# (4) ORDERING IS A REAL FLUX EDGE, NOT A COMMENT: the Flux Kustomization that
|
||||
# renders the Secret is a transitive `dependsOn` ancestor — with `wait: true`
|
||||
# — of the one that renders the realm import and the reconciler. The repo's
|
||||
# hard-won rule: a dependency that is only DOCUMENTED is not a dependency.
|
||||
# (5) THE RECONCILER IS WIRED AND SAFE: it is reconciled from BOTH cluster
|
||||
# entrypoints, mounts exactly the two Secrets it needs at the paths its code
|
||||
# defaults to, carries no ServiceAccount token, and contains no dollar sign
|
||||
# (it ships inside a substituted Flux path, where an unescaped dollar-brace
|
||||
# expansion is rewritten before any container runs).
|
||||
# (6) THE RECONCILER'S BEHAVIOUR IS PROVEN, NOT ASSERTED: this check EXTRACTS the
|
||||
# script from the ConfigMap and executes its `--selftest`, which drives the
|
||||
# real code paths against an in-process stub Keycloak and asserts
|
||||
# compare-then-write, idempotence (zero writes on a converged realm),
|
||||
# no-secret-logging, the endpoint allow-list (users/groups/roles are
|
||||
# unreachable) and fail-closed behaviour. What CI proves is byte-for-byte
|
||||
# what ships.
|
||||
# AND (7) THIS CHECK IS LOAD-BEARING: ci/l0.sh runs it unconditionally, ci/gate.sh
|
||||
# blocks the merge when the l0 stage fails, ci/touched-requirements.sh maps
|
||||
# platform/keycloak/ to these requirement IDs, and an L2 test exists that
|
||||
# audits the LIVE realm. An unreferenced test is precisely the "looks applied
|
||||
# and isn't" failure this repo keeps hitting.
|
||||
#
|
||||
# NOT proven here (named honestly, in the style of the sibling L0 checks):
|
||||
# - that the LIVE realm's stored secrets match git. That is cluster state; it is
|
||||
# conformance/tests/l2-behavioral/keycloak-client-secrets-match-git.
|
||||
# - that the SOPS ciphertext decrypts to a usable secret. Key PRESENCE is checkable
|
||||
# offline; the plaintext is not, and must not be.
|
||||
# - that the CronJob's image can be pulled or its schedule fires.
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT="$(cd "${DIR}/../../../.." && pwd)" # repo root
|
||||
PYTHON="${PYTHON:-python3}"
|
||||
|
||||
IMPORT_FILE="platform/keycloak/keycloak-realm-import.yaml"
|
||||
SOPS_FILE="platform/keycloak/keycloak-realm-clients.sops.yaml"
|
||||
RECON_FILE="platform/keycloak/client-secret-reconciler.yaml"
|
||||
|
||||
# Honest SKIP (never green) if the thing under test is absent entirely.
|
||||
for f in "$IMPORT_FILE" "$SOPS_FILE" "$RECON_FILE"; do
|
||||
if [ ! -f "${ROOT}/${f}" ]; then
|
||||
echo "check SKIPPED: ${f} absent — nothing to audit" >&2
|
||||
exit 3
|
||||
fi
|
||||
done
|
||||
|
||||
cd "${ROOT}"
|
||||
"${PYTHON}" - "${ROOT}" <<'PY'
|
||||
"""Static audit of the Keycloak client-secret declaration, its Flux ordering, and the
|
||||
reconciler that heals it. PyYAML (the same dependency tests/l0-static/no-plaintext-secrets
|
||||
and least-privilege-rbac already take) parses the manifests; lib/status.py is IMPORTED for
|
||||
the Flux graph walk rather than forked, so there is one definition of "what a Flux path
|
||||
renders" in the harness."""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError: # pragma: no cover
|
||||
print("check FAIL: PyYAML is required (same dependency as catalog/render.py)",
|
||||
file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
ROOT = Path(sys.argv[1])
|
||||
sys.dont_write_bytecode = True # keep __pycache__ out of the tracked tree
|
||||
sys.path.insert(0, str(ROOT / "conformance" / "lib"))
|
||||
import status as st # noqa: E402 — reuse the walk, don't fork it
|
||||
|
||||
IMPORT_FILE = "platform/keycloak/keycloak-realm-import.yaml"
|
||||
SOPS_FILE = "platform/keycloak/keycloak-realm-clients.sops.yaml"
|
||||
RECON_FILE = "platform/keycloak/client-secret-reconciler.yaml"
|
||||
SECRET_NAME = "keycloak-realm-clients"
|
||||
ADMIN_SECRET = "keycloak-admin-credentials"
|
||||
KEY_SUFFIX = "-client-secret"
|
||||
SECRETS_MOUNT = "/etc/keycloak-realm-clients"
|
||||
DOLLAR = chr(36)
|
||||
|
||||
FAILURES: list[str] = []
|
||||
CHECKS = 0
|
||||
|
||||
|
||||
def check(cond, msg):
|
||||
global CHECKS
|
||||
CHECKS += 1
|
||||
if not cond:
|
||||
FAILURES.append(msg)
|
||||
|
||||
|
||||
def load(rel):
|
||||
return list(yaml.safe_load_all((ROOT / rel).read_text()))
|
||||
|
||||
|
||||
# ================================================================= (1)-(3) declaration
|
||||
realm_import = next((d for d in load(IMPORT_FILE)
|
||||
if d and d.get("kind") == "KeycloakRealmImport"), None)
|
||||
check(realm_import is not None, f"{IMPORT_FILE}: no KeycloakRealmImport document found")
|
||||
if realm_import is None:
|
||||
print("CHECK FAIL: cannot parse the realm import", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
placeholders = realm_import["spec"].get("placeholders") or {}
|
||||
clients = realm_import["spec"]["realm"].get("clients") or []
|
||||
check(len(clients) > 0, f"{IMPORT_FILE}: parsed zero clients — the file shape changed and "
|
||||
f"this check would silently audit nothing")
|
||||
|
||||
# The operator's placeholder syntax, spelled without a literal dollar so this file is
|
||||
# itself safe to grep for one. Matches `<dollar>(env:NAME)`.
|
||||
PLACEHOLDER_RE = re.compile(re.escape(DOLLAR) + r"\(env:([A-Za-z_][A-Za-z0-9_]*)\)")
|
||||
|
||||
used_placeholders: set[str] = set()
|
||||
declared_pairs: dict[str, str] = {} # clientId -> secret key in the Secret
|
||||
|
||||
for c in clients:
|
||||
cid = c.get("clientId", "<unnamed>")
|
||||
secret = c.get("secret")
|
||||
public = bool(c.get("publicClient"))
|
||||
if public:
|
||||
# (3) a PKCE public client must not carry a secret at all.
|
||||
check(secret is None,
|
||||
f"{IMPORT_FILE}: client '{cid}' is publicClient but declares a secret — a "
|
||||
f"public client holds no secret; declaring one is a silent downgrade")
|
||||
continue
|
||||
# (1) every confidential client must declare one.
|
||||
check(isinstance(secret, str) and secret,
|
||||
f"{IMPORT_FILE}: confidential client '{cid}' declares NO secret. The operator "
|
||||
f"would generate a random one that exists nowhere in git, so nothing could "
|
||||
f"ever reconcile it and the consuming app's mirrored credential would be wrong")
|
||||
if not isinstance(secret, str):
|
||||
continue
|
||||
m = PLACEHOLDER_RE.fullmatch(secret.strip())
|
||||
check(m is not None,
|
||||
f"{IMPORT_FILE}: client '{cid}' has a secret that is not an (env:NAME) "
|
||||
f"placeholder. A literal secret in git violates R-IDP-04/R-SEC-01; the only "
|
||||
f"permitted form is a placeholder resolved from the {SECRET_NAME} Secret")
|
||||
if m is None:
|
||||
continue
|
||||
name = m.group(1)
|
||||
used_placeholders.add(name)
|
||||
# (1) the placeholder must be DEFINED. An undefined one is stored literally by the
|
||||
# operator, with a successful-looking import — the whole incident.
|
||||
ph = placeholders.get(name)
|
||||
check(ph is not None,
|
||||
f"{IMPORT_FILE}: client '{cid}' uses placeholder '{name}' but spec.placeholders "
|
||||
f"does not define it. The operator stores the LITERAL placeholder as the client "
|
||||
f"secret and STILL reports the import successful (measured 2026-09-06)")
|
||||
if ph is None:
|
||||
continue
|
||||
ref = (ph or {}).get("secret") or {}
|
||||
check(ref.get("name") == SECRET_NAME,
|
||||
f"{IMPORT_FILE}: placeholder '{name}' reads Secret '{ref.get('name')}', not "
|
||||
f"'{SECRET_NAME}'. The reconciler mounts only {SECRET_NAME}, so a second source "
|
||||
f"would be declared but never healed")
|
||||
key = ref.get("key")
|
||||
check(isinstance(key, str) and key,
|
||||
f"{IMPORT_FILE}: placeholder '{name}' names no Secret key")
|
||||
if not isinstance(key, str):
|
||||
continue
|
||||
# (2) the naming contract the reconciler derives its client set from.
|
||||
check(key == cid + KEY_SUFFIX,
|
||||
f"{IMPORT_FILE}: client '{cid}' reads Secret key '{key}', but the reconciler "
|
||||
f"derives the clientId from the key name, so it MUST be "
|
||||
f"'{cid}{KEY_SUFFIX}'. Rename the key (and the SOPS entry) or the reconciler "
|
||||
f"would look up a client that does not exist")
|
||||
declared_pairs[cid] = key
|
||||
|
||||
# (2) no orphan placeholder: a defined-but-unused placeholder is dead config that will
|
||||
# quietly rot out of sync with the Secret.
|
||||
for name in sorted(set(placeholders) - used_placeholders):
|
||||
check(False,
|
||||
f"{IMPORT_FILE}: spec.placeholders defines '{name}' but no client uses it — "
|
||||
f"dead declaration; remove it or wire it to the client that needs it")
|
||||
|
||||
# ---- the SOPS Secret: keys are cleartext in a SOPS-encrypted file; values are not.
|
||||
sops_doc = next((d for d in load(SOPS_FILE) if d and d.get("kind") == "Secret"), None)
|
||||
check(sops_doc is not None, f"{SOPS_FILE}: no Secret document found")
|
||||
sops_keys = set((sops_doc or {}).get("stringData") or {})
|
||||
check(len(sops_keys) > 0,
|
||||
f"{SOPS_FILE}: parsed zero stringData keys — the file shape changed and this check "
|
||||
f"would silently audit nothing")
|
||||
check((sops_doc or {}).get("metadata", {}).get("name") == SECRET_NAME,
|
||||
f"{SOPS_FILE}: the Secret is not named {SECRET_NAME}")
|
||||
|
||||
for cid, key in sorted(declared_pairs.items()):
|
||||
# (1) THE DEV FAILURE, made unmergeable: a client declared in the import whose key
|
||||
# is absent from the Secret imports "successfully" with a literal placeholder.
|
||||
check(key in sops_keys,
|
||||
f"{SOPS_FILE}: client '{cid}' needs key '{key}' and it is NOT in the Secret. "
|
||||
f"The realm import would still report SUCCESS and store the literal placeholder "
|
||||
f"as that client's secret — this is exactly how dev's oauth2-proxy broke")
|
||||
|
||||
# (2) no orphan Secret key: the reconciler treats every `<x>-client-secret` key as a
|
||||
# client that MUST exist, and fails loudly if it does not. Catch it here instead.
|
||||
for key in sorted(k for k in sops_keys if k.endswith(KEY_SUFFIX)):
|
||||
cid = key[: -len(KEY_SUFFIX)]
|
||||
check(cid in declared_pairs,
|
||||
f"{SOPS_FILE}: key '{key}' has no matching client in {IMPORT_FILE}. The "
|
||||
f"reconciler would look up client '{cid}', not find it, and fail every run")
|
||||
|
||||
# ===================================================== (4) ordering is a real Flux edge
|
||||
# Parse contract mirrors lib/status.py's flux_paths(): docs split on a lone `---`; a doc
|
||||
# is an entrypoint only if it carries the kustomize.toolkit apiVersion, `kind:
|
||||
# Kustomization` and a spec-level `path:`. Overlay `patch: |-` bodies carry Kustomizations
|
||||
# WITHOUT a path and are correctly skipped (they are patches, not entrypoints).
|
||||
_DEPENDSON_RE = re.compile(r"^(\s*)dependsOn:\s*$", re.M)
|
||||
_DEP_NAME_RE = re.compile(r"^\s*(?:-\s*)?name:\s*([^\s#]+)")
|
||||
_WAIT_RE = re.compile(r"^\s{2,6}wait:\s*true\s*(?:#.*)?$", re.M)
|
||||
|
||||
|
||||
def depends_on(doc: str) -> list[str]:
|
||||
m = _DEPENDSON_RE.search(doc)
|
||||
if not m:
|
||||
return []
|
||||
indent = len(m.group(1))
|
||||
out: list[str] = []
|
||||
for raw in doc[m.end():].splitlines():
|
||||
if raw.strip() == "" or raw.lstrip().startswith("#"):
|
||||
continue
|
||||
if len(raw) - len(raw.lstrip()) <= indent and not raw.lstrip().startswith("-"):
|
||||
break # dedented out of the list
|
||||
nm = _DEP_NAME_RE.match(raw)
|
||||
if nm:
|
||||
out.append(nm.group(1))
|
||||
return out
|
||||
|
||||
|
||||
kustomizations: dict[str, dict] = {}
|
||||
for yfile in sorted((ROOT / "clusters").rglob("*.yaml")):
|
||||
text = yfile.read_text()
|
||||
if st.FLUX_API not in text:
|
||||
continue
|
||||
for doc in st._DOC_SPLIT_RE.split(text):
|
||||
if st.FLUX_API not in doc or not re.search(r"^kind:\s*Kustomization\s*$", doc, re.M):
|
||||
continue
|
||||
pm = st._PATH_RE.search(doc)
|
||||
if not pm:
|
||||
continue
|
||||
nm = st._NAME_RE.search(doc)
|
||||
name = nm.group(1) if nm else "?"
|
||||
kustomizations[name] = {
|
||||
"file": str(yfile.relative_to(ROOT)),
|
||||
"path": pm.group(1),
|
||||
"dependsOn": depends_on(doc),
|
||||
"wait": bool(_WAIT_RE.search(doc)),
|
||||
}
|
||||
|
||||
check(len(kustomizations) > 0,
|
||||
"no Flux Kustomization with a spec.path found under clusters/ — the parser or the "
|
||||
"repo layout changed; this check would silently audit nothing")
|
||||
|
||||
|
||||
def direct_render(path_spec: str) -> set[str]:
|
||||
"""Files a Flux `spec.path` renders ITSELF — a kustomize walk that deliberately does
|
||||
NOT expand nested Flux Kustomizations.
|
||||
|
||||
lib/status.py's reconciled_files() follows every `spec.path` it discovers, which is
|
||||
right for "is this file applied to the cluster at all" but wrong for OWNERSHIP: the
|
||||
root Kustomization's path is clusters/<env>, so a transitive walk makes the root the
|
||||
owner of every file in the repo and the ordering assertion below becomes vacuous
|
||||
(measured: it reported the root as an owner of all three keycloak files). `resources:`
|
||||
parsing itself is st.kustomize_resources, not forked.
|
||||
"""
|
||||
rel = path_spec.lstrip("./") or "."
|
||||
out: set[str] = set()
|
||||
seen: set[str] = set()
|
||||
|
||||
def walk(d: str) -> None:
|
||||
if d in seen:
|
||||
return
|
||||
seen.add(d)
|
||||
kfile = ROOT / d / "kustomization.yaml"
|
||||
if not kfile.is_file():
|
||||
return
|
||||
out.add(str(kfile.relative_to(ROOT)))
|
||||
for item in st.kustomize_resources(kfile):
|
||||
if "://" in item:
|
||||
continue # remote base: not repo state
|
||||
target = (ROOT / d / item).resolve()
|
||||
try:
|
||||
trel = target.relative_to(ROOT).as_posix()
|
||||
except ValueError:
|
||||
continue # outside the repo
|
||||
if target.is_dir():
|
||||
walk(trel)
|
||||
elif target.is_file():
|
||||
out.add(trel)
|
||||
|
||||
if (ROOT / rel).is_dir():
|
||||
walk(rel)
|
||||
return out
|
||||
|
||||
|
||||
rendered_by = {n: direct_render(k["path"]) for n, k in kustomizations.items()}
|
||||
|
||||
|
||||
def owners(rel_file: str) -> list[str]:
|
||||
return sorted(n for n, files in rendered_by.items() if rel_file in files)
|
||||
|
||||
|
||||
def ancestors(name: str, seen=None) -> set[str]:
|
||||
"""Transitive dependsOn closure of `name` (its dependencies, direct and indirect)."""
|
||||
seen = seen if seen is not None else set()
|
||||
for dep in kustomizations.get(name, {}).get("dependsOn", []):
|
||||
if dep in seen:
|
||||
continue
|
||||
seen.add(dep)
|
||||
ancestors(dep, seen)
|
||||
return seen
|
||||
|
||||
|
||||
secret_owners = owners(SOPS_FILE)
|
||||
import_owners = owners(IMPORT_FILE)
|
||||
recon_owners = owners(RECON_FILE)
|
||||
|
||||
check(bool(secret_owners),
|
||||
f"{SOPS_FILE} is rendered by NO Flux Kustomization — the client secrets are in git "
|
||||
f"but never applied")
|
||||
check(bool(import_owners),
|
||||
f"{IMPORT_FILE} is rendered by NO Flux Kustomization")
|
||||
# (5) the reconciler must actually be reconciled, or it is decoration.
|
||||
check(bool(recon_owners),
|
||||
f"{RECON_FILE} is rendered by NO Flux Kustomization — the healer would sit in git "
|
||||
f"and never run, which is the 'looks applied and isn't' failure it exists to end")
|
||||
|
||||
for consumer_file, consumer_owners in ((IMPORT_FILE, import_owners),
|
||||
(RECON_FILE, recon_owners)):
|
||||
for co in consumer_owners:
|
||||
for so in secret_owners:
|
||||
if so == co:
|
||||
continue # same Kustomization: one atomic apply, ordering moot
|
||||
anc = ancestors(co)
|
||||
check(so in anc,
|
||||
f"ORDERING IS ONLY DOCUMENTED: '{co}' renders {consumer_file} but does "
|
||||
f"not (transitively) dependsOn '{so}', which renders the "
|
||||
f"{SECRET_NAME} Secret. Without that edge the consumer can reconcile "
|
||||
f"before the secret exists, and the Keycloak Operator's response to a "
|
||||
f"missing placeholder is to store it LITERALLY and report success")
|
||||
if so in anc:
|
||||
check(kustomizations[so]["wait"],
|
||||
f"'{so}' (which renders the {SECRET_NAME} Secret) does not set "
|
||||
f"wait: true, so '{co}' can proceed before it is applied")
|
||||
|
||||
# ------ both environments must reconcile the reconciler, not just dev.
|
||||
for env in ("dev", "prod"):
|
||||
entry = f"clusters/{env}"
|
||||
if not (ROOT / entry).is_dir():
|
||||
continue
|
||||
applied, _ = st.reconciled_files(ROOT, entry)
|
||||
check(RECON_FILE in applied,
|
||||
f"{RECON_FILE} is not reachable from the {entry} entrypoint — the healer would "
|
||||
f"exist on one cluster only, and prod is where all 12 clients were broken")
|
||||
check(IMPORT_FILE in applied, f"{IMPORT_FILE} is not reachable from {entry}")
|
||||
|
||||
# ============================================ (5) the reconciler manifest is safe + wired
|
||||
recon_docs = [d for d in load(RECON_FILE) if d]
|
||||
cm = next((d for d in recon_docs if d.get("kind") == "ConfigMap"), None)
|
||||
cj = next((d for d in recon_docs if d.get("kind") == "CronJob"), None)
|
||||
check(cm is not None, f"{RECON_FILE}: no ConfigMap carrying the reconciler script")
|
||||
check(cj is not None, f"{RECON_FILE}: no CronJob running the reconciler")
|
||||
|
||||
script = ""
|
||||
if cm is not None:
|
||||
data = cm.get("data") or {}
|
||||
check(len(data) == 1,
|
||||
f"{RECON_FILE}: the reconciler ConfigMap must carry exactly one script key "
|
||||
f"(found {sorted(data)}) — a second copy would be untested by this gate")
|
||||
script = next(iter(data.values()), "")
|
||||
check(len(script.splitlines()) > 50,
|
||||
f"{RECON_FILE}: the extracted script is implausibly short "
|
||||
f"({len(script.splitlines())} lines) — the ConfigMap shape changed and the "
|
||||
f"selftest below would prove nothing")
|
||||
# THE nextcloud TRAP. This file is rendered by a Flux Kustomization with
|
||||
# postBuild.substituteFrom, so envsubst rewrites it before any container runs. The
|
||||
# script is written with NO dollar sign at all, so the hazard is structural rather
|
||||
# than escaped and no future edit has to reason about escaping.
|
||||
check(DOLLAR not in script,
|
||||
f"{RECON_FILE}: the reconciler script contains a dollar sign. It ships inside a "
|
||||
f"SUBSTITUTED Flux path, where a dollar-brace expansion is rewritten at "
|
||||
f"reconcile time (see apps/wave-1/nextcloud.yaml). Rewrite without it — Python "
|
||||
f"never needs one; chr(36) spells it where a literal is genuinely required")
|
||||
|
||||
if cj is not None:
|
||||
pod = cj["spec"]["jobTemplate"]["spec"]["template"]["spec"]
|
||||
ctr = pod["containers"][0]
|
||||
vols = {v["name"]: v for v in pod.get("volumes", [])}
|
||||
mounts = {m["name"]: m for m in ctr.get("volumeMounts", [])}
|
||||
secret_vols = [v for v in vols.values()
|
||||
if (v.get("secret") or {}).get("secretName") == SECRET_NAME]
|
||||
check(len(secret_vols) == 1,
|
||||
f"{RECON_FILE}: the CronJob must mount the {SECRET_NAME} Secret — that mount IS "
|
||||
f"the desired state it reconciles towards")
|
||||
if secret_vols:
|
||||
mp = mounts.get(secret_vols[0]["name"], {}).get("mountPath")
|
||||
check(mp == SECRETS_MOUNT,
|
||||
f"{RECON_FILE}: {SECRET_NAME} is mounted at {mp}, but the script defaults "
|
||||
f"to {SECRETS_MOUNT} and no env var overrides it")
|
||||
cm_vols = [v for v in vols.values()
|
||||
if (v.get("configMap") or {}).get("name") == (cm or {}).get(
|
||||
"metadata", {}).get("name")]
|
||||
check(len(cm_vols) == 1,
|
||||
f"{RECON_FILE}: the CronJob does not mount the reconciler ConfigMap, so it "
|
||||
f"would run nothing")
|
||||
env_srcs = {e["name"]: e for e in ctr.get("env", [])}
|
||||
for var in ("KEYCLOAK_ADMIN_USER", "KEYCLOAK_ADMIN_PASSWORD"):
|
||||
ref = (env_srcs.get(var, {}).get("valueFrom") or {}).get("secretKeyRef") or {}
|
||||
check(ref.get("name") == ADMIN_SECRET,
|
||||
f"{RECON_FILE}: {var} must come from the {ADMIN_SECRET} Secret "
|
||||
f"(got {ref.get('name')!r}) — the admin credential is never inlined")
|
||||
for var in ("KEYCLOAK_BASE_URL", "KEYCLOAK_REALM"):
|
||||
check(env_srcs.get(var, {}).get("value"),
|
||||
f"{RECON_FILE}: {var} is not set; the script fails closed without it, but "
|
||||
f"the CronJob should not ship broken")
|
||||
# R-SEC-02: it talks only to Keycloak's REST API and needs no Kubernetes API rights.
|
||||
check(pod.get("automountServiceAccountToken") is False,
|
||||
f"{RECON_FILE}: the reconciler needs NO Kubernetes API access (its inputs are "
|
||||
f"mounted Secrets) — set automountServiceAccountToken: false")
|
||||
check(cj["spec"].get("concurrencyPolicy") == "Forbid",
|
||||
f"{RECON_FILE}: set concurrencyPolicy: Forbid so a run stalled on an "
|
||||
f"unavailable Keycloak is not lapped by the next tick")
|
||||
img = ctr.get("image", "")
|
||||
check(re.search(r":[0-9]", img),
|
||||
f"{RECON_FILE}: container image {img!r} is not pinned to an explicit version")
|
||||
|
||||
# ============================== (6) the reconciler's BEHAVIOUR, proven by its own selftest
|
||||
if script:
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
p = Path(td) / "reconcile.py"
|
||||
p.write_text(script)
|
||||
proc = subprocess.run([sys.executable, str(p), "--selftest"],
|
||||
capture_output=True, text=True, timeout=180)
|
||||
tail = (proc.stdout + proc.stderr).strip().splitlines()
|
||||
check(proc.returncode == 0,
|
||||
"the reconciler's --selftest FAILED (compare-then-write / idempotence / "
|
||||
"no-secret-logging / endpoint allow-list / fail-closed):\n "
|
||||
+ "\n ".join(tail[-12:]))
|
||||
if proc.returncode == 0:
|
||||
print(" reconciler --selftest: " + (tail[-1] if tail else "(no output)"))
|
||||
|
||||
# ======================================================= (7) this check is load-bearing
|
||||
L0 = (ROOT / "ci" / "l0.sh").read_text() if (ROOT / "ci" / "l0.sh").is_file() else ""
|
||||
GATE = (ROOT / "ci" / "gate.sh").read_text() if (ROOT / "ci" / "gate.sh").is_file() else ""
|
||||
TOUCHED = ((ROOT / "ci" / "touched-requirements.sh").read_text()
|
||||
if (ROOT / "ci" / "touched-requirements.sh").is_file() else "")
|
||||
check("keycloak-client-secret-reconciler/check.sh" in L0,
|
||||
"ci/l0.sh does not run keycloak-client-secret-reconciler/check.sh — the "
|
||||
"requirement-scoped conformance selection can filter this check out of a gate run, "
|
||||
"so l0.sh must invoke it directly and unconditionally or the gate is not enforcing "
|
||||
"it. An unreferenced test is the failure this whole change exists to end")
|
||||
check(re.search(r"^for s in .*\bl0\b.*; do", GATE, re.M) and re.search(r"fail\).*BLOCK=1", GATE),
|
||||
"ci/gate.sh does not set the merge block when the l0 stage fails")
|
||||
check(re.search(r"^\s*platform/keycloak/\*\)", TOUCHED, re.M),
|
||||
"ci/touched-requirements.sh has no platform/keycloak/* entry in its path->requirement "
|
||||
"map, so editing the realm import or this reconciler attributes NO requirement and "
|
||||
"the L2 keycloak tests are filtered out of the merge gate's touched subset")
|
||||
l2 = ROOT / "conformance/tests/l2-behavioral/keycloak-client-secrets-match-git/chainsaw-test.yaml"
|
||||
check(l2.is_file(),
|
||||
"the LIVE-realm counterpart (conformance/tests/l2-behavioral/"
|
||||
"keycloak-client-secrets-match-git) is missing — this static check cannot see a "
|
||||
"realm that drifted after apply, which is the whole failure mode")
|
||||
if l2.is_file():
|
||||
check("KEYCLOAK_CLIENT_SECRETS_AUDIT_OK" in l2.read_text(),
|
||||
"the L2 test does not assert the reconciler's audit sentinel, so it could pass "
|
||||
"on a Job that never ran the audit")
|
||||
|
||||
# ------------------------------------------------------------------------------ report
|
||||
print(f"realm import: {len(clients)} client(s), {len(declared_pairs)} confidential with a "
|
||||
f"declared secret, {len(placeholders)} placeholder(s)")
|
||||
print(f"{SECRET_NAME} Secret: {len(sops_keys)} key(s) in git (names only; values stay "
|
||||
f"SOPS-encrypted)")
|
||||
print("flux ordering: " + ", ".join(
|
||||
f"{f.split('/')[-1]} <- {owners(f)}" for f in (SOPS_FILE, IMPORT_FILE, RECON_FILE)))
|
||||
|
||||
print()
|
||||
if not FAILURES:
|
||||
print(f"check OK: keycloak-client-secret-reconciler — {CHECKS} assertions "
|
||||
f"(R-IDP-03, R-IDP-04)")
|
||||
raise SystemExit(0)
|
||||
print(f"CHECK FAIL (keycloak-client-secret-reconciler): {len(FAILURES)} failed, "
|
||||
f"{CHECKS - len(FAILURES)} passed", file=sys.stderr)
|
||||
for f in FAILURES:
|
||||
print(f" FAIL {f}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
PY
|
||||
@@ -1,68 +0,0 @@
|
||||
# =============================================================================
|
||||
# L0 STATIC — a placeholder value in cluster-vars is REFUSED by the guard that owns it
|
||||
# Catalog: docs/14 L0 row ("Pre-apply, in CI") — runs WITHOUT a cluster.
|
||||
#
|
||||
# THE BUG CLASS THIS CLOSES (it has now bitten three times in one file). A guard in
|
||||
# `bootstrap/standup.sh` keys on the SHAPE of a value that lives in
|
||||
# `clusters/base/cluster-vars.yaml`, and nothing tied the two together. On 2026-09-05 the
|
||||
# S3 placeholder changed from `TBD-PROD-S3-BACKUP-ENDPOINT` to
|
||||
# `https://TBD-R2-ACCOUNT-ID.r2.cloudflarestorage.com`; the three guards matched the glob
|
||||
# `TBD-*`, a PREFIX match, and `fnmatch('https://TBD-R2-…','TBD-*')` is False. All three
|
||||
# guards silently stopped matching, and `standup.sh resume` would have resumed `velero` on
|
||||
# prod against a nonexistent hostname: installs clean, reports healthy, NO BACKUPS EXIST.
|
||||
# It was found by accident. The guard is fixed (`s3_is_placeholder`, substring); the
|
||||
# COUPLING was untested, which is what makes the class reintroducible.
|
||||
#
|
||||
# GIVEN the repository as authored,
|
||||
# WHEN the REAL predicate is extracted from bootstrap/standup.sh and executed by bash
|
||||
# (never reimplemented — see check.sh "HOW THE REAL GUARD IS EXERCISED"),
|
||||
# THEN (1) every authored value of a guarded knob that is placeholder-shaped by an
|
||||
# INDEPENDENT detector is actually refused by that guard, in base and in every
|
||||
# environment overlay;
|
||||
# (2) no environment overlay carries a placeholder-shaped value for a knob no guard
|
||||
# refuses;
|
||||
# (3) every function in standup.sh that reads a guarded knob applies the guard to it
|
||||
# (the "three guards died at once" regression);
|
||||
# (4) the guard matches its own vocabulary ANYWHERE in a value, not just as a prefix
|
||||
# — the exact anchoring defect above — and still accepts a real endpoint;
|
||||
# AND (5) the extraction is faithful, and the check fails LOUDLY rather than silently
|
||||
# testing a copy if the guard stops being a self-contained predicate.
|
||||
#
|
||||
# STATUS: EXECUTABLE. `check.sh` is a pure offline assertion script (standard library +
|
||||
# PyYAML + bash); conformance/run.sh --layer l0 runs it directly and this Chainsaw Test
|
||||
# wraps the SAME script.
|
||||
#
|
||||
# NOT proven: that the endpoint is REACHABLE or that backups exist (runtime facts —
|
||||
# `standup.sh s3check` probes the former from a pod, l3-fault-injection/dr-restore proves
|
||||
# the latter); that guards for knobs OTHER than the placeholder-refusing ones are correct;
|
||||
# and that the guard runs at all in a real standup (the dispatch path is shell, executed
|
||||
# only against a live cluster). R-STATE-03 is tagged for the narrow static slice only: the
|
||||
# authored backup target cannot be a placeholder that the guard fails to recognise.
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: placeholder-guard-coupling
|
||||
labels:
|
||||
conformance.home.k3s/R-ENV-02: "true"
|
||||
conformance.home.k3s/R-STATE-03: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-ENV-02, R-STATE-03"
|
||||
conformance.home.k3s/layer: "L0"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
Every placeholder-shaped value authored in cluster-vars, for any environment
|
||||
overlay, is actually recognised as a placeholder by the real bootstrap guard that
|
||||
is supposed to refuse it — so a value change in one file cannot silently disarm a
|
||||
guard in another and enable velero against an endpoint that does not exist.
|
||||
spec:
|
||||
steps:
|
||||
- name: couple-authored-placeholders-to-the-real-guard
|
||||
description: >-
|
||||
Extract the real placeholder predicate from bootstrap/standup.sh, run it against
|
||||
every authored cluster-vars value the repo actually carries, and assert the guard
|
||||
refuses each one an independent detector calls a placeholder.
|
||||
try:
|
||||
- script:
|
||||
content: bash ./check.sh
|
||||
@@ -1,516 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# L0 static check (offline, no cluster) — placeholder-guard-coupling
|
||||
# Serves: R-ENV-02 ("Same GitOps config promotes dev/test -> prod (authored once)."
|
||||
# — spec docs/02-environments-and-provisioning.md)
|
||||
# R-STATE-03 ("Scheduled off-cluster backups + restore (DR)."
|
||||
# — spec docs/04-storage-and-state.md)
|
||||
#
|
||||
# WHY THIS EXISTS — A GUARD KEYED ON A VALUE THAT LIVES IN ANOTHER FILE
|
||||
# --------------------------------------------------------------------
|
||||
# `bootstrap/standup.sh` refuses to resume tiers whose configuration is still a
|
||||
# placeholder. Three of those guards keyed on the SHAPE of `S3_BACKUP_ENDPOINT`, which is
|
||||
# authored in `clusters/base/cluster-vars.yaml`. On 2026-09-05 the placeholder changed
|
||||
# from `TBD-PROD-S3-BACKUP-ENDPOINT` to `https://TBD-R2-ACCOUNT-ID.r2.cloudflarestorage.com`
|
||||
# (the MinIO -> Cloudflare R2 move). The guards matched the glob `TBD-*` — a PREFIX match —
|
||||
# and `fnmatch('https://TBD-R2-…','TBD-*')` is FALSE. All three guards silently stopped
|
||||
# matching. `standup.sh resume` would then have resumed `velero` on prod against a hostname
|
||||
# that does not exist: the HelmRelease installs clean, the Kustomization reports Ready, and
|
||||
# NO BACKUPS EXIST. The only reason it was not shipped is that someone noticed by accident.
|
||||
#
|
||||
# The predicate is fixed (`s3_is_placeholder`, substring match). The COUPLING is what was
|
||||
# never tested: a guard in one file keying on a value in another, with nothing tying them
|
||||
# together. This check is that tie, and it is derived from BOTH real files — no current
|
||||
# string is hardcoded here, so it does not rot when the placeholder or the guard changes.
|
||||
#
|
||||
# HOW THE REAL GUARD IS EXERCISED (and why not a copy)
|
||||
# ---------------------------------------------------
|
||||
# `bootstrap/standup.sh` is NOT sourceable: at top level it parses argv, hard-exits when
|
||||
# --kubeconfig/--expect-server/<step> are absent, `cd`s to the repo root, exports
|
||||
# KUBECONFIG, installs an EXIT trap and finally dispatches steps against a live cluster.
|
||||
# So this check EXTRACTS the predicate's own source lines from the file and sources ONLY
|
||||
# those, then calls the real function. That is the real bytes, not a reimplementation:
|
||||
# * every function whose name ends `_is_placeholder` is discovered, not named here;
|
||||
# * a duplicate definition of one is a failure (which of the two would run?);
|
||||
# * the extracted body must be a SELF-CONTAINED predicate — `case`/`return` and nothing
|
||||
# else, no command substitution, no call to another function. If the guard ever grows
|
||||
# a dependency, extraction stops being faithful and THIS CHECK FAILS LOUDLY, naming the
|
||||
# remedy (move the predicate into a small sourceable library that standup.sh sources)
|
||||
# rather than quietly falling back to testing a copy of the logic;
|
||||
# * a CANARY pins the extraction itself: the extracted function must refuse the empty
|
||||
# string and accept a realistic endpoint. An extraction that grabbed the wrong lines,
|
||||
# or a predicate degraded to always-true / always-false, fails there — this repo has
|
||||
# already been bitten by a check that passed when perturbed because it asked a question
|
||||
# that was always satisfiable.
|
||||
# There is deliberately NO second implementation of the matching logic anywhere in this file.
|
||||
#
|
||||
# THE INDEPENDENT DETECTOR (the other half of the comparison)
|
||||
# ----------------------------------------------------------
|
||||
# Asking "does the guard agree with the guard?" proves nothing, so the set of
|
||||
# placeholder-shaped values is decided by a detector written HERE, whose vocabulary is the
|
||||
# UNION of two independent sources:
|
||||
# * SEED — universal loud-placeholder idioms (TBD, TODO, FIXME, PLACEHOLDER, REPLACE-ME,
|
||||
# CHANGE-ME, FILL-ME, SET-ME, XXXX), plus `<angle-bracket>` templates and the empty
|
||||
# string. These are NOT derived from the guard, so DELETING a guard arm cannot make a
|
||||
# value stop being placeholder-shaped — it makes assertion (1) fail, which is the point.
|
||||
# A value using an idiom the guard does not know is a finding either way: fix it by
|
||||
# giving the knob a real value, or by teaching the guard the idiom.
|
||||
# * the guard's OWN vocabulary, read out of its case-arms with the glob anchors STRIPPED.
|
||||
# Stripping the anchors is what makes the comparison non-circular in the dimension that
|
||||
# actually broke: the detector matches a word ANYWHERE, case-insensitively, and the
|
||||
# guard must agree. A prefix-anchored guard cannot pass.
|
||||
#
|
||||
# GIVEN the repository as authored,
|
||||
# WHEN the real predicate(s) are extracted from bootstrap/standup.sh and executed,
|
||||
# THEN (1) VALUES: for every knob a guard is responsible for, every authored value — the
|
||||
# base default AND each environment overlay's value — that the independent
|
||||
# detector calls placeholder-shaped is REFUSED by the real guard, and every value
|
||||
# it calls usable is ACCEPTED (a guard that refuses a real endpoint leaves velero
|
||||
# suspended forever, which is the same class of silent failure);
|
||||
# (2) OVERLAYS: no environment overlay's EFFECTIVE cluster-vars carries a
|
||||
# placeholder-shaped value for a knob that no guard refuses. (Base alone is
|
||||
# exempt: it is not a bootstrap target — `flux bootstrap --path=clusters/<env>` —
|
||||
# and its loud sentinels exist precisely to be overridden. A base placeholder an
|
||||
# overlay forgets to override IS caught, because it becomes that overlay's
|
||||
# effective value.)
|
||||
# (3) CONSUMERS: every function in standup.sh that reads a guarded knob (via `cvar`,
|
||||
# following one hop of variable aliasing) applies a placeholder guard to it. This
|
||||
# is the "all three guards died at once" regression stated as a property: the
|
||||
# guard being correct is worthless if a consumer stops calling it.
|
||||
# (4) ANCHORING: for each word in the guard's own vocabulary, the guard refuses a
|
||||
# value carrying that word in the MIDDLE (not at the start) — the precise defect
|
||||
# of 2026-09-05 — while still accepting a realistic endpoint.
|
||||
# (5) HARNESS: exactly one definition per predicate, the extracted body is verbatim
|
||||
# from the file and self-contained, the canary holds, no divergent second copy of
|
||||
# the matching logic exists elsewhere in standup.sh, and a floor on the number of
|
||||
# assertions run keeps a pass from degenerating into a no-op.
|
||||
# AND (6) LOAD-BEARING: ci/l0.sh runs the L0 conformance layer on every gate run and
|
||||
# ci/gate.sh sets the merge block when that stage fails.
|
||||
#
|
||||
# WHAT THIS DOES *NOT* PROVE (honest gaps)
|
||||
# - Nothing about RUNTIME: not that the endpoint resolves, not that credentials work, not
|
||||
# that a backup exists. `standup.sh s3check` probes reachability from a pod (operator-
|
||||
# run, cluster-gated) and conformance/tests/l3-fault-injection/dr-restore proves restore.
|
||||
# - It does not execute standup.sh's dispatch path, so it cannot prove the guard is
|
||||
# REACHED in a real run — only that, when reached, it refuses what it must. The
|
||||
# consumer property (3) is a static approximation of reachability: it is a text-level
|
||||
# taint trace over one file, not an interpreter, and a guard whose result is computed
|
||||
# and then ignored would satisfy it.
|
||||
# - It says nothing about guards keyed on shapes OTHER than placeholders (e.g. the
|
||||
# EXTERNALDNS_TXT_OWNER_ID == 'default' test), nor about placeholder-shaped values
|
||||
# outside cluster-vars (the Ansible inventory's BLOCKED/NOT-A-DEVICE sentinels are
|
||||
# asserted by the node_baseline role itself, at play time).
|
||||
# - Stage 3b of ci/l0.sh scopes the conformance run to the TOUCHED requirements. An edit
|
||||
# to bootstrap/* attributes NO requirement in ci/touched-requirements.sh's path map, so
|
||||
# the touched set is empty and the FULL L0 layer runs (verified: ci/lib.sh
|
||||
# load_touched_requirements) — this check runs. An edit to clusters/* maps to
|
||||
# R-ENV-02, which this check carries, so a cluster-vars value change also selects it.
|
||||
# A change that touches standup.sh ALONGSIDE some unrelated mapped path (say
|
||||
# platform/cilium/) would scope the run away from this check; closing that needs a
|
||||
# `bootstrap/*` entry in the path map, which is not this check's file to edit.
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT="$(cd "${DIR}/../../../.." && pwd)" # repo root
|
||||
PYTHON="${PYTHON:-python3}"
|
||||
STANDUP_REL="${STANDUP_REL:-bootstrap/standup.sh}"
|
||||
|
||||
cd "${ROOT}"
|
||||
|
||||
if [ ! -f "${ROOT}/${STANDUP_REL}" ]; then
|
||||
echo "check SKIPPED (placeholder-guard-coupling): ${STANDUP_REL} is absent — nothing to couple." >&2
|
||||
exit 3
|
||||
fi
|
||||
|
||||
WORK="$(mktemp -d)"
|
||||
trap 'rm -rf "${WORK}"' EXIT
|
||||
|
||||
# ---------------------------------------------------------------------------------------
|
||||
# PHASE 1 (python): extract the real predicate(s), derive the corpus from the real files,
|
||||
# and run every assertion that does not need a shell. Writes:
|
||||
# ${WORK}/guards.sh the predicate source lines, verbatim, and nothing else
|
||||
# ${WORK}/corpus.tsv <guard>\t<expect: placeholder|usable>\t<origin>\t<value>
|
||||
# ${WORK}/findings.txt one line per failed assertion
|
||||
# ${WORK}/counts.txt <assertions-run>
|
||||
# ---------------------------------------------------------------------------------------
|
||||
"${PYTHON}" - "${ROOT}" "${WORK}" "${STANDUP_REL}" <<'PY'
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError: # pragma: no cover
|
||||
print("check FAIL: PyYAML is required (same dependency as catalog/render.py)",
|
||||
file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
ROOT, WORK, STANDUP_REL = Path(sys.argv[1]), Path(sys.argv[2]), sys.argv[3]
|
||||
FAILURES: list[str] = []
|
||||
CHECKS = 0
|
||||
|
||||
|
||||
def check(cond, msg):
|
||||
global CHECKS
|
||||
CHECKS += 1
|
||||
if not cond:
|
||||
FAILURES.append(msg)
|
||||
|
||||
|
||||
def bail(msg):
|
||||
"""A failure that makes the rest of the check meaningless: report and stop."""
|
||||
print(f"CHECK FAIL (placeholder-guard-coupling): {msg}", file=sys.stderr)
|
||||
(WORK / "findings.txt").write_text(msg + "\n")
|
||||
(WORK / "counts.txt").write_text("0\n")
|
||||
(WORK / "corpus.tsv").write_text("")
|
||||
(WORK / "guards.sh").write_text("")
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
TEXT = (ROOT / STANDUP_REL).read_text()
|
||||
LINES = TEXT.splitlines()
|
||||
|
||||
# ------------------------------------------------------------------ (5) extract the guard
|
||||
DEF_RE = re.compile(r"^([a-z0-9_]+_is_placeholder)\(\)\s*\{")
|
||||
defs: dict[str, list[int]] = {}
|
||||
for i, line in enumerate(LINES):
|
||||
m = DEF_RE.match(line)
|
||||
if m:
|
||||
defs.setdefault(m.group(1), []).append(i)
|
||||
|
||||
if not defs:
|
||||
bail(f"{STANDUP_REL} defines no *_is_placeholder predicate. Either the guard was renamed "
|
||||
"(update this check's discovery pattern in the SAME change) or it was DELETED, which "
|
||||
"is the 2026-09-05 failure with the guard removed instead of disarmed.")
|
||||
|
||||
# An "argument-shaped" call: <name> "<something>"
|
||||
CALL_RE = re.compile(r"\b([a-z0-9_]+_is_placeholder)\s+\"([^\"]*)\"")
|
||||
|
||||
guard_src: list[str] = []
|
||||
guard_words: dict[str, list[str]] = {}
|
||||
ARM_RE = re.compile(r"^(?P<pats>[^()]*?)\)\s*return\s+[01]\s*;;$")
|
||||
ALLOWED = (
|
||||
re.compile(r"^$"),
|
||||
re.compile(r"^#"),
|
||||
re.compile(r"^case\s+\"\$\{?[0-9A-Za-z_]+\}?\"\s+in$"),
|
||||
re.compile(r"^esac$"),
|
||||
re.compile(r"^return\s+[01]$"),
|
||||
ARM_RE,
|
||||
)
|
||||
|
||||
for name, at in sorted(defs.items()):
|
||||
check(len(at) == 1,
|
||||
f"{STANDUP_REL} defines {name}() {len(at)} times (lines "
|
||||
f"{', '.join(str(a + 1) for a in at)}) — which one runs depends on file order, and "
|
||||
"this check can no longer say which one it extracted. Keep exactly one definition.")
|
||||
if len(at) != 1:
|
||||
continue
|
||||
start = at[0]
|
||||
end = next((j for j in range(start + 1, len(LINES)) if LINES[j] == "}"), None)
|
||||
if end is None:
|
||||
bail(f"{name}() at {STANDUP_REL}:{start + 1} has no closing brace at column 0 — the "
|
||||
"extraction boundary this check relies on is gone. Restore the conventional "
|
||||
"layout, or move the predicate into a sourceable library and point this check at it.")
|
||||
body = LINES[start:end + 1]
|
||||
block = "\n".join(body)
|
||||
check(block in TEXT,
|
||||
f"extracted {name}() is not a verbatim substring of {STANDUP_REL} — extraction bug; "
|
||||
"the harness would be testing something the script does not contain")
|
||||
|
||||
words: list[str] = []
|
||||
for raw in body[1:-1]:
|
||||
s = raw.strip()
|
||||
if not any(rx.match(s) for rx in ALLOWED) or "$(" in s or "`" in s:
|
||||
bail(f"{name}() at {STANDUP_REL}:{start + 1} is no longer a SELF-CONTAINED "
|
||||
f"case/return predicate (offending line: {s!r}). This check exercises the REAL "
|
||||
"function by sourcing its own lines; a body that calls out to anything else "
|
||||
"cannot be extracted faithfully, and silently testing a COPY of the logic is "
|
||||
"exactly the failure mode this check exists to prevent. REMEDY: move the "
|
||||
"predicate into a tiny library that standup.sh sources (e.g. "
|
||||
"bootstrap/lib/placeholder.sh) and point STANDUP_REL/this extractor at it — do "
|
||||
"NOT reimplement the matching here.")
|
||||
m = ARM_RE.match(s)
|
||||
if not m:
|
||||
continue
|
||||
for pat in m.group("pats").split("|"):
|
||||
pat = pat.strip()
|
||||
if pat in ('""', "''", ""):
|
||||
continue # the empty-value arm: covered by the canary
|
||||
words.append(pat.strip("'\"").strip("*"))
|
||||
check(len(words) >= 1,
|
||||
f"{name}() has no non-empty placeholder pattern left — every arm that recognises a "
|
||||
"loud placeholder string has been removed, so the guard now only refuses the empty "
|
||||
"value and any 'TBD'-style endpoint would sail straight through it")
|
||||
guard_words[name] = words
|
||||
guard_src.append(block)
|
||||
|
||||
(WORK / "guards.sh").write_text("\n".join(guard_src) + "\n")
|
||||
|
||||
# ------------------------------------------------- no divergent second copy of the logic
|
||||
# Any of the guard's own vocabulary appearing in EXECUTABLE text outside the predicate means
|
||||
# a second, independently-drifting implementation of the same test (which is how three
|
||||
# guards came to disagree with one value in the first place).
|
||||
guard_line_nums = {j for name, at in defs.items() if len(at) == 1
|
||||
for j in range(at[0], next(k for k in range(at[0] + 1, len(LINES))
|
||||
if LINES[k] == "}") + 1)}
|
||||
all_words = sorted({w.upper() for ws in guard_words.values() for w in ws if w})
|
||||
divergent: list[str] = []
|
||||
for j, line in enumerate(LINES):
|
||||
if j in guard_line_nums or line.strip().startswith("#"):
|
||||
continue
|
||||
code = line.split("#", 1)[0]
|
||||
hit = [w for w in all_words if w in code.upper()]
|
||||
if hit:
|
||||
divergent.append(f"{STANDUP_REL}:{j + 1} ({', '.join(hit)}): {line.strip()!r}")
|
||||
check(not divergent,
|
||||
"the guard's own placeholder vocabulary appears in EXECUTABLE text outside the "
|
||||
f"predicate — a SECOND implementation of the same test, which is precisely how a value "
|
||||
f"change disarmed three guards at once. Call the predicate instead. Sites: "
|
||||
+ " | ".join(divergent))
|
||||
|
||||
# ------------------------------------------------------------- cluster-vars, as authored
|
||||
CV = ROOT / "clusters" / "base" / "cluster-vars.yaml"
|
||||
if not CV.is_file():
|
||||
bail("clusters/base/cluster-vars.yaml is absent — the authored half of the coupling is "
|
||||
"missing and this check cannot mean anything")
|
||||
|
||||
base: dict[str, str] = {}
|
||||
for d in yaml.safe_load_all(CV.read_text()):
|
||||
if isinstance(d, dict) and d.get("kind") == "ConfigMap" \
|
||||
and (d.get("metadata") or {}).get("name") == "cluster-vars":
|
||||
base = {str(k): "" if v is None else str(v) for k, v in (d.get("data") or {}).items()}
|
||||
check(len(base) > 5, f"clusters/base/cluster-vars.yaml yielded {len(base)} knob(s) — the "
|
||||
"ConfigMap parse failed, so every value assertion below would be vacuous")
|
||||
|
||||
overlays: dict[str, dict[str, str]] = {}
|
||||
for overlay in sorted((ROOT / "clusters").glob("*/kustomization.yaml")):
|
||||
env = overlay.parent.name
|
||||
if env == "base":
|
||||
continue
|
||||
eff = dict(base)
|
||||
try:
|
||||
k = yaml.safe_load(overlay.read_text()) or {}
|
||||
except Exception as exc: # noqa: BLE001
|
||||
check(False, f"clusters/{env}/kustomization.yaml does not parse as YAML ({exc}) — this "
|
||||
"check cannot compute that overlay's effective values")
|
||||
continue
|
||||
for p in (k.get("patches") or []):
|
||||
tgt = p.get("target") or {}
|
||||
if tgt.get("kind") != "ConfigMap" or tgt.get("name") != "cluster-vars":
|
||||
continue
|
||||
try:
|
||||
ops = yaml.safe_load(p.get("patch") or "") or []
|
||||
except Exception: # noqa: BLE001
|
||||
continue
|
||||
for op in ops if isinstance(ops, list) else []:
|
||||
if not isinstance(op, dict):
|
||||
continue
|
||||
m = re.match(r"^/data/([A-Za-z0-9_]+)$", str(op.get("path", "")))
|
||||
if m and str(op.get("op", "")) in ("replace", "add"):
|
||||
eff[m.group(1)] = "" if op.get("value") is None else str(op["value"])
|
||||
overlays[env] = eff
|
||||
check(len(overlays) >= 1,
|
||||
"no clusters/<env>/kustomization.yaml overlay was parsed — assertion (2) would be vacuous")
|
||||
|
||||
# ------------------------------------------------ which knobs is a guard responsible for?
|
||||
# Function bodies of standup.sh, then a one-hop taint trace from `x="$(cvar KEY)"` to a
|
||||
# `*_is_placeholder "${x}"` call. Both halves are read out of the file; nothing is named here.
|
||||
FUNC_RE = re.compile(r"^([a-z0-9_]+)\(\)\s*\{")
|
||||
funcs: dict[str, list[str]] = {}
|
||||
i = 0
|
||||
while i < len(LINES):
|
||||
m = FUNC_RE.match(LINES[i])
|
||||
if m:
|
||||
end = next((j for j in range(i + 1, len(LINES)) if LINES[j] == "}"), len(LINES) - 1)
|
||||
funcs[m.group(1)] = LINES[i + 1:end]
|
||||
i = end + 1
|
||||
continue
|
||||
i += 1
|
||||
check(len(funcs) > 5, f"parsed only {len(funcs)} function(s) out of {STANDUP_REL} — the "
|
||||
"consumer assertion (3) would be vacuous")
|
||||
|
||||
CVAR_RE = re.compile(r"([A-Za-z_][A-Za-z0-9_]*)=\"\$\(\s*cvar\s+([A-Z0-9_]+)\s*\)\"")
|
||||
ALIAS_RE = re.compile(r"([A-Za-z_][A-Za-z0-9_]*)=\"\$\{([A-Za-z_][A-Za-z0-9_]*)[:}]")
|
||||
|
||||
guarded_keys: set[str] = set()
|
||||
consumers: list[tuple[str, str, bool]] = [] # (func, key, guarded-in-this-func)
|
||||
for fname, body in funcs.items():
|
||||
text = "\n".join(body)
|
||||
keys_of_var: dict[str, set[str]] = {}
|
||||
for m in CVAR_RE.finditer(text):
|
||||
keys_of_var.setdefault(m.group(1), set()).add(m.group(2))
|
||||
# one hop of aliasing: ep="${live}" carries live's taint
|
||||
for _ in range(3):
|
||||
for m in ALIAS_RE.finditer(text):
|
||||
src = keys_of_var.get(m.group(2))
|
||||
if src:
|
||||
keys_of_var.setdefault(m.group(1), set()).update(src)
|
||||
called_on = {m.group(2) for m in CALL_RE.finditer(text)}
|
||||
called_vars = {re.sub(r"^\$\{?|\}?$", "", a) for a in called_on}
|
||||
# Aggregate per (function, knob): the knob is guarded HERE if ANY variable carrying its
|
||||
# value is passed to a predicate — `live="$(cvar K)"; ep="${live}"; guard "${ep}"` counts.
|
||||
guarded_here: dict[str, bool] = {}
|
||||
for var, keys in keys_of_var.items():
|
||||
for key in keys:
|
||||
guarded_here[key] = guarded_here.get(key, False) or (var in called_vars)
|
||||
for key, g in guarded_here.items():
|
||||
consumers.append((fname, key, g))
|
||||
if g:
|
||||
guarded_keys.add(key)
|
||||
|
||||
if not guarded_keys:
|
||||
bail(f"no cluster-vars knob in {STANDUP_REL} is passed to a *_is_placeholder predicate. "
|
||||
"Either the guards were removed, or the way the script reads cluster-vars changed and "
|
||||
"this check's derivation must be updated in the SAME change — it must never silently "
|
||||
"verify an empty set.")
|
||||
|
||||
# (3) CONSUMERS: every reader of a guarded knob must apply the guard to it.
|
||||
for fname, key, guarded in sorted(consumers):
|
||||
if key not in guarded_keys:
|
||||
continue # nothing claims responsibility for this knob at all
|
||||
check(guarded,
|
||||
f"{STANDUP_REL}: {fname}() reads cluster-vars knob {key} but never passes it to a "
|
||||
"*_is_placeholder guard, while other functions do. On 2026-09-05 all three consumers "
|
||||
"of S3_BACKUP_ENDPOINT were disarmed at once; a consumer that reads the knob and skips "
|
||||
"the guard is that failure in one function.")
|
||||
|
||||
# ---------------------------------------------------------------- independent detector
|
||||
# Deliberately NOT derived from the guard alone (see the header): SEED is universal
|
||||
# loud-placeholder vocabulary, and the guard's own words are added so that a guard whose
|
||||
# ANCHORING is wrong still has its vocabulary tested against real values.
|
||||
SEED = ["TBD", "TODO", "FIXME", "PLACEHOLDER", "REPLACE-ME", "REPLACEME",
|
||||
"CHANGE-ME", "CHANGEME", "FILL-ME", "FILLME", "SET-ME", "XXXX"]
|
||||
VOCAB = sorted({w.upper() for w in SEED} | set(all_words))
|
||||
ANGLE = re.compile(r"<[^>\s]+>")
|
||||
|
||||
|
||||
def placeholder_shaped(value: str) -> bool:
|
||||
if value.strip() == "":
|
||||
return True # an unset endpoint is not a target
|
||||
up = value.upper()
|
||||
return any(w in up for w in VOCAB) or bool(ANGLE.search(value))
|
||||
|
||||
|
||||
rows: list[tuple[str, str, str, str]] = [] # guard, expect, origin, value
|
||||
GUARD = sorted(guard_words)[0] if len(guard_words) == 1 else None
|
||||
|
||||
# (1) VALUES: base default and every overlay's value, for every guarded knob.
|
||||
for key in sorted(guarded_keys):
|
||||
seen: list[tuple[str, str]] = []
|
||||
if key in base:
|
||||
seen.append((f"clusters/base/cluster-vars.yaml:{key}", base[key]))
|
||||
for env, eff in sorted(overlays.items()):
|
||||
seen.append((f"clusters/{env} effective:{key}", eff.get(key, "")))
|
||||
check(bool(seen), f"guarded knob {key} is not defined in cluster-vars at all — the guard "
|
||||
"protects a knob nothing authors")
|
||||
for origin, value in seen:
|
||||
expect = "placeholder" if placeholder_shaped(value) else "usable"
|
||||
for name in sorted(guard_words):
|
||||
rows.append((name, expect, origin, value))
|
||||
|
||||
# (2) OVERLAYS: a placeholder-shaped effective value for an UNGUARDED knob.
|
||||
for env, eff in sorted(overlays.items()):
|
||||
for key, value in sorted(eff.items()):
|
||||
if key in guarded_keys or not placeholder_shaped(value):
|
||||
continue
|
||||
check(False,
|
||||
f"clusters/{env}: knob {key} resolves to the placeholder-shaped value {value!r}, "
|
||||
f"and NO guard in {STANDUP_REL} refuses it. Either give it a real value before it "
|
||||
"reaches an overlay, or guard it the way the S3 endpoint is guarded — an overlay "
|
||||
"that ships a placeholder no guard recognises is the 2026-09-05 failure with a "
|
||||
"different knob's name on it.")
|
||||
|
||||
# (4) ANCHORING: the guard's own vocabulary must match ANYWHERE, not just as a prefix,
|
||||
# and a realistic endpoint must still be accepted.
|
||||
for name, words in sorted(guard_words.items()):
|
||||
for w in sorted(set(words)):
|
||||
rows.append((name, "placeholder",
|
||||
f"anchoring:{name}:mid-string({w})",
|
||||
f"https://account-{w}-id.r2.cloudflarestorage.com"))
|
||||
rows.append((name, "placeholder", f"anchoring:{name}:tail({w})",
|
||||
f"https://r2-endpoint.example.invalid/{w}"))
|
||||
rows.append((name, "placeholder", f"canary:{name}:empty", ""))
|
||||
rows.append((name, "usable", f"canary:{name}:real-r2",
|
||||
"https://0123456789abcdef0123456789abcd.r2.cloudflarestorage.com"))
|
||||
rows.append((name, "usable", f"canary:{name}:real-minio", "http://192.168.1.9:9000"))
|
||||
|
||||
# Every guarded knob must reach the oracle once per authored source (base + each overlay),
|
||||
# so a knob silently dropping out of the corpus is a failure rather than a quiet pass.
|
||||
authored = [r for r in rows if not r[2].startswith(("anchoring:", "canary:"))]
|
||||
expected_rows = len(guarded_keys) * (1 + len(overlays)) * len(guard_words)
|
||||
check(len(authored) == expected_rows,
|
||||
f"{len(authored)} authored value(s) reached the guard, expected {expected_rows} "
|
||||
f"({len(guarded_keys)} guarded knob(s) x base+{len(overlays)} overlay(s) x "
|
||||
f"{len(guard_words)} predicate(s)) — a guarded knob is missing from cluster-vars or from "
|
||||
"an overlay's effective values, and the coupling is only partly exercised")
|
||||
|
||||
# ------------------------------------------------------- (6) the check is load-bearing
|
||||
L0 = (ROOT / "ci" / "l0.sh").read_text() if (ROOT / "ci" / "l0.sh").is_file() else ""
|
||||
GATE = (ROOT / "ci" / "gate.sh").read_text() if (ROOT / "ci" / "gate.sh").is_file() else ""
|
||||
check("--layer l0" in L0,
|
||||
"ci/l0.sh does not run the L0 conformance layer — this check would never execute in the gate")
|
||||
check(bool(re.search(r"^for s in .*\bl0\b.*; do", GATE, re.M)) and bool(re.search(r"fail\).*BLOCK=1", GATE)),
|
||||
"ci/gate.sh does not set the merge block when the l0 stage fails")
|
||||
|
||||
# --------------------------------------------------------------------------- handoff
|
||||
with (WORK / "corpus.tsv").open("w") as fh:
|
||||
for name, expect, origin, value in rows:
|
||||
fh.write(f"{name}\t{expect}\t{origin}\t{value}\n")
|
||||
(WORK / "findings.txt").write_text("".join(f"{f}\n" for f in FAILURES))
|
||||
(WORK / "counts.txt").write_text(f"{CHECKS}\n")
|
||||
|
||||
print(f"guards: {', '.join(sorted(guard_words))} "
|
||||
f"vocabulary: {', '.join(VOCAB)}")
|
||||
print(f"guarded knobs: {', '.join(sorted(guarded_keys))} "
|
||||
f"consumers traced: {len(consumers)} overlays: {', '.join(sorted(overlays))}")
|
||||
for name, expect, origin, value in rows:
|
||||
if not origin.startswith(("anchoring:", "canary:")):
|
||||
print(f" authored {origin} = {value!r} -> expect {expect}")
|
||||
PY
|
||||
|
||||
# ---------------------------------------------------------------------------------------
|
||||
# PHASE 2 (bash): source the REAL predicate lines and run the corpus through them.
|
||||
# ---------------------------------------------------------------------------------------
|
||||
# shellcheck disable=SC1090,SC1091
|
||||
source "${WORK}/guards.sh"
|
||||
|
||||
oracle_failures=0
|
||||
oracle_rows=0
|
||||
while IFS=$'\t' read -r guard expect origin value; do
|
||||
[ -n "${guard}" ] || continue
|
||||
oracle_rows=$((oracle_rows + 1))
|
||||
if "${guard}" "${value}"; then verdict=placeholder; else verdict=usable; fi
|
||||
if [ "${verdict}" != "${expect}" ]; then
|
||||
oracle_failures=$((oracle_failures + 1))
|
||||
if [ "${expect}" = placeholder ]; then
|
||||
printf '%s\n' "${guard}() in ${STANDUP_REL} ACCEPTS '${value}' (from ${origin}) as a usable value, but it is placeholder-shaped. This is the 2026-09-05 failure exactly: the guard is asked about a value it does not recognise, returns 'fine', and the tier it protects is enabled against something that does not exist. Fix the predicate (match the idiom anywhere, not as a prefix) or stop authoring that idiom." >> "${WORK}/findings.txt"
|
||||
else
|
||||
printf '%s\n' "${guard}() in ${STANDUP_REL} REFUSES '${value}' (from ${origin}), which is a usable value. An over-matching guard leaves the tier it protects suspended forever and reports it as an operator decision — a silent failure in the other direction." >> "${WORK}/findings.txt"
|
||||
fi
|
||||
fi
|
||||
done < "${WORK}/corpus.tsv"
|
||||
|
||||
# ------------------------------------------------------------------------------ verdict
|
||||
py_checks="$(cat "${WORK}/counts.txt")"
|
||||
total=$((py_checks + oracle_rows))
|
||||
findings="$(cat "${WORK}/findings.txt")"
|
||||
|
||||
# NO-OP GUARD: this repo has been bitten by a check that passed when perturbed because it
|
||||
# asked a question that was always satisfiable. The floors below are the minimum evidence
|
||||
# for the verdict to mean anything, not a style rule.
|
||||
if [ "${oracle_rows}" -lt 6 ]; then
|
||||
findings="${findings}only ${oracle_rows} value(s) were put to the real guard — the coupling is not actually being exercised
|
||||
"
|
||||
fi
|
||||
if [ "${total}" -lt 20 ]; then
|
||||
findings="${findings}only ${total} assertions ran — the check degenerated to a near-no-op and its pass would be meaningless
|
||||
"
|
||||
fi
|
||||
|
||||
if [ -n "${findings}" ]; then
|
||||
echo "CHECK FAIL (placeholder-guard-coupling): $(printf '%s' "${findings}" | grep -c . ) of ${total} assertions failed" >&2
|
||||
printf '%s' "${findings}" | sed 's/^/ - /' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "check OK: placeholder-guard-coupling — ${total} assertions; ${oracle_rows} value(s) run through the REAL extracted predicate(s) from ${STANDUP_REL} (no reimplementation), covering every authored cluster-vars value of every guarded knob in base and each overlay, the guard's own vocabulary matched mid-string, and the empty/real-endpoint canaries. Static slice only: refusal, not reachability — see the header."
|
||||
@@ -1,77 +0,0 @@
|
||||
# =============================================================================
|
||||
# L0 STATIC — prod is bare metal, and is not a development surface (R-ENV-03) [EXECUTABLE]
|
||||
# Catalog: docs/14 L0 row ("Pre-apply, in CI") — runs WITHOUT a cluster.
|
||||
# Spec: docs/00 R-ENV-03 ("prod is a 3× mini-PC bare-metal k3s cluster with local
|
||||
# storage; it MUST NOT be an active development surface until the roadmap
|
||||
# declares it ready"); docs/02 (prod = 3 physical nodes, local SSD/NVMe to
|
||||
# Longhorn, "the *same* Ansible bootstrap runs; only the machine-creation step
|
||||
# differs", "Never develop against prod"); roadmap/phases.md Phase 7.
|
||||
#
|
||||
# HOW A "MUST NOT" IS MADE TESTABLE — the same discipline as R-TEST-06's test: a
|
||||
# governance rule is verified through the property that ENFORCES it, never by observing
|
||||
# that nobody misbehaved today. Here that property is structural: prod receives the base
|
||||
# configuration plus environment-scoped overrides and nothing else, from the same branch
|
||||
# and the same Ansible code as dev, so there is no place to put prod-only work at all.
|
||||
#
|
||||
# ENVIRONMENT-AGNOSTIC: the prod inventory group is DERIVED (it is the group whose
|
||||
# group_vars retargets the k3s roles away from the global default), the prod overlay is
|
||||
# derived from CLUSTER_ENV, and the storage/replica expectations are compared against the
|
||||
# node count that inventory resolves to. No hostname, address or environment name is
|
||||
# written into the test.
|
||||
#
|
||||
# GIVEN the repository as git tracks it (plus the read-only spec repo when mounted),
|
||||
# WHEN the prod inventory, overlay, IaC module and readiness record are inspected — and
|
||||
# the shared provisioning chain is actually retargeted at prod with --list-hosts,
|
||||
# THEN prod resolves to exactly three nodes reached directly on their own sshd; no IaC
|
||||
# resource creates any of them (bare metal is imaged, not instantiated); prod's
|
||||
# Longhorn path is node-local with one replica per node and the default
|
||||
# StorageClass is the local driver; prod runs the same playbooks/roles as dev,
|
||||
# retargeted only by variables the shared code already defines; the prod overlay
|
||||
# carries only the base plus environment-scoped patches on the same Git ref dev
|
||||
# tracks, with no prod-only manifest anywhere in the shared configuration; and prod
|
||||
# artifacts exist only alongside the operator's recorded opening of the prod phase.
|
||||
#
|
||||
# STATUS: EXECUTABLE. `check.sh` is a pure offline assertion script (git, PyYAML, and —
|
||||
# where present — ansible-playbook in --list-hosts mode, which opens no connection);
|
||||
# conformance/run.sh --layer l0 and ci/l0.sh run it directly, and this Chainsaw Test
|
||||
# wraps the SAME script.
|
||||
#
|
||||
# KNOWN, DELIBERATE GAPS (enumerated in check.sh's header; this test claims none of them):
|
||||
# * THE HARDWARE. That the three machines are mini-PCs, are bare metal, and hold local
|
||||
# SSD/NVMe is operator-attested (infra/PROD-STANDUP.md, PROD-PROVISIONING.md); the
|
||||
# node-health/BIOS surveys that establish it write to gitignored artifacts, and the
|
||||
# Kubernetes API exposes no "am I virtual" fact, so nothing here asserts it.
|
||||
# * The LIVE prod cluster's shape (three Ready nodes, all control-plane+etcd, Longhorn
|
||||
# disks at the local path) — cluster state, belonging to R-PLAT-01's still-uncovered
|
||||
# HA test; asserting it here would also make this test environment-specific, since
|
||||
# the same suite must pass against dev.
|
||||
# * Git HISTORY: this is the working tree at HEAD.
|
||||
# * Whether the roadmap SHOULD have opened prod — only that prod work is accompanied by
|
||||
# the recorded declaration.
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: prod-baremetal-not-dev-surface
|
||||
labels:
|
||||
conformance.home.k3s/R-ENV-03: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-ENV-03"
|
||||
conformance.home.k3s/layer: "L0"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
Given the tracked tree, when prod's inventory, overlay, IaC and readiness record are
|
||||
inspected, then prod is authored as three bare-metal nodes with node-local storage,
|
||||
bootstrapped by dev's own code retargeted by variables, receiving only
|
||||
environment-scoped overrides on the same branch — so there is nowhere to develop
|
||||
against prod — and it exists only because the roadmap phase was opened.
|
||||
spec:
|
||||
steps:
|
||||
- name: then-prod-is-bare-metal-and-carries-no-development-of-its-own
|
||||
description: >-
|
||||
Run the offline assertion script over the tracked tree (same script ci/l0.sh and
|
||||
conformance/run.sh --layer l0 execute).
|
||||
try:
|
||||
- script:
|
||||
content: bash ./check.sh
|
||||
@@ -1,691 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# L0 static check (offline, no cluster) — prod-baremetal-not-dev-surface
|
||||
# Serves: R-ENV-03 ("prod is a 3× mini-PC bare-metal k3s cluster with local storage; it
|
||||
# MUST NOT be an active development surface until the roadmap declares it ready."
|
||||
# — spec docs/00-vision-and-scope.md. Detail in docs/02: prod substrate "3× mini-PC
|
||||
# bare metal", "3 physical nodes", storage "local SSD/NVMe presented to Longhorn",
|
||||
# "For prod bare-metal, the *same* Ansible bootstrap runs; only the machine-creation
|
||||
# step differs", and "**Never develop against prod.** Prod hardware is a target, not
|
||||
# a sandbox (R-ENV-03)". The roadmap gate is roadmap/phases.md Phase 7: entry
|
||||
# "operator declares prod hardware ready"; note "Do not begin building on prod
|
||||
# hardware before this phase is explicitly opened".)
|
||||
#
|
||||
# HOW A "MUST NOT" IS MADE TESTABLE
|
||||
# --------------------------------
|
||||
# The second clause is a governance rule, and the same discipline applies as to R-TEST-06
|
||||
# (tests/l0-static/spec-immutability-enforced): "nobody developed against prod today"
|
||||
# cannot be observed, but the PROPERTY THAT MAKES DEVELOPING AGAINST PROD IMPOSSIBLE can
|
||||
# be. In this repo that property is structural — prod receives the base configuration plus
|
||||
# environment-scoped overrides and NOTHING ELSE, so there is no place to put prod-only work
|
||||
# even if someone wanted to. This check asserts that, together with the first clause's
|
||||
# authored facts (three nodes, no VM IaC behind them, node-local storage, the same
|
||||
# bootstrap code as dev).
|
||||
#
|
||||
# The first clause's HARDWARE facts (that the three machines really are mini-PCs, really
|
||||
# are bare metal, really hold local NVMe) are operator-attested, not machine-checkable —
|
||||
# see the gap list. What IS checkable is that the repository describes prod that way and
|
||||
# that no IaC creates prod nodes as virtual machines.
|
||||
#
|
||||
# GIVEN the repository as git tracks it (plus the read-only spec repo when mounted),
|
||||
# WHEN the prod inventory, the prod overlay, the IaC module and the roadmap record are
|
||||
# inspected — and, where offline execution is possible, the provisioning chain is
|
||||
# actually retargeted at prod with `--list-hosts`,
|
||||
# THEN (1) prod resolves to EXACTLY THREE nodes, reached directly on their own sshd
|
||||
# (no NAT indirection, i.e. real machines on the LAN);
|
||||
# (2) NO IaC resource creates any prod node: the VM module's generated node names are
|
||||
# disjoint from the prod host set and no prod hostname appears anywhere under
|
||||
# infra/tofu — prod is imaged by the operator (docs/02 stage 1), not instantiated;
|
||||
# (3) prod's storage is NODE-LOCAL: the Longhorn data path for prod is an absolute
|
||||
# local path, and the replica count equals the node count (one replica per node's
|
||||
# own disk). A distinct bulk/media tier is explicitly permitted (R-STATE-04) and
|
||||
# is not counted against this;
|
||||
# (4) prod runs THE SAME bootstrap code as dev: the same playbooks/roles, retargeted
|
||||
# by variables only — every prod group_vars key is an existing role/global
|
||||
# variable (an invented prod-only knob would be a fork), and no prod-only role
|
||||
# or playbook exists;
|
||||
# (5) prod is NOT a development surface: clusters/prod carries only the base plus
|
||||
# sync.yaml, every patch it applies targets an ENVIRONMENT-SCOPED object
|
||||
# (cluster-vars, a Flux Kustomization's sops decryption, or the Git source),
|
||||
# prod's Git ref is the SAME ref dev tracks (no prod-only branch to develop on),
|
||||
# and no prod-only manifest exists anywhere in the rendered configuration — a file
|
||||
# whose NAME carries the environment token is excused ONLY when the tree proves it
|
||||
# is an OVERLAY-SELECTED half that every environment applies (see the four
|
||||
# conditions at "THE ONE STRUCTURAL EXCEPTION" below); anything else still fails;
|
||||
# (6) prod artifacts exist only WITH a declaration of readiness: the spec roadmap
|
||||
# carries the phase that opens prod build (checked when the spec repo is mounted),
|
||||
# and the repository records that the operator opened it.
|
||||
#
|
||||
# WHAT THIS DOES *NOT* PROVE (honest gaps — do not read this test as claiming them):
|
||||
# - The HARDWARE. That the three nodes are mini-PCs, are bare metal, and present local
|
||||
# SSD/NVMe is operator-attested in infra/PROD-STANDUP.md / PROD-PROVISIONING.md; the
|
||||
# node-health and BIOS surveys that establish it write to gitignored artifacts, so
|
||||
# nothing tracked can be asserted against. Cluster-side, the Kubernetes API exposes no
|
||||
# "am I virtual" fact either — a bare-metal check would need a privileged host-path
|
||||
# probe of DMI data, which is a heavier and more invasive test than the claim warrants.
|
||||
# - The LIVE prod cluster's shape (three Ready nodes, all control-plane+etcd, Longhorn
|
||||
# disks at the local path). That is cluster state and belongs to R-PLAT-01's
|
||||
# (currently uncovered) HA test; asserting it here would also make this test
|
||||
# environment-specific, since the same suite must pass against dev.
|
||||
# - That prod's HISTORY contains no development. This is the working tree at HEAD.
|
||||
# - Whether the roadmap SHOULD have opened prod (the entry-gate judgement). Only that
|
||||
# prod work is accompanied by the operator's recorded declaration.
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT="$(cd "${DIR}/../../../.." && pwd)" # repo root
|
||||
PYTHON="${PYTHON:-python3}"
|
||||
: "${SPEC_REPO:=${ROOT}/../home-network-k3s}"
|
||||
export SPEC_REPO
|
||||
|
||||
cd "${ROOT}"
|
||||
"${PYTHON}" - "${ROOT}" <<'PY'
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError: # pragma: no cover
|
||||
print("check FAIL: PyYAML is required (same dependency as catalog/render.py)",
|
||||
file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
ROOT = Path(sys.argv[1])
|
||||
FAILURES: list[str] = []
|
||||
NOTES: list[str] = []
|
||||
CHECKS = 0
|
||||
|
||||
|
||||
def check(cond, msg):
|
||||
global CHECKS
|
||||
CHECKS += 1
|
||||
if not cond:
|
||||
FAILURES.append(msg)
|
||||
|
||||
|
||||
def note(msg):
|
||||
NOTES.append(msg)
|
||||
|
||||
|
||||
# A LOCAL DEPENDENCY the repo deliberately does not carry — the vault passphrase (kept
|
||||
# outside the repo, R-SEC-01) or the Galaxy collections (gitignored; installed from
|
||||
# collections/requirements.yml). Their absence is an ENVIRONMENT gap, not a repo defect,
|
||||
# and is the ONLY tolerated reason the driven sub-assertions may not run. Deliberately
|
||||
# narrow: every other non-zero exit is a real failure. The parse assertions always run.
|
||||
ENV_ONLY_FAILURE = re.compile(
|
||||
r"vault password|vault-id|Attempting to decrypt"
|
||||
r"|No module named 'ansible_collections|couldn't resolve module/action", re.I)
|
||||
|
||||
ANS = ROOT / "infra" / "ansible"
|
||||
INV = ANS / "inventory" / "hosts.ini"
|
||||
GV = ANS / "inventory" / "group_vars"
|
||||
TOFU = ROOT / "infra" / "tofu"
|
||||
check(INV.is_file(), "infra/ansible/inventory/hosts.ini is missing — prod is not described anywhere")
|
||||
if not INV.is_file():
|
||||
print("CHECK FAIL (prod-baremetal-not-dev-surface): no inventory", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
# --------------------------------------------------------------- inventory parsing
|
||||
ini_groups: dict[str, list[dict]] = {}
|
||||
children: dict[str, list[str]] = {}
|
||||
cur = None
|
||||
for line in INV.read_text().splitlines():
|
||||
s = line.strip()
|
||||
if not s or s.startswith("#"):
|
||||
continue
|
||||
m = re.match(r"^\[([^\]]+)\]$", s)
|
||||
if m:
|
||||
cur = m.group(1)
|
||||
continue
|
||||
if cur is None or cur.endswith(":vars"):
|
||||
continue
|
||||
if cur.endswith(":children"):
|
||||
children.setdefault(cur.split(":")[0], []).append(s.split()[0])
|
||||
continue
|
||||
parts = s.split()
|
||||
host = {"name": parts[0]}
|
||||
for kv in parts[1:]:
|
||||
if "=" in kv:
|
||||
k, v = kv.split("=", 1)
|
||||
host[k] = v
|
||||
ini_groups.setdefault(cur, []).append(host)
|
||||
|
||||
|
||||
def hosts_in(group: str, seen=None) -> list[dict]:
|
||||
seen = seen or set()
|
||||
if group in seen:
|
||||
return []
|
||||
seen.add(group)
|
||||
out = list(ini_groups.get(group, []))
|
||||
for child in children.get(group, []):
|
||||
out += hosts_in(child, seen)
|
||||
return out
|
||||
|
||||
|
||||
def load_gv(group: str) -> dict:
|
||||
d: dict = {}
|
||||
for p in [GV / f"{group}.yml", GV / group / "vars.yml"]:
|
||||
if p.is_file():
|
||||
try:
|
||||
d.update(yaml.safe_load(p.read_text()) or {})
|
||||
except Exception:
|
||||
pass
|
||||
return d
|
||||
|
||||
|
||||
# WHICH GROUP IS PROD? Derived, never named here: it is the group whose group_vars
|
||||
# RETARGETS the k3s roles (k3s_primary_group / k3s_servers_group) away from the global
|
||||
# dev default. That is exactly the seam docs/02 + infra/ansible/README describe for prod.
|
||||
ALL_VARS = load_gv("all")
|
||||
dev_primary_group = str(ALL_VARS.get("k3s_primary_group", ""))
|
||||
candidates = []
|
||||
for entry in sorted(GV.iterdir()) if GV.is_dir() else []:
|
||||
g = entry.name[:-4] if entry.name.endswith(".yml") else entry.name
|
||||
if g == "all":
|
||||
continue
|
||||
v = load_gv(g)
|
||||
if v.get("k3s_primary_group") and str(v["k3s_primary_group"]) != dev_primary_group:
|
||||
candidates.append((g, v))
|
||||
check(len(candidates) == 1,
|
||||
f"expected exactly ONE environment whose group_vars retargets the k3s roles (prod); found "
|
||||
f"{[c[0] for c in candidates]} — R-ENV-03 describes a single prod estate reached through "
|
||||
"the same roles as dev")
|
||||
if not candidates:
|
||||
print("CHECK FAIL (prod-baremetal-not-dev-surface): no prod group_vars found", file=sys.stderr)
|
||||
for f in FAILURES:
|
||||
print(f" - {f}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
PROD_GROUP, PROD_VARS = candidates[0]
|
||||
PROD_PRIMARY = str(PROD_VARS["k3s_primary_group"])
|
||||
prod_hosts = hosts_in(PROD_GROUP)
|
||||
prod_names = sorted(h["name"] for h in prod_hosts)
|
||||
|
||||
# ------------------------------------------------- (1) exactly three, directly reachable
|
||||
check(len(prod_names) == 3,
|
||||
f"prod group {PROD_GROUP!r} resolves to {len(prod_names)} host(s) {prod_names} — R-ENV-03 "
|
||||
"says prod is a 3× mini-PC cluster")
|
||||
dev_hosts = hosts_in(str(ALL_VARS.get("k3s_servers_group", "k3s_servers")))
|
||||
for h in prod_hosts:
|
||||
# Real machines on the LAN answer their own sshd. Dev's hosts are reached through a
|
||||
# NAT port-forward (ansible_port 225x -> a VM's :22), which is precisely the
|
||||
# indirection a hypervisor-hosted estate needs and bare metal does not.
|
||||
check(str(h.get("ansible_port", "22")) == "22",
|
||||
f"prod node {h['name']} is reached on port {h.get('ansible_port')} — a forwarded port "
|
||||
"implies an indirection (NAT to a virtual estate), not a machine on the LAN")
|
||||
check(h.get("k3s_node_ip") == h.get("ansible_host"),
|
||||
f"prod node {h['name']} has ansible_host={h.get('ansible_host')} but "
|
||||
f"k3s_node_ip={h.get('k3s_node_ip')} — a split address is the dev NAT pattern; a "
|
||||
"bare-metal node's SSH address IS its cluster address")
|
||||
|
||||
# ------------------------------------------------------ (2) nothing creates prod as a VM
|
||||
TF = "\n".join(p.read_text(errors="replace") for p in sorted(TOFU.glob("*.tf"))) \
|
||||
if TOFU.is_dir() else ""
|
||||
check(len(TF) > 500, "infra/tofu/*.tf is empty — cannot show that it does NOT create prod")
|
||||
|
||||
|
||||
def tf_default(var: str):
|
||||
m = re.search(r'variable\s+"' + re.escape(var) + r'"\s*\{(.*?)\n\}', TF, re.S)
|
||||
if not m:
|
||||
return None
|
||||
d = re.search(r"default\s*=\s*(.+)", m.group(1))
|
||||
return d.group(1).strip() if d else None
|
||||
|
||||
|
||||
prefix = (tf_default("node_name_prefix") or "").strip('"')
|
||||
try:
|
||||
node_count = int((tf_default("node_count") or "0").strip())
|
||||
except ValueError:
|
||||
node_count = 0
|
||||
iac_names = {f"{prefix}{i + 1}" for i in range(node_count)}
|
||||
check(len(iac_names) > 0, "the IaC module generates no node names — cannot compare with prod")
|
||||
check(iac_names.isdisjoint(set(prod_names)),
|
||||
f"the IaC VM module would create {sorted(iac_names & set(prod_names))}, which are PROD nodes "
|
||||
"— prod is bare metal that the operator images (docs/02 stage 1), never instantiated by the "
|
||||
"hypervisor module")
|
||||
for name in prod_names:
|
||||
check(name not in TF,
|
||||
f"prod node {name} is named inside infra/tofu — prod hardware must not be described by "
|
||||
"the VM module at all")
|
||||
# The prod host addresses must not appear as IaC node addresses either (a renamed VM is
|
||||
# still a VM).
|
||||
tf_ips = set(re.findall(r'"(\d+\.\d+\.\d+\.\d+)"', tf_default("node_ips") or ""))
|
||||
prod_ips = {str(h.get("ansible_host")) for h in prod_hosts}
|
||||
check(tf_ips.isdisjoint(prod_ips),
|
||||
f"the IaC module addresses {sorted(tf_ips & prod_ips)}, which are prod node addresses")
|
||||
|
||||
# --------------------------------------------------------------- (3) node-local storage
|
||||
# Every environment's value for a cluster-vars knob (base default + each overlay patch).
|
||||
def env_values(key: str) -> dict[str, str]:
|
||||
vals: dict[str, str] = {}
|
||||
base = ROOT / "clusters" / "base" / "cluster-vars.yaml"
|
||||
if base.is_file():
|
||||
for d in yaml.safe_load_all(base.read_text()):
|
||||
if isinstance(d, dict) and d.get("kind") == "ConfigMap" \
|
||||
and (d.get("metadata") or {}).get("name") == "cluster-vars":
|
||||
v = (d.get("data") or {}).get(key)
|
||||
if v is not None:
|
||||
vals["base"] = str(v)
|
||||
for overlay in sorted((ROOT / "clusters").glob("*/kustomization.yaml")):
|
||||
env = overlay.parent.name
|
||||
if env == "base":
|
||||
continue
|
||||
try:
|
||||
k = yaml.safe_load(overlay.read_text()) or {}
|
||||
except Exception:
|
||||
continue
|
||||
for p in (k.get("patches") or []):
|
||||
tgt = p.get("target") or {}
|
||||
if tgt.get("kind") != "ConfigMap" or tgt.get("name") != "cluster-vars":
|
||||
continue
|
||||
try:
|
||||
ops = yaml.safe_load(p.get("patch") or "") or []
|
||||
except Exception:
|
||||
continue
|
||||
for op in ops if isinstance(ops, list) else []:
|
||||
if isinstance(op, dict) and str(op.get("path", "")) == f"/data/{key}":
|
||||
vals[env] = str(op.get("value"))
|
||||
return vals
|
||||
|
||||
|
||||
# The prod OVERLAY is the one whose CLUSTER_ENV is not the base default.
|
||||
cluster_envs = env_values("CLUSTER_ENV")
|
||||
prod_overlays = [e for e, v in cluster_envs.items()
|
||||
if e != "base" and v != cluster_envs.get("base")]
|
||||
check(len(prod_overlays) >= 1,
|
||||
"no clusters/ overlay sets its own CLUSTER_ENV — prod has no overlay of its own")
|
||||
prod_overlay = None
|
||||
for e in prod_overlays:
|
||||
if PROD_GROUP.replace("k3s_", "") in cluster_envs[e] or cluster_envs[e] in PROD_GROUP:
|
||||
prod_overlay = e
|
||||
check(prod_overlay is not None,
|
||||
f"no overlay's CLUSTER_ENV corresponds to the prod inventory group {PROD_GROUP!r} — the "
|
||||
"cluster config and the machine inventory disagree about which estate is prod")
|
||||
|
||||
data_paths = env_values("LONGHORN_DATA_PATH")
|
||||
replicas = env_values("LONGHORN_DEFAULT_REPLICA_COUNT")
|
||||
if prod_overlay:
|
||||
p = data_paths.get(prod_overlay, data_paths.get("base", ""))
|
||||
check(p.startswith("/"),
|
||||
f"prod's LONGHORN_DATA_PATH is {p!r} — R-ENV-03's 'local storage' requires an absolute "
|
||||
"node-local filesystem path")
|
||||
check(not re.match(r"^/(net|nfs|mnt/nfs)|:", p),
|
||||
f"prod's LONGHORN_DATA_PATH {p!r} looks like a network mount, not local SSD/NVMe")
|
||||
r = replicas.get(prod_overlay, replicas.get("base", ""))
|
||||
check(r.isdigit() and int(r) == len(prod_names),
|
||||
f"prod's LONGHORN_DEFAULT_REPLICA_COUNT is {r!r} for {len(prod_names)} node(s) — "
|
||||
"'local storage' on a 3-node estate means one replica per node's own disk")
|
||||
# The DEFAULT storage class must be the local (Longhorn) driver. A separate bulk/media tier
|
||||
# on network storage is explicitly a different requirement (R-STATE-04, docs/04) and is not
|
||||
# counted here.
|
||||
default_scs = []
|
||||
for f in subprocess.run(["git", "-C", str(ROOT), "ls-files", "-z", "*.yaml", "*.yml"],
|
||||
capture_output=True, text=True, check=True).stdout.split("\0"):
|
||||
if not f:
|
||||
continue
|
||||
try:
|
||||
for d in yaml.safe_load_all((ROOT / f).read_text(errors="replace")):
|
||||
if isinstance(d, dict) and d.get("kind") == "StorageClass" and \
|
||||
str(((d.get("metadata") or {}).get("annotations") or {})
|
||||
.get("storageclass.kubernetes.io/is-default-class", "")) == "true":
|
||||
default_scs.append((f, str(d.get("provisioner", ""))))
|
||||
except Exception:
|
||||
continue
|
||||
check(len(default_scs) == 1,
|
||||
f"expected exactly one default StorageClass, found {default_scs} — the tier prod's workloads "
|
||||
"land on by default must be unambiguous")
|
||||
for f, prov in default_scs:
|
||||
check(prov == "driver.longhorn.io",
|
||||
f"{f}: the default StorageClass provisioner is {prov!r}, not Longhorn — prod's default "
|
||||
"tier would not be its own local disks")
|
||||
|
||||
# ------------------------------------- (4) prod runs the SAME bootstrap code, retargeted
|
||||
PLAYBOOKS = ANS / "playbooks"
|
||||
ROLES = ANS / "roles"
|
||||
prod_token = PROD_GROUP.replace("k3s_", "")
|
||||
for d, what in ((ROLES, "role"), (PLAYBOOKS, "playbook")):
|
||||
for entry in sorted(d.iterdir()) if d.is_dir() else []:
|
||||
check(prod_token not in entry.name.lower(),
|
||||
f"infra/ansible/{d.name}/{entry.name} is a prod-specific {what} — docs/02 requires "
|
||||
"the SAME Ansible bootstrap for prod, differing only in the machine-creation step")
|
||||
# Every prod group_vars key must be a variable the shared code already knows: a role default,
|
||||
# a global, or an ansible built-in. An invented key is a fork of the role by another name.
|
||||
known: set[str] = set(ALL_VARS)
|
||||
for rd in sorted(ROLES.iterdir()) if ROLES.is_dir() else []:
|
||||
for p in (rd / "defaults").glob("*.yml") if (rd / "defaults").is_dir() else []:
|
||||
try:
|
||||
known |= set((yaml.safe_load(p.read_text()) or {}).keys())
|
||||
except Exception:
|
||||
pass
|
||||
check(len(known) > 20, f"only {len(known)} shared variables found — (4) would prove nothing")
|
||||
for k in sorted(PROD_VARS):
|
||||
check(k.startswith("ansible_") or k in known,
|
||||
f"prod group_vars sets {k!r}, which is not a role default or a global — prod-only "
|
||||
"configuration that the dev path never exercises is developing against prod")
|
||||
# ...and it must retarget, not re-implement: the group it names must exist in the inventory.
|
||||
check(PROD_PRIMARY in ini_groups or PROD_PRIMARY in children,
|
||||
f"prod group_vars names primary group {PROD_PRIMARY!r}, which the inventory does not define")
|
||||
|
||||
# ---------------------------------------- (5) prod is not a development surface (overlay)
|
||||
if prod_overlay:
|
||||
odir = ROOT / "clusters" / prod_overlay
|
||||
files = sorted(p.name for p in odir.iterdir() if p.is_file())
|
||||
check(files == ["kustomization.yaml", "sync.yaml"],
|
||||
f"clusters/{prod_overlay}/ contains {files} — a prod overlay may hold only its "
|
||||
"kustomization and its own root sync; anything else is configuration authored FOR PROD")
|
||||
k = yaml.safe_load((odir / "kustomization.yaml").read_text()) or {}
|
||||
check(sorted(str(r) for r in (k.get("resources") or [])) == ["../base", "sync.yaml"],
|
||||
f"clusters/{prod_overlay} resources are {k.get('resources')} — prod must render the base "
|
||||
"plus its own sync only (docs/02: 'apply the prod overlay (no base edits)')")
|
||||
# Every patch must be an ENVIRONMENT-SCOPED override, never a change to what runs.
|
||||
ALLOWED = {("ConfigMap", "cluster-vars"), ("GitRepository", None)}
|
||||
for p in (k.get("patches") or []):
|
||||
tgt = p.get("target") or {}
|
||||
body = p.get("patch") or ""
|
||||
kind = tgt.get("kind")
|
||||
if kind is None: # inline patch: kind comes from the body
|
||||
try:
|
||||
doc = yaml.safe_load(body) or {}
|
||||
except Exception:
|
||||
doc = {}
|
||||
kind = doc.get("kind")
|
||||
name = (doc.get("metadata") or {}).get("name")
|
||||
if kind == "Kustomization":
|
||||
# Flux Kustomization patches are allowed ONLY to wire sops decryption —
|
||||
# anything else (path, interval, images, values) changes what prod RUNS.
|
||||
keys = set((doc.get("spec") or {}).keys())
|
||||
check(keys <= {"decryption"},
|
||||
f"clusters/{prod_overlay}: patch of Kustomization/{name} touches {sorted(keys)} "
|
||||
"— only `decryption` (the per-cluster sops key) is an environment-scoped "
|
||||
"override; changing anything else is developing against prod")
|
||||
continue
|
||||
check((kind, None) in ALLOWED or (kind, name) in ALLOWED,
|
||||
f"clusters/{prod_overlay}: patch targets {kind}/{name}, which is not an "
|
||||
"environment-scoped object — prod must not carry configuration of its own")
|
||||
else:
|
||||
check((kind, tgt.get("name")) in ALLOWED or (kind, None) in ALLOWED,
|
||||
f"clusters/{prod_overlay}: patch targets {kind}/{tgt.get('name')}, which is not "
|
||||
"an environment-scoped object — prod must not carry configuration of its own")
|
||||
|
||||
# Same Git ref as the other environment(s): a prod-only branch is a development line
|
||||
# that only prod runs, which is exactly what R-ENV-03 forbids.
|
||||
def git_ref(overlay: str):
|
||||
kk = yaml.safe_load((ROOT / "clusters" / overlay / "kustomization.yaml").read_text()) or {}
|
||||
for p in (kk.get("patches") or []):
|
||||
body = p.get("patch") or ""
|
||||
try:
|
||||
doc = yaml.safe_load(body) or {}
|
||||
except Exception:
|
||||
continue
|
||||
if doc.get("kind") == "GitRepository":
|
||||
return (doc.get("spec") or {}).get("ref")
|
||||
return None
|
||||
|
||||
others = [o.name for o in (ROOT / "clusters").iterdir()
|
||||
if o.is_dir() and o.name not in ("base", prod_overlay)]
|
||||
prod_ref = git_ref(prod_overlay)
|
||||
check(prod_ref is not None,
|
||||
f"clusters/{prod_overlay} does not pin a Git ref — prod would follow whatever the base "
|
||||
"points at, with no record of what it runs")
|
||||
for o in others:
|
||||
check(git_ref(o) == prod_ref,
|
||||
f"clusters/{o} tracks {git_ref(o)} but prod tracks {prod_ref} — a prod-only ref is a "
|
||||
"development line only prod runs (R-ENV-03: prod is a target, not a sandbox)")
|
||||
|
||||
# No prod-only manifest anywhere in the rendered configuration.
|
||||
tracked = [f for f in subprocess.run(["git", "-C", str(ROOT), "ls-files", "-z"],
|
||||
capture_output=True, text=True, check=True
|
||||
).stdout.split("\0") if f]
|
||||
RENDERED_DIRS = ("platform/", "platform-config", "apps/", "catalog/", "clusters/base/")
|
||||
|
||||
# THE ONE STRUCTURAL EXCEPTION: AN OVERLAY-SELECTED ENVIRONMENT HALF.
|
||||
# ------------------------------------------------------------------
|
||||
# A filename carrying an environment token is a HEURISTIC for the harm R-ENV-03 names
|
||||
# (content only prod runs, in a path every environment renders). It is not the harm
|
||||
# itself. There is exactly one shape where a `-prod` name is the OPPOSITE of a prod-only
|
||||
# fork: a credential that is TWO Secret objects, BOTH applied to BOTH clusters as base
|
||||
# resources, where a cluster-vars knob decides only which one the shared consumer
|
||||
# REFERENCES (clusters/base/cluster-vars.yaml, BACKUP_CREDENTIALS_SUFFIX: dev backs up to
|
||||
# MinIO, prod to Cloudflare R2, and one shared Secret meant either environment's rotation
|
||||
# broke the other). Nothing is forked: every environment renders the identical manifests
|
||||
# and the identical object set, which is precisely the R-ENV-02 contract.
|
||||
#
|
||||
# So the exception is not a path allowlist — it is the PROPERTY that makes that shape
|
||||
# safe, asserted from the tree, and a file is excused ONLY when all of it holds:
|
||||
#
|
||||
# (A) it declares nothing but Secrets whose names end in the environment suffix — the
|
||||
# only kind of object this repo has a selection mechanism for;
|
||||
# (1) it is listed in its own directory's kustomization.yaml, so EVERY environment
|
||||
# applies it, and every environment's value of the selector resolves to an object
|
||||
# that some resource of that same kustomization declares (same namespace) — no
|
||||
# environment is left selecting a Secret that does not exist;
|
||||
# (2) nothing references the suffixed name literally: every reference is a substituted
|
||||
# expression `<base-name>${VAR}`, and the suffixed FILE is named only by the
|
||||
# kustomization that applies it — so no shared manifest hardcodes prod's half;
|
||||
# (3) that VAR is defined for the base AND for every overlay (StrictPostBuildSubstitutions
|
||||
# fails on an UNDEFINED variable, so an overlay that forgot it would break that
|
||||
# cluster's reconciliation), and prod's value is the one that selects THIS file.
|
||||
#
|
||||
# References are read from PARSED YAML VALUES, never from raw text: the mechanism is
|
||||
# documented by name in comments beside the resources (cluster-vars says so explicitly),
|
||||
# and a name in a comment references nothing. Anything that fails any condition still
|
||||
# fails, naming the condition it broke.
|
||||
ENV_SUFFIX = f"-{prod_token}"
|
||||
# Every environment that renders the shared configuration.
|
||||
ENVS = ["base"] + sorted(p.parent.name
|
||||
for p in (ROOT / "clusters").glob("*/kustomization.yaml")
|
||||
if p.parent.name != "base")
|
||||
|
||||
|
||||
def _docs(rel: str):
|
||||
p = ROOT / rel
|
||||
if not p.is_file():
|
||||
return None
|
||||
try:
|
||||
return [d for d in yaml.safe_load_all(p.read_text(errors="replace"))
|
||||
if isinstance(d, dict)]
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _strings(node):
|
||||
if isinstance(node, str):
|
||||
yield node
|
||||
elif isinstance(node, dict):
|
||||
for k, v in node.items():
|
||||
yield from _strings(k)
|
||||
yield from _strings(v)
|
||||
elif isinstance(node, (list, tuple)):
|
||||
for v in node:
|
||||
yield from _strings(v)
|
||||
|
||||
|
||||
RENDERED_YAML = [f for f in tracked
|
||||
if f.startswith(RENDERED_DIRS) and f.endswith((".yaml", ".yml"))]
|
||||
VALUE_STRINGS = {f: list(_strings(_docs(f) or [])) for f in RENDERED_YAML}
|
||||
|
||||
|
||||
def overlay_selected(f: str) -> list[str]:
|
||||
"""[] if f is an overlay-SELECTED environment half; else the conditions it breaks."""
|
||||
docs = _docs(f)
|
||||
objs = [((d.get("kind")), (d.get("metadata") or {}).get("name"),
|
||||
(d.get("metadata") or {}).get("namespace")) for d in (docs or [])]
|
||||
if not objs or any(k != "Secret" or not isinstance(n, str) or not n.endswith(ENV_SUFFIX)
|
||||
for k, n, _ in objs):
|
||||
return [f"(A) it does not consist solely of Secret objects whose names end in "
|
||||
f"{ENV_SUFFIX!r} — an overlay can select which SECRET a shared manifest "
|
||||
f"references, and nothing else, so any other prod-suffixed object in a "
|
||||
f"shared path is prod-only content"]
|
||||
|
||||
reasons: list[str] = []
|
||||
d = str(Path(f).parent)
|
||||
kfile = f"{d}/kustomization.yaml"
|
||||
kdocs = _docs(kfile) or []
|
||||
res = [str(r) for r in ((kdocs[0].get("resources") if kdocs else None) or [])]
|
||||
if Path(f).name not in res:
|
||||
reasons.append(f"(1) {kfile} does not list it in `resources`, so it is not applied "
|
||||
f"in every environment — one environment would carry an object the "
|
||||
f"other does not")
|
||||
|
||||
# Every Secret the applying kustomization declares: name -> namespace.
|
||||
declared: dict[str, object] = {}
|
||||
for r in res:
|
||||
for dd in (_docs(f"{d}/{r}") or []):
|
||||
if dd.get("kind") == "Secret":
|
||||
md = dd.get("metadata") or {}
|
||||
declared[str(md.get("name"))] = md.get("namespace")
|
||||
|
||||
for _, name, ns in objs:
|
||||
base_name = name[:-len(ENV_SUFFIX)]
|
||||
pat = re.escape(base_name) + r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}"
|
||||
sel = sorted({m for ss in VALUE_STRINGS.values() for s in ss
|
||||
for m in re.findall(pat, s)})
|
||||
# (2) literal references to the suffixed OBJECT — anywhere but this file, except
|
||||
# the applying kustomization naming this FILE (which is how it gets applied).
|
||||
lit = sorted({f2 for f2, ss in VALUE_STRINGS.items() for s in ss
|
||||
if name in s and f2 != f
|
||||
and not (f2 == kfile and s == Path(f).name)})
|
||||
if lit:
|
||||
reasons.append(f"(2) {', '.join(lit)} names the Secret {name!r} literally — a "
|
||||
f"shared manifest that hardcodes prod's half makes every "
|
||||
f"environment use prod's credential; reference "
|
||||
f"{base_name}${{...}} instead")
|
||||
# (2) literal references to the suffixed FILE from anywhere but its kustomization.
|
||||
stray = sorted({f2 for f2, ss in VALUE_STRINGS.items() for s in ss
|
||||
if Path(f).name in s and f2 not in (f, kfile)})
|
||||
if stray:
|
||||
reasons.append(f"(2) {', '.join(stray)} names the file {Path(f).name!r} "
|
||||
f"literally — only {kfile} may apply it")
|
||||
if not sel:
|
||||
reasons.append(f"(2) no manifest references {base_name}${{...}}: nothing selects "
|
||||
f"between the halves, so this file is prod-only content that "
|
||||
f"only prod could ever consume")
|
||||
continue
|
||||
if len(sel) > 1:
|
||||
reasons.append(f"(2) {base_name} is selected by more than one variable {sel} — "
|
||||
f"they can drift apart and leave one consumer on the other "
|
||||
f"environment's credential")
|
||||
var = sel[0]
|
||||
vals = env_values(var)
|
||||
missing = [e for e in ENVS if e not in vals]
|
||||
if missing:
|
||||
reasons.append(f"(3) ${{{var}}} is not defined for {missing} (defined for "
|
||||
f"{sorted(vals)}) — StrictPostBuildSubstitutions fails on an "
|
||||
f"undefined variable, so that environment would stop "
|
||||
f"reconciling, or silently inherit the other's credential")
|
||||
for e, v in sorted(vals.items()):
|
||||
want = base_name + v
|
||||
if want not in declared:
|
||||
reasons.append(f"(1) environment {e!r} selects Secret {want!r} via "
|
||||
f"${{{var}}}={v!r}, which no resource of {kfile} declares — "
|
||||
f"that environment renders a dangling reference")
|
||||
elif declared[want] != ns:
|
||||
reasons.append(f"(1) environment {e!r} selects Secret {want!r}, declared in "
|
||||
f"namespace {declared[want]!r}, but this half is in {ns!r} — "
|
||||
f"the halves are not interchangeable")
|
||||
if prod_overlay and prod_overlay in vals and base_name + vals[prod_overlay] != name:
|
||||
reasons.append(f"(3) the prod overlay's ${{{var}}}={vals[prod_overlay]!r} selects "
|
||||
f"{base_name + vals[prod_overlay]!r}, not this file's {name!r} — "
|
||||
f"prod's half is applied but no environment selects it")
|
||||
return reasons
|
||||
|
||||
|
||||
flagged = [f for f in tracked
|
||||
if f.startswith(RENDERED_DIRS)
|
||||
and re.search(rf"(^|[/_.-]){re.escape(prod_token)}([/_.-]|$)", f, re.I)]
|
||||
for f in flagged:
|
||||
why = overlay_selected(f)
|
||||
check(not why,
|
||||
f"{f} is a prod-specific file inside the shared configuration — every environment "
|
||||
f"renders these paths, so prod-only content here is development against prod. It is "
|
||||
f"not an overlay-selected environment half either: " + " AND ".join(why))
|
||||
if not why:
|
||||
note(f"{f} carries the {prod_token!r} token but is an OVERLAY-SELECTED half: both "
|
||||
f"halves are applied in every environment and only the cluster-vars selector "
|
||||
f"differs (R-ENV-02), so it is not prod-only content")
|
||||
check(len(tracked) > 50, f"git ls-files returned only {len(tracked)} paths — not a real checkout")
|
||||
|
||||
# --------------------------- (6) prod work exists only with a declaration of readiness
|
||||
# In-repo: the operator's recorded decision to open the prod phase.
|
||||
log = ROOT / "DECISIONS-LOG.md"
|
||||
check(log.is_file() and re.search(r"PHASE\s*7\b.*OPEN", log.read_text(errors="replace"), re.I),
|
||||
"DECISIONS-LOG.md carries no record of the operator OPENING the prod phase, yet prod "
|
||||
"inventory/overlay exist — R-ENV-03 allows prod build only once the roadmap declares it ready")
|
||||
# In the spec (read-only, may not be mounted): the roadmap phase that opens prod.
|
||||
spec = Path(os.environ.get("SPEC_REPO", ""))
|
||||
phases = spec / "roadmap" / "phases.md"
|
||||
if phases.is_file():
|
||||
txt = phases.read_text(errors="replace")
|
||||
check(re.search(r"##\s*Phase\s*7\b", txt) is not None and
|
||||
re.search(r"prod", txt, re.I) is not None,
|
||||
"the spec roadmap has no Phase 7 opening prod build — the declaration R-ENV-03 defers "
|
||||
"to does not exist")
|
||||
check(re.search(r"explicitly opened|declares prod hardware ready|operator declares", txt, re.I)
|
||||
is not None,
|
||||
"the spec roadmap's prod phase states no explicit entry declaration — the gate R-ENV-03 "
|
||||
"names would be unenforceable")
|
||||
else:
|
||||
note(f"spec repo not mounted at {spec} — the roadmap half of (6) was NOT checked "
|
||||
"(the in-repo declaration still was)")
|
||||
|
||||
# --------------------------- (4b) DRIVE the retarget offline: same chain, prod hosts
|
||||
if shutil.which("ansible-playbook") is None:
|
||||
note("ansible-playbook is not installed here (e.g. the in-cluster chainsaw image), so the "
|
||||
"prod retarget was PARSED but not DRIVEN; the merge gate's container has it")
|
||||
else:
|
||||
sec_groups = [g for g in children.get(PROD_GROUP, []) if g != PROD_PRIMARY]
|
||||
args = ["--list-hosts", "playbooks/site.yml",
|
||||
"-e", f"k3s_hosts={PROD_GROUP}",
|
||||
"-e", f"k3s_primary_hosts={PROD_PRIMARY}"]
|
||||
if sec_groups:
|
||||
args += ["-e", f"k3s_secondary_hosts={sec_groups[0]}"]
|
||||
lh = subprocess.run(["ansible-playbook", *args], cwd=ANS, capture_output=True, text=True)
|
||||
out = (lh.stderr or "") + (lh.stdout or "")
|
||||
if lh.returncode != 0 and ENV_ONLY_FAILURE.search(out):
|
||||
# See the same narrow, self-policing skip in dev-nodes-provisioned-as-code.
|
||||
note("ansible-playbook lacks a LOCAL dependency kept outside the repo by design (the vault "
|
||||
"passphrase and/or the gitignored Galaxy collections), so the prod retarget was not "
|
||||
"driven; every parse assertion above still ran")
|
||||
else:
|
||||
check(lh.returncode == 0,
|
||||
f"retargeting the SHARED chain at prod failed: {out.strip()[:400]} — docs/02 requires "
|
||||
"the same Ansible bootstrap to run for prod")
|
||||
seen: set[str] = set()
|
||||
in_hosts = False
|
||||
for line in lh.stdout.splitlines():
|
||||
if re.match(r"\s*hosts \(\d+\):", line):
|
||||
in_hosts = True
|
||||
continue
|
||||
if re.search(r"play #\d+", line):
|
||||
in_hosts = False
|
||||
continue
|
||||
if in_hosts and line.strip():
|
||||
seen.add(line.strip())
|
||||
check(seen == set(prod_names),
|
||||
f"the shared chain retargeted at prod resolves to {sorted(seen)}, not the three prod "
|
||||
f"nodes {prod_names} — prod would be bootstrapped by something other than dev's path")
|
||||
|
||||
# --------------------------------------------------------------------------- verdict
|
||||
check(CHECKS >= 25,
|
||||
f"only {CHECKS} assertions ran — the check degenerated to a near-no-op and its pass "
|
||||
"would be meaningless")
|
||||
|
||||
if FAILURES:
|
||||
print(f"CHECK FAIL (prod-baremetal-not-dev-surface): {len(FAILURES)} of {CHECKS} assertions failed",
|
||||
file=sys.stderr)
|
||||
for f in FAILURES:
|
||||
print(f" - {f}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
for n in NOTES:
|
||||
print(f" note: {n}")
|
||||
print(f"check OK: prod-baremetal-not-dev-surface — {CHECKS} assertions; prod group {PROD_GROUP!r} = "
|
||||
f"{prod_names} (3 nodes, none created by IaC), overlay clusters/{prod_overlay} renders the base "
|
||||
f"plus environment-scoped patches only, local Longhorn path "
|
||||
f"{data_paths.get(prod_overlay, data_paths.get('base'))!r} with "
|
||||
f"{replicas.get(prod_overlay, replicas.get('base'))} replica(s). R-ENV-03's AUTHORED clauses "
|
||||
"only — the hardware facts (mini-PC, bare metal, local NVMe) are operator-attested and the "
|
||||
"live cluster's shape is NOT asserted here (see the header's gap list).")
|
||||
PY
|
||||
@@ -1,69 +0,0 @@
|
||||
# =============================================================================
|
||||
# L0 STATIC — certificate lifecycle needs no manual step (R-TLS-02) [EXECUTABLE]
|
||||
# Catalog: docs/14 L0 row ("Pre-apply, in CI") — runs WITHOUT a cluster.
|
||||
# Spec: docs/00 R-TLS-02 ("Certificates MUST be automatically issued, renewed, and
|
||||
# revoked without manual steps"); mechanism in docs/08 + ADR-0006.
|
||||
#
|
||||
# R-TLS-02 is one adverbial clause plus three runtime verbs. This test asserts the
|
||||
# clause — "without manual steps" — which in a GitOps platform is a property of the
|
||||
# REPOSITORY: a human step in the certificate path leaves evidence in git (committed
|
||||
# cert material, a Gateway serving a Secret nothing declares, an issuer with no
|
||||
# automated solver, a credential never delivered, auto-approval switched off, or CRs
|
||||
# nothing reconciles). The three verbs are observed by the companion L2 test.
|
||||
#
|
||||
# ENVIRONMENT-AGNOSTIC (R-ENV-02): the issuer a Certificate names is `${ACME_CLUSTER_ISSUER}`,
|
||||
# so the check resolves it against the BASE cluster-vars default AND every overlay's value and
|
||||
# requires each resolved issuer to exist. dev's issuer and prod's issuer are both verified from
|
||||
# one authored assertion, with no hostname, address or environment name written down.
|
||||
#
|
||||
# GIVEN the repository as git tracks it,
|
||||
# WHEN every serving-TLS reference, Certificate, Issuer and the cert-manager release
|
||||
# is inspected offline,
|
||||
# THEN every Gateway listener that terminates TLS serves a Secret a tracked Certificate
|
||||
# produces; no certificate material is committed anywhere; every Certificate's
|
||||
# issuer resolves — in EVERY environment — to a tracked ACME issuer with an
|
||||
# automated dns01/http01 solver whose credential is SOPS-delivered from git;
|
||||
# cert-manager's automatic request approval is not disabled; and every one of
|
||||
# those objects is rendered by a Flux Kustomization, so nothing in the chain
|
||||
# depends on a human running kubectl.
|
||||
#
|
||||
# STATUS: EXECUTABLE. `check.sh` is a pure offline assertion script (git + PyYAML, no
|
||||
# cluster, no network); conformance/run.sh --layer l0 and ci/l0.sh run it directly, and
|
||||
# this Chainsaw Test wraps the SAME script.
|
||||
#
|
||||
# KNOWN, DELIBERATE GAPS (named in check.sh's header too — this test claims NONE of them):
|
||||
# * ISSUED / RENEWED as runtime events: L2 (l2-behavioral/cert-lifecycle-automated).
|
||||
# * REVOKED: not proven at any layer today. cert-manager performs no ACME revocation,
|
||||
# and the spec's own reading (docs/08: "removing/rotating a certificate is a
|
||||
# declarative change") needs `enableCertificateOwnerRef`, which the tracked
|
||||
# cert-manager HelmRelease does not set — so a removed Certificate leaves its
|
||||
# serving Secret behind. The L2 test asserts the withdrawal behaviour and is
|
||||
# EXPECTED TO FAIL until that knob is set; escalated, not softened (R-TEST-06).
|
||||
# * That the live cluster is serving the tracked Certificate (cluster state).
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: tls-lifecycle-automated
|
||||
labels:
|
||||
conformance.home.k3s/R-TLS-02: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-TLS-02"
|
||||
conformance.home.k3s/layer: "L0"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
Given the tracked tree, when the certificate path is inspected offline, then no
|
||||
manual step exists anywhere in it: serving Secrets are cert-manager-produced, no
|
||||
cert material is committed, every environment's issuer is a tracked ACME issuer
|
||||
with an automated solver and a git-delivered credential, auto-approval is on, and
|
||||
the whole chain is Flux-reconciled.
|
||||
spec:
|
||||
steps:
|
||||
- name: then-no-manual-step-exists-in-the-certificate-path
|
||||
description: >-
|
||||
Run the offline assertion script over the tracked tree (same script ci/l0.sh
|
||||
and conformance/run.sh --layer l0 execute).
|
||||
try:
|
||||
- script:
|
||||
content: bash ./check.sh
|
||||
@@ -1,478 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# L0 static check (offline, no cluster) — tls-lifecycle-automated
|
||||
# Serves: R-TLS-02 ("Certificates MUST be automatically **issued, renewed, and revoked**
|
||||
# without manual steps." — spec docs/00-vision-and-scope.md; mechanism in
|
||||
# docs/08-tls-and-certificates.md, ADR-0006.)
|
||||
#
|
||||
# WHICH CLAUSE THIS LAYER CAN PROVE — AND WHICH IT CANNOT
|
||||
# ------------------------------------------------------
|
||||
# R-TLS-02 has one adverbial clause ("without manual steps") and three verbs
|
||||
# ("issued", "renewed", "revoked"). In a GitOps platform the *without manual steps*
|
||||
# clause IS a repository property, and it is the half that can be proven with no
|
||||
# cluster: if a human had to mint, install, or approve a serving certificate, the
|
||||
# evidence of that human step would be IN THE REPO — a committed `tls.crt`, a Gateway
|
||||
# pointing at a Secret nothing declares, an issuer that no automated solver can
|
||||
# satisfy, a per-cert credential handed over out of band, or an admission path that
|
||||
# waits for a manual CertificateRequest approval. This check asserts that none of
|
||||
# those exist and that the automated path is completely wired instead.
|
||||
#
|
||||
# The three VERBS are runtime events. They are asserted by the companion L2 test
|
||||
# (conformance/tests/l2-behavioral/cert-lifecycle-automated), which observes an
|
||||
# issuance, an unattended renewal, and the withdrawal of a removed certificate. This
|
||||
# check does NOT claim them; see "WHAT THIS DOES *NOT* PROVE" below.
|
||||
#
|
||||
# GIVEN the repository as git tracks it,
|
||||
# WHEN every serving-TLS reference, Certificate, Issuer and the cert-manager release
|
||||
# are inspected offline,
|
||||
# THEN (1) at least one cert-manager `Certificate` is authored (vacuity floor: a repo
|
||||
# with no Certificate cannot pass this check by having nothing to check);
|
||||
# (2) every Gateway HTTPS listener that TERMINATES TLS references ONLY Secrets
|
||||
# that a tracked Certificate declares as its `secretName` — so the serving
|
||||
# credential is produced by the controller, never dropped in by hand;
|
||||
# (3) NO certificate material is tracked anywhere: no Secret manifest of type
|
||||
# kubernetes.io/tls, no tls.crt/tls.key/ca.crt key in any committed Secret
|
||||
# (plain OR SOPS-encrypted), and no PEM certificate body in any tracked
|
||||
# YAML — i.e. issuance is not, and cannot silently become, a human step;
|
||||
# (4) every Certificate's `issuerRef` resolves to a TRACKED issuer IN EVERY
|
||||
# ENVIRONMENT — the `${ACME_CLUSTER_ISSUER}` token is resolved against the
|
||||
# base cluster-vars default AND every overlay's value, so an environment
|
||||
# whose issuer does not exist is caught here rather than at prod bring-up;
|
||||
# (5) every issuer so referenced is an ACME issuer with a NON-EMPTY solver list
|
||||
# whose every solver is an automated challenge type (dns01 / http01) — the
|
||||
# same code path serves issuance AND renewal, so a solver a human must
|
||||
# complete would defeat both;
|
||||
# (6) each dns01 solver's provider credential is referenced BY NAME and that
|
||||
# Secret is delivered SOPS-encrypted from git via a kustomization — the
|
||||
# operator's one-time token hand-off (docs/08 OQ-4), never a per-cert step;
|
||||
# (7) cert-manager's automatic CertificateRequest approval is NOT disabled
|
||||
# (`disableAutoApproval`), and any `approveSignerNames` narrowing still
|
||||
# covers the ClusterIssuers actually used — a request that waits for a
|
||||
# human approver is the definition of a manual step;
|
||||
# (8) the whole chain is reconciled from git — cert-manager's HelmRelease, the
|
||||
# ClusterIssuers and the Certificate are each included in a kustomization
|
||||
# that a Flux Kustomization renders — so a deleted Certificate is
|
||||
# re-created and re-issued with no human action;
|
||||
# (9) no Certificate defeats renewal arithmetically (`renewBefore` >= `duration`
|
||||
# would mean the renewal instant never arrives).
|
||||
#
|
||||
# WHAT THIS DOES *NOT* PROVE (honest gaps — do not read this test as claiming them):
|
||||
# - ISSUED: that an issuance actually happens. Asserted at L2 (step 2 of
|
||||
# l2-behavioral/cert-lifecycle-automated) and, for the real ACME/DNS-01 path,
|
||||
# evidenced by l2-behavioral/trusted-tls serving a publicly-trusted cert.
|
||||
# - RENEWED: that a renewal actually happens unattended. docs/08's acceptance gate
|
||||
# is explicit that this must be "observed across a renewal" — L2 step 3 does that
|
||||
# by fast-forwarding a certificate's renewal window; nothing static can.
|
||||
# - REVOKED: NOT PROVEN AT ANY LAYER TODAY, and not asserted here. Two distinct
|
||||
# readings, neither established: (a) ACME revocation — cert-manager does not
|
||||
# revoke certificates at all, so no repo state could show it; (b) the spec's own
|
||||
# reading in docs/08 ("removing/rotating a certificate is a declarative change")
|
||||
# — withdrawal of a REMOVED certificate's serving material requires cert-manager's
|
||||
# `enableCertificateOwnerRef`, which the tracked HelmRelease does NOT set (chart
|
||||
# default false). L2 step 4 asserts the withdrawal behaviour and is expected to
|
||||
# FAIL until that knob is set; escalated to the PM rather than softened (R-TEST-06).
|
||||
# - That the tracked Certificate is the one the live Gateway is actually serving
|
||||
# (cluster state), or that the ACME account/DNS credential is still valid.
|
||||
# - Certificate EXPIRY ALERTING, which docs/08 delegates to R-OBS-04 (see
|
||||
# l3-fault-injection/alerting).
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT="$(cd "${DIR}/../../../.." && pwd)" # repo root
|
||||
PYTHON="${PYTHON:-python3}"
|
||||
|
||||
cd "${ROOT}"
|
||||
"${PYTHON}" - "${ROOT}" <<'PY'
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError: # pragma: no cover
|
||||
print("check FAIL: PyYAML is required (same dependency as catalog/render.py)",
|
||||
file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
ROOT = Path(sys.argv[1])
|
||||
FAILURES: list[str] = []
|
||||
CHECKS = 0
|
||||
|
||||
|
||||
def check(cond, msg):
|
||||
global CHECKS
|
||||
CHECKS += 1
|
||||
if not cond:
|
||||
FAILURES.append(msg)
|
||||
|
||||
|
||||
def tracked() -> list[str]:
|
||||
out = subprocess.run(["git", "-C", str(ROOT), "ls-files", "-z"],
|
||||
capture_output=True, text=True, check=True).stdout
|
||||
return [p for p in out.split("\0") if p]
|
||||
|
||||
|
||||
FILES = tracked()
|
||||
check(len(FILES) > 50, f"git ls-files returned only {len(FILES)} paths — not a real checkout")
|
||||
YAML_FILES = [f for f in FILES if re.search(r"\.(ya?ml)$", f)]
|
||||
|
||||
|
||||
def docs_of(rel: str):
|
||||
"""Parsed YAML documents of a tracked file, or [] when it is not parseable YAML."""
|
||||
try:
|
||||
return [d for d in yaml.safe_load_all((ROOT / rel).read_text(errors="replace"))
|
||||
if isinstance(d, dict)]
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
DOCS: list[tuple[str, dict]] = [(rel, d) for rel in YAML_FILES for d in docs_of(rel)]
|
||||
|
||||
|
||||
def of_kind(kind: str, group_prefix: str = "") -> list[tuple[str, dict]]:
|
||||
out = []
|
||||
for rel, d in DOCS:
|
||||
if d.get("kind") != kind:
|
||||
continue
|
||||
if group_prefix and not str(d.get("apiVersion", "")).startswith(group_prefix):
|
||||
continue
|
||||
out.append((rel, d))
|
||||
return out
|
||||
|
||||
|
||||
def ns_of(d: dict, default: str = "default") -> str:
|
||||
return (d.get("metadata") or {}).get("namespace") or default
|
||||
|
||||
|
||||
def name_of(d: dict) -> str:
|
||||
return (d.get("metadata") or {}).get("name") or "<unnamed>"
|
||||
|
||||
|
||||
# ------------------------------------------------------------ cluster-vars resolution
|
||||
# Every environment's value for a ${TOKEN}: the base default plus each overlay's JSON6902
|
||||
# `- op: replace / path: /data/KEY / value: ...` patch on the cluster-vars ConfigMap. This
|
||||
# is the same mechanism flux-postbuild-substitution audits; here we need the VALUES, so an
|
||||
# issuer name can be resolved per environment instead of being asserted for dev only.
|
||||
def env_values(key: str) -> dict[str, str]:
|
||||
vals: dict[str, str] = {}
|
||||
base = ROOT / "clusters" / "base" / "cluster-vars.yaml"
|
||||
if base.is_file():
|
||||
for d in yaml.safe_load_all(base.read_text()):
|
||||
if isinstance(d, dict) and d.get("kind") == "ConfigMap" \
|
||||
and name_of(d) == "cluster-vars":
|
||||
v = (d.get("data") or {}).get(key)
|
||||
if v is not None:
|
||||
vals["base"] = str(v)
|
||||
for overlay in sorted((ROOT / "clusters").glob("*/kustomization.yaml")):
|
||||
env = overlay.parent.name
|
||||
if env == "base":
|
||||
continue
|
||||
try:
|
||||
k = yaml.safe_load(overlay.read_text()) or {}
|
||||
except Exception:
|
||||
continue
|
||||
for p in (k.get("patches") or []):
|
||||
tgt = p.get("target") or {}
|
||||
if tgt.get("kind") != "ConfigMap" or tgt.get("name") != "cluster-vars":
|
||||
continue
|
||||
try:
|
||||
ops = yaml.safe_load(p.get("patch") or "") or []
|
||||
except Exception:
|
||||
continue
|
||||
for op in ops if isinstance(ops, list) else []:
|
||||
if isinstance(op, dict) and str(op.get("path", "")) == f"/data/{key}":
|
||||
vals[env] = str(op.get("value"))
|
||||
return vals
|
||||
|
||||
|
||||
TOKEN_RE = re.compile(r"^\$\{([A-Z0-9_]+)\}$")
|
||||
|
||||
|
||||
def resolve_per_env(value: str) -> dict[str, str]:
|
||||
"""A literal resolves to itself in every environment; a ${TOKEN} to each env's value."""
|
||||
m = TOKEN_RE.match(value.strip())
|
||||
if not m:
|
||||
return {"(literal)": value.strip()}
|
||||
return env_values(m.group(1))
|
||||
|
||||
|
||||
# --------------------------------------------- what Flux actually renders (reachability)
|
||||
# A manifest is DELIVERED only if it is reachable from a Flux Kustomization's `path` through
|
||||
# the kustomize resource graph: <path>/kustomization.yaml -> resources -> (file | directory
|
||||
# -> that directory's kustomization.yaml -> ...). Membership is deliberately computed by
|
||||
# walking that graph rather than by asking "is this file's DIRECTORY mentioned somewhere":
|
||||
# the loose form silently passes when an entry is deleted from the very kustomization.yaml
|
||||
# that the directory reference renders (verified — it made two assertions here no-ops).
|
||||
KUST_RESOURCES: dict[str, list[str]] = {}
|
||||
for rel in [f for f in YAML_FILES if f.endswith("kustomization.yaml")]:
|
||||
refs: list[str] = []
|
||||
for d in docs_of(rel):
|
||||
for r in (d.get("resources") or []) + (d.get("components") or []):
|
||||
if not isinstance(r, str) or "://" in r: # skip remote bases
|
||||
continue
|
||||
try:
|
||||
p = (ROOT / rel).parent.joinpath(r).resolve()
|
||||
refs.append(str(p.relative_to(ROOT)))
|
||||
except Exception:
|
||||
continue
|
||||
KUST_RESOURCES[rel] = refs
|
||||
|
||||
FLUX_PATHS: set[str] = set()
|
||||
for _rel, _d in DOCS:
|
||||
if _d.get("kind") == "Kustomization" and \
|
||||
str(_d.get("apiVersion", "")).startswith("kustomize.toolkit.fluxcd.io/"):
|
||||
p = str((_d.get("spec") or {}).get("path", "")).lstrip("./").rstrip("/")
|
||||
if p:
|
||||
FLUX_PATHS.add(p)
|
||||
|
||||
|
||||
def _walk(entry: str, seen: set[str], out: set[str]) -> None:
|
||||
if entry in seen:
|
||||
return
|
||||
seen.add(entry)
|
||||
kust = entry if entry.endswith("kustomization.yaml") else f"{entry}/kustomization.yaml"
|
||||
if kust in KUST_RESOURCES: # a directory (or a kustomization)
|
||||
for r in KUST_RESOURCES[kust]:
|
||||
_walk(r, seen, out)
|
||||
return
|
||||
out.add(entry) # a leaf manifest
|
||||
|
||||
|
||||
RENDERED: set[str] = set()
|
||||
for _p in FLUX_PATHS:
|
||||
_walk(_p, set(), RENDERED)
|
||||
|
||||
|
||||
def delivered(rel: str) -> bool:
|
||||
"""True iff Flux renders this file from one of its Kustomization paths."""
|
||||
return rel in RENDERED
|
||||
|
||||
|
||||
# --------------------------------------------------- (1) there is something to check
|
||||
CERTS = of_kind("Certificate", "cert-manager.io/")
|
||||
check(len(CERTS) >= 1,
|
||||
"no cert-manager.io Certificate is authored anywhere in the tracked tree — every "
|
||||
"assertion below would be vacuous, so this is a FAILURE, not a pass (R-TLS-02)")
|
||||
|
||||
ISSUERS = of_kind("ClusterIssuer", "cert-manager.io/") + of_kind("Issuer", "cert-manager.io/")
|
||||
check(len(ISSUERS) >= 1,
|
||||
"no cert-manager ClusterIssuer/Issuer is authored — nothing could issue automatically")
|
||||
|
||||
# ---------------------------- (2) every terminating listener serves a controller cert
|
||||
CERT_SECRETS: dict[tuple[str, str], str] = { # (namespace, secretName) -> file
|
||||
(ns_of(d, "default"), str((d.get("spec") or {}).get("secretName", ""))): rel
|
||||
for rel, d in CERTS
|
||||
}
|
||||
listeners_checked = 0
|
||||
for rel, gw in of_kind("Gateway", "gateway.networking.k8s.io/"):
|
||||
gw_ns = ns_of(gw)
|
||||
for lis in ((gw.get("spec") or {}).get("listeners") or []):
|
||||
tls = lis.get("tls") or {}
|
||||
if str(tls.get("mode", "")) != "Terminate":
|
||||
continue
|
||||
listeners_checked += 1
|
||||
refs = tls.get("certificateRefs") or []
|
||||
check(len(refs) > 0,
|
||||
f"{rel}: Gateway {gw_ns}/{name_of(gw)} listener {lis.get('name')!r} terminates "
|
||||
"TLS with NO certificateRefs — it can only serve a certificate someone puts "
|
||||
"there by hand")
|
||||
for r in refs:
|
||||
kind = str(r.get("kind", "Secret"))
|
||||
key = (str(r.get("namespace") or gw_ns), str(r.get("name")))
|
||||
check(kind == "Secret",
|
||||
f"{rel}: listener {lis.get('name')!r} certificateRef kind is {kind!r}; only "
|
||||
"Secret refs can be traced to a cert-manager Certificate")
|
||||
check(key in CERT_SECRETS,
|
||||
f"{rel}: listener {lis.get('name')!r} serves Secret {key[0]}/{key[1]}, which "
|
||||
"NO tracked cert-manager Certificate declares as its secretName — that "
|
||||
"credential would have to be created by hand (R-TLS-02 'without manual steps')")
|
||||
check(listeners_checked > 0,
|
||||
"no Gateway listener with tls.mode=Terminate found in the tracked tree — there is no "
|
||||
"serving-TLS reference to trace, so (2) proved nothing")
|
||||
|
||||
# --------------------------------- (3) no certificate material is tracked, anywhere
|
||||
TLS_KEYS = {"tls.crt", "tls.key", "ca.crt", "tls-combined.pem"}
|
||||
for rel, d in DOCS:
|
||||
if d.get("kind") != "Secret":
|
||||
continue
|
||||
stype = str(d.get("type", ""))
|
||||
check(stype != "kubernetes.io/tls",
|
||||
f"{rel}: a Secret of type kubernetes.io/tls is COMMITTED (metadata.name="
|
||||
f"{name_of(d)}) — a serving certificate authored by a human, not issued")
|
||||
for field in ("data", "stringData"):
|
||||
for k in (d.get(field) or {}):
|
||||
check(str(k) not in TLS_KEYS,
|
||||
f"{rel}: committed Secret {name_of(d)} carries {field}.{k} — certificate "
|
||||
"material in git means a manual issuance step exists (R-TLS-02)")
|
||||
|
||||
PEM_RE = re.compile(r"-----BEGIN (?:[A-Z ]+ )?CERTIFICATE-----")
|
||||
for rel in YAML_FILES:
|
||||
body = (ROOT / rel).read_text(errors="replace")
|
||||
# Scoped to YAML manifests deliberately: prose/docs may legitimately SHOW a PEM header
|
||||
# in an example, and a .md file is not applied to any cluster. A manifest cannot.
|
||||
check(not PEM_RE.search(body),
|
||||
f"{rel}: contains a PEM CERTIFICATE body — a hand-minted certificate in a manifest")
|
||||
|
||||
# --------------------------- (4)+(5)+(6) the issuer chain is automated in EVERY env
|
||||
# The cert-manager release's namespace IS the controller's `--cluster-resource-namespace`
|
||||
# (chart default), i.e. where a ClusterIssuer's solver credential must live. Derived from the
|
||||
# HelmRelease, never written down here, so this holds if the release namespace ever changes.
|
||||
cm_releases = [(rel, d) for rel, d in of_kind("HelmRelease") if name_of(d) == "cert-manager"]
|
||||
CM_NS = ns_of(cm_releases[0][1], "cert-manager") if cm_releases else "cert-manager"
|
||||
|
||||
ISSUER_INDEX = {(d.get("kind"), name_of(d)): (rel, d) for rel, d in ISSUERS}
|
||||
solver_creds: list[tuple[str, str, str, str]] = [] # (issuer, ns, secretName, ref field)
|
||||
issuers_verified: set[str] = set()
|
||||
for rel, cert in CERTS:
|
||||
spec = cert.get("spec") or {}
|
||||
ref = spec.get("issuerRef") or {}
|
||||
kind = str(ref.get("kind", "Issuer"))
|
||||
per_env = resolve_per_env(str(ref.get("name", "")))
|
||||
check(len(per_env) > 0,
|
||||
f"{rel}: Certificate {name_of(cert)} issuerRef.name is a token with NO value in "
|
||||
"clusters/base/cluster-vars.yaml or any overlay — it would render empty and never issue")
|
||||
for env, iname in sorted(per_env.items()):
|
||||
hit = ISSUER_INDEX.get((kind, iname))
|
||||
check(hit is not None,
|
||||
f"{rel}: Certificate {name_of(cert)} resolves issuerRef to {kind}/{iname} in "
|
||||
f"environment {env!r}, but no such issuer is tracked — that environment cannot "
|
||||
"issue anything without a hand-created issuer (R-TLS-02, R-ENV-02)")
|
||||
if hit is None:
|
||||
continue
|
||||
irel, issuer = hit
|
||||
issuers_verified.add(iname)
|
||||
ispec = issuer.get("spec") or {}
|
||||
acme = ispec.get("acme")
|
||||
check(isinstance(acme, dict),
|
||||
f"{irel}: {kind}/{iname} (used by {name_of(cert)} in env {env!r}) is not an ACME "
|
||||
f"issuer — its issuance/renewal path is {sorted(ispec)}; docs/08/ADR-0006 mandate "
|
||||
"ACME so both are unattended")
|
||||
if not isinstance(acme, dict):
|
||||
continue
|
||||
check(str(acme.get("server", "")).startswith("https://"),
|
||||
f"{irel}: {kind}/{iname} has no https ACME directory endpoint")
|
||||
check(bool((acme.get("privateKeySecretRef") or {}).get("name")),
|
||||
f"{irel}: {kind}/{iname} declares no privateKeySecretRef — the ACME account key "
|
||||
"would have to be provisioned by hand")
|
||||
solvers = acme.get("solvers") or []
|
||||
check(len(solvers) > 0,
|
||||
f"{irel}: {kind}/{iname} declares NO solvers — every order would sit unsolved "
|
||||
"until a human completed the challenge")
|
||||
for s in solvers:
|
||||
types = [t for t in ("dns01", "http01") if t in s]
|
||||
check(len(types) == 1,
|
||||
f"{irel}: {kind}/{iname} has a solver with challenge types {sorted(s)} — only "
|
||||
"the automated dns01/http01 solvers satisfy 'without manual steps'")
|
||||
for provider, cfg in (s.get("dns01") or {}).items():
|
||||
if not isinstance(cfg, dict):
|
||||
continue
|
||||
for field, val in cfg.items():
|
||||
if isinstance(val, dict) and str(field).endswith(("SecretRef", "secretRef")):
|
||||
n = str(val.get("name", ""))
|
||||
check(bool(n),
|
||||
f"{irel}: {kind}/{iname} dns01 {provider}.{field} has no secret name")
|
||||
if n:
|
||||
# A ClusterIssuer resolves secret refs in the controller's
|
||||
# cluster-resource-namespace; a namespaced Issuer in its own.
|
||||
want_ns = CM_NS if kind == "ClusterIssuer" else ns_of(issuer, CM_NS)
|
||||
solver_creds.append((iname, want_ns, n, f"{provider}.{field}"))
|
||||
|
||||
check(len(issuers_verified) > 0, "no issuer was reachable from any Certificate — (5) proved nothing")
|
||||
|
||||
# (6) the solver credential is delivered, SOPS-encrypted, from git — never per-cert by hand.
|
||||
check(len(solver_creds) > 0,
|
||||
"no dns01 solver credential reference found — docs/08 (OQ-4) requires the scoped "
|
||||
"Cloudflare token to drive DNS-01, so this check would otherwise be vacuous")
|
||||
# Keyed by (namespace, name): a same-named Secret in ANOTHER namespace is NOT this credential.
|
||||
# (Verified: keying by name alone let the external-dns copy of `cloudflare-api-token` satisfy
|
||||
# the cert-manager reference, so deleting cert-manager's own copy passed — a silent no-op.)
|
||||
SECRET_DOCS = {(ns_of(d, CM_NS), name_of(d)): (rel, d)
|
||||
for rel, d in DOCS if d.get("kind") == "Secret"}
|
||||
for iname, sns, sname, field in solver_creds:
|
||||
hit = SECRET_DOCS.get((sns, sname))
|
||||
check(hit is not None,
|
||||
f"{iname}: dns01 {field} references Secret {sname!r}, which no tracked manifest "
|
||||
f"delivers into namespace {sns!r} (the issuer's secret-resolution namespace) — the "
|
||||
"credential would be created by hand in-cluster (R-TLS-02)")
|
||||
if hit is None:
|
||||
continue
|
||||
srel, sdoc = hit
|
||||
check("sops" in sdoc,
|
||||
f"{srel}: Secret {sname} is the DNS-01 credential but is not SOPS-encrypted (R-SEC-01)")
|
||||
check(delivered(srel),
|
||||
f"{srel}: Secret {sname} is not rendered by any Flux Kustomization path — Flux would "
|
||||
"never deliver it, so issuance would wait on a manual kubectl apply")
|
||||
|
||||
# ------------------------------- (7) automatic CertificateRequest approval is intact
|
||||
check(len(cm_releases) == 1,
|
||||
f"expected exactly one cert-manager HelmRelease, found {[r for r, _ in cm_releases]} — "
|
||||
"the controller that issues and renews must itself be GitOps-managed")
|
||||
for rel, hr in cm_releases:
|
||||
values = ((hr.get("spec") or {}).get("values") or {})
|
||||
check(values.get("disableAutoApproval") in (None, False),
|
||||
f"{rel}: cert-manager values set disableAutoApproval={values.get('disableAutoApproval')!r} "
|
||||
"— every CertificateRequest would then wait for an approver, which is a manual step")
|
||||
signers = values.get("approveSignerNames")
|
||||
if signers is not None:
|
||||
pats = [str(s) for s in (signers if isinstance(signers, list) else [signers])]
|
||||
for iname in sorted(issuers_verified):
|
||||
ok = any(p in ("*", "*/*") or "clusterissuers.cert-manager.io" in p and
|
||||
(p.endswith("/*") or p.endswith(f"/{iname}")) for p in pats)
|
||||
check(ok,
|
||||
f"{rel}: approveSignerNames={pats} does not cover ClusterIssuer {iname} — "
|
||||
"requests for it would never be auto-approved")
|
||||
# The chart's own CRD install must be on, else the CRs below cannot even be applied.
|
||||
check((values.get("crds") or {}).get("enabled") is True,
|
||||
f"{rel}: cert-manager values do not enable CRD installation — the Certificate CRD "
|
||||
"would have to be installed by hand")
|
||||
|
||||
# ------------------------------- (8) the whole chain is continuously reconciled from git
|
||||
check(len(FLUX_PATHS) > 0 and len(RENDERED) > 20,
|
||||
f"the kustomize resource graph resolved {len(RENDERED)} manifest(s) from {len(FLUX_PATHS)} "
|
||||
"Flux Kustomization path(s) — too few to be a real graph, so (8) would prove nothing")
|
||||
for rel, _ in cm_releases + CERTS + ISSUERS:
|
||||
check(delivered(rel),
|
||||
f"{rel} is not rendered by any Flux Kustomization path — nothing reconciles it, so a "
|
||||
"deleted Certificate/issuer/controller would stay deleted until a human re-applied it")
|
||||
|
||||
# --------------------------------------------- (9) renewal is not defeated arithmetically
|
||||
def to_seconds(v) -> int | None:
|
||||
m = re.fullmatch(r"\s*(\d+)\s*([smh])\s*", str(v))
|
||||
if not m:
|
||||
return None
|
||||
return int(m.group(1)) * {"s": 1, "m": 60, "h": 3600}[m.group(2)]
|
||||
|
||||
|
||||
renew_pairs = 0
|
||||
for rel, cert in CERTS:
|
||||
spec = cert.get("spec") or {}
|
||||
dur, rb = to_seconds(spec.get("duration")), to_seconds(spec.get("renewBefore"))
|
||||
if dur is None or rb is None:
|
||||
continue
|
||||
renew_pairs += 1
|
||||
check(rb < dur,
|
||||
f"{rel}: Certificate {name_of(cert)} sets renewBefore={spec.get('renewBefore')} >= "
|
||||
f"duration={spec.get('duration')} — the renewal instant never arrives")
|
||||
|
||||
# --------------------------------------------------------------------------- verdict
|
||||
# NO-OP GUARD: this repo has been bitten by checks that degrade to a successful no-op.
|
||||
# The floor below is a count of assertions that MUST have run for the verdict to mean
|
||||
# anything; it is not a style rule.
|
||||
check(CHECKS >= 20,
|
||||
f"only {CHECKS} assertions ran — the check degenerated to a near-no-op and its pass "
|
||||
"would be meaningless")
|
||||
|
||||
if FAILURES:
|
||||
print(f"CHECK FAIL (tls-lifecycle-automated): {len(FAILURES)} of {CHECKS} assertions failed",
|
||||
file=sys.stderr)
|
||||
for f in FAILURES:
|
||||
print(f" - {f}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
print(f"check OK: tls-lifecycle-automated — {CHECKS} assertions; {len(CERTS)} Certificate(s), "
|
||||
f"{len(ISSUERS)} issuer(s), {len(issuers_verified)} issuer(s) reached from a Certificate "
|
||||
f"across every environment, {listeners_checked} terminating listener(s), "
|
||||
f"{len(solver_creds)} dns01 credential ref(s), {renew_pairs} duration/renewBefore pair(s). "
|
||||
"R-TLS-02 'without manual steps' clause only — issued/renewed/revoked are RUNTIME verbs "
|
||||
"(see l2-behavioral/cert-lifecycle-automated; 'revoked' is NOT proven at any layer).")
|
||||
PY
|
||||
@@ -1,391 +0,0 @@
|
||||
# =============================================================================
|
||||
# L2 BEHAVIORAL — certificate lifecycle is automatic (R-TLS-02)
|
||||
# [RUNNABLE / NOT YET EXECUTED — see STATUS]
|
||||
# Catalog: docs/14 "Trusted TLS (R-TLS-01/02)"
|
||||
# Spec: docs/00 R-TLS-02 — "Certificates MUST be automatically **issued, renewed,
|
||||
# and revoked** without manual steps."
|
||||
# docs/08 §"Renewal & revocation (R-TLS-02)" — "cert-manager renews well
|
||||
# before expiry automatically; no human action." / "Revocation: removing/
|
||||
# rotating a certificate is a declarative change; cert-manager handles
|
||||
# issuance of replacements."
|
||||
# docs/08 acceptance gate — "Certificates renew automatically ahead of expiry
|
||||
# with no manual step (**observed across a renewal**)."
|
||||
# Impl: platform/cert-manager/**, platform/gateway/wildcard-certificate.yaml.
|
||||
# Sibling: tests/l0-static/tls-lifecycle-automated proves the "without manual steps"
|
||||
# clause statically (in the merge gate). This test observes the three VERBS.
|
||||
#
|
||||
# ENVIRONMENT-AGNOSTIC (R-ENV-02): NOTHING about either environment is written down. The
|
||||
# serving certificate is DISCOVERED — find the Gateway whose listener terminates TLS, follow
|
||||
# its certificateRefs Secret, and read that Secret's `cert-manager.io/certificate-name`
|
||||
# annotation to reach the Certificate. No hostname, no namespace list, no issuer name, no
|
||||
# address: dev proves dev's cert and prod proves prod's from the same authored assertions.
|
||||
#
|
||||
# GIVEN cert-manager reconciled by Flux and a Gateway terminating TLS,
|
||||
# WHEN the serving Certificate's provenance and renewal schedule are read, and a NEW
|
||||
# Certificate is then requested, left to renew itself, and removed,
|
||||
# THEN 1. the serving credential was produced by the CONTROLLER, not a human: its
|
||||
# Secret carries cert-manager's certificate-name annotation, its
|
||||
# CertificateRequest was submitted by cert-manager's own ServiceAccount and
|
||||
# was APPROVED automatically, and its issuer is a Ready ACME ClusterIssuer;
|
||||
# 2. an unattended RENEWAL IS SCHEDULED: status.renewalTime is in the future and
|
||||
# strictly before status.notAfter, so no human is in the renewal path;
|
||||
# 3. ISSUED — a newly declared Certificate becomes Ready and its Secret appears
|
||||
# with certificate material, with no step between declaring and having it;
|
||||
# 4. RENEWED — that certificate, given a renewal window that opens one minute
|
||||
# after issuance, REISSUES ITSELF at least twice (status.revision >= 3) and
|
||||
# the serving bytes actually change, with no human action. This is docs/08's
|
||||
# "observed across a renewal", fast-forwarded;
|
||||
# 5. REMOVED/REVOKED — deleting the Certificate withdraws its serving material.
|
||||
#
|
||||
# WHY A HERMETIC SELF-SIGNED ISSUER FOR STEPS 3-5, AND WHAT THAT COSTS
|
||||
# -------------------------------------------------------------------
|
||||
# Steps 3-5 need a certificate lifecycle that completes in minutes and repeats on every
|
||||
# run. Driving that through the real ACME issuer would fire a public Let's Encrypt order
|
||||
# (and a Cloudflare _acme-challenge write) per conformance run, burning issuance rate
|
||||
# limits for evidence that is not about ACME: cert-manager's renewal scheduler and
|
||||
# ownership logic are issuer-independent. So steps 3-5 use a `selfSigned` Issuer created
|
||||
# in this test's OWN namespace, and the ACME/DNS-01 path is proven elsewhere — step 1-2
|
||||
# read the REAL wildcard certificate the Gateway serves, and l2-behavioral/trusted-tls
|
||||
# proves a publicly-trusted LE certificate is actually served (R-TLS-01/03).
|
||||
# STATED PLAINLY: steps 3-5 do NOT prove the ACME order path renews; they prove the
|
||||
# CONTROLLER's issue/renew/withdraw automation runs unattended in this cluster, while
|
||||
# steps 1-2 prove the real serving certificate is on exactly that automated path.
|
||||
#
|
||||
# STATUS: RUNNABLE, NOT YET EXECUTED. Steps 1-2 are read-only. Steps 3-5 CREATE an
|
||||
# Issuer/Certificate (and DELETE the Certificate they created) inside chainsaw's
|
||||
# per-test namespace — the same class of mutation every other L2 test here performs,
|
||||
# self-cleaning via namespace teardown, touching no platform object. WS-TEST authored
|
||||
# them against live dev/prod read-only inspection under an explicit instruction not to
|
||||
# mutate either cluster, so THE MUTATING STEPS HAVE NOT BEEN RUN. Nothing about the
|
||||
# suite gates on them: ci/conformance.sh only runs L2/L3 when CLUSTER_AVAILABLE=true.
|
||||
#
|
||||
# EXPECTED FAILURE, DELIBERATELY NOT SOFTENED (R-TEST-06)
|
||||
# ------------------------------------------------------
|
||||
# Step 5 WILL FAIL as the platform is authored today. cert-manager only garbage-collects
|
||||
# a Certificate's Secret when the controller runs with `enableCertificateOwnerRef: true`;
|
||||
# platform/cert-manager/helmrelease.yaml does not set it (chart default false), and the
|
||||
# live wildcard Secret on both clusters has NO ownerReferences (verified read-only,
|
||||
# 2026-09-05). So a removed Certificate leaves its credential in place, and "revoked
|
||||
# without manual steps" does not hold. That is a finding about the platform, not a
|
||||
# defect in this test: the assertion stays as the requirement reads and is escalated to
|
||||
# the PM. (ACME revocation proper is not implemented by cert-manager at all.)
|
||||
#
|
||||
# KNOWN, DELIBERATE GAPS (this test claims none of these):
|
||||
# * Not proven: that the CA would refuse a revoked certificate — there is no
|
||||
# revocation request to make, so nothing observable exists to assert.
|
||||
# * Not proven: renewal of the REAL wildcard cert (its window opens 30 days before a
|
||||
# 90-day expiry; a conformance run cannot wait, and forcing it would rotate the
|
||||
# platform's serving credential). Step 2 asserts the schedule; step 4 asserts the
|
||||
# mechanism that will act on it.
|
||||
# * Not proven: certificate expiry ALERTING — docs/08 delegates that to R-OBS-04
|
||||
# (l3-fault-injection/alerting).
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: cert-lifecycle-automated
|
||||
labels:
|
||||
conformance.home.k3s/R-TLS-02: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-TLS-02"
|
||||
conformance.home.k3s/layer: "L2"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
Given cert-manager and a TLS-terminating Gateway, when the serving certificate's
|
||||
provenance and renewal schedule are read and a new certificate is requested, left
|
||||
to renew, and removed, then issuance, renewal and withdrawal all happen with no
|
||||
human step — the serving cert was controller-issued and auto-approved, its
|
||||
renewal is scheduled ahead of expiry, and a declared certificate issues and
|
||||
reissues itself unattended.
|
||||
spec:
|
||||
timeouts:
|
||||
apply: 60s
|
||||
assert: 3m
|
||||
exec: 10m
|
||||
steps:
|
||||
# ---------------------------------------------------------------- READ-ONLY HALF
|
||||
- name: given-cert-manager-is-reconciled-and-healthy
|
||||
description: >-
|
||||
The controller that issues and renews must be running, and must have been
|
||||
installed by Flux rather than by hand — otherwise "automatic" has no engine.
|
||||
try:
|
||||
- script:
|
||||
content: |
|
||||
set -u
|
||||
# Discover cert-manager's Deployments by chart label, in whatever namespace
|
||||
# the release lives (nothing about the environment is assumed).
|
||||
sel='app.kubernetes.io/instance=cert-manager'
|
||||
rows="$(kubectl get deploy -A -l "$sel" \
|
||||
-o custom-columns='NS:.metadata.namespace,N:.metadata.name,RDY:.status.readyReplicas' \
|
||||
--no-headers 2>/dev/null || true)"
|
||||
echo "$rows"
|
||||
[ -n "$rows" ] || { echo "FAIL: no cert-manager Deployment found ($sel)"; exit 1; }
|
||||
n=0
|
||||
echo "$rows" | while read -r ns name rdy; do
|
||||
[ "${rdy:-0}" -ge 1 ] || { echo "FAIL: $ns/$name has no ready replica"; exit 1; }
|
||||
done || exit 1
|
||||
n="$(echo "$rows" | wc -l)"
|
||||
# controller + webhook + cainjector: without the webhook a Certificate is
|
||||
# rejected, without cainjector the webhook has no CA — all three or nothing.
|
||||
[ "$n" -ge 3 ] || { echo "FAIL: expected >=3 cert-manager Deployments, saw $n"; exit 1; }
|
||||
# GitOps-managed, not hand-installed: Flux labels what it applies.
|
||||
ns="$(echo "$rows" | head -1 | awk '{print $1}')"
|
||||
lbl="$(kubectl get deploy -n "$ns" -l "$sel" \
|
||||
-o jsonpath='{.items[0].metadata.labels.helm\.toolkit\.fluxcd\.io/name}' 2>/dev/null || true)"
|
||||
[ -n "$lbl" ] || { echo "FAIL: cert-manager is not owned by a Flux HelmRelease"; exit 1; }
|
||||
echo "CERT_MANAGER_HEALTHY_OK ($n deployments, flux HelmRelease $lbl)"
|
||||
check:
|
||||
(contains($stdout, 'CERT_MANAGER_HEALTHY_OK')): true
|
||||
|
||||
- name: then-the-serving-certificate-was-issued-by-the-controller
|
||||
description: >-
|
||||
Follow the Gateway's terminating listener to the Secret it serves, and prove
|
||||
that Secret's provenance is cert-manager — annotation, CertificateRequest
|
||||
requester, automatic approval, and a Ready ACME issuer.
|
||||
try:
|
||||
- script:
|
||||
content: |
|
||||
set -u
|
||||
# --- discover the serving Secret from the Gateway itself -------------------
|
||||
gw="$(kubectl get gateway -A \
|
||||
-o jsonpath='{range .items[*]}{.metadata.namespace}{" "}{.metadata.name}{"\n"}{end}' \
|
||||
2>/dev/null | head -1)"
|
||||
[ -n "$gw" ] || { echo "FAIL: no Gateway found — nothing terminates TLS"; exit 1; }
|
||||
gns="${gw% *}"; gname="${gw#* }"
|
||||
sec="$(kubectl -n "$gns" get gateway "$gname" \
|
||||
-o jsonpath='{.spec.listeners[?(@.tls.mode=="Terminate")].tls.certificateRefs[0].name}' \
|
||||
2>/dev/null | awk '{print $1}')"
|
||||
[ -n "$sec" ] || { echo "FAIL: Gateway $gns/$gname terminates no TLS listener"; exit 1; }
|
||||
echo "gateway=$gns/$gname serving-secret=$sec"
|
||||
|
||||
# --- provenance (1): only the controller writes this annotation ------------
|
||||
cert="$(kubectl -n "$gns" get secret "$sec" \
|
||||
-o jsonpath='{.metadata.annotations.cert-manager\.io/certificate-name}' 2>/dev/null || true)"
|
||||
[ -n "$cert" ] || { echo "FAIL: Secret $gns/$sec carries no cert-manager.io/certificate-name annotation — it was not produced by cert-manager"; exit 1; }
|
||||
crt="$(kubectl -n "$gns" get secret "$sec" -o jsonpath='{.data.tls\.crt}' 2>/dev/null || true)"
|
||||
[ -n "$crt" ] || { echo "FAIL: Secret $gns/$sec holds no tls.crt"; exit 1; }
|
||||
|
||||
# --- provenance (2): the Certificate is Ready and has a revision -----------
|
||||
ready="$(kubectl -n "$gns" get certificate "$cert" \
|
||||
-o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)"
|
||||
rev="$(kubectl -n "$gns" get certificate "$cert" -o jsonpath='{.status.revision}' 2>/dev/null || true)"
|
||||
echo "certificate=$gns/$cert ready=$ready revision=$rev"
|
||||
[ "$ready" = "True" ] || { echo "FAIL: Certificate $cert is not Ready"; exit 1; }
|
||||
[ -n "$rev" ] && [ "$rev" -ge 1 ] || { echo "FAIL: Certificate $cert has no issued revision"; exit 1; }
|
||||
|
||||
# --- provenance (3): the request was made BY THE CONTROLLER and auto-approved
|
||||
# spec.username is the identity that created the CertificateRequest. A human
|
||||
# operator would appear as a user/OIDC identity, not a ServiceAccount.
|
||||
cr="$(kubectl -n "$gns" get certificaterequest \
|
||||
-o jsonpath="{range .items[?(@.metadata.annotations.cert-manager\.io/certificate-name=='${cert}')]}{.metadata.name}{' '}{.spec.username}{'\n'}{end}" \
|
||||
2>/dev/null | tail -1)"
|
||||
[ -n "$cr" ] || { echo "FAIL: no CertificateRequest for $cert — cannot show who requested it"; exit 1; }
|
||||
crname="${cr% *}"; requester="${cr#* }"
|
||||
echo "certificaterequest=$crname requester=$requester"
|
||||
case "$requester" in
|
||||
system:serviceaccount:*) : ;;
|
||||
*) echo "FAIL: CertificateRequest $crname was submitted by '$requester', not a controller ServiceAccount"; exit 1 ;;
|
||||
esac
|
||||
appr="$(kubectl -n "$gns" get certificaterequest "$crname" \
|
||||
-o jsonpath='{.status.conditions[?(@.type=="Approved")].status}' 2>/dev/null || true)"
|
||||
[ "$appr" = "True" ] || { echo "FAIL: CertificateRequest $crname is not Approved=True (approval would be a manual step)"; exit 1; }
|
||||
|
||||
# --- provenance (4): an automated ACME issuer, and it is Ready -------------
|
||||
ikind="$(kubectl -n "$gns" get certificate "$cert" -o jsonpath='{.spec.issuerRef.kind}')"
|
||||
iname="$(kubectl -n "$gns" get certificate "$cert" -o jsonpath='{.spec.issuerRef.name}')"
|
||||
echo "issuer=$ikind/$iname"
|
||||
if [ "$ikind" = "ClusterIssuer" ]; then
|
||||
iready="$(kubectl get clusterissuer "$iname" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)"
|
||||
iacme="$(kubectl get clusterissuer "$iname" -o jsonpath='{.spec.acme.server}' 2>/dev/null || true)"
|
||||
else
|
||||
iready="$(kubectl -n "$gns" get issuer "$iname" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)"
|
||||
iacme="$(kubectl -n "$gns" get issuer "$iname" -o jsonpath='{.spec.acme.server}' 2>/dev/null || true)"
|
||||
fi
|
||||
[ "$iready" = "True" ] || { echo "FAIL: issuer $iname is not Ready — renewal would need a human to fix it"; exit 1; }
|
||||
[ -n "$iacme" ] || { echo "FAIL: issuer $iname is not an ACME issuer"; exit 1; }
|
||||
echo "SERVING_CERT_CONTROLLER_ISSUED_OK"
|
||||
check:
|
||||
(contains($stdout, 'SERVING_CERT_CONTROLLER_ISSUED_OK')): true
|
||||
|
||||
- name: then-an-unattended-renewal-is-already-scheduled
|
||||
description: >-
|
||||
cert-manager writes status.renewalTime only after computing when it will renew.
|
||||
Asserting it is in the future and before expiry proves the renewal is scheduled
|
||||
and that nothing is waiting on a human (docs/08 "renews well before expiry").
|
||||
try:
|
||||
- script:
|
||||
content: |
|
||||
set -u
|
||||
gw="$(kubectl get gateway -A \
|
||||
-o jsonpath='{range .items[*]}{.metadata.namespace}{" "}{.metadata.name}{"\n"}{end}' | head -1)"
|
||||
gns="${gw% *}"; gname="${gw#* }"
|
||||
sec="$(kubectl -n "$gns" get gateway "$gname" \
|
||||
-o jsonpath='{.spec.listeners[?(@.tls.mode=="Terminate")].tls.certificateRefs[0].name}' | awk '{print $1}')"
|
||||
cert="$(kubectl -n "$gns" get secret "$sec" \
|
||||
-o jsonpath='{.metadata.annotations.cert-manager\.io/certificate-name}')"
|
||||
renew="$(kubectl -n "$gns" get certificate "$cert" -o jsonpath='{.status.renewalTime}')"
|
||||
after="$(kubectl -n "$gns" get certificate "$cert" -o jsonpath='{.status.notAfter}')"
|
||||
[ -n "$renew" ] || { echo "FAIL: Certificate $cert has no status.renewalTime — no renewal is scheduled"; exit 1; }
|
||||
[ -n "$after" ] || { echo "FAIL: Certificate $cert has no status.notAfter"; exit 1; }
|
||||
now_s="$(date -u +%s)"
|
||||
r_s="$(date -u -d "$renew" +%s)"
|
||||
a_s="$(date -u -d "$after" +%s)"
|
||||
echo "now=$(date -u -d "@$now_s" +%FT%TZ) renewalTime=$renew notAfter=$after"
|
||||
echo "renewal is in $(( (r_s - now_s) / 86400 ))d; expiry in $(( (a_s - now_s) / 86400 ))d"
|
||||
[ "$r_s" -gt "$now_s" ] || { echo "FAIL: renewalTime is in the PAST — the scheduled renewal did not happen"; exit 1; }
|
||||
[ "$r_s" -lt "$a_s" ] || { echo "FAIL: renewalTime is not before notAfter — renewal would come too late"; exit 1; }
|
||||
echo "RENEWAL_SCHEDULED_OK"
|
||||
check:
|
||||
(contains($stdout, 'RENEWAL_SCHEDULED_OK')): true
|
||||
|
||||
# ------------------------------------------------- MUTATING HALF (hermetic namespace)
|
||||
- name: when-a-certificate-is-declared-then-it-is-issued-with-no-human-step
|
||||
description: >-
|
||||
Declare an Issuer and a Certificate in this test's own namespace, with a renewal
|
||||
window that opens one minute after issuance. Nothing else is done to them for the
|
||||
rest of the test — every later transition is the controller acting on its own.
|
||||
try:
|
||||
- apply:
|
||||
resource:
|
||||
# selfSigned: no external CA, no rate limit, no DNS write. See the header for
|
||||
# what this proves (controller automation) and what it does not (the ACME path).
|
||||
apiVersion: cert-manager.io/v1
|
||||
kind: Issuer
|
||||
metadata:
|
||||
name: conformance-lifecycle
|
||||
spec:
|
||||
selfSigned: {}
|
||||
- apply:
|
||||
resource:
|
||||
apiVersion: cert-manager.io/v1
|
||||
kind: Certificate
|
||||
metadata:
|
||||
name: conformance-lifecycle
|
||||
spec:
|
||||
secretName: conformance-lifecycle-tls
|
||||
# 1h is cert-manager's minimum duration; renewing 59m before expiry puts the
|
||||
# renewal instant ~1 minute after each issuance, so an unattended renewal is
|
||||
# observable inside a test run instead of 30 days from now.
|
||||
duration: 1h
|
||||
renewBefore: 3540s
|
||||
commonName: conformance-lifecycle.invalid
|
||||
dnsNames:
|
||||
- conformance-lifecycle.invalid
|
||||
privateKey:
|
||||
rotationPolicy: Always
|
||||
issuerRef:
|
||||
kind: Issuer
|
||||
name: conformance-lifecycle
|
||||
- assert:
|
||||
resource:
|
||||
apiVersion: cert-manager.io/v1
|
||||
kind: Certificate
|
||||
metadata:
|
||||
name: conformance-lifecycle
|
||||
status:
|
||||
conditions:
|
||||
- type: Ready
|
||||
status: "True"
|
||||
revision: 1
|
||||
- script:
|
||||
content: |
|
||||
set -u
|
||||
crt="$(kubectl -n "$NAMESPACE" get secret conformance-lifecycle-tls \
|
||||
-o jsonpath='{.data.tls\.crt}' 2>/dev/null || true)"
|
||||
key="$(kubectl -n "$NAMESPACE" get secret conformance-lifecycle-tls \
|
||||
-o jsonpath='{.data.tls\.key}' 2>/dev/null || true)"
|
||||
[ -n "$crt" ] && [ -n "$key" ] || { echo "FAIL: no certificate material was issued into conformance-lifecycle-tls"; exit 1; }
|
||||
# Record the serving bytes so the renewal step can prove they CHANGED.
|
||||
echo "$crt" | cksum > /tmp/conformance-cert-r1.cksum
|
||||
echo "revision-1 cert fingerprint: $(cat /tmp/conformance-cert-r1.cksum)"
|
||||
echo "CERT_ISSUED_OK"
|
||||
check:
|
||||
(contains($stdout, 'CERT_ISSUED_OK')): true
|
||||
|
||||
- name: then-it-renews-itself-before-expiry-with-no-human-step
|
||||
description: >-
|
||||
Nothing touches the Certificate here. Poll until the controller has reissued it
|
||||
at least twice on its own and the serving bytes differ from revision 1 — docs/08's
|
||||
"observed across a renewal", and every request auto-approved.
|
||||
try:
|
||||
- script:
|
||||
content: |
|
||||
set -u
|
||||
want=3 # revision 1 = issuance; >=3 means TWO unattended renewals
|
||||
for i in $(seq 1 30); do
|
||||
rev="$(kubectl -n "$NAMESPACE" get certificate conformance-lifecycle \
|
||||
-o jsonpath='{.status.revision}' 2>/dev/null || echo 0)"
|
||||
echo "poll $i: revision=${rev:-0}"
|
||||
[ "${rev:-0}" -ge "$want" ] && break
|
||||
sleep 20
|
||||
done
|
||||
rev="$(kubectl -n "$NAMESPACE" get certificate conformance-lifecycle \
|
||||
-o jsonpath='{.status.revision}' 2>/dev/null || echo 0)"
|
||||
[ "${rev:-0}" -ge "$want" ] || { echo "FAIL: revision reached only ${rev:-0} — cert-manager did not renew unattended"; exit 1; }
|
||||
|
||||
# The serving material must actually have rotated, not just the revision count.
|
||||
now="$(kubectl -n "$NAMESPACE" get secret conformance-lifecycle-tls \
|
||||
-o jsonpath='{.data.tls\.crt}' | cksum)"
|
||||
was="$(cat /tmp/conformance-cert-r1.cksum)"
|
||||
echo "r1=$was now=$now"
|
||||
[ "$now" != "$was" ] || { echo "FAIL: the Secret still holds the revision-1 certificate — nothing was actually reissued"; exit 1; }
|
||||
|
||||
# Every renewal request was approved by the controller, never by a person.
|
||||
bad=0
|
||||
for cr in $(kubectl -n "$NAMESPACE" get certificaterequest -o name 2>/dev/null); do
|
||||
st="$(kubectl -n "$NAMESPACE" get "$cr" \
|
||||
-o jsonpath='{.status.conditions[?(@.type=="Approved")].status}' 2>/dev/null || true)"
|
||||
who="$(kubectl -n "$NAMESPACE" get "$cr" -o jsonpath='{.spec.username}' 2>/dev/null || true)"
|
||||
echo " $cr approved=$st requester=$who"
|
||||
[ "$st" = "True" ] || bad=1
|
||||
case "$who" in system:serviceaccount:*) : ;; *) bad=1 ;; esac
|
||||
done
|
||||
[ "$bad" = 0 ] || { echo "FAIL: a CertificateRequest was not auto-approved, or was not submitted by a controller"; exit 1; }
|
||||
echo "CERT_RENEWED_UNATTENDED_OK (revision=$rev)"
|
||||
check:
|
||||
(contains($stdout, 'CERT_RENEWED_UNATTENDED_OK')): true
|
||||
|
||||
- name: then-removing-the-certificate-withdraws-the-serving-material
|
||||
description: >-
|
||||
docs/08 reads revocation as "removing/rotating a certificate is a declarative
|
||||
change". The declarative change is the deletion; withdrawal of the credential
|
||||
must follow with no manual cleanup. EXPECTED TO FAIL TODAY — cert-manager needs
|
||||
`enableCertificateOwnerRef: true` for the Secret to be garbage-collected, and the
|
||||
tracked HelmRelease does not set it. Kept as the requirement reads (R-TEST-06).
|
||||
try:
|
||||
- delete:
|
||||
ref:
|
||||
apiVersion: cert-manager.io/v1
|
||||
kind: Certificate
|
||||
name: conformance-lifecycle
|
||||
- script:
|
||||
content: |
|
||||
set -u
|
||||
for i in $(seq 1 15); do
|
||||
if ! kubectl -n "$NAMESPACE" get secret conformance-lifecycle-tls >/dev/null 2>&1; then
|
||||
echo "CERT_WITHDRAWN_OK (serving material gone after $((i * 4))s)"
|
||||
exit 0
|
||||
fi
|
||||
sleep 4
|
||||
done
|
||||
owners="$(kubectl -n "$NAMESPACE" get secret conformance-lifecycle-tls \
|
||||
-o jsonpath='{.metadata.ownerReferences}' 2>/dev/null || true)"
|
||||
echo "FAIL: the Certificate was deleted but Secret conformance-lifecycle-tls still"
|
||||
echo " holds usable certificate material (ownerReferences=${owners:-none})."
|
||||
echo " Removing a certificate therefore does NOT withdraw the credential, so"
|
||||
echo " R-TLS-02's 'revoked ... without manual steps' does not hold. FIX:"
|
||||
echo " set enableCertificateOwnerRef: true in platform/cert-manager values"
|
||||
echo " (owner decision — WS-TEST does not weaken the assertion instead)."
|
||||
exit 1
|
||||
check:
|
||||
(contains($stdout, 'CERT_WITHDRAWN_OK')): true
|
||||
finally:
|
||||
# Namespace teardown removes both objects anyway; explicit so a leftover Secret
|
||||
# from the expected failure above is never left holding key material.
|
||||
- script:
|
||||
content: |
|
||||
kubectl -n "$NAMESPACE" delete secret conformance-lifecycle-tls --ignore-not-found --wait=false || true
|
||||
kubectl -n "$NAMESPACE" delete certificate conformance-lifecycle --ignore-not-found --wait=false || true
|
||||
kubectl -n "$NAMESPACE" delete issuer conformance-lifecycle --ignore-not-found --wait=false || true
|
||||
rm -f /tmp/conformance-cert-r1.cksum || true
|
||||
@@ -153,20 +153,3 @@ spec:
|
||||
finally:
|
||||
- script:
|
||||
content: 'echo "cleanup handled by chainsaw namespace teardown"'
|
||||
# -----------------------------------------------------------------------------------
|
||||
# PROOF-OF-PASS SENTINEL (R-SEC-03). Every step above is an assert/Job-status gate, so
|
||||
# chainsaw already fails the test before reaching this step if any of them fails — but a
|
||||
# green Chainsaw JUnit carries NO stdout, and the L2/L3 tests in this suite can self-skip
|
||||
# GREEN, so lib/record_run.py refuses to treat a bare `passed` as evidence and records
|
||||
# `unknown-no-sentinel-declared` instead. Without a declared token this test could pass
|
||||
# forever and NEVER become a recorded proof of R-NET-01 — the isolation half of docs/07's audit proof list
|
||||
# — which is precisely what docs/07 asks for ("These proofs are part of the platform
|
||||
# acceptance suite, not one-time checks"). This step exists only to emit that token, and it
|
||||
# is reached only when everything above has passed. Asserted by
|
||||
# tests/l0-static/auditability-proofs-producible.
|
||||
- name: then-emit-the-proof-of-pass-sentinel
|
||||
try:
|
||||
- script:
|
||||
content: echo DEFAULT_DENY_PROVEN_OK
|
||||
check:
|
||||
(contains($stdout, 'DEFAULT_DENY_PROVEN_OK')): true
|
||||
@@ -3,18 +3,11 @@
|
||||
# Catalog: docs/14 "DNS/exposure (R-DNS-*)"
|
||||
# Spec: docs/09 (ExternalDNS auto-manages records; split-horizon foundation)
|
||||
#
|
||||
# ENVIRONMENT-AGNOSTIC (R-ENV-02): the hostname is built from the ${BASE_DOMAIN} of the CLUSTER
|
||||
# UNDER TEST (read from flux-system/cluster-vars via a test-level chainsaw binding), and the
|
||||
# expected answer is the LB IP that THIS Service was actually assigned (read back from its own
|
||||
# status) — so neither the name nor the address is authored per environment, and no run-time flag
|
||||
# can be forgotten. Requesting a record under the cluster's own suffix is also what keeps the two
|
||||
# environments from ever describing the same public name (the 2026-09-04 incident precondition).
|
||||
#
|
||||
# GIVEN a LoadBalancer Service annotated with an external-dns hostname,
|
||||
# WHEN ExternalDNS reconciles (source=service, provider=cloudflare, policy=sync),
|
||||
# THEN a PUBLIC DNS record for that hostname is auto-created, pointing at the
|
||||
# Service's Cilium-assigned LB IP (R-DNS-02) — asserted as an EXACT match against that
|
||||
# Service's own `status.loadBalancer.ingress[0].ip`, i.e. clients reach the cluster endpoint.
|
||||
# Service's Cilium-assigned LB IP (R-DNS-02). The record resolves to the
|
||||
# cluster LB IP (Cilium L2 `shared` pool 192.168.1.53-.60), i.e. clients reach the cluster endpoint.
|
||||
#
|
||||
# CLEANUP: chainsaw deletes the per-test namespace (Service included) at teardown;
|
||||
# ExternalDNS policy=sync then REMOVES the record from Cloudflare on its next pass.
|
||||
@@ -45,13 +38,6 @@ spec:
|
||||
apply: 90s
|
||||
assert: 5m
|
||||
exec: 6m
|
||||
# -- R-ENV-02: the environment's hostname suffix is READ FROM THE CLUSTER UNDER TEST.
|
||||
# flux-system/cluster-vars is the ConfigMap Flux substitutes ${BASE_DOMAIN} from, so this
|
||||
# binding is the same source of truth the platform is rendered from — dev asks dev, prod
|
||||
# asks prod, and there is no per-environment fork and no flag to forget.
|
||||
bindings:
|
||||
- name: baseDomain
|
||||
value: (x_k8s_get($client, 'v1', 'ConfigMap', 'flux-system', 'cluster-vars').data.BASE_DOMAIN)
|
||||
steps:
|
||||
- name: given-a-service-requesting-a-dns-name
|
||||
try:
|
||||
@@ -62,8 +48,7 @@ spec:
|
||||
metadata:
|
||||
name: dns-probe-svc
|
||||
annotations:
|
||||
# conformance-dns.<this cluster's BASE_DOMAIN> — never a literal apex.
|
||||
external-dns.alpha.kubernetes.io/hostname: (join('.', ['conformance-dns', $baseDomain]))
|
||||
external-dns.alpha.kubernetes.io/hostname: conformance-dns.home.keithjackson.dev
|
||||
spec:
|
||||
type: LoadBalancer
|
||||
# No real backend needed — Cilium LB-IPAM assigns a VIP regardless, and
|
||||
@@ -74,27 +59,7 @@ spec:
|
||||
- name: http
|
||||
port: 80
|
||||
targetPort: 80
|
||||
# Wait for Cilium LB-IPAM to actually assign the VIP: it is the value ExternalDNS will
|
||||
# publish and the value the probe below asserts EXACTLY. `|| ` + empty-array literal
|
||||
# keeps the expression nil-safe while the address is still absent, so the assert POLLS
|
||||
# (fails and retries) instead of erroring out on the first evaluation.
|
||||
- assert:
|
||||
resource:
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: dns-probe-svc
|
||||
status:
|
||||
(length(loadBalancer.ingress || `[]`)): 1
|
||||
- name: then-externaldns-creates-the-public-record
|
||||
# Step-scoped binding: read back the LB IP THIS Service was assigned (the previous step's
|
||||
# assert guarantees it exists by now — step bindings are evaluated when the step runs).
|
||||
# Expecting that exact address is both stricter than the old hardcoded 192.168.1.53-.60
|
||||
# range and correct in every environment: dev's pool is 192.168.1.53-.60, prod's is
|
||||
# 192.168.2.20-.30, and neither is written down here.
|
||||
bindings:
|
||||
- name: lbIp
|
||||
value: (x_k8s_get($client, 'v1', 'Service', $namespace, 'dns-probe-svc').status.loadBalancer.ingress[0].ip)
|
||||
try:
|
||||
- apply:
|
||||
resource:
|
||||
@@ -111,27 +76,20 @@ spec:
|
||||
containers:
|
||||
- name: probe
|
||||
image: busybox:1.36.1
|
||||
# Both values come from the cluster under test: the suffix from
|
||||
# flux-system/cluster-vars, the address from this Service's own status.
|
||||
env:
|
||||
- name: BASE_DOMAIN
|
||||
value: ($baseDomain)
|
||||
- name: LB_IP
|
||||
value: ($lbIp)
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
set -u
|
||||
host="conformance-dns.${BASE_DOMAIN}"
|
||||
host="conformance-dns.home.keithjackson.dev"
|
||||
# Poll PUBLIC DNS (1.1.1.1) until the auto-created A record appears and
|
||||
# points at THIS Service's assigned LB IP. Proves ExternalDNS created it.
|
||||
# points into the cluster LB range (Cilium L2 `shared` pool 192.168.1.53-.60). Proves ExternalDNS created it.
|
||||
for i in $(seq 1 28); do
|
||||
ip="$(nslookup "$host" 1.1.1.1 2>/dev/null | awk '/^Address: /{print $2}' | grep -Fx "${LB_IP}" | head -1)"
|
||||
ip="$(nslookup "$host" 1.1.1.1 2>/dev/null | grep -oE '192\.168\.1\.(5[3-9]|60)' | head -1)"
|
||||
if [ -n "$ip" ]; then echo "RECORD_FOUND $host -> $ip"; exit 0; fi
|
||||
echo "poll $i: record not yet published as ${LB_IP}"; sleep 10
|
||||
echo "poll $i: record not yet published"; sleep 10
|
||||
done
|
||||
echo "FAIL: ExternalDNS did not publish ${host} -> ${LB_IP} within timeout"; exit 1
|
||||
echo "FAIL: ExternalDNS did not publish the record within timeout"; exit 1
|
||||
- assert:
|
||||
resource:
|
||||
apiVersion: batch/v1
|
||||
|
||||
@@ -2,24 +2,17 @@
|
||||
# L2 BEHAVIORAL — split-horizon INTERNAL resolution (R-DNS-01 / R-DNS-03) [RUNNABLE]
|
||||
# Catalog: docs/14 "DNS/exposure (R-DNS-*)"
|
||||
# Spec: docs/09 "Split-horizon resolution (R-DNS-03)" — internal clients resolve
|
||||
# <svc>.${BASE_DOMAIN} to the INTERNAL Gateway VIP, not the public hairpin.
|
||||
# <svc>.home.keithjackson.dev to the INTERNAL Gateway VIP, not the public hairpin.
|
||||
# Impl: platform/coredns/coredns-custom.yaml (k3s CoreDNS `import *.server`; template plugin
|
||||
# answers the whole ${BASE_DOMAIN} zone with ${GATEWAY_VIP}).
|
||||
#
|
||||
# ENVIRONMENT-AGNOSTIC (R-ENV-02): this test asks the CLUSTER IT IS POINTED AT for its own
|
||||
# ${BASE_DOMAIN} and ${GATEWAY_VIP} — the same flux-system/cluster-vars ConfigMap Flux
|
||||
# substitutes from — via test-level chainsaw bindings (see `spec.bindings`). Nothing here is
|
||||
# per-environment and no flag is passed at run time, so the suite cannot be run against the
|
||||
# wrong expectation: dev asserts dev's zone/VIP, prod asserts prod's.
|
||||
# answers the whole home.keithjackson.dev zone with ${GATEWAY_VIP}).
|
||||
#
|
||||
# GIVEN the split-horizon coredns-custom ConfigMap is reconciled into k3s CoreDNS,
|
||||
# WHEN an in-cluster pod resolves a platform hostname over its PUBLIC name
|
||||
# (keycloak.${BASE_DOMAIN}) using the default cluster resolver (CoreDNS 10.43.0.10),
|
||||
# THEN the answer is EXACTLY that cluster's ${GATEWAY_VIP} (the address coredns-custom is
|
||||
# templated with, and the `shared` Gateway's own status address) — NOT a public/off-cluster
|
||||
# IP. This is what lets pod->public-hostname traffic hairpin through the
|
||||
# Gateway with the correct SNI (*.${BASE_DOMAIN}) so the wildcard cert validates
|
||||
# (R-DNS-01 "services reachable at <svc>.${BASE_DOMAIN}"; R-DNS-03 split-horizon).
|
||||
# (keycloak.home.keithjackson.dev) using the default cluster resolver (CoreDNS 10.43.0.10),
|
||||
# THEN the answer is the in-cluster Gateway VIP (the Cilium L2 `shared` gateway pool 192.168.1.53-.60) — NOT a
|
||||
# public/off-cluster IP. This is what lets pod->public-hostname traffic hairpin through the
|
||||
# Gateway with the correct SNI (*.home.keithjackson.dev) so the wildcard cert validates
|
||||
# (R-DNS-01 "services reachable at <svc>.home..."; R-DNS-03 split-horizon).
|
||||
#
|
||||
# Complements dns-exposure (R-DNS-02, the EXTERNAL/public-record side). Together they cover the
|
||||
# split-horizon pair: external answer = public record (R-DNS-02); internal answer = Gateway VIP
|
||||
@@ -41,23 +34,13 @@ metadata:
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
An in-cluster pod resolving keycloak.${BASE_DOMAIN} via cluster DNS gets that cluster's
|
||||
internal Gateway VIP, not the public hairpin — split-horizon in-cluster resolution.
|
||||
An in-cluster pod resolving keycloak.home.keithjackson.dev via cluster DNS gets the internal
|
||||
Gateway VIP (192.168.1.53-.60), not the public hairpin — split-horizon in-cluster resolution.
|
||||
spec:
|
||||
timeouts:
|
||||
apply: 60s
|
||||
assert: 2m
|
||||
exec: 2m
|
||||
# -- R-ENV-02: the environment's identity is READ FROM THE CLUSTER UNDER TEST, never authored
|
||||
# here. flux-system/cluster-vars is the ConfigMap Flux itself substitutes from
|
||||
# (clusters/base/cluster-vars.yaml + the dev/prod overlay patch), so these bindings are the
|
||||
# same source of truth the platform is rendered from. Test-level => in scope for every step
|
||||
# (unlike step `outputs`, which are step-scoped).
|
||||
bindings:
|
||||
- name: baseDomain
|
||||
value: (x_k8s_get($client, 'v1', 'ConfigMap', 'flux-system', 'cluster-vars').data.BASE_DOMAIN)
|
||||
- name: gatewayVip
|
||||
value: (x_k8s_get($client, 'v1', 'ConfigMap', 'flux-system', 'cluster-vars').data.GATEWAY_VIP)
|
||||
steps:
|
||||
- name: then-cluster-dns-answers-with-the-internal-gateway-vip
|
||||
try:
|
||||
@@ -76,32 +59,23 @@ spec:
|
||||
containers:
|
||||
- name: probe
|
||||
image: busybox:1.36.1
|
||||
# The cluster's OWN identity, resolved from flux-system/cluster-vars by the
|
||||
# test-level bindings — never a literal (R-ENV-02).
|
||||
env:
|
||||
- name: BASE_DOMAIN
|
||||
value: ($baseDomain)
|
||||
- name: GATEWAY_VIP
|
||||
value: ($gatewayVip)
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
set -u
|
||||
host="keycloak.${BASE_DOMAIN}"
|
||||
host="keycloak.home.keithjackson.dev"
|
||||
# Resolve via the DEFAULT cluster resolver (CoreDNS, /etc/resolv.conf ->
|
||||
# 10.43.0.10). Split-horizon must answer with THIS cluster's Gateway VIP
|
||||
# (the address coredns-custom is templated with), proving pods hairpin
|
||||
# through the Gateway. Exact match on ${GATEWAY_VIP}: stricter than the
|
||||
# old hardcoded 192.168.1.53-.60 range AND correct in both environments.
|
||||
# 10.43.0.10). Split-horizon must answer with the in-cluster Gateway VIP
|
||||
# (Cilium L2 `shared` pool 192.168.1.53-.60), proving pods hairpin through the Gateway.
|
||||
out="$(nslookup "$host" 2>/dev/null)"
|
||||
echo "$out"
|
||||
ip="$(echo "$out" | awk '/^Address: /{print $2}' | grep -Fx "${GATEWAY_VIP}" | head -1)"
|
||||
ip="$(echo "$out" | awk '/^Address: /{print $2}' | grep -E '^192\.168\.1\.(5[3-9]|60)$' | head -1)"
|
||||
if [ -n "$ip" ]; then
|
||||
echo "INTERNAL_RESOLUTION_OK $host -> $ip (in-cluster Gateway VIP)"
|
||||
exit 0
|
||||
fi
|
||||
echo "FAIL: $host did not resolve to this cluster's Gateway VIP (${GATEWAY_VIP});"
|
||||
echo "FAIL: $host did not resolve to the in-cluster Gateway VIP (192.168.1.53-.60);"
|
||||
echo " split-horizon coredns-custom not in effect (got a public/off-cluster answer)."
|
||||
exit 1
|
||||
- assert:
|
||||
|
||||
@@ -1,199 +0,0 @@
|
||||
# =============================================================================
|
||||
# L2 BEHAVIORAL — the default egress path is the WAN; the VPN is opt-in (R-NET-04) [RUNNABLE]
|
||||
# Catalog: docs/14 L2 row ("real runtime behavior, probes against a cluster")
|
||||
# Spec: docs/00 R-NET-04 ("The platform MUST support **multiple egress paths** selected by
|
||||
# routing rules ... realized as two paths — the local `internet` path and the
|
||||
# in-cluster `vpn` path").
|
||||
# docs/03 §"Egress routing: two paths, one physical WAN": "the `internet` path is
|
||||
# simply the default route out".
|
||||
# docs/03 §"VPN egress model": "**Default is the WAN; the VPN is strictly opt-in.**
|
||||
# Every pod — and the nodes themselves — egress the local/private WAN by default and
|
||||
# **never** the VPN. Only services labeled `egress-class=vpn` are steered to the
|
||||
# tunnel"; and the tunnel "is **split-tunnel**: ... only what Cilium points at `tun0`
|
||||
# uses the VPN."
|
||||
#
|
||||
# WHAT THIS PROVES THAT tests/l2-behavioral/egress-class DOES NOT
|
||||
# --------------------------------------------------------------
|
||||
# That test (R-NET-05/07) compares two LABELLED pods — `internet` vs `vpn` — and shows their
|
||||
# public IPs differ, then flips a label. Both of its pods declared a class, so it says nothing
|
||||
# about the case R-NET-04 actually turns on: a pod that NO routing rule selects. This test adds
|
||||
# that pod, and asserts the two claims that make a rule-selected path set complete:
|
||||
#
|
||||
# TOTALITY — an UNLABELLED pod egresses the WAN, i.e. the same public IP as the
|
||||
# explicitly-`internet` pod. Absence of a matching rule IS a path, and it is
|
||||
# the local one. If the steering rule ever widened (an empty selector, a
|
||||
# matchExpression, a second rule), this pod's IP would move to the tunnel and
|
||||
# this assertion is what would notice.
|
||||
# COEXISTENCE — the WAN path and the VPN path are live AT THE SAME TIME, observed in one
|
||||
# pass: three pods, two distinct public IPs. "Multiple egress paths" is a
|
||||
# simultaneity claim, not a claim that one switch has two positions.
|
||||
# PER-DESTINATION SELECTION — the vpn-class pod still reaches an in-cluster/RFC1918
|
||||
# destination over the LOCAL path while its public traffic is tunnelled. The
|
||||
# tunnel is destination-scoped (excludedCIDRs), not a default route, so ONE pod
|
||||
# uses BOTH paths at once.
|
||||
#
|
||||
# ENVIRONMENT-AGNOSTIC (R-ENV-02): no address, hostname or range appears. Every assertion is a
|
||||
# RELATION between observed values (equal / not-equal), so it holds against dev and prod alike
|
||||
# and survives the WAN or the VPN provider rotating addresses.
|
||||
#
|
||||
# GIVEN one unlabelled, one `internet`-class and one `vpn`-class pod — the same workload three
|
||||
# times, differing only in the class they declare (or do not),
|
||||
# WHEN each asks an external echo for the public IP it egressed from, and the vpn-class pod
|
||||
# also resolves a name through the in-cluster resolver,
|
||||
# THEN the unlabelled pod's public IP EQUALS the internet-class pod's (the default route out
|
||||
# is the WAN, and the VPN is strictly opt-in);
|
||||
# AND the vpn-class pod's public IP DIFFERS from both (a second, simultaneously-live path);
|
||||
# AND the vpn-class pod still resolves in-cluster (its private-destination traffic was not
|
||||
# swallowed by the tunnel), so the same pod is using both paths at once.
|
||||
#
|
||||
# WHAT THIS DOES *NOT* PROVE (honest gaps — do not read this test as claiming them):
|
||||
# - THAT NO OTHER RULE EXISTS. Three pods cannot enumerate a rule set. That exactly one
|
||||
# steering rule is authored, that it selects only the vpn class, that no `ip rule` is
|
||||
# unscoped and no nftables DROP unqualified is the L0 half, in
|
||||
# tests/l0-static/egress-paths-selected-by-rules (same requirement label).
|
||||
# - WHICH INTERFACE the packets left by. The observation is the public IP an external echo
|
||||
# reports; it distinguishes the paths without naming them. Asserting "out tun0" would need
|
||||
# host-level datapath inspection (Hubble/tcpdump on the gateway node), which no test in
|
||||
# this suite does.
|
||||
# - THE KILL-SWITCH. Tunnel-down behaviour is R-NET-06's, at L3 (vpn-killswitch): `vpn`
|
||||
# fails closed. Its mirror — that `internet` is never blackholed — is asserted statically
|
||||
# by the L0 half, not here.
|
||||
# - NODE traffic. docs/03 also says "the nodes themselves" egress the WAN by default. That
|
||||
# is a host-level fact this test cannot see from a pod.
|
||||
# - EXTERNAL DEPENDENCY, stated plainly: the public-IP echo (api.ipify.org) is off-cluster,
|
||||
# so a WAN outage or the echo being down fails this test. That is the same dependency
|
||||
# tests/l2-behavioral/egress-class already carries; the alternative — trusting the
|
||||
# cluster's own account of which path it used — would not be a behavioural test at all.
|
||||
#
|
||||
# CLEANUP: chainsaw deletes the per-test namespace (all three pods with it). Nothing outside
|
||||
# that namespace is created or changed — no label on a platform object, no policy edit.
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: egress-default-path
|
||||
labels:
|
||||
conformance.home.k3s/R-NET-04: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-NET-04"
|
||||
conformance.home.k3s/layer: "L2"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
Given an unlabelled, an internet-class and a vpn-class pod, when each observes the public
|
||||
IP it egressed from, then the unlabelled pod's IP equals the internet-class pod's (the
|
||||
default route out is the WAN, the VPN is opt-in) while the vpn-class pod's differs — two
|
||||
paths live at once — and the vpn-class pod still reaches in-cluster destinations over the
|
||||
local path, so one pod uses both paths simultaneously.
|
||||
spec:
|
||||
timeouts:
|
||||
apply: 60s
|
||||
assert: 2m
|
||||
exec: 3m
|
||||
steps:
|
||||
# -----------------------------------------------------------------------------------
|
||||
- name: given-three-pods-differing-only-in-their-declared-egress-class
|
||||
try:
|
||||
- apply:
|
||||
file: fixtures/three-egress-pods.yaml
|
||||
# Bare pods (restartPolicy: Never) — Ready == Running here.
|
||||
- assert:
|
||||
resource:
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata: { name: egress-unlabelled }
|
||||
status: { phase: Running }
|
||||
- assert:
|
||||
resource:
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata: { name: egress-internet }
|
||||
status: { phase: Running }
|
||||
- assert:
|
||||
resource:
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata: { name: egress-vpn }
|
||||
status: { phase: Running }
|
||||
# -----------------------------------------------------------------------------------
|
||||
- name: then-the-unlabelled-pod-takes-the-wan-and-the-vpn-path-is-live-too
|
||||
try:
|
||||
- script:
|
||||
content: |
|
||||
set -eu
|
||||
ns="$NAMESPACE"
|
||||
get_ip() {
|
||||
# Echo the pod's observed public IP, or empty on failure.
|
||||
kubectl -n "$ns" exec "$1" -c curl -- \
|
||||
curl -fsS --max-time 10 https://api.ipify.org 2>/dev/null || true
|
||||
}
|
||||
|
||||
# Poll until all three answer AND the vpn class has diverged from the WAN:
|
||||
# Cilium may still be programming the egress datapath just after the pods go
|
||||
# Ready. Bounded by the loop, never a bare sleep-and-hope.
|
||||
none="" inet="" vpn=""
|
||||
for i in $(seq 1 12); do
|
||||
[ -n "$none" ] || none="$(get_ip egress-unlabelled)"
|
||||
[ -n "$inet" ] || inet="$(get_ip egress-internet)"
|
||||
[ -n "$vpn" ] || vpn="$(get_ip egress-vpn)"
|
||||
if [ -n "$none" ] && [ -n "$inet" ] && [ -n "$vpn" ]; then
|
||||
[ "$vpn" != "$inet" ] && break
|
||||
# Steering may not have converged; re-read the vpn pod only.
|
||||
vpn=""
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
echo "unlabelled_ip=$none internet_ip=$inet vpn_ip=$vpn"
|
||||
|
||||
[ -n "$none" ] || { echo "FAIL: the UNLABELLED pod never returned a public IP — a pod that declares no egress class must still reach the internet over the default WAN path (docs/03: 'the internet path is simply the default route out'); no path at all means the default was blackholed or steered into a dead tunnel" >&2; exit 1; }
|
||||
[ -n "$inet" ] || { echo "FAIL: the internet-class pod never returned a public IP — the declared default path is down, so no comparison is possible" >&2; exit 1; }
|
||||
[ -n "$vpn" ] || { echo "FAIL: the vpn-class pod never returned a public IP — the second egress path is not carrying traffic, so the platform is not offering MULTIPLE paths (R-NET-04)" >&2; exit 1; }
|
||||
|
||||
# TOTALITY: no rule selects the unlabelled pod, so it must land on the WAN —
|
||||
# the same WAN the explicitly-internet pod uses.
|
||||
if [ "$none" != "$inet" ]; then
|
||||
echo "FAIL: the UNLABELLED pod egressed $none but the internet-class pod egressed $inet. They must be the same path: docs/03 — 'Default is the WAN; the VPN is strictly opt-in. Every pod ... egress the local/private WAN by default and never the VPN.' A different IP means some rule is selecting a pod that never opted in." >&2
|
||||
exit 1
|
||||
fi
|
||||
# COEXISTENCE: and the vpn path is a genuinely different path, live right now.
|
||||
if [ "$vpn" = "$inet" ]; then
|
||||
echo "FAIL: the vpn-class pod egressed the same public IP as the WAN ($vpn) — there is only ONE live egress path, so 'multiple egress paths selected by routing rules' (R-NET-04) does not hold" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "EGRESS_DEFAULT_PATH_IS_WAN_OK"
|
||||
check:
|
||||
(contains($stdout, 'EGRESS_DEFAULT_PATH_IS_WAN_OK')): true
|
||||
# -----------------------------------------------------------------------------------
|
||||
- name: then-the-vpn-class-pod-still-reaches-private-destinations-locally
|
||||
try:
|
||||
- script:
|
||||
# PER-DESTINATION SELECTION. The steering rule tunnels 0.0.0.0/0 MINUS all
|
||||
# RFC1918 space, so a vpn-class pod's private-destination traffic must stay on the
|
||||
# local datapath. The cluster resolver is an in-cluster (RFC1918) address reached
|
||||
# from the pod's own /etc/resolv.conf — no literal is written here — so a
|
||||
# successful lookup from the vpn-class pod proves the tunnel did not swallow its
|
||||
# local traffic, i.e. ONE pod is using BOTH paths at the same time.
|
||||
content: |
|
||||
set -eu
|
||||
ns="$NAMESPACE"
|
||||
resolver="$(kubectl -n "$ns" exec egress-vpn -c curl -- \
|
||||
sh -c 'awk "/^nameserver/{print \$2; exit}" /etc/resolv.conf' 2>/dev/null || true)"
|
||||
[ -n "$resolver" ] || { echo "FAIL: the vpn-class pod has no nameserver in /etc/resolv.conf, so its local-path reachability cannot be observed" >&2; exit 1; }
|
||||
echo "vpn_pod_resolver=$resolver (in-cluster, RFC1918 — excluded from the tunnel)"
|
||||
|
||||
# A cluster-internal name, resolvable only by the in-cluster resolver: reaching
|
||||
# it proves the private-destination path works from a vpn-class pod.
|
||||
out=""
|
||||
for i in $(seq 1 6); do
|
||||
out="$(kubectl -n "$ns" exec egress-vpn -c curl -- \
|
||||
nslookup kubernetes.default.svc.cluster.local 2>/dev/null || true)"
|
||||
case "$out" in *"Address"*) break ;; esac
|
||||
sleep 5
|
||||
done
|
||||
case "$out" in
|
||||
*Address*) : ;;
|
||||
*) echo "FAIL: the vpn-class pod could not resolve kubernetes.default.svc.cluster.local via its in-cluster resolver ($resolver). Its RFC1918 traffic is being steered into the tunnel (or blackholed) instead of taking the local path — the two paths are not coexisting per destination, and the tunnel is behaving as a default route (docs/03: split-tunnel, 'only what Cilium points at tun0 uses the VPN')" >&2; exit 1 ;;
|
||||
esac
|
||||
echo "EGRESS_PATHS_COEXIST_PER_DESTINATION_OK"
|
||||
check:
|
||||
(contains($stdout, 'EGRESS_PATHS_COEXIST_PER_DESTINATION_OK')): true
|
||||
@@ -1,67 +0,0 @@
|
||||
# Three pods for the R-NET-04 default-path test — the SAME workload three times, differing
|
||||
# ONLY in how (or whether) they declare an egress class:
|
||||
#
|
||||
# egress-unlabelled NO egress-class label at all -> selected by NO routing rule
|
||||
# egress-internet egress-class: internet -> the declared default path
|
||||
# egress-vpn egress-class: vpn -> the only class any rule steers
|
||||
#
|
||||
# WHY AN UNLABELLED POD IS THE POINT. docs/03: "Every pod — and the nodes themselves — egress
|
||||
# the local/private WAN by default and **never** the VPN." The unlabelled pod is the case no
|
||||
# other test covers: nothing selects it, so the DATAPATH's default must carry it, and it must
|
||||
# come out the same WAN as the explicitly-`internet` pod. That is the totality half of
|
||||
# R-NET-04's "selected by routing rules" — absence of a rule is itself a path.
|
||||
#
|
||||
# The unlabelled pod is admissible here only because chainsaw's per-test namespace is not an
|
||||
# app namespace: Kyverno's require-egress-class-label (R-NET-05) rejects an unlabelled pod in
|
||||
# a namespace labelled homelab.keithjackson.dev/app-namespace=true. That admission guarantee
|
||||
# is R-NET-05's, and it is exactly why this test must run outside an app namespace to observe
|
||||
# the datapath's own default at all.
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: egress-unlabelled
|
||||
labels:
|
||||
app: egress-default-path-test
|
||||
spec:
|
||||
containers:
|
||||
- name: curl
|
||||
image: curlimages/curl:8.11.1
|
||||
command: ["sleep", "3600"]
|
||||
resources:
|
||||
requests: { cpu: 10m, memory: 16Mi }
|
||||
limits: { memory: 32Mi }
|
||||
restartPolicy: Never
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: egress-internet
|
||||
labels:
|
||||
app: egress-default-path-test
|
||||
homelab.keithjackson.dev/egress-class: internet
|
||||
spec:
|
||||
containers:
|
||||
- name: curl
|
||||
image: curlimages/curl:8.11.1
|
||||
command: ["sleep", "3600"]
|
||||
resources:
|
||||
requests: { cpu: 10m, memory: 16Mi }
|
||||
limits: { memory: 32Mi }
|
||||
restartPolicy: Never
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: egress-vpn
|
||||
labels:
|
||||
app: egress-default-path-test
|
||||
homelab.keithjackson.dev/egress-class: vpn
|
||||
spec:
|
||||
containers:
|
||||
- name: curl
|
||||
image: curlimages/curl:8.11.1
|
||||
command: ["sleep", "3600"]
|
||||
resources:
|
||||
requests: { cpu: 10m, memory: 16Mi }
|
||||
limits: { memory: 32Mi }
|
||||
restartPolicy: Never
|
||||
@@ -68,15 +68,6 @@ spec:
|
||||
timeouts:
|
||||
apply: 60s
|
||||
assert: 180s
|
||||
# -- R-ENV-02: the fixture's route hostnames MUST fall under the Gateway's wildcard listener
|
||||
# (*.${BASE_DOMAIN}) or they never attach — so the suffix is READ FROM THE CLUSTER UNDER TEST
|
||||
# (flux-system/cluster-vars, the ConfigMap Flux substitutes ${BASE_DOMAIN} from) rather than
|
||||
# authored here. Test-level => in scope for the fixture apply AND every probe step below
|
||||
# (step `outputs`, by contrast, are step-scoped — see the gwvip re-capture comments).
|
||||
# fixtures/app-and-probe.yaml consumes it via chainsaw resource templating.
|
||||
bindings:
|
||||
- name: baseDomain
|
||||
value: (x_k8s_get($client, 'v1', 'ConfigMap', 'flux-system', 'cluster-vars').data.BASE_DOMAIN)
|
||||
steps:
|
||||
# -- GIVEN: the exposed-app fixture (default-deny + gateway-ingress allow + backend +
|
||||
# Service + declared/undeclared HTTPRoutes) and a probe namespace. Assert the shared
|
||||
@@ -148,14 +139,12 @@ spec:
|
||||
env:
|
||||
- name: GWVIP
|
||||
value: ($gwvip)
|
||||
- name: BASE_DOMAIN
|
||||
value: ($baseDomain)
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
set -u
|
||||
host="gw-declared-conformance.${BASE_DOMAIN}"
|
||||
host="gw-declared-conformance.home.keithjackson.dev"
|
||||
last=000
|
||||
i=0
|
||||
while [ "$i" -lt 30 ]; do
|
||||
@@ -218,14 +207,12 @@ spec:
|
||||
env:
|
||||
- name: GWVIP
|
||||
value: ($gwvip)
|
||||
- name: BASE_DOMAIN
|
||||
value: ($baseDomain)
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
set -u
|
||||
host="gw-undeclared-conformance.${BASE_DOMAIN}"
|
||||
host="gw-undeclared-conformance.home.keithjackson.dev"
|
||||
reached_gw=no
|
||||
i=0
|
||||
while [ "$i" -lt 8 ]; do
|
||||
@@ -313,20 +300,3 @@ spec:
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
name: conformance-gw-probe
|
||||
# -----------------------------------------------------------------------------------
|
||||
# PROOF-OF-PASS SENTINEL (R-SEC-03). Every step above is an assert/Job-status gate, so
|
||||
# chainsaw already fails the test before reaching this step if any of them fails — but a
|
||||
# green Chainsaw JUnit carries NO stdout, and the L2/L3 tests in this suite can self-skip
|
||||
# GREEN, so lib/record_run.py refuses to treat a bare `passed` as evidence and records
|
||||
# `unknown-no-sentinel-declared` instead. Without a declared token this test could pass
|
||||
# forever and NEVER become a recorded proof of R-NET-01/02/03 — north-south reachability under default-deny, part of the isolation proof
|
||||
# — which is precisely what docs/07 asks for ("These proofs are part of the platform
|
||||
# acceptance suite, not one-time checks"). This step exists only to emit that token, and it
|
||||
# is reached only when everything above has passed. Asserted by
|
||||
# tests/l0-static/auditability-proofs-producible.
|
||||
- name: then-emit-the-proof-of-pass-sentinel
|
||||
try:
|
||||
- script:
|
||||
content: echo GATEWAY_INGRESS_ISOLATION_PROVEN_OK
|
||||
check:
|
||||
(contains($stdout, 'GATEWAY_INGRESS_ISOLATION_PROVEN_OK')): true
|
||||
+2
-11
@@ -10,13 +10,6 @@
|
||||
# The namespace opts into the shared Gateway (gateway-access: shared) exactly like a
|
||||
# rendered app namespace, so its HTTPRoutes attach to the real platform Gateway and the
|
||||
# probe exercises the true north-south path (Gateway Envoy -> backend), not a stand-in.
|
||||
#
|
||||
# ENVIRONMENT-AGNOSTIC HOSTNAMES (R-ENV-02): the two HTTPRoute hostnames are NOT literals. They
|
||||
# are chainsaw-templated from the `baseDomain` binding declared in ../chainsaw-test.yaml, which
|
||||
# reads BASE_DOMAIN out of the flux-system/cluster-vars ConfigMap of the cluster under test — the
|
||||
# same ConfigMap Flux substitutes ${BASE_DOMAIN} from. They MUST fall under the shared Gateway's
|
||||
# wildcard listener (*.${BASE_DOMAIN}) or the routes never attach, which is precisely why a
|
||||
# hardcoded suffix silently broke this test on any cluster that did not hold the apex.
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
@@ -141,8 +134,7 @@ spec:
|
||||
namespace: gateway
|
||||
sectionName: https
|
||||
hostnames:
|
||||
# gw-declared-conformance.<this cluster's BASE_DOMAIN> — see the header.
|
||||
- (join('.', ['gw-declared-conformance', $baseDomain]))
|
||||
- gw-declared-conformance.home.keithjackson.dev
|
||||
rules:
|
||||
- matches:
|
||||
- path:
|
||||
@@ -166,8 +158,7 @@ spec:
|
||||
namespace: gateway
|
||||
sectionName: https
|
||||
hostnames:
|
||||
# gw-undeclared-conformance.<this cluster's BASE_DOMAIN> — see the header.
|
||||
- (join('.', ['gw-undeclared-conformance', $baseDomain]))
|
||||
- gw-undeclared-conformance.home.keithjackson.dev
|
||||
rules:
|
||||
- matches:
|
||||
- path:
|
||||
|
||||
@@ -1,379 +0,0 @@
|
||||
# =============================================================================
|
||||
# L2 BEHAVIORAL — the cluster under test IS an HA k3s control plane (R-PLAT-01) [RUNNABLE]
|
||||
# Catalog: docs/14 L2 row ("real runtime behavior, probes against a cluster")
|
||||
# Spec: docs/00 R-PLAT-01 ("The platform MUST run on k3s across 3 nodes with
|
||||
# production-grade HA control plane in prod; dev/test MAY run fewer nodes but MUST
|
||||
# exercise the same multi-node topology.")
|
||||
# docs/01 "Kubernetes distro | **k3s** (HA, embedded etcd) | R-PLAT-01/02"
|
||||
# docs/02 table: "Control plane | HA embedded etcd (k3s server on all 3) | HA
|
||||
# embedded etcd (k3s server on all 3)"
|
||||
# "Node count | 3 VMs (mirror prod topology) | 3 physical nodes"
|
||||
# docs/02 §"k3s topology decisions": "HA control plane with **embedded etcd** across
|
||||
# all 3 nodes (no external datastore)"; "Nodes are both control-plane and workload
|
||||
# (3-node converged cluster)".
|
||||
#
|
||||
# WHY L2 — AND WHAT L0 ALREADY DID
|
||||
# --------------------------------
|
||||
# tests/l0-static/ha-control-plane-as-code (same requirement label) proves the AUTHORED
|
||||
# half offline: the repo builds a 3-server embedded-etcd control plane in both environments,
|
||||
# one founder, no external datastore, a leader-elected control-plane-only VIP in the serving
|
||||
# cert's SANs. It cannot prove that any of it is TRUE OF A RUNNING CLUSTER — an inventory
|
||||
# entry is not a node, and a templated DaemonSet is not a floating VIP. That is this test:
|
||||
# every assertion below is read from the cluster the suite is pointed at.
|
||||
#
|
||||
# ENVIRONMENT-AGNOSTIC (R-ENV-02): no address, hostname, node count or range is written
|
||||
# here. The expected node count comes from the cluster's OWN declared identity
|
||||
# (flux-system/cluster-vars CLUSTER_ENV, via a test-level chainsaw binding — the same
|
||||
# ConfigMap Flux substitutes from); the control-plane VIP is read out of the kube-vip
|
||||
# DaemonSet the cluster is actually running; the server addresses come from the Node
|
||||
# objects; and the LoadBalancer pool the VIP must stay outside of is read from cluster-vars
|
||||
# too. R-PLAT-01 itself distinguishes prod from dev/test ("3 nodes ... in prod; dev/test MAY
|
||||
# run fewer"), so the assertion is necessarily environment-AWARE — but it takes the
|
||||
# environment from the cluster, never from a flag or a literal, which is precisely the
|
||||
# mechanism the tests/README.md convention prescribes.
|
||||
#
|
||||
# GIVEN the cluster this suite is pointed at,
|
||||
# WHEN its Nodes, its kube-vip DaemonSet and its kube-apiserver endpoints are inspected,
|
||||
# THEN DISTRO: every node's kubelet is a k3s build (`+k3s`) — the distro really is k3s;
|
||||
# SERVERS: every Ready node is BOTH control-plane AND etcd (the converged 3-node
|
||||
# topology docs/02 fixes — no agent-only node, no control-plane-only node),
|
||||
# the count is ODD and quorum-capable (tolerates >= 1 loss), and it is
|
||||
# EXACTLY three when the cluster declares itself prod;
|
||||
# ETCD: every one of those nodes is a REGISTERED embedded-etcd member (k3s stamps
|
||||
# `etcd.k3s.cattle.io/node-name` on a node that joined the etcd cluster), so
|
||||
# the datastore is embedded and replicated across all of them;
|
||||
# VIP: kube-vip is Ready on EVERY control-plane node, is control-plane-only
|
||||
# (svc_enable=false — it must not contend with Cilium LB-IPAM for ARP), and
|
||||
# the address it floats is neither a node address, nor the ingress Gateway
|
||||
# VIP, nor inside the Cilium LoadBalancer pool;
|
||||
# ENDPOINT: the kube-apiserver answers `/readyz` THROUGH THAT VIP over TLS that
|
||||
# validates against the cluster CA for the VIP itself — i.e. the VIP really
|
||||
# fronts the API and the serving cert really carries it;
|
||||
# EVERY SERVER SERVES: the apiserver also answers `/readyz` on EVERY server's own
|
||||
# address, again with a CA-valid cert for that address. This is the assertion
|
||||
# that separates a real HA control plane from one server plus two joiners:
|
||||
# if only the founder ran an apiserver, the other addresses would refuse.
|
||||
#
|
||||
# THE PROBE RUNS IN-CLUSTER, DELIBERATELY. An earlier draft dialled the VIP and the node
|
||||
# addresses from the RUNNER. That works from a host on the node LAN (verified against prod)
|
||||
# and FAILS against dev, whose nodes sit behind a NAT the runner reaches only via a
|
||||
# forwarded port — i.e. it would have made the test a property of where it was launched
|
||||
# from. A Job on the cluster is on the node network in both environments, so the probe is
|
||||
# the same probe everywhere. The Job is created imperatively (kubectl heredoc) rather than
|
||||
# as a fixture file because its address list is DISCOVERED from the cluster at run time —
|
||||
# writing it into a fixture is exactly the per-environment literal R-ENV-02 forbids.
|
||||
#
|
||||
# WHAT THIS DOES *NOT* PROVE (honest gaps — do not read this test as claiming them):
|
||||
# - FAILOVER / SURVIVAL. Nothing here removes a node. That etcd keeps quorum, that the VIP
|
||||
# migrates, and that the API stays reachable when a server dies is an L3 fault-injection
|
||||
# claim and is proven NOWHERE in this suite yet (tests/l3-fault-injection/node-loss
|
||||
# drains a node for R-STATE-02's data-availability claim and asserts nothing about the
|
||||
# control plane). This is the largest remaining R-PLAT-01 gap.
|
||||
# - THE NEXT environment. That the repo would build an HA control plane again is the L0
|
||||
# half (tests/l0-static/ha-control-plane-as-code).
|
||||
# - "PRODUCTION-GRADE" as a capacity/performance judgement (operator's, docs/02 + §2 of
|
||||
# infra/PROD-PROVISIONING.md).
|
||||
# - THAT THE RUNNING cilium-agents ACTUALLY DIAL THIS VIP. As of 2026-09-06 the repository
|
||||
# authors `K3S_API_HOST` = the control-plane VIP in both environments (asserted at L0 by
|
||||
# tests/l0-static/ha-control-plane-as-code, on both sides of the [parity] contract), but
|
||||
# this test does not read the agents' live `KUBERNETES_SERVICE_HOST`. Until the operator
|
||||
# applies that change the LIVE agents still carry the PRIMARY NODE's address — measured
|
||||
# 2026-09-06: all three prod agents on 192.168.2.10 while kube-vip held 192.168.2.9. An
|
||||
# in-cluster assertion on the agents' endpoint belongs here and is NOT yet written; it is
|
||||
# the obvious follow-up once the change is applied and would catch drift the L0 check
|
||||
# cannot see (a hand-edited DaemonSet, a Helm value overridden out of band).
|
||||
# - etcd health beyond membership registration (raft leader, DB size, alarms) — k3s exposes
|
||||
# it only on the node-local etcd metrics port, which is not reachable through the
|
||||
# Kubernetes API this test is allowed to use.
|
||||
#
|
||||
# CLEANUP: the probe Job lives in chainsaw's per-test namespace and is deleted with it
|
||||
# (config/chainsaw-config.yaml skipDelete: false). Everything else is read-only: this test
|
||||
# creates nothing outside its own namespace and mutates no platform object.
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: ha-control-plane
|
||||
labels:
|
||||
conformance.home.k3s/R-PLAT-01: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-PLAT-01"
|
||||
conformance.home.k3s/layer: "L2"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
Given the cluster under test, when its Nodes, kube-vip DaemonSet and apiserver
|
||||
endpoints are probed, then every node is a k3s server that is both control-plane and a
|
||||
registered embedded-etcd member, the count is odd and quorum-capable (exactly three
|
||||
when the cluster declares itself prod), kube-vip floats a control-plane-only VIP that
|
||||
collides with no allocator, and the apiserver answers /readyz through that VIP AND on
|
||||
every server's own address with a CA-valid cert.
|
||||
spec:
|
||||
timeouts:
|
||||
apply: 60s
|
||||
assert: 2m
|
||||
exec: 3m
|
||||
# -- R-ENV-02: the environment's identity and its address allocations are READ FROM THE
|
||||
# CLUSTER UNDER TEST. flux-system/cluster-vars is the ConfigMap Flux itself substitutes
|
||||
# from, so these are the same values the platform was rendered with. Test-level, so
|
||||
# they are in scope for every step (step outputs/bindings are step-scoped).
|
||||
bindings:
|
||||
- name: clusterEnv
|
||||
value: (x_k8s_get($client, 'v1', 'ConfigMap', 'flux-system', 'cluster-vars').data.CLUSTER_ENV)
|
||||
- name: gatewayVip
|
||||
value: (x_k8s_get($client, 'v1', 'ConfigMap', 'flux-system', 'cluster-vars').data.GATEWAY_VIP)
|
||||
- name: lbPoolStart
|
||||
value: (x_k8s_get($client, 'v1', 'ConfigMap', 'flux-system', 'cluster-vars').data.CILIUM_LB_POOL_START)
|
||||
- name: lbPoolStop
|
||||
value: (x_k8s_get($client, 'v1', 'ConfigMap', 'flux-system', 'cluster-vars').data.CILIUM_LB_POOL_STOP)
|
||||
steps:
|
||||
# -----------------------------------------------------------------------------------
|
||||
- name: then-the-node-set-is-a-quorum-capable-converged-k3s-control-plane
|
||||
description: >-
|
||||
Read the Node objects: k3s on every node, every node both control-plane and a
|
||||
registered embedded-etcd member, an odd quorum-capable count, exactly three in prod.
|
||||
try:
|
||||
- script:
|
||||
env:
|
||||
- name: CLUSTER_ENV
|
||||
value: ($clusterEnv)
|
||||
content: |
|
||||
set -eu
|
||||
fail() { echo "FAIL: $*" >&2; exit 1; }
|
||||
|
||||
total="$(kubectl get nodes -o name | grep -c . || true)"
|
||||
[ "$total" -ge 1 ] || fail "the cluster reports NO nodes at all"
|
||||
|
||||
# --- k3s, on every node (R-PLAT-01 names the distro; docs/01 ADR-0001) -----
|
||||
# A non-k3s kubelet here would mean the platform is not running on k3s.
|
||||
notk3s="$(kubectl get nodes \
|
||||
-o jsonpath='{range .items[*]}{.metadata.name}{"="}{.status.nodeInfo.kubeletVersion}{"\n"}{end}' \
|
||||
| grep -v '+k3s' | grep -c . || true)"
|
||||
[ "$notk3s" = "0" ] \
|
||||
|| fail "$notk3s node(s) do not run a k3s kubelet — R-PLAT-01 requires the platform to run on k3s"
|
||||
|
||||
# --- every node is BOTH control-plane AND etcd (docs/02 converged topology) -
|
||||
# Counted by label EXISTENCE, so it does not matter whether k3s writes "true"
|
||||
# or "". A control-plane node without the etcd role would be a server with an
|
||||
# external/absent datastore; an etcd node without control-plane would be an
|
||||
# isolated datastore member. Both are excluded by requiring all three counts
|
||||
# to be equal.
|
||||
cp="$(kubectl get nodes -l node-role.kubernetes.io/control-plane -o name | grep -c . || true)"
|
||||
etcd="$(kubectl get nodes -l node-role.kubernetes.io/etcd -o name | grep -c . || true)"
|
||||
echo "nodes_total=$total control_plane=$cp etcd=$etcd cluster_env=${CLUSTER_ENV:-<unset>}"
|
||||
[ "$cp" = "$total" ] \
|
||||
|| fail "$cp of $total nodes carry node-role.kubernetes.io/control-plane — docs/02 fixes a converged cluster where every node is a control-plane member ('k3s server on all 3')"
|
||||
[ "$etcd" = "$total" ] \
|
||||
|| fail "$etcd of $total nodes carry node-role.kubernetes.io/etcd — docs/02 requires embedded etcd 'across all 3 nodes (no external datastore)'"
|
||||
|
||||
# --- every node is a REGISTERED embedded-etcd member -----------------------
|
||||
# k3s stamps etcd.k3s.cattle.io/node-name on a node that actually joined the
|
||||
# embedded-etcd cluster. The role label alone is a label; this annotation is
|
||||
# the datastore's own record of membership.
|
||||
members="$(kubectl get nodes \
|
||||
-o jsonpath='{range .items[*]}{.metadata.annotations.etcd\.k3s\.cattle\.io/node-name}{"\n"}{end}' \
|
||||
| grep -c . || true)"
|
||||
[ "$members" = "$total" ] \
|
||||
|| fail "$members of $total nodes are registered embedded-etcd members (etcd.k3s.cattle.io/node-name) — the datastore is not replicated across every server"
|
||||
|
||||
# --- Ready ---------------------------------------------------------------
|
||||
notready="$(kubectl get nodes \
|
||||
-o jsonpath='{range .items[*]}{.metadata.name}{"="}{range .status.conditions[?(@.type=="Ready")]}{.status}{end}{"\n"}{end}' \
|
||||
| grep -v '=True$' | grep -c . || true)"
|
||||
[ "$notready" = "0" ] \
|
||||
|| fail "$notready node(s) are not Ready — a control plane missing a member is not the HA topology under test"
|
||||
|
||||
# --- quorum arithmetic ---------------------------------------------------
|
||||
# Embedded etcd needs floor(N/2)+1 members to commit. Fault tolerance is
|
||||
# N - quorum; 'production-grade HA' must survive at least one loss, which
|
||||
# rules out 1 and 2 members. An EVEN count buys no extra tolerance and adds
|
||||
# a tie, so it is also rejected.
|
||||
quorum=$(( total / 2 + 1 ))
|
||||
tolerance=$(( total - quorum ))
|
||||
echo "quorum=$quorum tolerance=$tolerance"
|
||||
[ "$tolerance" -ge 1 ] \
|
||||
|| fail "$total etcd member(s) give quorum $quorum and tolerate $tolerance loss(es) — an HA control plane must survive losing a server"
|
||||
[ $(( total % 2 )) -eq 1 ] \
|
||||
|| fail "$total etcd members is EVEN: quorum is the same as for $(( total - 1 )) members, so the extra member adds a tie instead of tolerance"
|
||||
|
||||
# --- the count R-PLAT-01 itself fixes, per environment --------------------
|
||||
# The environment is the cluster's OWN CLUSTER_ENV (cluster-vars), never a
|
||||
# flag: 'k3s across 3 nodes ... in prod; dev/test MAY run fewer nodes but MUST
|
||||
# exercise the same multi-node topology'.
|
||||
case "${CLUSTER_ENV:-}" in
|
||||
prod)
|
||||
[ "$total" = "3" ] \
|
||||
|| fail "this cluster declares CLUSTER_ENV=prod and has $total nodes — R-PLAT-01 requires 'k3s across 3 nodes ... in prod' (docs/02: '3 physical nodes')" ;;
|
||||
"")
|
||||
fail "flux-system/cluster-vars has no CLUSTER_ENV — the cluster does not declare which environment it is, so R-PLAT-01's prod clause has no referent (and no literal may be substituted for it)" ;;
|
||||
*)
|
||||
[ "$total" -ge 2 ] \
|
||||
|| fail "this cluster declares CLUSTER_ENV=${CLUSTER_ENV} and has $total node(s) — dev/test may run fewer nodes than prod but MUST 'exercise the same multi-node topology'" ;;
|
||||
esac
|
||||
echo "HA_TOPOLOGY_OK"
|
||||
check:
|
||||
(contains($stdout, 'HA_TOPOLOGY_OK')): true
|
||||
# -----------------------------------------------------------------------------------
|
||||
- name: then-a-control-plane-only-vip-is-floated-on-every-server
|
||||
description: >-
|
||||
Read the running kube-vip DaemonSet: Ready on every control-plane node,
|
||||
control-plane-only, and floating an address that collides with no other allocator.
|
||||
try:
|
||||
- script:
|
||||
env:
|
||||
- name: GATEWAY_VIP
|
||||
value: ($gatewayVip)
|
||||
- name: LB_POOL_START
|
||||
value: ($lbPoolStart)
|
||||
- name: LB_POOL_STOP
|
||||
value: ($lbPoolStop)
|
||||
content: |
|
||||
set -eu
|
||||
fail() { echo "FAIL: $*" >&2; exit 1; }
|
||||
# dotted quad -> integer, for range comparison without ipcalc/python.
|
||||
n() { echo "$1" | awk -F. '{printf "%d", ($1*16777216)+($2*65536)+($3*256)+$4}'; }
|
||||
|
||||
# The DaemonSet is found by the label roles/kube_vip stamps on it, not by a
|
||||
# hardcoded name in a namespace this test guessed.
|
||||
ds="$(kubectl -n kube-system get ds -l app.kubernetes.io/name=kube-vip-ds -o name | head -1)"
|
||||
[ -n "$ds" ] \
|
||||
|| fail "no kube-vip DaemonSet is running in kube-system — the kube-apiserver has no floating endpoint, so losing the founder takes the API endpoint with it"
|
||||
|
||||
desired="$(kubectl -n kube-system get "$ds" -o jsonpath='{.status.desiredNumberScheduled}')"
|
||||
ready="$(kubectl -n kube-system get "$ds" -o jsonpath='{.status.numberReady}')"
|
||||
cp="$(kubectl get nodes -l node-role.kubernetes.io/control-plane -o name | grep -c . || true)"
|
||||
vip="$(kubectl -n kube-system get "$ds" \
|
||||
-o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="address")].value}')"
|
||||
cp_enable="$(kubectl -n kube-system get "$ds" \
|
||||
-o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="cp_enable")].value}')"
|
||||
svc_enable="$(kubectl -n kube-system get "$ds" \
|
||||
-o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="svc_enable")].value}')"
|
||||
echo "kube_vip=$ds ready=$ready/$desired control_plane_nodes=$cp vip=$vip cp_enable=$cp_enable svc_enable=$svc_enable"
|
||||
|
||||
[ -n "$vip" ] \
|
||||
|| fail "the kube-vip DaemonSet announces no address — nothing is floated"
|
||||
[ "$cp_enable" = "true" ] \
|
||||
|| fail "kube-vip runs with cp_enable=$cp_enable — control-plane mode is off, so it floats no API VIP"
|
||||
[ "$svc_enable" = "false" ] \
|
||||
|| fail "kube-vip runs with svc_enable=$svc_enable — it would also serve Service LoadBalancers and contend with Cilium LB-IPAM for ARP on the same addresses (ADR-0002 gives Service LB to Cilium)"
|
||||
[ "$desired" = "$cp" ] \
|
||||
|| fail "kube-vip is scheduled on $desired node(s) but there are $cp control-plane nodes — the VIP must be able to move to ANY server, or its failover set is smaller than the control plane"
|
||||
[ "$ready" = "$desired" ] \
|
||||
|| fail "kube-vip is Ready on $ready of $desired nodes — a VIP holder that is not running cannot take the address over"
|
||||
|
||||
# --- the VIP collides with nothing --------------------------------------
|
||||
node_ips="$(kubectl get nodes \
|
||||
-o jsonpath='{range .items[*]}{.status.addresses[?(@.type=="InternalIP")].address}{"\n"}{end}' \
|
||||
| grep -c "^${vip}$" || true)"
|
||||
[ "$node_ips" = "0" ] \
|
||||
|| fail "the floated VIP $vip is ALSO a node's own InternalIP — a VIP that is a node's address cannot float off that node"
|
||||
[ "$vip" != "${GATEWAY_VIP:-}" ] \
|
||||
|| fail "the control-plane VIP and the ingress Gateway VIP are the same address ($vip) — kube-vip and Cilium would both ARP for it"
|
||||
if [ -n "${LB_POOL_START:-}" ] && [ -n "${LB_POOL_STOP:-}" ]; then
|
||||
v="$(n "$vip")"; lo="$(n "$LB_POOL_START")"; hi="$(n "$LB_POOL_STOP")"
|
||||
[ "$lo" -le "$hi" ] \
|
||||
|| fail "the cluster's Cilium LB pool ${LB_POOL_START}-${LB_POOL_STOP} is inverted, so the VIP cannot be proven to be outside it"
|
||||
if [ "$v" -ge "$lo" ] && [ "$v" -le "$hi" ]; then
|
||||
fail "the control-plane VIP $vip lies INSIDE this cluster's Cilium LoadBalancer pool ${LB_POOL_START}-${LB_POOL_STOP} — LB-IPAM could hand the kube-API's VIP to a Service"
|
||||
fi
|
||||
else
|
||||
fail "flux-system/cluster-vars declares no CILIUM_LB_POOL_START/STOP — the separation between the control-plane VIP and the Service-LB allocator cannot be checked, and no literal range may be substituted for it (R-ENV-02)"
|
||||
fi
|
||||
echo "HA_VIP_CONTROL_PLANE_ONLY_OK"
|
||||
check:
|
||||
(contains($stdout, 'HA_VIP_CONTROL_PLANE_ONLY_OK')): true
|
||||
# -----------------------------------------------------------------------------------
|
||||
- name: then-the-apiserver-answers-through-the-vip-and-on-every-server
|
||||
description: >-
|
||||
From inside the cluster, probe /readyz on the floated VIP and on every server's own
|
||||
address, validating TLS against the cluster CA for the address dialled.
|
||||
try:
|
||||
- script:
|
||||
# The address list is DISCOVERED here (Nodes + the kube-vip DaemonSet), so the
|
||||
# Job cannot be a static fixture — writing the addresses down is the
|
||||
# per-environment literal R-ENV-02 forbids. Hence the imperative heredoc, the
|
||||
# same pattern tests/l3-fault-injection/dr-restore uses for run-time-shaped
|
||||
# fixtures. The Job lives in chainsaw's per-test namespace and is torn down with
|
||||
# it; nothing outside that namespace is created or changed.
|
||||
content: |
|
||||
set -eu
|
||||
ns="$NAMESPACE"
|
||||
fail() { echo "FAIL: $*" >&2; exit 1; }
|
||||
|
||||
ds="$(kubectl -n kube-system get ds -l app.kubernetes.io/name=kube-vip-ds -o name | head -1)"
|
||||
vip="$(kubectl -n kube-system get "$ds" \
|
||||
-o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="address")].value}')"
|
||||
servers="$(kubectl get nodes -l node-role.kubernetes.io/control-plane \
|
||||
-o jsonpath='{range .items[*]}{.status.addresses[?(@.type=="InternalIP")].address}{" "}{end}')"
|
||||
[ -n "$vip" ] || fail "no floated VIP to probe"
|
||||
[ -n "$servers" ] || fail "no control-plane node addresses to probe"
|
||||
echo "probing vip=$vip servers=$servers"
|
||||
|
||||
# The probe pod authenticates with its own ServiceAccount token and validates
|
||||
# TLS against the cluster CA, both from the projected SA volume. /readyz is
|
||||
# authorized for any authenticated identity via the built-in
|
||||
# system:public-info-viewer role — anonymous access is NOT relied on (verified:
|
||||
# an unauthenticated /readyz on this platform answers 401).
|
||||
kubectl -n "$ns" apply -f - <<EOF
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: ha-api-endpoint-probe
|
||||
spec:
|
||||
backoffLimit: 1
|
||||
activeDeadlineSeconds: 150
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
containers:
|
||||
- name: probe
|
||||
image: curlimages/curl:8.11.1
|
||||
env:
|
||||
- name: VIP
|
||||
value: "${vip}"
|
||||
- name: SERVERS
|
||||
value: "${servers}"
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
set -u
|
||||
SA=/var/run/secrets/kubernetes.io/serviceaccount
|
||||
probe() { # \$1 = address to dial
|
||||
curl -sS --max-time 15 \\
|
||||
--cacert "\$SA/ca.crt" \\
|
||||
-H "Authorization: Bearer \$(cat \$SA/token)" \\
|
||||
"https://\$1:6443/readyz"
|
||||
}
|
||||
rc=0
|
||||
out="\$(probe "\$VIP" 2>&1)" || rc=1
|
||||
echo "vip \$VIP -> [\$out]"
|
||||
if [ "\$rc" != "0" ] || [ "\$out" != "ok" ]; then
|
||||
echo "FAIL: the kube-apiserver did not answer /readyz through the floated VIP \$VIP with a CA-valid cert for that address — the VIP does not front the API (or it is missing from the serving cert's SANs)"
|
||||
exit 1
|
||||
fi
|
||||
for s in \$SERVERS; do
|
||||
rc=0
|
||||
out="\$(probe "\$s" 2>&1)" || rc=1
|
||||
echo "server \$s -> [\$out]"
|
||||
if [ "\$rc" != "0" ] || [ "\$out" != "ok" ]; then
|
||||
echo "FAIL: server \$s does not answer /readyz with a CA-valid cert for its own address — it is not running a kube-apiserver, so the control plane is not HA across every node"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
echo "HA_API_ENDPOINT_OK"
|
||||
EOF
|
||||
|
||||
# Bounded wait; on failure surface the probe's own output before failing.
|
||||
if ! kubectl -n "$ns" wait --for=condition=complete --timeout=140s job/ha-api-endpoint-probe; then
|
||||
kubectl -n "$ns" logs job/ha-api-endpoint-probe --tail=40 || true
|
||||
fail "the in-cluster API-endpoint probe did not complete — see its log above"
|
||||
fi
|
||||
logs="$(kubectl -n "$ns" logs job/ha-api-endpoint-probe --tail=40)"
|
||||
echo "$logs"
|
||||
case "$logs" in
|
||||
*HA_API_ENDPOINT_OK*) echo "HA_API_ENDPOINT_REACHABLE_OK" ;;
|
||||
*) fail "the probe completed without reporting success" ;;
|
||||
esac
|
||||
check:
|
||||
(contains($stdout, 'HA_API_ENDPOINT_REACHABLE_OK')): true
|
||||
@@ -1,212 +0,0 @@
|
||||
# =============================================================================
|
||||
# L2 BEHAVIORAL — every OIDC client's STORED secret equals the one git declares
|
||||
# (R-IDP-02, R-IDP-04) [RUNNABLE]
|
||||
# Spec: docs/15 identity-and-access; ADR-0014.
|
||||
# Impl: platform/keycloak/client-secret-reconciler.yaml (the CronJob + its script).
|
||||
#
|
||||
# WHY THIS TEST EXISTS
|
||||
# --------------------
|
||||
# The KeycloakRealmImport resolves each confidential client's secret from a `(env:NAME)`
|
||||
# placeholder mapped to a key of the SOPS-encrypted keycloak-realm-clients Secret. When
|
||||
# the Keycloak Operator cannot resolve a placeholder it stores the LITERAL placeholder
|
||||
# text as the client secret AND THE IMPORT JOB STILL REPORTS SUCCESS; the CR then rests
|
||||
# at Done=True / Started=False and never re-runs. Measured 2026-09-06: prod had all 12
|
||||
# clients holding the literal placeholder — every OIDC login on the cluster failed with
|
||||
# "Invalid client or Invalid client credentials" — and dev had one (oauth2-proxy), whose
|
||||
# Secret key was added after dev's import had already completed.
|
||||
#
|
||||
# Every SIGNAL the cluster offered was green: the import Job succeeded, the CR read
|
||||
# Done=True, the Keycloak pods were healthy, and the existing keycloak-config-as-code
|
||||
# test (which asserts Done=True) PASSED throughout. The only observable was a human
|
||||
# failing to log in. This test is that observable, automated: it compares STORED against
|
||||
# DECLARED for every client and fails on any difference, so the class cannot sit
|
||||
# undetected for months again.
|
||||
#
|
||||
# GIVEN the `homelab` realm and the SOPS-encrypted keycloak-realm-clients Secret,
|
||||
# WHEN the reconciler's script is run in --audit mode (READ-ONLY: it cannot write, and
|
||||
# its endpoint allow-list makes users/groups/roles unreachable),
|
||||
# -> THEN every declared client's stored secret matches git, and none holds an
|
||||
# unresolved import placeholder. Emits KEYCLOAK_CLIENT_SECRETS_AUDIT_OK.
|
||||
#
|
||||
# WHY IT RUNS THE SHIPPED SCRIPT rather than re-implementing the comparison: a detector
|
||||
# that duplicates the healer's logic drifts from it. This audits with the same code, in
|
||||
# the same mode the L0 gate selftests, so a passing audit is evidence about the healer
|
||||
# too. It runs as a Job IN the keycloak namespace (not in the ephemeral chainsaw
|
||||
# namespace) because the desired state is the mounted keycloak-realm-clients Secret and
|
||||
# the admin credential is keycloak-admin-credentials — both namespace-local, which is
|
||||
# also why neither this Job nor the CronJob needs any Kubernetes API access.
|
||||
#
|
||||
# chainsaw v0.2.12: Job + status assert; the leftover-Job delete keeps a re-run from
|
||||
# colliding with an immutable Job left by a previous failure.
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: keycloak-client-secrets-match-git
|
||||
labels:
|
||||
conformance.home.k3s/R-IDP-02: "true"
|
||||
conformance.home.k3s/R-IDP-04: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-IDP-02, R-IDP-04"
|
||||
conformance.home.k3s/layer: "L2"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
Every OIDC client's stored secret equals the SOPS-declared one and none holds an
|
||||
unresolved import placeholder — the observable that was missing when all 12 prod
|
||||
clients were broken behind a green realm import.
|
||||
spec:
|
||||
timeouts:
|
||||
apply: 60s
|
||||
assert: 5m
|
||||
exec: 5m
|
||||
steps:
|
||||
- name: given-the-reconciler-is-reconciled-onto-the-cluster
|
||||
description: >-
|
||||
The audit Job runs the CronJob's own script from its ConfigMap; if Flux has not
|
||||
applied the reconciler, there is nothing to audit WITH and that is the failure.
|
||||
try:
|
||||
- assert:
|
||||
resource:
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: keycloak-client-secret-reconciler
|
||||
namespace: keycloak
|
||||
- assert:
|
||||
resource:
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: keycloak-client-secret-reconciler
|
||||
namespace: keycloak
|
||||
spec:
|
||||
concurrencyPolicy: Forbid
|
||||
- name: then-every-stored-client-secret-matches-git
|
||||
try:
|
||||
- script:
|
||||
content: |
|
||||
kubectl -n keycloak delete job keycloak-client-secret-audit \
|
||||
--ignore-not-found --wait=true
|
||||
- apply:
|
||||
resource:
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: keycloak-client-secret-audit
|
||||
namespace: keycloak
|
||||
labels:
|
||||
app.kubernetes.io/name: keycloak-client-secret-audit
|
||||
app.kubernetes.io/part-of: conformance
|
||||
spec:
|
||||
backoffLimit: 0
|
||||
activeDeadlineSeconds: 240
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: keycloak-client-secret-audit
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
automountServiceAccountToken: false
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 65532
|
||||
runAsGroup: 65532
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: audit
|
||||
image: python:3.13.15-alpine3.23
|
||||
command:
|
||||
- python3
|
||||
- /opt/reconciler/reconcile-client-secrets.py
|
||||
# READ-ONLY. --audit never issues a write on any path; the
|
||||
# script's --selftest proves that (L0 gate).
|
||||
- --audit
|
||||
env:
|
||||
- name: KEYCLOAK_BASE_URL
|
||||
value: http://keycloak-service.keycloak.svc.cluster.local:8080
|
||||
- name: KEYCLOAK_REALM
|
||||
value: homelab
|
||||
- name: PYTHONDONTWRITEBYTECODE
|
||||
value: "1"
|
||||
- name: PYTHONUNBUFFERED
|
||||
value: "1"
|
||||
- name: KEYCLOAK_ADMIN_USER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: keycloak-admin-credentials
|
||||
key: admin-user
|
||||
- name: KEYCLOAK_ADMIN_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: keycloak-admin-credentials
|
||||
key: admin-password
|
||||
volumeMounts:
|
||||
- name: script
|
||||
mountPath: /opt/reconciler
|
||||
readOnly: true
|
||||
- name: client-secrets
|
||||
mountPath: /etc/keycloak-realm-clients
|
||||
readOnly: true
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 48Mi
|
||||
limits:
|
||||
memory: 128Mi
|
||||
volumes:
|
||||
- name: script
|
||||
configMap:
|
||||
name: keycloak-client-secret-reconciler
|
||||
defaultMode: 0444
|
||||
- name: client-secrets
|
||||
secret:
|
||||
secretName: keycloak-realm-clients
|
||||
defaultMode: 0400
|
||||
- assert:
|
||||
resource:
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: keycloak-client-secret-audit
|
||||
namespace: keycloak
|
||||
status:
|
||||
succeeded: 1
|
||||
# The Job's exit status already encodes the verdict, but the sentinel is what the
|
||||
# run LEDGER records (conformance/runs/*/run.log): a JUnit alone cannot tell a real
|
||||
# audit from a Job that succeeded without auditing anything.
|
||||
- script:
|
||||
content: |
|
||||
set -u
|
||||
out="$(kubectl -n keycloak logs job/keycloak-client-secret-audit \
|
||||
--tail=-1 2>&1 || true)"
|
||||
echo "$out"
|
||||
echo "$out" | grep -q 'KEYCLOAK_CLIENT_SECRETS_AUDIT_OK' \
|
||||
|| { echo "FAIL: the audit did not report every client secret matching git"; exit 1; }
|
||||
echo "$out" | grep -q 'UNRESOLVED IMPORT PLACEHOLDER' \
|
||||
&& { echo "FAIL: a client still holds the literal import placeholder"; exit 1; }
|
||||
echo KEYCLOAK_CLIENT_SECRET_DRIFT_NONE_OK
|
||||
check:
|
||||
(contains($stdout, 'KEYCLOAK_CLIENT_SECRET_DRIFT_NONE_OK')): true
|
||||
catch:
|
||||
- describe:
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
namespace: keycloak
|
||||
name: keycloak-client-secret-audit
|
||||
- podLogs:
|
||||
namespace: keycloak
|
||||
selector: app.kubernetes.io/name=keycloak-client-secret-audit
|
||||
tail: 60
|
||||
finally:
|
||||
- script:
|
||||
content: |
|
||||
kubectl -n keycloak delete job keycloak-client-secret-audit \
|
||||
--ignore-not-found --wait=false
|
||||
@@ -11,18 +11,6 @@
|
||||
#
|
||||
# Goes GREEN once Keycloak is up + the realm client-secret hand-off (keycloak-realm-clients) is filled
|
||||
# so the placeholder resolves and the import completes. chainsaw v0.2.12: script + block contains check.
|
||||
#
|
||||
# NOT PROVEN HERE — READ THIS BEFORE TRUSTING A GREEN RUN (added 2026-09-06).
|
||||
# Done=True means the import JOB EXITED 0. It does NOT mean the placeholders resolved: when the
|
||||
# operator cannot resolve one it stores the LITERAL `(env:...)` text as the client secret and the
|
||||
# import still reports success, then rests at Done=True / Started=False and never re-runs. THIS
|
||||
# TEST PASSED THROUGHOUT the 2026-09-06 outage in which all 12 prod clients held the literal
|
||||
# placeholder and every OIDC login on the cluster failed. Done=True is a necessary, badly
|
||||
# insufficient condition.
|
||||
# The sufficient check is tests/l2-behavioral/keycloak-client-secrets-match-git, which compares
|
||||
# each client's STORED secret against the SOPS-declared one; the static half is
|
||||
# tests/l0-static/keycloak-client-secret-reconciler. Do not "simplify" by folding those into this
|
||||
# test: the whole lesson is that the import's own success signal is not evidence.
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
|
||||
@@ -3,15 +3,10 @@
|
||||
# Spec: docs/15 identity-and-access; ADR-0014. Impl: Grafana as a Keycloak OIDC client
|
||||
# (platform/monitoring/kube-prometheus-stack.yaml auth.generic_oauth) — the self-contained SSO proof.
|
||||
#
|
||||
# ENVIRONMENT-AGNOSTIC (R-ENV-02): the expected IdP host is built from the ${BASE_DOMAIN} of the
|
||||
# CLUSTER UNDER TEST, read from flux-system/cluster-vars (the ConfigMap Flux substitutes from) by
|
||||
# a test-level chainsaw binding — so this asserts each cluster redirects to ITS OWN Keycloak, which
|
||||
# is exactly what would catch a cross-environment OIDC issuer left pointing at the other cluster.
|
||||
#
|
||||
# GIVEN Grafana is deployed and configured as an OIDC client of the Keycloak `homelab` realm,
|
||||
# WHEN an unauthenticated browser starts the OIDC login (GET /login/generic_oauth)
|
||||
# -> THEN Grafana redirects (302) to the CENTRAL IdP's authorize endpoint on
|
||||
# keycloak.${BASE_DOMAIN}/realms/homelab — proving SSO is driven by the one IdP
|
||||
# keycloak.home.keithjackson.dev/realms/homelab — proving SSO is driven by the one IdP
|
||||
# (R-IDP-01: central Identity Provider as the single user directory).
|
||||
#
|
||||
# Goes GREEN once the grafana-oidc client-secret hand-off is filled and Keycloak + Grafana are up.
|
||||
@@ -35,11 +30,6 @@ spec:
|
||||
apply: 60s
|
||||
assert: 3m
|
||||
exec: 3m
|
||||
# -- R-ENV-02: the IdP hostname suffix is READ FROM THE CLUSTER UNDER TEST
|
||||
# (flux-system/cluster-vars — the ConfigMap Flux substitutes ${BASE_DOMAIN} from).
|
||||
bindings:
|
||||
- name: baseDomain
|
||||
value: (x_k8s_get($client, 'v1', 'ConfigMap', 'flux-system', 'cluster-vars').data.BASE_DOMAIN)
|
||||
steps:
|
||||
- name: given-grafana-is-an-oidc-client
|
||||
try:
|
||||
@@ -69,9 +59,6 @@ spec:
|
||||
containers:
|
||||
- name: probe
|
||||
image: busybox:1.36.1
|
||||
env:
|
||||
- name: BASE_DOMAIN
|
||||
value: ($baseDomain)
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
@@ -81,14 +68,13 @@ spec:
|
||||
# busybox wget has no --max-redirect; -S dumps every response header
|
||||
# (incl. the initial 302 Location to the IdP) before following, so the
|
||||
# grep below still matches the Keycloak authorize URL. (Grafana verified
|
||||
# live: 302 -> keycloak.${BASE_DOMAIN}/realms/homelab/.../auth.)
|
||||
idp="keycloak.${BASE_DOMAIN}/realms/homelab"
|
||||
# live: 302 -> keycloak.home.keithjackson.dev/realms/homelab/.../auth.)
|
||||
out="$(wget -S -O /dev/null "$svc/login/generic_oauth" 2>&1 || true)"
|
||||
echo "$out"
|
||||
if echo "$out" | grep -qiF "$idp"; then
|
||||
echo "GRAFANA_OIDC_REDIRECT_OK -> ${idp}"
|
||||
if echo "$out" | grep -qi 'keycloak.home.keithjackson.dev/realms/homelab'; then
|
||||
echo GRAFANA_OIDC_REDIRECT_OK
|
||||
else
|
||||
echo "FAIL: Grafana did not redirect to ${idp} (this cluster's Keycloak homelab realm)"; exit 1
|
||||
echo "FAIL: Grafana did not redirect to the Keycloak homelab realm"; exit 1
|
||||
fi
|
||||
- assert:
|
||||
resource:
|
||||
|
||||
@@ -3,13 +3,8 @@
|
||||
# Catalog: docs/14 onboarding (auth wiring) + docs/15 identity-and-access.
|
||||
# Impl: platform/oauth2-proxy (forward-auth against the Keycloak `homelab` realm), ADR-0014.
|
||||
#
|
||||
# ENVIRONMENT-AGNOSTIC (R-ENV-02): this test names NO hostname. It talks to oauth2-proxy over its
|
||||
# in-cluster Service DNS and asserts HTTP behaviour, so the same authored test is correct on dev and
|
||||
# on prod. The issuer host below is written as ${BASE_DOMAIN} because that is what the platform
|
||||
# substitutes; it is prose, not an input — nothing here needs parameterizing.
|
||||
#
|
||||
# GIVEN oauth2-proxy is deployed and has validated the Keycloak OIDC issuer (it only becomes Available
|
||||
# after successfully discovering https://keycloak.${BASE_DOMAIN}/realms/homelab — this is
|
||||
# after successfully discovering https://keycloak.home.keithjackson.dev/realms/homelab — this is
|
||||
# the proxy->Keycloak flow working),
|
||||
# WHEN an UNAUTHENTICATED request hits the forward-auth decision endpoint (/oauth2/auth)
|
||||
# -> THEN it is challenged: HTTP 401 (no valid Keycloak session => access denied), and a normal
|
||||
|
||||
@@ -3,13 +3,8 @@
|
||||
# Catalog: docs/14 "Trusted TLS (R-TLS-01/02)"
|
||||
# Spec: docs/09 (Gateway API + DNS-01 wildcard), ADR-0006 (LE issuers)
|
||||
#
|
||||
# ENVIRONMENT-AGNOSTIC (R-ENV-02): the hostnames are built from the ${BASE_DOMAIN} of the CLUSTER
|
||||
# UNDER TEST, read from flux-system/cluster-vars (the ConfigMap Flux substitutes from) by a
|
||||
# test-level chainsaw binding. The same authored test therefore verifies dev's wildcard on dev and
|
||||
# prod's on prod, with nothing to pass at run time.
|
||||
#
|
||||
# GIVEN the shared Gateway terminating TLS with the cert-manager wildcard cert,
|
||||
# WHEN an in-cluster probe dials it by hostname (SNI *.${BASE_DOMAIN}) and
|
||||
# WHEN an in-cluster probe dials it by hostname (SNI *.home.keithjackson.dev) and
|
||||
# verifies the chain against the SYSTEM trust store,
|
||||
# THEN the served cert is non-self-signed, unexpired, covers the wildcard SAN, and
|
||||
# chains to a PUBLICLY-TRUSTED CA root (R-TLS-01). The wildcard SAN also covers
|
||||
@@ -41,11 +36,6 @@ metadata:
|
||||
that chains to a publicly-trusted CA root (LE production); the wildcard SAN
|
||||
covers internal-only hostnames too.
|
||||
spec:
|
||||
# -- R-ENV-02: the hostname suffix is READ FROM THE CLUSTER UNDER TEST (flux-system/cluster-vars
|
||||
# — the ConfigMap Flux substitutes ${BASE_DOMAIN} from), never authored per environment.
|
||||
bindings:
|
||||
- name: baseDomain
|
||||
value: (x_k8s_get($client, 'v1', 'ConfigMap', 'flux-system', 'cluster-vars').data.BASE_DOMAIN)
|
||||
steps:
|
||||
- name: then-gateway-cert-chains-to-trusted-root
|
||||
try:
|
||||
@@ -64,16 +54,13 @@ spec:
|
||||
containers:
|
||||
- name: probe
|
||||
image: curlimages/curl:8.11.1
|
||||
env:
|
||||
- name: BASE_DOMAIN
|
||||
value: ($baseDomain)
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
set -eu
|
||||
# Probe TWO wildcard hostnames — one "public", one "internal-only". Both
|
||||
# match *.${BASE_DOMAIN}, so a clean verify against the SYSTEM CA
|
||||
# match *.home.keithjackson.dev, so a clean verify against the SYSTEM CA
|
||||
# bundle for BOTH proves R-TLS-01 (chains to a publicly-trusted root) AND
|
||||
# R-TLS-03 (the same trusted wildcard covers internal-only names). Names
|
||||
# resolve via in-cluster split-horizon DNS to the Gateway VIP. curl (no
|
||||
@@ -81,7 +68,7 @@ spec:
|
||||
# the gateway's 404 once TLS+hostname verify; it exits 60 if the chain or
|
||||
# hostname fails. TLS terminates at the gateway before HTTP routing, so no
|
||||
# backend is needed.
|
||||
for host in "conformance.${BASE_DOMAIN}" "internal-only.${BASE_DOMAIN}"; do
|
||||
for host in conformance.home.keithjackson.dev internal-only.home.keithjackson.dev; do
|
||||
curl -sS --retry 5 --retry-connrefused --retry-delay 3 --max-time 20 \
|
||||
-o /dev/null "https://${host}/"
|
||||
echo "TLS_TRUSTED ${host}"
|
||||
@@ -93,20 +80,3 @@ spec:
|
||||
kind: Job
|
||||
metadata: { name: tls-probe }
|
||||
status: { succeeded: 1 }
|
||||
# -----------------------------------------------------------------------------------
|
||||
# PROOF-OF-PASS SENTINEL (R-SEC-03). Every step above is an assert/Job-status gate, so
|
||||
# chainsaw already fails the test before reaching this step if any of them fails — but a
|
||||
# green Chainsaw JUnit carries NO stdout, and the L2/L3 tests in this suite can self-skip
|
||||
# GREEN, so lib/record_run.py refuses to treat a bare `passed` as evidence and records
|
||||
# `unknown-no-sentinel-declared` instead. Without a declared token this test could pass
|
||||
# forever and NEVER become a recorded proof of R-TLS-01/03 — the TLS half of docs/07's audit proof list
|
||||
# — which is precisely what docs/07 asks for ("These proofs are part of the platform
|
||||
# acceptance suite, not one-time checks"). This step exists only to emit that token, and it
|
||||
# is reached only when everything above has passed. Asserted by
|
||||
# tests/l0-static/auditability-proofs-producible.
|
||||
- name: then-emit-the-proof-of-pass-sentinel
|
||||
try:
|
||||
- script:
|
||||
content: echo TLS_TRUSTED_PROVEN_OK
|
||||
check:
|
||||
(contains($stdout, 'TLS_TRUSTED_PROVEN_OK')): true
|
||||
@@ -40,17 +40,10 @@
|
||||
# alternative source for the data.
|
||||
#
|
||||
# KNOWN, DELIBERATE GAPS (asserted nowhere — do not read this test as claiming them):
|
||||
# * OFF-HOST is NOT proven, only OFF-CLUSTER — TRUE AS RUN, and being fixed. On the
|
||||
# dev estate the k3s nodes are VMs on the same Proxmox host that runs the
|
||||
# versitygw/MinIO S3 target (192.168.1.9), so this drill does not survive loss of
|
||||
# that hypervisor: one machine takes the cluster AND its backups.
|
||||
# That is exactly why the operator moved off-cluster DR to Cloudflare R2 on
|
||||
# 2026-09-05 (commit 32d6766 — S3_BACKUP_ENDPOINT is now an R2 endpoint). Once that
|
||||
# merges and this drill is RE-RUN against R2, off-host stops being a gap and becomes
|
||||
# proven, because the target no longer shares a failure domain with the cluster.
|
||||
# UNTIL THEN THIS TEST'S PASS IS AGAINST MinIO. R-STATE-03 is currently proven only
|
||||
# for the same-host target; re-proving it against R2 is a prerequisite for claiming
|
||||
# off-host DR, and the ledger row that proves R-STATE-03 predates the move.
|
||||
# * OFF-HOST is NOT proven, only OFF-CLUSTER. On the dev estate the k3s nodes are
|
||||
# VMs on the same Proxmox host that runs the versitygw S3 target (192.168.1.9),
|
||||
# so this drill does not survive loss of that hypervisor. Off-site/off-host
|
||||
# replication is an operator concern outside R-STATE-03's wording.
|
||||
# * The test does NOT read the backup objects out of the bucket with an S3 client.
|
||||
# Doing so would require handing the conformance runner the backup credential,
|
||||
# widening a privileged test identity for weaker evidence than (1)-(4) above.
|
||||
|
||||
@@ -1,337 +0,0 @@
|
||||
# Bulk NFS storage — the media library and the legacy Nextcloud archive
|
||||
|
||||
The operator-facing companion to the `class: bulk` storage entries in `catalog/{plex,sonarr,radarr,nextcloud}.yaml`.
|
||||
Everything here was **measured** on the storage host on 2026-09-07, over read-only SSH; nothing is
|
||||
assumed. It resolves the long-standing **OQ-2** stub (`platform/longhorn/storageclasses.yaml`) for the
|
||||
case that actually exists here — attaching datasets that already hold data — and leaves dynamic bulk
|
||||
provisioning unimplemented, because nothing needs it.
|
||||
|
||||
---
|
||||
|
||||
## 1. The storage host, and why prod cannot reach it
|
||||
|
||||
The bulk data lives on the **Proxmox host's ZFS pool**. That host has exactly one interface:
|
||||
|
||||
```
|
||||
vmbr0 192.168.1.9/24 default via 192.168.1.1 (pfSense LAN)
|
||||
```
|
||||
|
||||
| | nodes | reaches 192.168.1.9? |
|
||||
|---|---|---|
|
||||
| **dev** | 192.168.1.40 / .41 / .42 | **Yes** — same L2 segment, and it already holds an NFS export from this host (`/mnt/Storage/k8s-volumes`, in `/etc/exports`) |
|
||||
| **prod** | 192.168.2.10 / .11 / .12 | **No** — 192.168.2.0/24 is pfSense's **WAN**. Ports 22 and 8006 answer only because pfSense port-forwards them via 192.168.2.102. 2049 and 111 do not. |
|
||||
|
||||
Opening the storage path for prod is a **pending operator decision**. Its two candidate outcomes give
|
||||
different values for the `BULK_NFS_SERVER` cluster-var:
|
||||
|
||||
| outcome | `BULK_NFS_SERVER` for prod |
|
||||
|---|---|
|
||||
| port-forward `192.168.2.102:2049` → `192.168.1.9:2049` | `192.168.2.102` |
|
||||
| a WAN→LAN route to 192.168.1.0/24 | `192.168.1.9` |
|
||||
|
||||
Until it lands, `clusters/prod` holds `TBD-PROD-BULK-NFS-SERVER`. **See §7 for why that placeholder
|
||||
behaves differently from the repo's other ones.**
|
||||
|
||||
### It only needs ONE port
|
||||
|
||||
The PVs mount `nfsvers=4.1`. **NFSv4 carries mount and lock in-protocol**, so the only thing that
|
||||
must be reachable is **tcp/2049** — no rpcbind/111, no mountd, no statd. NFSv3 would have needed
|
||||
four ports, three of them on ephemeral numbers, which is a materially worse firewall story. The
|
||||
server advertises `+3 +4 +4.1 +4.2` (`/proc/fs/nfsd/versions`), so v4.1 is available and is the
|
||||
conservative floor.
|
||||
|
||||
### The export's client list is the SAME under both outcomes — measured, not assumed
|
||||
|
||||
**pfSense DNATs the existing port-forward WITHOUT source NAT.** Connecting from this workstation
|
||||
(192.168.2.132) to `192.168.2.102:22`, the storage host logged the peer as `192.168.2.132`:
|
||||
|
||||
```
|
||||
SSH_CONNECTION=192.168.2.132 39768 192.168.1.9 22
|
||||
ss -tn -> 192.168.1.9:22 192.168.2.132:39768
|
||||
```
|
||||
|
||||
The original WAN-side source address survives the translation (return traffic finds its way back via
|
||||
the host's default route, 192.168.1.1). So under **either** outcome the NFS server will see prod's
|
||||
**real** node addresses — 192.168.2.10/.11/.12 — and the `sharenfs` change in §5 is the same either
|
||||
way. **It does not have to wait for the routing decision.**
|
||||
|
||||
---
|
||||
|
||||
## 2. The layout, as found
|
||||
|
||||
### `Storage/plex` → `/mnt/Storage/plex` — 8.33 T used, 6.91 T avail
|
||||
|
||||
```
|
||||
/mnt/Storage/plex/ drwxr-xr-x 1000:3000
|
||||
├── downloads/ drwxr-xr-x 1000:1000
|
||||
│ ├── complete/ drwxr-xr-x 1000:3000
|
||||
│ │ ├── radarr/ drwxr-xr-x 1000:3000
|
||||
│ │ └── tv-sonarr/ drwxr-xr-x 1000:3000
|
||||
│ ├── incomplete/ drwxrwxr-x 1000:3000
|
||||
│ ├── downloads -> /shared/downloads (symlink OUTSIDE the export — see §8)
|
||||
│ └── <in-flight release dirs> drwxr-xr-x 1000:1000
|
||||
├── movies/ 1165 entries drwxr-xr-x 1000:3000
|
||||
├── tv/ 107 entries drwxr-xr-x 1000:3000
|
||||
└── photos/ drwxr-xr-x 1000:3000
|
||||
```
|
||||
|
||||
**The single most important fact about this layout: `downloads/` is a SIBLING of `tv/` and `movies/`
|
||||
inside the same dataset.** Sonarr's import source and its library are on one filesystem on the
|
||||
server, and so are Radarr's. That is what makes hardlinked / atomically-renamed imports possible at
|
||||
all — and §4 is about not throwing it away on the client side.
|
||||
|
||||
Note the download client's completed folders are already named `radarr` and `tv-sonarr`, i.e. this
|
||||
tree has been driven by an arr stack for a long time. The k3s arrs are joining an existing layout,
|
||||
not defining one.
|
||||
|
||||
### `Storage/nextcloud` → `/mnt/Storage/nextcloud` — 270 G used
|
||||
|
||||
```
|
||||
/mnt/Storage/nextcloud/ drwxr-xr-x root:root
|
||||
└── nextcloud/ drwxr-xr-x 1000:3000
|
||||
├── config/ 103 M drwxr-xr-x 1000:3000
|
||||
├── config_old/ 3.4 G drwxr-xr-x 1000:3000
|
||||
└── config_backup/ 267 G drwxr-xr-x 1000:3000
|
||||
```
|
||||
|
||||
**There is no `data/` directory.** 99% of the dataset is `config_backup`. This is the old, non-k3s
|
||||
Nextcloud's configuration plus a backup blob — an **archive**, not a live user-data tree. That is
|
||||
why it is mounted read-only; see §6.
|
||||
|
||||
---
|
||||
|
||||
## 3. What each app mounts, and why
|
||||
|
||||
| app | export | mountPath | subPath | mode | access mode |
|
||||
|---|---|---|---|---|---|
|
||||
| plex | `/mnt/Storage/plex` | `/media` | — | **ro** | ReadOnlyMany |
|
||||
| sonarr | `/mnt/Storage/plex` | `/media` | — | rw | ReadWriteMany |
|
||||
| radarr | `/mnt/Storage/plex` | `/media` | — | rw | ReadWriteMany |
|
||||
| nextcloud | `/mnt/Storage/nextcloud` | `/mnt/legacy-nextcloud` | `nextcloud` | **ro** | ReadOnlyMany |
|
||||
|
||||
In-app paths that follow: Plex libraries at `/media/movies`, `/media/tv`, `/media/photos`; Sonarr
|
||||
Root Folder `/media/tv`; Radarr Root Folder `/media/movies`; both importing from
|
||||
`/media/downloads/complete/{tv-sonarr,radarr}`.
|
||||
|
||||
**All three media consumers use the identical mountPath on purpose.** When qbittorrent is eventually
|
||||
given the same export (it is not in this change), the arrs and the download client will already agree
|
||||
on absolute paths, which is the second precondition for hardlinked imports — otherwise every import
|
||||
needs a Remote Path Mapping and quietly falls back to copying when one is missing.
|
||||
|
||||
**`/media` and not `/data`, and that is not cosmetic.** The arrs' Longhorn config PVC renders at the
|
||||
renderer default `/data/config`. A media mount at `/data` would be mounted **over** it and mask the
|
||||
config volume entirely. See §8 for the separate latent bug that discovery exposed.
|
||||
|
||||
---
|
||||
|
||||
## 4. Why the arrs mount the export ROOT, not two subPaths
|
||||
|
||||
This is the design decision most likely to be "simplified" later by someone who has not read this.
|
||||
|
||||
Sonarr and Radarr import a finished download into the library by **hardlinking** it (preserving the
|
||||
seeding copy) or by an **atomic rename**. Both are same-filesystem operations, and **to the kernel,
|
||||
"same filesystem" means "same mount."**
|
||||
|
||||
The tempting narrower shape is two subPath volumes — `subPath: tv` and `subPath: downloads`. Those
|
||||
are **two separate mounts** of the same export. `link()` across them returns **EXDEV** no matter that
|
||||
the server knows perfectly well they are one dataset. Sonarr does not fail loudly at that: it
|
||||
degrades to copy-then-delete. Every import then rewrites the entire file over NFS and destroys the
|
||||
seeding copy. The symptom is "imports are slow and my ratio is gone", weeks later.
|
||||
|
||||
The smallest subtree containing both `tv/` and `downloads/` **is the export root**. So the root mount
|
||||
is the *minimal* mount that preserves hardlinks: least privilege bounded below by a hard requirement,
|
||||
not a shortcut.
|
||||
|
||||
The cost is stated rather than hidden: Sonarr can also write `/media/movies` and Radarr `/media/tv`.
|
||||
Narrowing that would cost hardlinks, which is the worse trade.
|
||||
|
||||
---
|
||||
|
||||
## 5. The export change the operator must apply — STRICTLY ADDITIVE
|
||||
|
||||
**Do not apply this from the repo.** It is an operator step on the storage host, recorded here so the
|
||||
exact value is reviewable. The existing consumers **must keep working**: 192.168.1.6, 10.123.1.10 and
|
||||
192.168.1.18 on the media tree, and 192.168.1.6 and 192.168.1.5 on the Nextcloud dataset, are the
|
||||
live non-k3s prod stack.
|
||||
|
||||
Current values (`zfs get sharenfs Storage`):
|
||||
|
||||
```
|
||||
Storage/plex rw=@192.168.1.6:@10.123.1.10:@192.168.1.18,no_subtree_check,no_root_squash
|
||||
Storage/nextcloud rw=@192.168.1.6:@192.168.1.5,no_subtree_check,no_root_squash
|
||||
```
|
||||
|
||||
### `Storage/plex` — append the six k3s node addresses to the existing `rw=` list
|
||||
|
||||
```sh
|
||||
zfs set sharenfs='rw=@192.168.1.6:@10.123.1.10:@192.168.1.18:@192.168.1.40:@192.168.1.41:@192.168.1.42:@192.168.2.10:@192.168.2.11:@192.168.2.12,no_subtree_check,no_root_squash' Storage/plex
|
||||
```
|
||||
|
||||
Every pre-existing host is preserved verbatim and no option changes; only clients are added.
|
||||
|
||||
### `Storage/nextcloud` — add the k3s nodes to a **`ro=`** list, not to `rw=`
|
||||
|
||||
```sh
|
||||
zfs set sharenfs='rw=@192.168.1.6:@192.168.1.5,ro=@192.168.1.40:@192.168.1.41:@192.168.1.42:@192.168.2.10:@192.168.2.11:@192.168.2.12,no_subtree_check,no_root_squash' Storage/nextcloud
|
||||
```
|
||||
|
||||
This is the one place read-only can be enforced **server-side**, and it is worth taking: only
|
||||
Nextcloud consumes this dataset from k3s, so the export itself can refuse writes. Even a
|
||||
misconfigured client cannot then damage the archive.
|
||||
|
||||
**The media tree cannot get the same treatment.** Plex is read-only and the arrs are read-write, but
|
||||
all three may be scheduled on any node, so the server sees the same source addresses for all of
|
||||
them and cannot distinguish the reader from the writers. **Plex's read-only posture is therefore a
|
||||
client-side guarantee** (`spec.nfs.readOnly: true` on `plex-media`, plus the volumeMount flag) — an
|
||||
honest limitation, not an oversight. If it ever needs to be server-enforced, the lever is a
|
||||
dedicated node pool for Plex, which is a much larger change.
|
||||
|
||||
Per-node addresses rather than `@192.168.2.0/24` follows the precedent already on this host
|
||||
(`/etc/exports` lists the dev nodes individually, as `/32`). A `/24` would work and would survive
|
||||
adding a node; it also grants the whole subnet. Per-node is the tighter default.
|
||||
|
||||
### Verifying, without changing anything
|
||||
|
||||
```sh
|
||||
exportfs -v # the new client lines appear, existing ones unchanged
|
||||
zfs get sharenfs Storage/plex Storage/nextcloud
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 6. uid / gid — the failure this design exists to avoid
|
||||
|
||||
The export uses `sec=sys` and `no_root_squash`, so **the client's numeric uid is passed through
|
||||
verbatim** and the server's on-disk ownership is the entire access story. The media tree is
|
||||
`0755 1000:3000` throughout (some directories `1000:1000`), so **only uid 1000 can write it**.
|
||||
|
||||
**`fsGroup` cannot fix this, and reaching for it is how this gets misdiagnosed.** The in-tree NFS
|
||||
volume plugin performs **no ownership management**, so `podSecurityContext.fsGroup` is a **no-op on
|
||||
an NFS volume**. There is no recursive chown, no `g+rwX`, nothing. The container's own uid has to be
|
||||
right.
|
||||
|
||||
| app | container uid | writes? | outcome |
|
||||
|---|---|---|---|
|
||||
| plex | 1000 (`PLEX_UID`/`PLEX_GID`, already set) | no | reads fine — the tree is 0755, so world-readable anyway |
|
||||
| sonarr | **1000 via `PUID`** (added) | yes | matches the owner ⇒ writes fine |
|
||||
| radarr | **1000 via `PUID`** (added) | yes | matches the owner ⇒ writes fine |
|
||||
| nextcloud | 33 (`www-data`) | no | reads and traverses 0755 fine |
|
||||
|
||||
**Without the added `PUID`/`PGID`, sonarr and radarr would have failed exactly the way this tier
|
||||
always fails: the mount succeeds, the pod is Ready, and every import fails EACCES.** LinuxServer
|
||||
images default to uid/gid **911**, which owns nothing on that tree.
|
||||
|
||||
`PGID: 3000` is chosen over 1000 so files these write are group-owned by the dominant group already
|
||||
on the tree, keeping them readable to the live non-k3s consumers.
|
||||
|
||||
`podSecurityContext.fsGroup: 3000` is also added to sonarr and radarr — **for the Longhorn config
|
||||
PVC, not for the NFS mount.** fsGroup does work on a Longhorn filesystem, and it makes `/data/config`
|
||||
writable by the now-uid-1000 process without depending on the image re-chowning it after `PUID`
|
||||
changed.
|
||||
|
||||
Nextcloud needs no uid work at all, which is one more reason its mount is read-only.
|
||||
|
||||
---
|
||||
|
||||
## 7. The prod placeholder does NOT fail at apply time — read this before merging
|
||||
|
||||
Every other `TBD-PROD-*` in `clusters/prod/kustomization.yaml` is rejected loudly by CRD validation
|
||||
(Cilium, LB-IPAM, CNPG, Velero) because it is not a valid IP or URL. **This one is not.**
|
||||
|
||||
A PersistentVolume's `spec.nfs.server` is an **unvalidated string**. The API server accepts
|
||||
`TBD-PROD-BULK-NFS-SERVER`, the PV is created, and the PVC **binds** to it — binding is pure
|
||||
control-plane and never touches the server. The failure appears only when a pod mounts:
|
||||
|
||||
```
|
||||
MountVolume.SetUp failed ... Failed to resolve server TBD-PROD-BULK-NFS-SERVER
|
||||
```
|
||||
|
||||
so **plex, sonarr, radarr and nextcloud would sit in ContainerCreating on prod.**
|
||||
|
||||
**Consequently: set `BULK_NFS_SERVER` in `clusters/prod/kustomization.yaml` BEFORE prod reconciles a
|
||||
revision that carries these bulk volumes.** Dev renders a working value and is where these manifests
|
||||
are meant to be proven first.
|
||||
|
||||
### The static guard this convention leans on is missing from `main`
|
||||
|
||||
`bootstrap/standup.sh` — which carries the `*_is_placeholder` guard family — **does not exist on
|
||||
`main`** (verified: `git ls-tree main bootstrap/` lists only `README.md`, `flux/`, `gitea/`, `sops/`;
|
||||
the file lives on unmerged prod branches). So
|
||||
`conformance/tests/l0-static/placeholder-guard-coupling/check.sh` exits **3 SKIPPED**, and **nothing
|
||||
in the gate currently refuses a placeholder-shaped overlay value.**
|
||||
|
||||
When `standup.sh` lands, that check's assertions (2) and (3) cover this knob automatically — its
|
||||
independent detector's SEED vocabulary already matches `TBD` — **provided** the guard gains an arm
|
||||
for `BULK_NFS_SERVER` and a consumer that calls it. Without the arm, assertion (2) is what fails, by
|
||||
design.
|
||||
|
||||
---
|
||||
|
||||
## 8. Loose ends and things deliberately not done
|
||||
|
||||
1. **`/data/config` is very likely the wrong home for the arrs' config PVC — a pre-existing latent
|
||||
bug, not fixed here.** LinuxServer's arr images run with `-data=/config`, so the app writes
|
||||
`/config`, which is the container's **ephemeral layer**, while the Longhorn PVC mounted at
|
||||
`/data/config` sits unused. This is the exact failure `catalog/plex.yaml`'s `mountPath` comment
|
||||
records having already been bitten by ("the PVC sat unused and Plex wrote to the container's
|
||||
EPHEMERAL layer"). Plex was fixed; sonarr, radarr, qbittorrent and jackett were not. Fixing it
|
||||
means those apps start from an empty config volume, so it belongs in its own change with the
|
||||
operator present. **This change does not make it worse** — that is why the media mount is `/media`.
|
||||
2. **qbittorrent has no bulk mount.** Today the arrs' import source is written by the **legacy
|
||||
non-k3s** download client, which is coherent while the legacy stack stays live. Giving the k3s
|
||||
qbittorrent the same export at the same `/media` path is the natural follow-up, and the paths were
|
||||
chosen so it needs no Remote Path Mapping.
|
||||
3. **`/mnt/Storage/plex/downloads/downloads` is a symlink to `/shared/downloads`**, which is outside
|
||||
the export. Over NFS it is a dangling symlink on every client. Harmless, but expect Sonarr to log
|
||||
it while scanning `downloads/`.
|
||||
4. **`nfsvers=4.2` was not chosen**, though the server advertises it. Nothing in these workloads uses
|
||||
what 4.2 adds (server-side copy, sparse-file `SEEK`/`ALLOCATE`), and 4.1 is the version the
|
||||
existing `bulk` StorageClass stub already named.
|
||||
5. **`nconnect` was not set.** It would likely help throughput on the media tree, but it is a tuning
|
||||
change that should be measured, not guessed, and it changes client behaviour under server
|
||||
failure.
|
||||
6. **Nothing was verified against a live cluster.** No `kubectl` context is usable from this
|
||||
workstation (browser-OIDC only), and this change is manifest-authoring under an explicit
|
||||
do-not-apply constraint. Every claim about the *host* is measured; every claim about *binding and
|
||||
mounting* is reasoned from the manifests.
|
||||
7. **The bulk-tier requirement ID is deliberately not typed on any non-`.md` line of this change.**
|
||||
`ci/touched-requirements.sh` attributes requirements from R-IDs appearing in added lines, and the
|
||||
in-cluster conformance test for the bulk tier is still a gap, so typing it would select a
|
||||
requirement the gate cannot satisfy — the hazard `catalog/qbittorrent.yaml` already warns about.
|
||||
Scaffolding that test needs a cluster and is the correct follow-up.
|
||||
|
||||
---
|
||||
|
||||
## 9. Validating on dev, once the export is extended
|
||||
|
||||
Dev can prove all of this before prod's network path exists. Read-only checks first:
|
||||
|
||||
```sh
|
||||
infra/devcontainer/dev.sh run kubectl --context k3s-dev get pv
|
||||
infra/devcontainer/dev.sh run kubectl --context k3s-dev get pvc -A | grep -E 'media|legacy-archive'
|
||||
infra/devcontainer/dev.sh run kubectl --context k3s-dev -n sonarr get pod -o wide
|
||||
```
|
||||
|
||||
Expect **four** PVs — `plex-media`, `sonarr-media`, `radarr-media`, `nextcloud-legacy-archive` (three
|
||||
of them naming the *same* export, one per namespace, which is the cross-namespace model) — each
|
||||
`Bound` to the same-named PVC, with `RECLAIM POLICY Retain` and an empty `STORAGECLASS`.
|
||||
|
||||
Then the two things that actually matter, from inside a pod:
|
||||
|
||||
```sh
|
||||
# (a) the writer can write — this is the uid check, and it is the one that fails in practice
|
||||
infra/devcontainer/dev.sh run kubectl --context k3s-dev -n sonarr exec deploy/sonarr -- \
|
||||
sh -c 'id; touch /media/tv/.k3s-write-probe && rm /media/tv/.k3s-write-probe && echo WRITE-OK'
|
||||
|
||||
# (b) hardlinks work BETWEEN the library and the download path — the root-mount check
|
||||
infra/devcontainer/dev.sh run kubectl --context k3s-dev -n sonarr exec deploy/sonarr -- \
|
||||
sh -c 'set -e; d=/media/downloads/complete/tv-sonarr/.probe; t=/media/tv/.probe-link;
|
||||
echo x > "$d"; ln "$d" "$t"; ls -i "$d" "$t"; rm -f "$d" "$t"; echo HARDLINK-OK'
|
||||
|
||||
# (c) the reader really is read-only
|
||||
infra/devcontainer/dev.sh run kubectl --context k3s-dev -n plex exec deploy/plex -- \
|
||||
sh -c 'touch /media/.probe 2>&1 | head -1' # expect "Read-only file system"
|
||||
```
|
||||
|
||||
`(b)` printing two identical inode numbers is the proof that §4's argument holds in practice. If it
|
||||
prints `EXDEV`, something has split the single mount back into two.
|
||||
@@ -1,319 +0,0 @@
|
||||
# Decommission Proxmox `192.168.2.102` → TrueNAS SCALE on the N5 Air
|
||||
|
||||
**Status:** plan. **Drafted** 2026-09-06, **revised** 2026-09-06 after the operator settled the
|
||||
target architecture.
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| **N5 Air** | TrueNAS SCALE. **Storage only** — no VMs, no hypervisor. |
|
||||
| **Compute** | k3s **prod** cluster (`192.168.2.10/.11/.12`). Plex, sonarr, radarr, jackett, qbittorrent, seerr and nextcloud are already modelled in `apps/wave-0/`. |
|
||||
| **Dev k3s** | Does not move to the NAS. |
|
||||
| **Proxmox host** | **Decommissioned.** |
|
||||
| **Drives** | Physically relocated. None purchased. |
|
||||
|
||||
All figures measured on the live host, not inferred.
|
||||
|
||||
---
|
||||
|
||||
## Already done — 2026-09-06
|
||||
|
||||
**Reclaimed 3.56 TB.** `Storage/plex@pre-cleanup-20260828` destroyed after confirming no holds and
|
||||
no clones. Async free drained in ~90 seconds.
|
||||
|
||||
| | before | after |
|
||||
|---|---|---|
|
||||
| `Storage` allocated | 12.3 T (75%) | **8.76 T (53%)** |
|
||||
| `Storage` free | 4.04 T | **7.60 T** |
|
||||
| `Storage/plex` used | 11.9 T | 8.33 T (0 B in snapshots) |
|
||||
|
||||
That free space is not a nicety — **it is what lets this whole migration happen with no network
|
||||
transfer at all.** See §3.
|
||||
|
||||
---
|
||||
|
||||
## 1. The plan changed shape
|
||||
|
||||
The first draft carried the VMs across on their own zvols and put Proxmox on the NAS. With no
|
||||
hypervisor on the far side, that inverts:
|
||||
|
||||
> **Every zvol is now dead weight, and every service's data has to be drained onto a plain dataset
|
||||
> before the Proxmox box dies.**
|
||||
|
||||
Dead-weight zvols, once their contents are extracted:
|
||||
|
||||
| zvol | size | after extraction |
|
||||
|---|---|---|
|
||||
| `Storage:vm-101/102/105-disk-0,-disk-1` (dev k3s) | 585 G | delete |
|
||||
| `Storage:vm-112-disk-0` (docker-services) | 65 G | delete |
|
||||
| `Storage:vm-103-disk-0` (nextcloud boot) | 81 G | delete |
|
||||
| `Storage2:vm-103-disk-0` (**nextcloud data**) | 4.06 T | extract, then delete |
|
||||
|
||||
The good news is the shape of the drain. Everything that must be rescued is either already on
|
||||
`Storage`, or can be copied onto `Storage` **locally, inside the same chassis, onto the very drives
|
||||
that are moving.** With 7.60 T free there is room. **Nothing crosses the network.**
|
||||
|
||||
---
|
||||
|
||||
## 2. Findings
|
||||
|
||||
### 2a. "The path lies" — two confirmed instances, assume a third
|
||||
|
||||
**Games — 422 GB, not on the Storage pool.** Challenged and re-verified; the test is decisive:
|
||||
|
||||
```
|
||||
findmnt -T /mnt/Storage/Games → TARGET / SOURCE rpool/ROOT/pve-1
|
||||
findmnt -T /mnt/Storage/plex → TARGET /mnt/Storage/plex SOURCE Storage/plex
|
||||
```
|
||||
|
||||
`Games` is a plain directory on the Proxmox **root** filesystem that merely sits inside the Storage
|
||||
mount tree. The pool accounting closes on it exactly:
|
||||
|
||||
```
|
||||
rpool ALLOC 540 G = ROOT 431 G + data 105 G + var-lib-vz 3.75 G
|
||||
ROOT 431 G = Games 422 G + /usr 3.5 G + /var 5 G + /boot 0.4 G
|
||||
```
|
||||
|
||||
The two Microns are a **mirror**, so the pool is ~888 G usable, not ~1.9 T — 422 G is 48% of it, and
|
||||
`zpool list` agrees at 60% capacity overall. Surprising, but the numbers do match.
|
||||
|
||||
**Gitea — 13 MB in an anonymous Docker volume, not where its compose file points.**
|
||||
`/root/docker-compose.yml` in LXC 106 bind-mounts `/opt/gitea/data:/var/lib/data`. That path holds
|
||||
**512 bytes**. The real Gitea state is in an unnamed Docker volume:
|
||||
|
||||
```
|
||||
.../volumes/8a1d2616…/_data/git/repositories/ 12 M home-network.git, home-network-uds.git, jakman85/…
|
||||
.../volumes/8a1d2616…/_data/data/gitea.db 11 M SQLite, mtime today 19:00 — live
|
||||
```
|
||||
|
||||
Copying `/opt/gitea` would migrate an empty directory and silently lose all three repositories.
|
||||
|
||||
**Rule for the rest of the drain:** for every service, verify where the bytes actually are before
|
||||
trusting the path in a compose file, an export list or a mount table. Two for two so far.
|
||||
|
||||
### 2b. Retiring pfSense is easier than rehoming it
|
||||
|
||||
`pfsense` is VM 100 and **is** `192.168.1.1`. Decommissioning the host takes the entire
|
||||
`192.168.1.0/24` with it:
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| `.1` pfSense · `.4` iDRAC · `.9` the Proxmox host itself | die with the box |
|
||||
| `.5` nextcloud · `.6` plex · `.7` gitea · `.18` docker-services/Pi-hole | data drains, service moves to k3s |
|
||||
| `.8` unifi · `.20` monitoring · `.21` auth · `.10` semaphore | rehome or retire (§2e) |
|
||||
| `.49` kube-vip VIP · `.50/.51/.52` dev k3s | dev cluster is not moving; retire with the box |
|
||||
|
||||
**But pfSense exists only to contain that lab LAN.** Prod k3s is already on `192.168.2.0/24` —
|
||||
pfSense's *WAN* side — and the NAS will be too. Once the lab is gone, nothing lives behind pfSense.
|
||||
**Retire it; do not rehome it.** That is only clean because everything else is leaving `1.0/24`
|
||||
anyway, which is exactly what this migration does.
|
||||
|
||||
**A documented blocker dissolves with it.** `infra/PROD-STANDUP.md` records that prod k3s cannot
|
||||
reach the S3 backup endpoint at `192.168.1.9:9000` because prod sits on pfSense's WAN side and the
|
||||
crossing rule does not exist. With the NAS on `192.168.2.0/24`, prod reaches its storage directly
|
||||
and no pfSense rule is ever needed.
|
||||
|
||||
**Three things to close before pfSense goes:**
|
||||
|
||||
1. The SOHO router NATs inbound **443** to `192.168.2.102` for the legacy stack. It must be
|
||||
repointed at prod's Gateway (`192.168.2.20`). `infra/PROD-PROVISIONING.md` §10a already flags
|
||||
that the apex flip and the legacy cutover are the same event — **this migration is that event.**
|
||||
2. **VLANs 5 and 123** (minecraft; the arr VM, which reaches Plex media as `10.123.1.10`) are
|
||||
pfSense-side constructs. They disappear; the k3s apps need no VLAN.
|
||||
3. The **OpenVPN split-tunnel egress gateway** currently runs on `k3s-dev-3` (ADR-0015). It is
|
||||
already modelled for prod in `platform/vpn-egress/`, but confirm prod's gateway is live before
|
||||
the dev node goes away.
|
||||
|
||||
### 2c. Nextcloud is the critical path
|
||||
|
||||
Its user data is **inside a filesystem, inside LVM, inside a 4 TB zvol, inside a VM**:
|
||||
|
||||
```
|
||||
Storage2/vm-103-disk-0 → zd224 → PV /dev/zd224 → VG vg_storage → LV lv_storage (<4.00 T)
|
||||
```
|
||||
|
||||
With no hypervisor on the far side this must be copied out **at the file level, while the VM is
|
||||
still running** — it gates the decommission and it is the longest single task in the plan.
|
||||
|
||||
**One number is missing and I could not get it.** VM 103's QEMU guest agent is not running
|
||||
(`qm agent 103 ping` → *"QEMU guest agent is not running"*) and its SSH does not accept the
|
||||
`home_network` key. The zvol shows `referenced 3.94 T`, but it is attached **without `discard=on`**
|
||||
and is thick-provisioned (`refreservation 4.06 T`), so deletions inside the guest were never
|
||||
returned to ZFS — 3.94 T is an **upper bound**, possibly far above the truth.
|
||||
|
||||
**Run this inside the Nextcloud VM and the last open question closes:**
|
||||
|
||||
```bash
|
||||
df -h / # or wherever lv_storage is mounted
|
||||
lsblk -f /dev/mapper/vg_storage-lv_storage
|
||||
```
|
||||
|
||||
Separately: `Storage/nextcloud` is 270 G, and **267 G of it is a directory called `config_backup`**
|
||||
(`config` itself is 103 M). Almost certainly a stale dump — likely another easy reclaim, but confirm
|
||||
before deleting.
|
||||
|
||||
### 2d. Storage2 — and the two oldest drives — can probably be retired
|
||||
|
||||
After the drain and the zvol deletions, `Storage` would hold roughly:
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| plex | 8.33 T |
|
||||
| nextcloud data | ≤ 3.94 T (real number pending §2c) |
|
||||
| Games | 0.42 T |
|
||||
| docker-volumes, other_containers, k8s-volumes, gitea | ~0.02 T |
|
||||
| **total** | **≤ 12.7 T of 16.4 T (≤ 77%)** |
|
||||
|
||||
If the real Nextcloud figure comes in under ~2 T — which the missing-TRIM situation makes plausible
|
||||
— everything fits on the 18 TB mirror at a comfortable ~65%, and **`Storage2` can be retired
|
||||
entirely.** That would remove the two **6.0-year-old** Seagates, by a wide margin the most
|
||||
failure-prone hardware in the estate, and free two bays. This is the single best safety outcome
|
||||
available, and it costs nothing.
|
||||
|
||||
### 2e. Services that simply end
|
||||
|
||||
Decommissioning kills these; none has a home in the target architecture unless you give it one:
|
||||
|
||||
| guest | note |
|
||||
|---|---|
|
||||
| `docker-services` Pi-hole (`192.168.1.18`) | **LAN DNS.** Confirm what `192.168.2.0/24` uses for DNS before the box dies. |
|
||||
| `unifi` (VM 121) | Network controller — needs a home or a cloud key. |
|
||||
| `auth` (VM 122) | Superseded by Keycloak on prod k3s. |
|
||||
| `monitoring` (VM 120) | Superseded by the k3s platform monitoring stack. |
|
||||
| `semaphore` (LXC 110) | 432 MB resident of an 8 GB reservation; low value. |
|
||||
| `minecraft` (LXC 108) | Operator call. Its 3.5 G subvol is on `Storage` and rides along. |
|
||||
| **versitygw S3** on `192.168.1.9:9000` | **Velero + CNPG backup target.** Dies with the box. TrueNAS must take this role (MinIO app) or prod loses its backup destination — which `infra/PROD-STANDUP.md` already records as uncovered. |
|
||||
|
||||
### 2f. Still true from the first draft
|
||||
|
||||
- **No VM backups exist.** `/etc/pve/jobs.cfg` empty, no PBS, newest dump log April 2025.
|
||||
- **`ata-WDC_…3FGHAB2T` has 36 ZFS read errors** on pristine SMART (0 reallocated / pending /
|
||||
uncorrectable, `PASSED`) — points at cable or controller, not platter. Re-verify after the move.
|
||||
- Drive ages: WD 18 TB pair **4.75 y**, Seagate 8 TB pair **6.0 y**.
|
||||
|
||||
---
|
||||
|
||||
## 3. TrueNAS SCALE specifics
|
||||
|
||||
**ZFS compatibility: verified, not assumed.** The only non-default features *active* on either pool
|
||||
are `block_cloning`, `head_errlog`, `vdev_zaps_v2`, `zilsaxattr`, `zstd_compress`, `log_spacemap`,
|
||||
`project_quota`, `spacemap_v2` — all OpenZFS **2.2** era or older. Nothing 2.3- or 2.4-only is
|
||||
active, even though the pools were last written by `zfs-2.4.3-pve1`. Any TrueNAS SCALE from 24.10
|
||||
onward imports them cleanly.
|
||||
|
||||
**Mountpoints mostly already match TrueNAS convention** — a lucky accident worth knowing:
|
||||
|
||||
| dataset | current mountpoint | TrueNAS wants | |
|
||||
|---|---|---|---|
|
||||
| `Storage/plex`, `/nextcloud`, `/docker-volumes`, `/other_containers`, `/k8s-volumes`, `/k3s-backups` | `/mnt/Storage/…` | `/mnt/Storage/…` | already correct |
|
||||
| `Storage` (pool root) | `/Storage` | `/mnt/Storage` | fix on import |
|
||||
| `Storage2` (pool root) | `/Storage2` | `/mnt/Storage2` | fix on import |
|
||||
| `Storage/subvol-108-disk-0` | `/Storage/subvol-108-disk-0` | — | rename or delete |
|
||||
|
||||
**Clear the ZFS `sharenfs` properties.** `Storage/plex`, `Storage/nextcloud` and
|
||||
`Storage/docker-volumes` carry exports as ZFS properties; `Storage/k8s-volumes` is in
|
||||
`/etc/exports`. TrueNAS generates exports from its own middleware database and will fight
|
||||
properties it did not set. Wipe them and recreate the shares in the UI.
|
||||
|
||||
**Fix the export targets while you are there.** Today's lists are stale: `/mnt/k3s` →
|
||||
`192.168.1.25/.26/.27` and `k8s-volumes` → `192.168.1.40/.41/.42` are hosts that do not answer ARP.
|
||||
The new exports target **`192.168.2.10/.11/.12`**.
|
||||
|
||||
---
|
||||
|
||||
## 4. Sequence
|
||||
|
||||
### Phase A — drain, locally, no downtime, no network
|
||||
|
||||
Everything below is a copy from somewhere else in the chassis onto `Storage`, which is moving.
|
||||
|
||||
1. **Games** — `rpool` → new dataset.
|
||||
```bash
|
||||
zfs create Storage/games
|
||||
rsync -aHAX --info=progress2 /mnt/Storage/Games/ /mnt/Storage/games/
|
||||
```
|
||||
2. **Gitea** — copy the *Docker volume*, not `/opt/gitea` (§2a). Stop the container first so
|
||||
`gitea.db` is quiesced.
|
||||
```bash
|
||||
pct exec 106 -- docker stop gitea
|
||||
zfs create Storage/gitea
|
||||
rsync -aHAX /rpool/data/subvol-106-disk-0/var/lib/docker/volumes/8a1d2616*/_data/ /mnt/Storage/gitea/
|
||||
```
|
||||
3. **Nextcloud** — the long one. Copy `lv_storage`'s contents to a new dataset at file level from
|
||||
inside the running VM, over its existing NFS mount of the host. Verify §2c's size first so you
|
||||
know whether it fits (it will, in 7.60 T free).
|
||||
4. **Confirm `config_backup`** (267 G) is disposable; drop it if so.
|
||||
5. **Verify each service's real data location before copying it** — the §2a rule.
|
||||
|
||||
### Phase B — prove the compute side before the storage moves
|
||||
|
||||
Bring the `apps/wave-0` manifests up on prod k3s while the old box is still alive, so the cutover is
|
||||
a storage swap against known-good workloads rather than two migrations at once.
|
||||
|
||||
Wrinkle: prod cannot reach `192.168.1.9` today (§2b). Either add a temporary pfSense rule for the
|
||||
test, or accept that app bring-up happens after the NAS is live on `192.168.2.0/24` — in which case
|
||||
stage the app configs now and keep the window short.
|
||||
|
||||
### Phase C — cutover
|
||||
|
||||
1. Stop all guests; reverse `startup:` order, pfSense last.
|
||||
2. Export cleanly — **never just cut power:**
|
||||
```bash
|
||||
zpool export Storage && zpool export Storage2
|
||||
zpool status # both must be gone before power-off
|
||||
```
|
||||
3. **Label every drive** with pool and serial before it leaves the chassis
|
||||
(`3FGJ6M9T`/`3FGHAB2T` = Storage; `WKD1S0TR`/`WKD1S10M` = Storage2).
|
||||
4. Move the four HDDs — cold, upright. The 6-year-old Seagates are the fragile ones.
|
||||
5. Install TrueNAS SCALE on the shipped 64 GB M.2 (Slot A). The five bays take the four HDDs with
|
||||
one spare; the Micron SSDs are **not needed** in the target — leave them in the old chassis.
|
||||
6. **Import read-only first**, verify, then import for real:
|
||||
```bash
|
||||
zpool import # both pools listed
|
||||
zpool import -o readonly=on Storage
|
||||
zfs list # verify datasets, sizes
|
||||
zpool export Storage && zpool import Storage
|
||||
```
|
||||
|
||||
### Phase D — repoint, reclaim, retire
|
||||
|
||||
1. Recreate NFS shares in the TrueNAS UI targeting `192.168.2.10/.11/.12`; clear the old `sharenfs`
|
||||
properties.
|
||||
2. Point the `apps/wave-0` PVs at the NAS; bring Plex, the arr stack and Nextcloud up on prod k3s.
|
||||
3. Stand up MinIO on TrueNAS to replace versitygw, and re-enable Velero + CNPG backups (§2e).
|
||||
4. **Only once everything is verified from k3s:** delete the dead zvols (§1) — ~731 G on `Storage`,
|
||||
4.06 T on `Storage2`.
|
||||
5. Re-evaluate §2d. If it fits, evacuate `Storage2`, retire the 6-year-old Seagates, keep two bays
|
||||
free.
|
||||
6. Repoint the SOHO 443 NAT, then power off the Proxmox host.
|
||||
7. `zpool status` — confirm the 36 read errors did **not** follow `3FGHAB2T` into the new chassis.
|
||||
Full scrub on the new hardware.
|
||||
|
||||
---
|
||||
|
||||
## 5. Rollback, and the one-way door
|
||||
|
||||
Until Phase D step 4, the old chassis is intact: `rpool`, the Proxmox install, both Microns and
|
||||
every `/etc/pve` config untouched. Put the four HDDs back and it boots into the previous world.
|
||||
|
||||
> ### Do not run `zpool upgrade`
|
||||
>
|
||||
> `zpool status Storage2` already nags — *"Some supported and requested features are not enabled…
|
||||
> action: Enable all features using 'zpool upgrade'"* — and TrueNAS will offer it in the UI after
|
||||
> import. It is the only irreversible step here: run it and the pools can never go back to the
|
||||
> Proxmox host. Leave it until the NAS has been serving k3s for weeks.
|
||||
|
||||
Note the asymmetry that makes this safe: TrueNAS's OpenZFS **reads** these pools happily, because
|
||||
newer ZFS always reads older pools. The reverse is not true.
|
||||
|
||||
---
|
||||
|
||||
## 6. Open items
|
||||
|
||||
1. **The Nextcloud number** (§2c) — one `df -h` inside VM 103. It decides whether `Storage2` and the
|
||||
two oldest drives can be retired.
|
||||
2. **Is `config_backup` (267 G) disposable?**
|
||||
3. **What serves DNS on `192.168.2.0/24`** once Pi-hole dies?
|
||||
4. **UniFi controller** — rehome, cloud key, or drop?
|
||||
5. **Minecraft** — keep or retire?
|
||||
6. **Gitea** — its whole state is 3 repos and an 11 MB SQLite DB, and prod k3s already runs its own
|
||||
Gitea. Is the legacy one worth migrating at all, or does it just end?
|
||||
+4
-20
@@ -84,9 +84,6 @@ toolchain: yamllint, check-jsonschema, kyverno, chainsaw, yq) lives in the image
|
||||
infra/devcontainer/dev.sh build # once
|
||||
infra/devcontainer/dev.sh run make -C ci gate # exactly what CI runs
|
||||
infra/devcontainer/dev.sh # interactive shell at /work
|
||||
|
||||
infra/devcontainer/dev.sh run kubectl get nodes # DEV (the default)
|
||||
CLUSTER=prod infra/devcontainer/dev.sh run kubectl get nodes # PROD
|
||||
```
|
||||
|
||||
Unlike the libvirt era, **tofu and ansible now run fine inside the container** — the
|
||||
@@ -98,23 +95,10 @@ which is what broke the old `dmacvicar/libvirt` provider in a container.
|
||||
file, and the sibling spec repo read-only at `/home-network-k3s` so the spec-immutability
|
||||
check runs instead of silently skipping.
|
||||
|
||||
> **Closed (was a known gap): `dev.sh` can now target either cluster.** It used to hard-wire
|
||||
> `KUBECONFIG` to `dev.kubeconfig`, leaving an undocumented — and misleadingly named —
|
||||
> `DEV_KUBECONFIG=<prod path>` as the only route to prod. Select with **`CLUSTER=dev|prod`**;
|
||||
> **dev remains the default**, so every existing invocation means what it always meant. Inside the
|
||||
> container `KUBECONFIG` is a chain of a tiny per-invocation selector file over a merged config
|
||||
> carrying both clusters as contexts **`k3s-dev`** and **`k3s-prod`** (never `default`), regenerated
|
||||
> each run by [`infra/devcontainer/kubeconfig.sh`](devcontainer/kubeconfig.sh) into the gitignored
|
||||
> `infra/ansible/.cache/clusters.kubeconfig`. The merged file deliberately carries **no**
|
||||
> `current-context`: it is shared by every container, and stamping the current cluster into it made
|
||||
> concurrent `CLUSTER=dev` and `CLUSTER=prod` runs silently swap clusters mid-command. Naming matters: k3s writes *both* source kubeconfigs
|
||||
> with context, cluster and user all called `default`, and kubectl's merge keeps the first
|
||||
> duplicate — so a plain `KUBECONFIG=dev:prod` would silently resolve `default` to **dev** while you
|
||||
> believed you were on prod. Requesting a cluster whose kubeconfig is absent is a hard error, never
|
||||
> a fallback. `DEV_KUBECONFIG` still works and still wins, but is **superseded** — it is opaque
|
||||
> about its target and yields an ambiguous `default` context. `~/.kube` is still not mounted and the
|
||||
> OIDC context still cannot work in a container (no browser); use the break-glass cert kubeconfigs
|
||||
> above. Details: [`devcontainer/README.md`](devcontainer/README.md).
|
||||
> **Known gap.** `dev.sh` defaults `KUBECONFIG` to `/work/infra/ansible/.cache/dev.kubeconfig`,
|
||||
> a libvirt-era path that no longer exists, and does not mount `~/.kube`. In-container `kubectl`
|
||||
> therefore needs an explicit `DEV_KUBECONFIG=<path to a break-glass cert kubeconfig>` — the OIDC
|
||||
> context cannot work there anyway (no browser). Not yet fixed.
|
||||
|
||||
## Off-repo state you must not lose
|
||||
|
||||
|
||||
+267
-204
@@ -28,9 +28,9 @@ Fill a row per node as it lands. Node 1 is as-found on 2026-08-31.
|
||||
| Model | HP EliteDesk 800 G5 Desktop Mini | HP EliteDesk 800 G5 Desktop Mini | HP EliteDesk 800 G5 Desktop Mini |
|
||||
| CPU | i7-9700T, 8c/8t, VT-x | i7-9700T, 8c/8t, VT-x | i7-9700T, 8c/8t, VT-x |
|
||||
| RAM | 31 GiB (2×16 GB Micron) | **2.6 GiB usable of 16 GB — see §0b** | 31.1 GiB usable of 32 GB (2×16 GB Crucial `CT16G4SFD8266.C16FE`, **a matched pair**) |
|
||||
| Disks | 1× 466 GB SATA SSD (`Samsung SSD 850 EVO 500GB`) — **replaced 2026-09-07, was `C400-MTFDDAC512M` (~2011); see §0c** | 1× 477 GB **NVMe** (`PM9A1 Samsung`) | 1× 466 GB SATA SSD (`WDC WDS500G2B0A-00SM50`) |
|
||||
| Drive health | PASSED, **0 reallocated / 0 grown bad blocks**, `Wear_Leveling_Count` raw 106, but **77,047 power-on hours** (~8.8 y) over only 219 power cycles | **96% life**, 2142 h, PASSED | **100% life**, 3213 h, PASSED, 0 grown bad blocks / 0 erase / 0 program fails, 0 CRC |
|
||||
| OS | Debian 13 (trixie), kernel **7.1.8+deb13-amd64** | Debian 13 (trixie), kernel **7.1.8+deb13-amd64** | Debian 13 (trixie), kernel **7.1.8+deb13-amd64** |
|
||||
| Disks | 1× 477 GB SATA SSD (`C400-MTFDDAC512M`, ~2011) | 1× 477 GB **NVMe** (`PM9A1 Samsung`) | 1× 466 GB SATA SSD (`WDC WDS500G2B0A-00SM50`) |
|
||||
| Drive health | 69% life, 15 grown bad blocks, 187 erase fails | **96% life**, 2142 h, PASSED | **100% life**, 3213 h, PASSED, 0 grown bad blocks / 0 erase / 0 program fails, 0 CRC |
|
||||
| OS | Debian 13 (trixie), kernel 6.12 | Debian 13 (trixie), kernel 6.12 | Debian 13 (trixie), kernel 6.12.107+deb13-amd64 |
|
||||
| Firmware | HP R21 Ver. 02.26.00 | HP R21 Ver. 02.26.00 | HP R21 Ver. 02.26.00 |
|
||||
| `/dev/kvm` | present | present | present |
|
||||
| Serial | MXL1052NGN | MXL0073949 | 8CC015307N |
|
||||
@@ -48,47 +48,6 @@ modern NVMe; node 3 a third distinct model, a WD Blue 3D SATA SSD in excellent h
|
||||
an identical disk layout across all three because Longhorn replica placement assumes symmetry — so
|
||||
this is a live input to the disk decision, not a footnote. Node 2's NVMe is the drive node 1 should
|
||||
have had; node 3 does not have it either. Node 3 is also the only node with **two matched DIMMs**
|
||||
|
||||
### §0c — node 1's drive was replaced (2026-09-07)
|
||||
|
||||
**The paragraph above is history, not current state.** On 2026-09-07 node 1's kubelet stopped
|
||||
posting status while the host stayed up and SSH-able. Root cause was **etcd starved of disk I/O**,
|
||||
not memory: `slow fdatasync` at 3.5–6.7 s against an expected 1 s, `apply request took too long` at
|
||||
8.3 s against 100 ms, then `etcdserver: too many requests`. No OOM, MCE or hung task in the kernel
|
||||
log. All three nodes place etcd's WAL (`/data/k3s`) and every Longhorn replica (`/data/longhorn`) on
|
||||
the SAME partition; node 1's was the 2011 C400 with 61,440 reallocated sectors and ~31% of its
|
||||
endurance consumed, and its `/proc/pressure/io` `full avg300` ran **11.36** against ~1% steady state
|
||||
on the other two. The failure is self-reinforcing: the node dropping out degrades Longhorn volumes,
|
||||
whose rebuilds add write load to the very disk etcd needs.
|
||||
|
||||
The operator replaced the drive with a **Samsung SSD 850 EVO 500GB** and reinstalled Debian.
|
||||
Measured under the same load class — an active 18-replica Longhorn rebuild, with node 1 as the
|
||||
*write target*, i.e. the heavier side:
|
||||
|
||||
| node | drive | `full avg300` during rebuild |
|
||||
|---|---|---|
|
||||
| `k3s-prod-1` | Samsung 850 EVO (new) | **3.79** |
|
||||
| `k3s-prod-2` | Samsung PM9A1 NVMe | 0.33 |
|
||||
| `k3s-prod-3` | WDC WDS500G2B0A | 2.47 |
|
||||
| *`k3s-prod-1` before* | *C400-MTFDDAC512MAM* | ***11.36*** |
|
||||
|
||||
Roughly a **3× improvement**, moving node 1 from ~4× worse than its peers into the same band as
|
||||
node 3, and it stayed `Ready` throughout. **The 850 EVO is not new**: 77,047 power-on hours over 219
|
||||
power cycles — it evidently idled in an always-on system — but its wear tells the opposite story to
|
||||
the C400's (0 reallocated vs 61,440; wear-leveling 106 vs 949).
|
||||
|
||||
**§2's exit condition is NOT met by this replacement.** etcd and Longhorn still share one SATA
|
||||
spindle. The contention is reduced, not eliminated; giving etcd its own device remains the fix, and
|
||||
node 1 still has exactly one disk. The three nodes also remain asymmetric — now 850 EVO / PM9A1
|
||||
NVMe / WD Blue.
|
||||
|
||||
The rebuild itself surfaced two things worth carrying forward. Longhorn refused the new disk with
|
||||
`DiskFilesystemChanged` — "record diskUUID doesn't match the one on the disk" — because its node CR
|
||||
still held the old drive's `diskUUID`; the remedy is disable → remove the disk entry → re-add, in
|
||||
that order (the validating webhook enforces it). And the Ansible inventory still listed node 1 in
|
||||
`[k3s_prod_primary]`, which decides both which play gets `--cluster-init` *and* what `k3s_api_host`
|
||||
resolves to — so re-provisioning it unchanged would have pointed a rebuilt, empty node at itself and
|
||||
founded a second cluster beside the live one. The founder now sits on node 2.
|
||||
(2× Crucial 2Rx8 DDR4-2666) and the only one with its **full** RAM complement of 32 GB actually
|
||||
usable. **That is luck, not immunity:** node 3's firmware publishes an EFI Memory Attributes Table
|
||||
too (`MEMATTR=0xbeece018`), the kernel simply parsed it cleanly on this boot, so node 3 is exposed to
|
||||
@@ -820,74 +779,6 @@ disk each, already fully partitioned, no free slot verified populated.
|
||||
physically) and flip `node_baseline_etcd_source` back to `device` — the role still
|
||||
supports that path unchanged, and it is the dev shape.
|
||||
|
||||
#### THE MEASUREMENT CAME BACK — 2026-09-07. It does not hold on node 1.
|
||||
|
||||
Not under a deliberate test: in production, 2 days 7 hours after the cluster came up.
|
||||
**06:05:21Z `k3s-prod-1` went NotReady** (`Kubelet stopped posting node status`) while the
|
||||
host stayed up, pingable and SSH-able. From the k3s journal:
|
||||
|
||||
```
|
||||
slow fdatasync took=3.56s / 6.71s / 4.95s / 3.40s (etcd expects 1s)
|
||||
apply request took too long took=8.30s (expects 100ms)
|
||||
etcdserver: too many requests
|
||||
Failed to check local etcd status for learner management: context deadline exceeded
|
||||
```
|
||||
|
||||
No OOM, no MCE, no hung task in the kernel log — **this is not §0b and not memory**. Recovery
|
||||
required a manual `systemctl restart k3s`; 18 Longhorn volumes then rebuilt, and I/O pressure
|
||||
only fell once the rebuilds finished. The failure is **self-reinforcing**: the node dropping
|
||||
out degrades volumes → Longhorn rebuilds → more write load on the disk etcd needs → etcd
|
||||
stalls further.
|
||||
|
||||
`/proc/pressure/io` **full avg300**, all three nodes, during the incident:
|
||||
|
||||
| node | device | drive | IO PSI full avg300 |
|
||||
|---|---|---|---|
|
||||
| k3s-prod-1 | `/dev/sda5` | C400-MTFDDAC512MAM (2011 consumer SATA SSD, no PLP) | **11.36** |
|
||||
| k3s-prod-2 | `/dev/nvme0n1p5` | Samsung PM9A1 NVMe | 0.75 |
|
||||
| k3s-prod-3 | `/dev/sda5` | WDC WDS500G2B0A (WD Blue SATA) | 1.29 |
|
||||
|
||||
The prediction in the paragraph above — "every prod node is SSD or NVMe end-to-end, where
|
||||
concurrent rebuild I/O costs latency rather than collapsing it" — was **right about node 2 and
|
||||
wrong about node 1**. A 2011 consumer SATA SSD with 61,440 reallocated sectors, 31% of its
|
||||
endurance consumed and no power-loss protection is not the "SSD" that reasoning assumed. §0's
|
||||
"three nodes, three different drive models" is the fault line, exactly where §0 said it would
|
||||
be.
|
||||
|
||||
**The exit condition is therefore OPEN and the second bullet applies.** Replacing node 1's
|
||||
drive, or giving etcd its own device, is the fix, and it is the operator's.
|
||||
|
||||
Pending that, three SOFTWARE mitigations shipped on 2026-09-07 (branch
|
||||
`fix/etcd-io-priority`). **None of them is the fix**, and each says so where it lives:
|
||||
|
||||
1. **Observability first** — `NodeIOPressureHigh` / `NodeIOPressureCritical` in
|
||||
`platform/monitoring/alerts.yaml`, plus an I/O row on the node-pressure dashboard. The
|
||||
stack alerted on CPU and memory PSI and had **no I/O alert at all**, so the first signal
|
||||
of this outage was a node already out of the cluster. Thresholds (5% / 10% full IO PSI)
|
||||
come from the numbers in the table above. Note the etcd histograms this section's first
|
||||
bullet asks for are **not available**: kube-prometheus-stack's `kubeEtcd` scrape job is
|
||||
deliberately disabled for k3s (`platform/monitoring/kube-prometheus-stack.yaml`), so PSI
|
||||
is the signal that actually exists. Enabling the etcd scrape would make that bullet
|
||||
directly measurable and is worth doing.
|
||||
2. **Longhorn concurrency ceilings** — `platform/longhorn/helmrelease.yaml`. Honestly:
|
||||
`concurrentReplicaRebuildPerNodeLimit: 1` was **already in force during this outage** and
|
||||
held; the 18 rebuilds ran one at a time and the node stalled anyway. Longhorn offers no
|
||||
further rebuild lever for the v1 data engine (`replicaRebuildingBandwidthLimit` is
|
||||
v2/SPDK only; `rebuildConcurrentSyncLimit` is already at its floor of 1). What was
|
||||
tightened is every *other* Longhorn I/O source — restore, snapshot purge, backup, engine
|
||||
upgrade — each of which defaulted to 5 concurrent operations per node and each of which
|
||||
could reproduce this on its own.
|
||||
3. **Block-layer arbitration** — `roles/etcd_io_priority`, enabled for `[k3s_prod]`, applied
|
||||
only when the operator runs `playbooks/etcd-io-priority.yml`. BFQ + `IOWeight` so etcd's
|
||||
cgroup outweighs `kubepods.slice`. Its README is explicit that most of the obvious knobs
|
||||
here (`IOSchedulingClass`, `io.latency`, bare `IOWeight`) are **inert on these nodes**,
|
||||
and that even this one cannot arbitrate the ext4 journal both tenants share.
|
||||
|
||||
The last point is the one that matters for this section: **no cgroup or scheduler control
|
||||
separates two tenants on one filesystem.** etcd's `fdatasync` waits on a jbd2 journal commit,
|
||||
jbd2 is a kernel thread in the root cgroup, and Longhorn's writes dirty the same journal. The
|
||||
software mitigations raise the load level at which this happens. They do not remove it.
|
||||
|
||||
**~~Known, separate consequence — `/var` is only 12 GiB.~~ RESOLVED 2026-09-04 by moving
|
||||
k3s's data dir.** This was recorded here on 2026-09-03 as a capacity item "to watch or
|
||||
repartition". Re-measuring it against dev turned it from a watch item into a blocker: the
|
||||
@@ -1281,7 +1172,7 @@ no extra vars is byte-for-byte the dev behaviour it always had (verified by a be
|
||||
| `192.168.1.55` (crafty) | `platform/crafty/service-l4.yaml` | prod-LAN address |
|
||||
| `192.168.1.54` (UniFi VM) | `apps/vms/unifi/50-service-l4.yaml` | prod-LAN address |
|
||||
| `192.168.1.9:9000` (S3 backup) | `platform/keycloak/postgres-cluster.yaml`, `apps/wave-0/{nextcloud,semaphore}-db.yaml` | reachable prod backup target |
|
||||
| ~~`home.keithjackson.dev` (every hostname)~~ | ~~HTTPRoutes, Gateway, wildcard cert, coredns-custom, Keycloak realm, oauth2-proxy, Homepage, `catalog/`~~ | **CLOSED 2026-09-04 — see §10.** This one was not merely "would build wrong": it **already destroyed dev's public DNS**. `BASE_DOMAIN` was a declared-but-DEAD knob; it is now real, and prod serves `prod.home.keithjackson.dev`. |
|
||||
| ~~`home.keithjackson.dev` (every hostname)~~ | ~~HTTPRoutes, Gateway, wildcard cert, coredns-custom, Keycloak realm, oauth2-proxy, Homepage, `catalog/`~~ | **CLOSED 2026-09-04 — see §10.** This one was not merely "would build wrong": it **already destroyed dev's public DNS**. `BASE_DOMAIN` was a declared-but-DEAD knob; it is now real. Prod's value was inverted the same day to the APEX `home.keithjackson.dev`, with dev moving to `dev.home.keithjackson.dev` — see §10a, which is authored but NOT yet applied. |
|
||||
|
||||
- [ ] **Tokenize each into a `cluster-vars` knob** with a base default, then set the value in *both*
|
||||
overlays. Adding a knob before referencing `${KNOB}` is the sanctioned mechanism (see the
|
||||
@@ -1392,12 +1283,20 @@ single-server-etcd window, and avoids a later member-add dance.
|
||||
specifically.
|
||||
- [ ] Monitoring and alerts live on prod.
|
||||
- [ ] The conformance suite passes green **against the prod cluster**, not just dev (R-TEST-03).
|
||||
Blocked on parameterizing the L2 tests that hardcode `home.keithjackson.dev` — see §10 phase 2.
|
||||
- [ ] **Prod serves its own DNS/TLS identity** and dev's records are provably untouched (§10):
|
||||
distinct `EXTERNALDNS_TXT_OWNER_ID`, prod wildcard cert `*.prod.home.keithjackson.dev`, and
|
||||
prod's own Keycloak realm + per-service client secrets.
|
||||
Blocked on parameterizing the L2 tests that hardcode `home.keithjackson.dev` — see §10a
|
||||
"The L2 tests hardcode the apex".
|
||||
- [ ] **Prod serves the apex DNS/TLS identity** and dev's records are provably untouched (§10/§10a):
|
||||
distinct `EXTERNALDNS_TXT_OWNER_ID` (`prod` vs `dev`), prod wildcard cert
|
||||
`*.home.keithjackson.dev`, dev migrated to `*.dev.home.keithjackson.dev` with its old apex
|
||||
records retired (§10a step D8), and prod's own Keycloak realm + per-service client secrets.
|
||||
|
||||
## 10. DNS/TLS environment split — **the 2026-09-04 incident, and phase 1 of the fix**
|
||||
## 10. DNS/TLS environment split — **the 2026-09-04 incident, and the target architecture**
|
||||
|
||||
> **Current state (2026-09-04):** the target below is **AUTHORED, NOT APPLIED.** Prod's
|
||||
> `platform` tier and its 10 downstream tiers are SUSPENDED. Dev is **live on the apex**. The
|
||||
> ordered cutover is **§10a**, and merging the authored change to `main` is step **D3** of it,
|
||||
> not step one — dev self-reconciles `main`, so a merge outside the runbook window executes
|
||||
> the migration unattended.
|
||||
|
||||
### What happened
|
||||
|
||||
@@ -1429,43 +1328,82 @@ values; for DNS and TLS that promise was false, so `clusters/prod` was *authored
|
||||
public identity. Two clusters that describe the same DNS name will fight over it no matter how the
|
||||
DNS controller is configured.
|
||||
|
||||
### The architecture decision (operator)
|
||||
### The architecture decision (operator) — **INVERTED 2026-09-04, this is the current target**
|
||||
|
||||
```
|
||||
dev -> *.dev.home.keithjackson.dev gitea.dev.home.keithjackson.dev
|
||||
prod -> *.prod.home.keithjackson.dev gitea.prod.home.keithjackson.dev
|
||||
apex -> CNAME <svc>.home.keithjackson.dev -> <svc>.prod.home.keithjackson.dev
|
||||
dev -> *.dev.home.keithjackson.dev gitea.dev.home.keithjackson.dev
|
||||
prod -> *.home.keithjackson.dev gitea.home.keithjackson.dev (THE APEX)
|
||||
```
|
||||
|
||||
### PHASE 1 — **prod only. Dev did not move.** (this change)
|
||||
**Prod *is* the apex. There is no `prod.` label and no CNAME layer at all** —
|
||||
`<svc>.home.keithjackson.dev` is prod's real name, not an alias for one. This **supersedes** the
|
||||
interim split committed earlier the same day (`c8c1d29`), which put prod on
|
||||
`*.prod.home.keithjackson.dev` behind apex CNAMEs pointing at it. That layer is deleted: fewer
|
||||
moving parts, no CNAME chain to keep in step, and prod ends up on the names users already type.
|
||||
|
||||
`BASE_DOMAIN` is now real: every hostname in every Flux-reconciled manifest derives from it.
|
||||
The cost of the inversion is that **dev must move**, and dev is the live cluster. The interim split
|
||||
was chosen precisely because it moved only the unbuilt environment; this one does not have that
|
||||
property, so the whole of §10's remaining risk now sits on dev's side, and it is sequenced below.
|
||||
|
||||
| | `BASE_DOMAIN` | `EXTERNALDNS_TXT_OWNER_ID` | `EXTERNALDNS_DOMAIN_FILTER` |
|
||||
|---|---|---|---|
|
||||
| base default | `home.keithjackson.dev` | `unset-see-cluster-vars` (non-colliding sentinel) | `keithjackson.dev` |
|
||||
| **dev** | `home.keithjackson.dev` (**unchanged**) | `default` (**must not change**) | `keithjackson.dev` |
|
||||
| **prod** | `prod.home.keithjackson.dev` | `prod` | `prod.home.keithjackson.dev` |
|
||||
### The values, per environment
|
||||
|
||||
**Dev's effective hostname is deliberately still the apex `home.keithjackson.dev`.** Moving prod
|
||||
alone ends the collision and unblocks prod, and it is the whole of the risk that had to be taken
|
||||
while the operator is away. Dev's rendered output is **byte-identical** except for one added line,
|
||||
`txtOwnerId: default`, which is the value dev's external-dns was already using implicitly — verified
|
||||
by rendering all 12 `substituteFrom` paths through `kubectl kustomize | flux envsubst --strict` for
|
||||
both overlays and diffing.
|
||||
| | `BASE_DOMAIN` | `EXTERNALDNS_TXT_OWNER_ID` | `EXTERNALDNS_DOMAIN_FILTER` | Ansible `k3s_base_domain` |
|
||||
|---|---|---|---|---|
|
||||
| base default | `dev.home.keithjackson.dev` | `unset-see-cluster-vars` (non-colliding sentinel) | `keithjackson.dev` | `dev.home.keithjackson.dev` (role default) |
|
||||
| **dev** | `dev.home.keithjackson.dev` | `dev` | `keithjackson.dev` | (inherits the role default) |
|
||||
| **prod** | `home.keithjackson.dev` | `prod` | `keithjackson.dev` | `home.keithjackson.dev` (`group_vars/k3s_prod`) |
|
||||
|
||||
**Two independent guards now, either sufficient alone:**
|
||||
`[parity]`: `k3s_base_domain` must equal the same cluster's `BASE_DOMAIN`. The kube-apiserver's
|
||||
`oidc-issuer-url` is a k3s **server flag** in `/etc/rancher/k3s/config.yaml`, not a manifest, so Flux
|
||||
post-build substitution can never reach it — hence one fact in two places. If they disagree the
|
||||
apiserver validates tokens against a different Keycloak than the cluster runs: kubectl SSO fails
|
||||
while the cert-based admin kubeconfig keeps working, so it presents as "SSO is broken", not as skew.
|
||||
|
||||
1. **`txtOwnerId`** — per-environment identity. A cluster only ever touches records it owns. This is
|
||||
the direct fix for the incident. Dev **must** render exactly `default`: its live records are
|
||||
already registered under that owner, and any other value would make dev's own external-dns stop
|
||||
recognising them and re-create duplicates beside the orphans.
|
||||
2. **`domainFilters`** — per-environment name scope. Prod cannot see a name outside its own subtree.
|
||||
Defence in depth, added because the first guard is a single string in a single file.
|
||||
**`EXTERNALDNS_DOMAIN_FILTER` is the Cloudflare ZONE for BOTH, and must stay that way.** FINDING
|
||||
F-DNS1 (DECISIONS-LOG 2026-08-17): the Cloudflare provider uses this value for **zone selection** as
|
||||
well as record matching, so a filter that is a strict subdomain of the hosted zone matches **no
|
||||
zone** — external-dns generates the record and then skips it with *"no hosted zone matching record
|
||||
DNS Name"*, publishing **nothing**. This has been "hardened" and reverted more than once. It is not
|
||||
defence in depth; it is a total outage of DNS publication. Isolation comes from the two guards below,
|
||||
which a domain filter cannot provide anyway: both environments live in the one zone regardless.
|
||||
|
||||
The base default for the owner id is a **non-colliding sentinel, deliberately not `default`**: a
|
||||
future environment that forgets the knob must not silently inherit dev's identity — that is exactly
|
||||
what happened here. With the sentinel such a cluster owns nothing, so it can delete nothing.
|
||||
**The two guards that actually work:**
|
||||
|
||||
1. **`txtOwnerId`, per-environment identity** — a cluster only ever reconciles records its own owner
|
||||
id claims in the TXT registry. This is the direct fix for the incident.
|
||||
2. **Disjoint hostname sets** — after the migration the two clusters never describe the same record,
|
||||
so there is nothing to fight over. *This guard does not exist during the migration window*, which
|
||||
is the entire reason the ordering below is mandatory.
|
||||
|
||||
#### Why dev's owner id changed from `default` to `dev`
|
||||
|
||||
Until this migration dev **had** to render `default`: its live apex records were already registered
|
||||
in Cloudflare's TXT registry under that owner, and renaming would have orphaned every one of them.
|
||||
**The migration dissolves that constraint** — on `*.dev.home.keithjackson.dev` dev publishes an
|
||||
entirely new record set with no prior registry entries to recognise, so the id is free and is chosen
|
||||
on its merits. `default` has none: it is the value external-dns uses when you **forget** the flag,
|
||||
which is exactly how both clusters came to share it, and an environment whose identity is the
|
||||
forget-value can never be told apart from a misconfigured newcomer. Consequences, both deliberate:
|
||||
|
||||
- Dev's **old apex records stay owned by `default`**, which no environment now claims, so nothing
|
||||
deletes them implicitly on some future reconcile. Retiring them is an explicit step (**D7**) and
|
||||
prod's first publish is gated on that retirement being *confirmed* — not on two reconcile loops
|
||||
racing each other.
|
||||
- **Prod stays `prod` and must never be given `default`.** Prod is taking over the apex *names*, and
|
||||
dev's existing apex records are registered under `default` — so `default` is the one value that
|
||||
would let prod adopt them, and under `policy: sync` delete the ones no prod source claims, on a
|
||||
cluster dev is still serving. That is the 2026-09-04 incident re-armed with the roles swapped.
|
||||
Prod inherits the apex **names**; it must never inherit the apex **owner id**.
|
||||
|
||||
All three values differ, and the base default remains a sentinel so an environment that forgets the
|
||||
knob owns nothing and can therefore delete nothing.
|
||||
|
||||
### PHASE 1 — prod tokenized and moved off dev's names (`c8c1d29`, done; suffix since inverted)
|
||||
|
||||
`BASE_DOMAIN` was a **dead knob** — declared in `clusters/base/cluster-vars.yaml` and referenced by
|
||||
no reconciled manifest, because every hostname was a hardcoded literal. `c8c1d29` made it real (141
|
||||
references across the reconciled set) and gave prod its own suffix. That work stands; only the
|
||||
*value* prod holds has changed, from `prod.home.keithjackson.dev` to the apex.
|
||||
|
||||
### What could NOT be tokenized, and why
|
||||
|
||||
@@ -1474,70 +1412,195 @@ what happened here. With the sentinel such a cluster owns nothing, so it can del
|
||||
Re-encrypting needs the operator's age key, rewrites the MAC over every value in the file, and
|
||||
**cannot be verified by the offline render harness** (`kustomize` does not decrypt), so a mistake
|
||||
would break Headlamp SSO on the live dev cluster with no pre-flight proof. Left as-is deliberately.
|
||||
**Prod action required:** prod's Headlamp will otherwise authenticate against **dev's** Keycloak.
|
||||
Prod needs its own copy of this Secret with `keycloak.prod.home.keithjackson.dev` and a client
|
||||
secret from prod's own realm — which it needs regardless, since every `*.sops.yaml` client secret
|
||||
is per-realm.
|
||||
**Both environments now need action:** dev's copy must move to `keycloak.dev.home.keithjackson.dev`
|
||||
as part of step **D5**, and prod needs its own copy with `keycloak.home.keithjackson.dev` and a
|
||||
client secret from prod's own realm — which it needs regardless, since every `*.sops.yaml` client
|
||||
secret is realm-scoped.
|
||||
- **`oidc-issuer-url` in the kube-apiserver flags** (`roles/k3s_server/tasks/main.yml`) — a k3s
|
||||
**server flag** in `/etc/rancher/k3s/config.yaml`, not a manifest, so Flux post-build substitution
|
||||
can never reach it. Correctly tokenized on the **Ansible** side instead: new role default
|
||||
`k3s_base_domain` (dev value) + derived `k3s_oidc_issuer_url`, overridden for prod in
|
||||
`inventory/group_vars/k3s_prod/vars.yml` — the same inventory seam as `k3s_data_dir`/`k3s_api_host`.
|
||||
`[parity]`: `k3s_base_domain` **must** equal the same cluster's `BASE_DOMAIN`, or the apiserver
|
||||
validates tokens against a different Keycloak than the cluster runs and kubectl SSO fails while the
|
||||
cert-based admin kubeconfig keeps working (so it looks like "SSO is broken", not like config skew).
|
||||
server flag, not a manifest. Handled on the **Ansible** seam instead: role default
|
||||
`k3s_base_domain` (dev's value) + derived `k3s_oidc_issuer_url`, overridden for prod in
|
||||
`inventory/group_vars/k3s_prod/vars.yml` — the same inventory seam as `k3s_data_dir`/`k3s_api_host`,
|
||||
so a fourth prod server inherits it from group membership. Verified per host by a read-only probe
|
||||
(`connection: local`, zero SSH, `changed=0`): all three dev servers resolve
|
||||
`https://keycloak.dev.home.keithjackson.dev/realms/homelab`, all three prod servers resolve
|
||||
`https://keycloak.home.keithjackson.dev/realms/homelab`.
|
||||
- **`spec.url` of dev's `GitRepository`** (`clusters/dev/kustomization.yaml`) — see the strand below.
|
||||
It *cannot* be tokenized: the `flux-system` root Kustomization that renders `./clusters/dev`
|
||||
declares no `postBuild.substituteFrom`, so `gitea.${BASE_DOMAIN}` there would ship the **literal
|
||||
token** and break the source outright (the L0 `flux-postbuild-substitution` check asserts exactly
|
||||
this). It is a hand-maintained literal that must move in lockstep with `BASE_DOMAIN`.
|
||||
|
||||
### A pre-existing prod problem this incidentally FIXED
|
||||
---
|
||||
|
||||
§4 recorded that prod nodes use **public** resolvers, so the kube-apiserver would resolve its OIDC
|
||||
issuer publicly and get **dev's** Gateway VIP (192.168.1.53) — the wrong cluster on the wrong LAN,
|
||||
making prod kubectl SSO impossible, and requiring an `/etc/hosts` pin as a workaround. With prod on
|
||||
its own subdomain the public record for `keycloak.prod.home.keithjackson.dev` is published by
|
||||
**prod's own** external-dns pointing at **prod's** Gateway VIP (192.168.2.20), so the public answer
|
||||
and the split-horizon answer agree. **No `/etc/hosts` pin is needed.** This holds only once prod's
|
||||
external-dns has actually published the record; until then use the break-glass cert kubeconfig.
|
||||
## 10a. THE MIGRATION RUNBOOK — ordering is load-bearing
|
||||
|
||||
### Verify on prod's first reconcile
|
||||
> **Prod's `platform` tier and its 10 downstream tiers are SUSPENDED and must stay suspended until
|
||||
> step P1.** Nothing below is safe to run out of order, and nothing below should be run while the
|
||||
> operator is away.
|
||||
|
||||
- [ ] **`domainFilters` zone selection.** The Cloudflare provider uses this value for *zone* lookup
|
||||
as well as record matching, and in 2026-08 a strict-subdomain filter dropped the parent zone
|
||||
("no hosted zone matching record DNS Name"). If prod's external-dns logs no matching zone it
|
||||
will create **nothing** — fail-closed, never destructive. Fallback: set prod's
|
||||
`EXTERNALDNS_DOMAIN_FILTER` back to `keithjackson.dev` and rely on `txtOwnerId`, which is
|
||||
sufficient on its own.
|
||||
- [ ] **Prod's wildcard cert is `*.prod.home.keithjackson.dev`** and solves DNS-01 without touching
|
||||
dev's `_acme-challenge` records.
|
||||
- [ ] **Dev's records are untouched** after prod's tiers are resumed. Confirm dev's TXT registry
|
||||
entries still read owner `default` and that prod's read owner `prod`.
|
||||
- [ ] **Prod's own Keycloak realm + per-service client secrets** exist (every `*.sops.yaml` client
|
||||
secret is realm-scoped; prod cannot reuse dev's).
|
||||
### The two ways this goes wrong
|
||||
|
||||
### PHASE 2 — dev migration + apex CNAMEs (**deferred, not done**)
|
||||
**(1) BOTH CLUSTERS ON THE APEX — the 2026-09-04 incident, recreated.** If prod publishes
|
||||
`*.home.keithjackson.dev` before dev has vacated it, two clusters describe the same records in the
|
||||
same zone again. Distinct `txtOwnerId`s stop prod from *deleting* dev's records — that guard holds —
|
||||
but they do not stop the second failure mode: both clusters requesting a `letsencrypt-prod` wildcard
|
||||
for the **same name set** and racing each other's `_acme-challenge` TXT records, so neither reliably
|
||||
solves DNS-01. **Rule: dev vacates the apex first, and the vacate is verified against Cloudflare
|
||||
before prod is unsuspended.**
|
||||
|
||||
Deliberately not attempted here: dev is **live** (18 Longhorn volumes, working SSO) and it hosts the
|
||||
Gitea that is currently this repo's GitOps source. Moving it is a migration, not a variable change.
|
||||
Required, roughly in order:
|
||||
**(2) THE STRAND — dev's GitOps source is the thing that moves.** `spec.url` of dev's
|
||||
`GitRepository` is `https://gitea.home.keithjackson.dev/...`, which is **both** dev's GitOps source
|
||||
**and** the PM's push target. Flipping dev's `BASE_DOMAIN` moves Gitea's HTTPRoute, its public DNS
|
||||
record and the wildcard cert covering it to `gitea.dev.home.keithjackson.dev` all at once. If that
|
||||
host does not already resolve and serve a valid, Ready cert at that instant, **dev stops reconciling
|
||||
and cannot be fixed by pushing** — the broken thing *is* the path Flux uses to read pushes. Recovery
|
||||
is then hands-on and needs LAN access to the cluster.
|
||||
|
||||
1. **`BASE_DOMAIN` → `dev.home.keithjackson.dev`** in `clusters/dev`, plus `k3s_base_domain` for the
|
||||
dev inventory group.
|
||||
2. **A new `*.dev.home.keithjackson.dev` wildcard cert** issued and Ready *before* cutover.
|
||||
3. **Keycloak issuer change.** The realm's issuer URL is baked into every token; every OIDC client
|
||||
(`grafana`, `gitea`, `nextcloud`, `semaphore`, `headlamp`, `crafty`, `pihole`, `jackett`,
|
||||
`qbittorrent`, `radarr`, `sonarr`, and the `kubernetes` public client) needs its redirect URIs
|
||||
updated in the same change, and the k3s apiserver `oidc-issuer-url` with it — which means a
|
||||
**rolling k3s restart** on the dev servers.
|
||||
4. **kubectl SSO re-issue** for every user (issuer changed).
|
||||
5. **`gitea.dev.home.keithjackson.dev`** — this is the GitOps source URL in
|
||||
`clusters/dev/kustomization.yaml`. Flux must be able to resolve and trust the new host *before*
|
||||
the URL is switched, or dev stops reconciling and the change cannot be pushed to fix itself.
|
||||
Sequence this last, and keep the bootstrap Gitea available as the recovery path (docs/05).
|
||||
6. **`EXTERNALDNS_TXT_OWNER_ID` for dev.** Once dev's names move, its existing `default`-owned
|
||||
records are for names dev no longer serves; retire them deliberately rather than letting a
|
||||
renamed owner orphan them.
|
||||
7. **Apex CNAMEs** `<svc>.home.keithjackson.dev` → `<svc>.prod.home.keithjackson.dev`. This is what
|
||||
actually **moves user traffic to prod**, and it must not land before prod serves those services.
|
||||
8. **The L2 behavioural conformance tests hardcode `home.keithjackson.dev`** (`dns-exposure`,
|
||||
`dns-internal-resolution`, `gateway-ingress-reachable`, `oidc-sso-grafana`, `sso-forward-auth`,
|
||||
`trusted-tls`). They are test inputs, not reconciled manifests, so they were left alone in phase 1
|
||||
— but they will fail against prod and must be parameterized before §9's "suite passes green
|
||||
against the prod cluster" can be claimed.
|
||||
**(2b) AND THE MERGE ITSELF IS THE TRIGGER.** `clusters/dev/sync.yaml` is a self-managing root
|
||||
Kustomization reconciling `./clusters/dev` from **`main`** every 10 minutes, and prod tracks `main`
|
||||
too. **A merge to `main` therefore *executes* this migration on the live dev cluster, unattended.**
|
||||
The authored change is a *target*, not a deployment. **Do not merge these files to `main` outside the
|
||||
runbook window.** Park them on a branch until the operator is at the keyboard and step **D1** has
|
||||
been done.
|
||||
|
||||
### D — dev vacates the apex (operator present, in this order)
|
||||
|
||||
- [ ] **D0. Snapshot the escape routes before touching anything.** Confirm a working break-glass
|
||||
**cert-based** admin kubeconfig for dev (not the OIDC one — OIDC breaks in D6). Confirm the
|
||||
retained bootstrap Gitea can be started (`bootstrap/gitea/bootstrap-gitea.sh`; dev's historical
|
||||
recovery host was `192.168.2.103:3000`, prod's current bootstrap is `192.168.2.132:3000`).
|
||||
Record dev's current Cloudflare record set (names + TXT registry owners) so D7 has a checklist
|
||||
and so an accidental deletion is detectable.
|
||||
- [ ] **D1. Break the automatic trigger.** Either `flux suspend kustomization flux-system -n
|
||||
flux-system` on dev (stops the root from self-applying the merge), **or** re-point dev's
|
||||
`GitRepository` url at the bootstrap Gitea for the window. Do this **before** the merge, not
|
||||
after. This is what converts "merging strands dev" into "merging is inert until you say go".
|
||||
- [ ] **D2. Publish and prove the new host set *while dev still serves the apex*.** The new names
|
||||
must exist before anything depends on them:
|
||||
- a `*.dev.home.keithjackson.dev` + `dev.home.keithjackson.dev` **wildcard Certificate issued
|
||||
and `Ready`** (LE prod, DNS-01) — see the ACME note below before ordering it;
|
||||
- public DNS for `gitea.dev.home.keithjackson.dev` resolving to dev's Gateway VIP
|
||||
(192.168.1.53), **checked against Cloudflare / a public resolver, never through the pfSense
|
||||
override** — that mistake was made on 2026-09-04 and produced a false healthy reading;
|
||||
- `curl -sS https://gitea.dev.home.keithjackson.dev/api/v1/version` succeeding with a
|
||||
publicly-trusted cert from a host on the dev LAN.
|
||||
Until D2 is green, D3 is not safe.
|
||||
- [ ] **D3. Merge the authored change to `main`** (`clusters/base/cluster-vars.yaml`,
|
||||
`clusters/dev/`, `clusters/prod/`, the two Ansible files, `platform/externaldns`,
|
||||
`catalog/render.py`). With D1 in place nothing moves yet.
|
||||
- [ ] **D4. Release dev's reconciliation deliberately.** Resume the root Kustomization (or restore
|
||||
the in-cluster url, now pointing at `gitea.dev.home.keithjackson.dev`) and watch
|
||||
`flux get kustomizations -A` converge. Dev's Gateway listeners, HTTPRoutes, coredns-custom zone
|
||||
and Homepage links all move here. Dev's external-dns publishes the new record set under owner
|
||||
`dev`.
|
||||
- [ ] **D5. Keycloak: issuer + every client.** Dev's realm issuer becomes
|
||||
`https://keycloak.dev.home.keithjackson.dev/realms/homelab`. The issuer is baked into every
|
||||
token, and **~12 OIDC clients** need their redirect URIs moved in the same change: `grafana`,
|
||||
`gitea`, `nextcloud`, `semaphore`, `headlamp`, `crafty`, `pihole`, `jackett`, `qbittorrent`,
|
||||
`radarr`, `sonarr`, plus the `kubernetes` public client. Most are reconciled from
|
||||
`platform/keycloak/keycloak-realm-import.yaml` and move with D4; **headlamp's is the exception**
|
||||
— its issuer/callback live in a SOPS-encrypted Secret and must be re-encrypted by hand.
|
||||
- [ ] **D6. Rolling k3s restart + kubectl SSO re-issue.** `k3s_base_domain` now renders a different
|
||||
`oidc-issuer-url` into `/etc/rancher/k3s/config.yaml`, so the `k3s_server` role is **no longer
|
||||
a no-op on dev**: it rewrites the file and fires the Restart handler on all three servers. Run
|
||||
it **one server at a time**, confirming `kubectl get nodes` between each. Every user's OIDC
|
||||
kubeconfig is invalidated and must be re-issued; the cert-based admin kubeconfig keeps working
|
||||
throughout, which is what makes this step recoverable. **Do not reboot nodes** — restart the
|
||||
`k3s` service only.
|
||||
- [ ] **D7. Retire dev's old apex records.** They are owned by `default`, which no environment now
|
||||
claims, so nothing removes them automatically. Delete the `<svc>.home.keithjackson.dev` A
|
||||
records, the apex record, **and their TXT registry entries** (`external-dns-*` TXT), plus any
|
||||
stale `_acme-challenge.home.keithjackson.dev`. *Alternative, if you prefer external-dns to do
|
||||
its own cleanup:* set dev's `EXTERNALDNS_TXT_OWNER_ID` back to `default` for exactly one
|
||||
reconcile after D4 — `policy: sync` will then garbage-collect the apex set it no longer has
|
||||
sources for — and only then switch it to `dev`. That is two commits and needs watching; the
|
||||
direct deletion is preferred because it is verifiable in one place.
|
||||
- [ ] **D8. Verify the apex is EMPTY, against Cloudflare.** `dig +short <svc>.home.keithjackson.dev
|
||||
@1.1.1.1` returns nothing for every service name, and no TXT registry entry for those names
|
||||
remains. **This is the gate on the whole P sequence.** A pfSense/PiHole override is not
|
||||
evidence about public DNS.
|
||||
|
||||
### P — prod takes the apex
|
||||
|
||||
- [ ] **P1. Only now unsuspend prod**, in dependency order: `flux resume kustomization platform`,
|
||||
then the 10 downstream tiers. Confirm D8 first, in Cloudflare, not from a local resolver.
|
||||
- [ ] **P2. Prod's external-dns publishes under owner `prod`.** Check the logs for a matching hosted
|
||||
zone (F-DNS1 fail-closed mode: no zone selected ⇒ nothing created — which is safe, but means
|
||||
the filter was narrowed against instruction). Confirm the new TXT registry entries read owner
|
||||
`prod` and that no record owned by `dev` was touched.
|
||||
- [ ] **P3. Prod's wildcard cert `*.home.keithjackson.dev` + `home.keithjackson.dev` is `Ready`** and
|
||||
solved DNS-01 without colliding with any `_acme-challenge` dev still holds.
|
||||
- [ ] **P4. Prod's kubectl SSO works without an `/etc/hosts` pin.** Prod nodes resolve
|
||||
`keycloak.home.keithjackson.dev` through **public** upstreams (§4), so this only becomes true
|
||||
once P2 has published that name at prod's Gateway VIP (192.168.2.20). Before then a prod node
|
||||
resolving its own issuer reaches **dev**. Use the break-glass cert kubeconfig until P2 is done.
|
||||
- [ ] **P5. Cut prod's GitOps source over** from the workstation bootstrap Gitea
|
||||
(`http://192.168.2.132:3000/...`) to `https://gitea.home.keithjackson.dev/...` once prod runs
|
||||
its own Gitea — the block in `clusters/prod/kustomization.yaml` owns that procedure. Note the
|
||||
inversion **simplified** this: prod's apex Gitea name and dev's `gitea.dev.home...` are now
|
||||
different hosts, so prod pointing at the apex means "prod's own Gitea", never dev's.
|
||||
|
||||
### ACME: the duplicate-certificate limit
|
||||
|
||||
Let's Encrypt rate-limits **duplicate certificates — 5 per week for an identical set of names.**
|
||||
Prod's wildcard is `*.home.keithjackson.dev` + `home.keithjackson.dev`, which is **exactly the name
|
||||
set dev holds today**. Every prod order therefore counts against the same bucket as dev's historical
|
||||
renewals of that pair, and a retry loop during P3 can exhaust it and leave prod with **no trusted
|
||||
cert for up to a week**. Mitigations:
|
||||
|
||||
- Order prod's cert **after** D8, so dev is no longer renewing the same pair.
|
||||
- Let dev's old `*.home.keithjackson.dev` `Certificate`/Secret be **deleted** in D4/D7 rather than
|
||||
left to renew in the background.
|
||||
- Rehearse against **letsencrypt-staging** first (`ACME_CLUSTER_ISSUER`) — staging has its own,
|
||||
far looser limits — and only switch prod to `letsencrypt-prod` when the DNS-01 solve is proven.
|
||||
- Do **not** delete-and-recreate prod's `Certificate` to "retry"; each recreation is a fresh order.
|
||||
|
||||
### The L2 tests hardcode the apex
|
||||
|
||||
Six behavioural (L2) chainsaw tests embed `home.keithjackson.dev` as a literal:
|
||||
|
||||
| test | file |
|
||||
|---|---|
|
||||
| `dns-exposure` | `conformance/tests/l2-behavioral/dns-exposure/chainsaw-test.yaml` |
|
||||
| `dns-internal-resolution` | `conformance/tests/l2-behavioral/dns-internal-resolution/chainsaw-test.yaml` |
|
||||
| `gateway-ingress-reachable` | `conformance/tests/l2-behavioral/gateway-ingress-reachable/chainsaw-test.yaml` (+ `fixtures/app-and-probe.yaml`) |
|
||||
| `oidc-sso-grafana` | `conformance/tests/l2-behavioral/oidc-sso-grafana/chainsaw-test.yaml` |
|
||||
| `sso-forward-auth` | `conformance/tests/l2-behavioral/sso-forward-auth/chainsaw-test.yaml` |
|
||||
| `trusted-tls` | `conformance/tests/l2-behavioral/trusted-tls/chainsaw-test.yaml` |
|
||||
|
||||
They are **test inputs, not reconciled manifests**, so they are deliberately left alone by the
|
||||
authoring change: they are the only thing still asserting the apex, and while dev is live on the apex
|
||||
they are *correct*. Under the new direction the literal happens to become right for **prod** and
|
||||
wrong for **dev**, so they must be **parameterized per environment** (not merely edited) before
|
||||
either §9's *"suite passes green against the prod cluster"* or a green L2 run against migrated dev
|
||||
can be claimed. Until then, expect the dev L2 suite to fail between D4 and that parameterization —
|
||||
a known, scheduled failure, not a regression.
|
||||
|
||||
The L1 *rendering* tests need no change: their goldens carry `${BASE_DOMAIN}` as a token, and the
|
||||
literal hostnames in them come from each test's own fixture `ingressRoutes[].host`, which is
|
||||
environment-independent by construction.
|
||||
|
||||
### Offline proof carried out for the authoring change (no cluster touched)
|
||||
|
||||
All 12 `substituteFrom` paths were rendered for **both** overlays through the real toolchain
|
||||
(`kubectl kustomize --load-restrictor LoadRestrictionsNone <spec.path> | flux envsubst --strict`,
|
||||
with each overlay's effective `cluster-vars` taken from `kubectl kustomize clusters/<env>`), before
|
||||
and after the change.
|
||||
|
||||
- **Dev's diff is *only* the hostname move.** Proven by inverse substitution rather than by reading
|
||||
the diff: applying `s/dev.home.keithjackson.dev/home.keithjackson.dev/` to the new render restores
|
||||
the baseline **byte-for-byte** in all 12 paths, plus 2 declared `txtOwnerId: default → dev` lines.
|
||||
145 changed line pairs, all classified: 143 hostname-move, 2 declared owner-id, **0 unexplained**,
|
||||
and **0 bare-apex remnants**. The 2 paths with no change (`cilium`, `velero`) provably contain no
|
||||
hostname at all (`grep -c keithjackson.dev` = 0).
|
||||
- **Prod lands on the bare apex:** **0** occurrences of `prod.home.keithjackson.dev` and **0** of
|
||||
`dev.home.keithjackson.dev` anywhere in prod's rendered output; 143 apex names.
|
||||
- **Certs:** dev `*.dev.home.keithjackson.dev` + `dev.home.keithjackson.dev`; prod
|
||||
`*.home.keithjackson.dev` + `home.keithjackson.dev`.
|
||||
- **Positive controls** (the harness is provably not blind): perturbing an unrelated knob
|
||||
(`GATEWAY_VIP`) surfaces 1 unexplained line; perturbing `BASE_DOMAIN` to a wrong value fails 10 of
|
||||
12 paths; injecting a single un-moved hostname is caught by the independent bare-apex scan (which
|
||||
the byte-identity check alone cannot see — which is why both exist).
|
||||
- Gates: `ci/l0.sh`, `ci/l1.sh`, `ci/render-golden.sh`, `ci/lint.sh`,
|
||||
`flux-postbuild-substitution/check.sh` (254 assertions), `catalog/render.py --selftest`,
|
||||
`ci/spec-immutability.sh` — all **PASS**.
|
||||
+9
-47
@@ -27,7 +27,6 @@ ansible-playbook playbooks/site.yml # core bring-up, in order (baseline
|
||||
# — or step by step —
|
||||
ansible-playbook playbooks/node-baseline.yml # 0. packages, kernel modules, Longhorn disk, swap-off
|
||||
ansible-playbook playbooks/k3s.yml # 1. HA k3s (primary --cluster-init, then joins)
|
||||
ansible-playbook playbooks/etcd-io-priority.yml # 1b. shared etcd/Longhorn device I/O weights (PROD ONLY; a no-op on dev)
|
||||
ansible-playbook playbooks/cilium.yml # 2. Cilium CNI (cluster is NotReady until this)
|
||||
ansible-playbook playbooks/verify.yml # 3. gate #1: 3 nodes Ready, Hubble up, no kube-proxy
|
||||
ansible-playbook playbooks/flux-bootstrap.yml # 4. hand off to bootstrap/ (SOPS then Flux)
|
||||
@@ -72,7 +71,6 @@ should return a PIA exit IP (not the home ISP).
|
||||
|---|---|
|
||||
| `k3s.yml` / `k3s_server` | k3s server install with `--flannel-backend=none --disable-kube-proxy --disable-network-policy --disable=traefik,servicelb`; primary `--cluster-init`, secondaries join via the primary's node-token (fetched, never pre-stored) |
|
||||
| `cilium.yml` / `cilium` | Gateway API CRDs, then Helm `cilium` with `kubeProxyReplacement`, `k8sServiceHost/Port`, `gatewayAPI`, `egressGateway`, `hubble`, `l2announcements` |
|
||||
| `etcd-io-priority.yml` / `etcd_io_priority` | **prod only** (master switch defaults false; `group_vars/k3s_prod` sets it true). Switches the disk carrying `k3s_data_dir` to **BFQ**, drops `nr_requests` to 32, and gives `system.slice` + `k3s.service` `IOWeight=10000` so the embedded etcd outweighs `kubepods.slice` — then reads every value back out of the kernel and **fails** if it did not land. Mitigation for the 2026-09-07 k3s-prod-1 etcd WAL-fsync starvation. Restarts nothing. NVMe-backed nodes are skipped, not decorated. See `roles/etcd_io_priority/README.md` for the mechanisms in this space that are **inert** here and must not be "fixed" back in |
|
||||
| `verify.yml` / `cluster_verify` | waits nodes Ready + Cilium/Hubble rollout; asserts kube-proxy absent |
|
||||
| `flux-bootstrap.yml` / `flux_bootstrap` | calls `bootstrap/` (SOPS age key, then `flux bootstrap --path=clusters/dev`) |
|
||||
|
||||
@@ -81,30 +79,11 @@ Ansible installs Cilium; the Flux `platform/cilium` HelmRelease adopts config
|
||||
afterward. The parity-critical values **must match** between this role's
|
||||
`templates/cilium-values.yaml.j2` and that HelmRelease:
|
||||
- `kubeProxyReplacement: true`
|
||||
- `k8sServiceHost: {{ k3s_apiserver_endpoint }}` — **the kube-vip CONTROL-PLANE VIP**
|
||||
(`= kube_vip_address`: dev `192.168.1.49`, prod `192.168.2.9`), **not** `k3s_api_host`
|
||||
- `k8sServicePort: {{ k3s_api_port }}` (6443 in both environments)
|
||||
- `k8sServiceHost: {{ k3s_api_host }}` (= primary node IP, `10.73.0.11` by default)
|
||||
- `k8sServicePort: 6443`
|
||||
|
||||
### Two API addresses, split 2026-09-06 — do not re-merge them
|
||||
`k3s_api_host` and `k3s_apiserver_endpoint` were one variable, and the conflation put every
|
||||
prod `cilium-agent` on `192.168.2.10` (`k3s-prod-1`'s own address) on a cluster whose point is
|
||||
HA — verified live. Lose that node and the datapath keeps serving from Cilium's cache while no
|
||||
service or policy update ever lands, which is a silent failure, not an outage.
|
||||
|
||||
| variable | value | consumed by | why it must be that address |
|
||||
|---|---|---|---|
|
||||
| `k3s_api_host` | the founder's own node IP (dev `192.168.1.50`, prod `192.168.2.10`) | `roles/k3s_server`: the secondary `--server https://…:6443` join URL, and one `k3s_tls_sans` entry | on a **cold** build no VIP exists yet (kube-vip is a k3s auto-deploy manifest, so it needs a cluster first). The founder just ran `--cluster-init` and is up |
|
||||
| `k3s_apiserver_endpoint` | `{{ kube_vip_address }}` (dev `192.168.1.49`, prod `192.168.2.9`) | `roles/cilium` → `k8sServiceHost`; any steady-state client | it must survive losing any single server |
|
||||
|
||||
Consequences already handled: `playbooks/site.yml` imports `kube-vip.yml` **between** `k3s.yml`
|
||||
and `cilium.yml` (the VIP has to be floating before the CNI dials it), and `roles/kube_vip`'s
|
||||
last task blocks until the VIP answers on 6443. `k3s_tls_sans` already carried both addresses,
|
||||
so no cert reissue and no k3s restart are involved.
|
||||
|
||||
The Flux side of the same contract is `${K3S_API_HOST}` in `platform/cilium/helmrelease.yaml`,
|
||||
set per overlay in `clusters/<env>/kustomization.yaml`; `[parity]` means it must equal
|
||||
`k3s_apiserver_endpoint` for the SAME environment. Asserted by
|
||||
`conformance/tests/l0-static/ha-control-plane-as-code`.
|
||||
The HelmRelease currently has `k8sServiceHost: "REPLACE-ME"` — WS-NET must set it
|
||||
to the same value this role uses (surfaced by `tofu output primary_node_ip`).
|
||||
|
||||
## Interface assumed of `bootstrap/` (authored by other workers, NOT here)
|
||||
`roles/flux_bootstrap` invokes two entrypoints via overridable command vars:
|
||||
@@ -122,20 +101,10 @@ for PM reconciliation.
|
||||
## Targeting prod (R-ENV-02: same playbooks, same roles, scoped overrides)
|
||||
|
||||
`inventory/hosts.ini` carries a `[k3s_prod_primary]` / `[k3s_prod_secondary]` /
|
||||
`[k3s_prod:children]` section for the three bare-metal nodes — `k3s-prod-1` `192.168.2.10`,
|
||||
`k3s-prod-2` `.11`, `k3s-prod-3` `.12` — with its own `group_vars/k3s_prod/` (vaulted
|
||||
`ansible_become_password` + non-secret overrides).
|
||||
|
||||
**All three are live in the inventory.** This paragraph used to say `k3s-prod-2`/`-3` "are
|
||||
addressed but not yet racked, so their inventory lines stay commented out"; that stopped being
|
||||
true on 2026-09-03 when node 3 came online, and the group has been complete since. Corrected
|
||||
2026-09-07 because the stale sentence is the kind that gets used to reason about blast radius:
|
||||
`-e k3s_hosts=k3s_prod` targets **three** hosts, not one. Confirm it, every time, before running
|
||||
anything:
|
||||
|
||||
```bash
|
||||
ansible-playbook playbooks/<play>.yml -e k3s_hosts=k3s_prod --limit k3s-prod-3 --list-hosts
|
||||
```
|
||||
`[k3s_prod:children]` section for the three planned bare-metal nodes (`k3s-prod-1` at
|
||||
`192.168.2.10` exists and is reachable today; `k3s-prod-2`/`-3` at `.11`/`.12` are addressed but
|
||||
not yet racked, so their inventory lines stay commented out), with its own `group_vars/k3s_prod/`
|
||||
(vaulted `ansible_become_password` + non-secret overrides).
|
||||
|
||||
**Nothing here is forked for prod.** Every node-host play takes its target group from a variable
|
||||
that defaults to the dev group, so the dev commands above are unchanged, and prod is the same
|
||||
@@ -165,11 +134,6 @@ ansible-playbook playbooks/kube-vip.yml -e k3s_hosts=k3s_prod
|
||||
ansible-playbook playbooks/verify.yml -e k3s_hosts=k3s_prod
|
||||
ansible-playbook playbooks/flux-bootstrap.yml -e k3s_hosts=k3s_prod # see caveat below
|
||||
|
||||
# prod — I/O arbitration for the shared etcd/Longhorn disk. ONE NODE AT A TIME, node 3 first
|
||||
# (WD Blue, IO PSI 1.29 during the incident) before node 1 (the 2011 C400, 11.36).
|
||||
ansible-playbook playbooks/etcd-io-priority.yml -e k3s_hosts=k3s_prod --limit k3s-prod-3 --list-hosts
|
||||
ansible-playbook playbooks/etcd-io-priority.yml -e k3s_hosts=k3s_prod --limit k3s-prod-3
|
||||
|
||||
# prod — k3s.yml needs both halves of the primary/secondary split
|
||||
ansible-playbook playbooks/k3s.yml \
|
||||
-e k3s_primary_hosts=k3s_prod_primary -e k3s_secondary_hosts=k3s_prod_secondary
|
||||
@@ -184,9 +148,7 @@ ansible-playbook playbooks/site.yml \
|
||||
What the roles then resolve **by themselves** for a prod host, from `group_vars/k3s_prod/vars.yml`:
|
||||
`k3s_primary_group=k3s_prod_primary` (the `--cluster-init` founder, the node-token `delegate_to`
|
||||
target, and the node Cilium fetches the kubeconfig from), `k3s_servers_group=k3s_prod` (the tls-san
|
||||
node-IP list), `k3s_api_host=192.168.2.10` (the founder, bootstrap-join only),
|
||||
`kube_vip_address=192.168.2.9` — and therefore `k3s_apiserver_endpoint=192.168.2.9`, which is what
|
||||
Cilium dials — and
|
||||
node-IP list), `k3s_api_host=192.168.2.10`, `kube_vip_address=192.168.2.9`, and
|
||||
`cluster_kubeconfig_name=prod.kubeconfig` — so a prod run can never overwrite the
|
||||
`.cache/dev.kubeconfig` that `infra/devcontainer/dev.sh` exports as `$KUBECONFIG`.
|
||||
|
||||
|
||||
@@ -27,85 +27,19 @@ k3s_servers_group: k3s_servers
|
||||
# a kubeconfig is a credential.)
|
||||
cluster_kubeconfig_name: dev.kubeconfig
|
||||
|
||||
# --- TWO API ADDRESSES, AND THEY ARE NOT INTERCHANGEABLE (split 2026-09-06) ----------
|
||||
# This used to be ONE variable, k3s_api_host, consumed by BOTH the cold-bootstrap join and
|
||||
# Cilium's k8sServiceHost. That conflation is what pointed every cilium-agent on the LIVE
|
||||
# prod cluster at 192.168.2.10 — k3s-prod-1's own address — on a cluster whose entire
|
||||
# purpose is HA: verified 2026-09-06, all three agents carried
|
||||
# KUBERNETES_SERVICE_HOST=192.168.2.10 while kube-vip held 192.168.2.9. Lose that one node
|
||||
# and no service/policy update reaches the CNI until it returns (the existing datapath
|
||||
# survives on Cilium's cache, so it fails SILENTLY). The two consumers have genuinely
|
||||
# different requirements, so they now have two variables.
|
||||
#
|
||||
# k3s_api_host THE FOUNDER'S OWN ADDRESS. Bootstrap-time only.
|
||||
# k3s_apiserver_endpoint THE HA ENDPOINT (the kube-vip VIP). Steady-state.
|
||||
#
|
||||
# DO NOT "simplify" these back into one. See each block below for why each half must be
|
||||
# the address it is.
|
||||
|
||||
# THE FOUNDER'S OWN NODE IP — a BOOTSTRAP-TIME address, deliberately NOT the VIP.
|
||||
# The kube-apiserver endpoint the whole stack agrees on: the primary's REAL node IP.
|
||||
# NOTE: ansible_host is now the pfSense NAT SSH target (192.168.2.102:225x), NOT the node
|
||||
# IP — so we read the real 192.168.1.x from k3s_node_ip instead.
|
||||
#
|
||||
# EXACTLY TWO CONSUMERS, and both of them need the founder specifically:
|
||||
# 1. roles/k3s_server — the secondary `--server https://<host>:<port>` JOIN URL. On a COLD
|
||||
# build the VIP DOES NOT EXIST YET at join time: kube-vip is a k3s auto-deploy manifest
|
||||
# that only lands once a cluster exists, so a join aimed at the VIP is a join aimed at
|
||||
# an unanswered address and the cold build is unjoinable. The founder is up (it just ran
|
||||
# --cluster-init), which is the whole reason this variable is the founder.
|
||||
# 2. roles/k3s_server — one entry of k3s_tls_sans, so the founder's own address is a SAN
|
||||
# alongside the VIP and every other server's IP.
|
||||
#
|
||||
# ANYTHING THAT RUNS AGAINST A CLUSTER THAT ALREADY EXISTS MUST NOT USE THIS. It is a
|
||||
# single point of failure by construction. Use k3s_apiserver_endpoint below.
|
||||
# IP — so we read the real 192.168.1.x from k3s_node_ip instead. Used for k3s --tls-san /
|
||||
# secondary --server AND for Cilium k8sServiceHost so the Ansible bootstrap and the Flux
|
||||
# HelmRelease stay in PARITY (docs/02 contract).
|
||||
k3s_api_host: "{{ hostvars[groups[k3s_primary_group][0]]['k3s_node_ip'] }}"
|
||||
|
||||
# Shared by both halves: k3s serves the API on 6443 in every environment, so the port is
|
||||
# not a per-environment knob and is not duplicated per address.
|
||||
k3s_api_port: 6443
|
||||
|
||||
# THE HA kube-apiserver ENDPOINT — a VIRTUAL address that reaches the cluster wherever the
|
||||
# control plane currently is, not a machine. OPERATOR DECISION 2026-09-06: in-cluster
|
||||
# components dial the VIP, as kube-vip was always intended to provide.
|
||||
#
|
||||
# DERIVED from kube_vip_address (below), never restated: an environment overrides the VIP in
|
||||
# ONE place (group_vars/k3s_prod/vars.yml sets kube_vip_address: 192.168.2.9) and this
|
||||
# follows automatically, so prod needs no override of its own and a fourth environment
|
||||
# cannot set one of the two and forget the other.
|
||||
#
|
||||
# NAMED FOR THE ROLE, NOT THE MECHANISM. Consumers want "the address that reaches the
|
||||
# apiserver, HA"; that it is currently floated by kube-vip is an implementation detail. If
|
||||
# the mechanism is ever something else (an external LB, a DNS name), this one line moves and
|
||||
# no consumer changes.
|
||||
#
|
||||
# CONSUMERS:
|
||||
# - roles/cilium -> cilium-values.yaml.j2 -> k8sServiceHost. [parity] with the Flux side:
|
||||
# clusters/base/cluster-vars.yaml K3S_API_HOST (and each overlay's patch) MUST equal
|
||||
# THIS value for that environment, or Flux and the bootstrap fight over the CNI's
|
||||
# control-plane path. See platform/cilium/helmrelease.yaml's PARITY CONTRACT header.
|
||||
#
|
||||
# TLS IS ALREADY COVERED — verified, not assumed: k3s_tls_sans (roles/k3s_server/defaults)
|
||||
# has always included kube_vip_address, and on 2026-09-06 the prod apiserver was reached
|
||||
# through https://192.168.2.9:6443 with FULL CA verification and tls-server-name=192.168.2.9
|
||||
# (infra/ansible/.cache/prod.kubeconfig, no insecure-skip-tls-verify). So the serving cert
|
||||
# needs no change and no k3s restart for this switch.
|
||||
#
|
||||
# ORDERING CONSEQUENCE, and it is why playbooks/site.yml changed too: Cilium now needs the
|
||||
# VIP to be UP before it installs. site.yml therefore imports kube-vip.yml BETWEEN k3s.yml
|
||||
# and cilium.yml, and roles/kube_vip waits for the VIP to answer before returning. kube-vip
|
||||
# can precede the CNI because its DaemonSet is hostNetwork with
|
||||
# KUBERNETES_SERVICE_HOST=127.0.0.1 — it needs neither a CNI nor kube-proxy.
|
||||
k3s_apiserver_endpoint: "{{ kube_vip_address }}"
|
||||
|
||||
# Control-plane HA VIP floated by kube-vip (roles/kube_vip). A stable failover
|
||||
# endpoint for the kube-apiserver, fronted by an operator pfSense port-forward at
|
||||
# 192.168.1.49:6443. Shared here so THREE things agree on the one address:
|
||||
# - roles/kube_vip — what to ARP and lease-elect for;
|
||||
# - roles/k3s_server — the tls-san entry (the serving cert must carry this IP or
|
||||
# TLS via the VIP fails to validate);
|
||||
# - k3s_apiserver_endpoint — above; what in-cluster components actually dial.
|
||||
# THIS is the one line an environment overrides (prod: 192.168.2.9, in
|
||||
# group_vars/k3s_prod/vars.yml); all three follow.
|
||||
# 192.168.1.49:6443. Shared here so BOTH kube_vip AND the k3s_server tls-san list
|
||||
# agree on the address (the serving cert must carry this IP or TLS via the VIP
|
||||
# fails to validate).
|
||||
kube_vip_address: "192.168.1.49"
|
||||
|
||||
# Controller (dev container) reaches the kube-API DIRECTLY via the pfSense NAT
|
||||
|
||||
@@ -51,18 +51,12 @@ k3s_servers_group: k3s_prod
|
||||
# than no kubeconfig at all). .cache/ is gitignored; a kubeconfig is a credential.
|
||||
cluster_kubeconfig_name: prod.kubeconfig
|
||||
|
||||
# --- k3s API host — THE FOUNDER'S ADDRESS, bootstrap-time only --------------------
|
||||
# --- k3s API host — safe to set now, the primary's real IP is already known -------
|
||||
# Dev's k3s_api_host (group_vars/all.yml) reads a separate k3s_node_ip hostvar because
|
||||
# dev's ansible_host is the pfSense NAT SSH target, not the node's real address. Prod
|
||||
# has NO NAT hop — ansible_host in hosts.ini IS already the node's real, routable LAN
|
||||
# address (see the KEY DIFFERENCE note there) — so point straight at it instead of
|
||||
# needing a k3s_node_ip split.
|
||||
#
|
||||
# WHAT THIS IS *NOT*, since 2026-09-06: it is not the endpoint anything steady-state
|
||||
# dials. That is k3s_apiserver_endpoint (group_vars/all.yml), which derives from
|
||||
# kube_vip_address below and therefore needs NO override here. k3s_api_host is consumed
|
||||
# only by the secondary `--server` cold-join URL and by one tls-san entry. Read the
|
||||
# TWO API ADDRESSES block at the top of group_vars/all.yml before touching either.
|
||||
k3s_api_host: "{{ hostvars[groups[k3s_primary_group][0]]['ansible_host'] }}"
|
||||
# cluster_controller_tls_server_name (group_vars/all.yml) derives from kube_vip_address
|
||||
# by template reference, so it needs no separate override here — it picks up the real
|
||||
@@ -81,15 +75,8 @@ k3s_api_host: "{{ hostvars[groups[k3s_primary_group][0]]['ansible_host'] }}"
|
||||
# round (control-plane .49, ingress pool .53-.60).
|
||||
#
|
||||
# This is the address kube-vip floats for the API and that k3s tls-san's onto its
|
||||
# serving cert. Dev's equivalent is kube_vip_address: 192.168.1.49 in group_vars/all.yml.
|
||||
#
|
||||
# CORRECTION 2026-09-06: an earlier version of this comment claimed "secondaries use it as
|
||||
# their --server join target". They do not, and they must not — on a COLD build the VIP is
|
||||
# not up when the secondaries join (kube-vip is a k3s auto-deploy manifest and needs a
|
||||
# cluster first), so the join uses k3s_api_host, the founder. What DOES use this address is
|
||||
# k3s_apiserver_endpoint (group_vars/all.yml, derived from this line), i.e. Cilium's
|
||||
# k8sServiceHost and any other steady-state client. Overriding kube_vip_address here moves
|
||||
# BOTH the VIP and prod's Cilium endpoint together, which is the point of deriving it.
|
||||
# serving cert; secondaries use it as their --server join target. Dev's equivalent is
|
||||
# kube_vip_address: 192.168.1.49 in group_vars/all.yml.
|
||||
#
|
||||
# HISTORY: an earlier pass here wrongly set this to 192.168.2.20 — that is the INGRESS
|
||||
# VIP, not the control-plane VIP. Corrected. Keep the two distinct.
|
||||
@@ -306,35 +293,47 @@ node_baseline_dns_servers:
|
||||
# k3s_base_domain, below). It reaches the issuer through the NODE resolver, i.e. through
|
||||
# these PUBLIC upstreams — not through CoreDNS's split-horizon answer.
|
||||
#
|
||||
# THIS USED TO BE BROKEN, AND THE 2026-09-04 SUBDOMAIN SPLIT FIXED IT. While prod was
|
||||
# authored to serve dev's hostnames, a prod node resolving keycloak.home.keithjackson.dev
|
||||
# publicly got DEV's Gateway VIP (192.168.1.53) — the wrong cluster, on the wrong LAN, so
|
||||
# kubectl OIDC SSO on prod could not work at all. Now prod's issuer is
|
||||
# keycloak.prod.home.keithjackson.dev, and the public record for that name is published by
|
||||
# PROD's own external-dns pointing at PROD's Gateway VIP (192.168.2.20). The public answer
|
||||
# and the split-horizon answer therefore AGREE, and no /etc/hosts pin is needed.
|
||||
#
|
||||
# STILL TO VERIFY on prod bring-up (PROD-PROVISIONING.md §10): this holds only once prod's
|
||||
# external-dns has actually published the record. Until then the issuer does not resolve
|
||||
# from the nodes and kubectl SSO fails — the cert-based admin kubeconfig is the break-glass
|
||||
# path, and it always works. Ordering, not a design gap.
|
||||
# ORDERING HAZARD, NOT YET CLEARED. Prod's issuer is keycloak.home.keithjackson.dev — an APEX
|
||||
# name, because prod IS the apex under the 2026-09-04 inversion (k3s_base_domain below).
|
||||
# That is the name the LIVE DEV CLUSTER answers today, and its public A record currently
|
||||
# points at DEV's Gateway VIP (192.168.1.53) — the wrong cluster, on the wrong LAN. A prod
|
||||
# node resolving its own issuer through these public upstreams therefore reaches dev until:
|
||||
# (1) dev has migrated to *.dev.home.keithjackson.dev and RETIRED its apex records, and
|
||||
# (2) PROD's own external-dns has published keycloak.home.keithjackson.dev at PROD's
|
||||
# Gateway VIP (192.168.2.20).
|
||||
# Only then do the public answer and the in-cluster split-horizon answer AGREE, and no
|
||||
# /etc/hosts pin is needed. Until then kubectl OIDC SSO on prod cannot work; the cert-based
|
||||
# admin kubeconfig is the break-glass path and always works. Ordering, not a design gap —
|
||||
# the ordered steps are infra/PROD-PROVISIONING.md §10.
|
||||
node_baseline_dns_search: []
|
||||
|
||||
# --- platform hostname suffix, ANSIBLE SIDE (R-ENV-02, R-DNS-01) --------------------
|
||||
# PROD SERVES ITS OWN SUBDOMAIN. [parity] MUST equal BASE_DOMAIN in clusters/prod/
|
||||
# kustomization.yaml — the same one-fact-in-two-places contract as k3s_data_dir and
|
||||
# k3s_api_host. The Flux side cannot supply this: the kube-apiserver's oidc-issuer-url is a
|
||||
# k3s server FLAG written to /etc/rancher/k3s/config.yaml, not a manifest, so
|
||||
# postBuild.substituteFrom never sees it. Full rationale in
|
||||
# roles/k3s_server/defaults/main.yml, where k3s_oidc_issuer_url is derived from this.
|
||||
# PROD **IS** THE APEX. [parity] MUST equal BASE_DOMAIN in clusters/prod/kustomization.yaml —
|
||||
# the same one-fact-in-two-places contract as k3s_data_dir and k3s_api_host. The Flux side
|
||||
# cannot supply this: the kube-apiserver's oidc-issuer-url is a k3s server FLAG written to
|
||||
# /etc/rancher/k3s/config.yaml, not a manifest, so postBuild.substituteFrom never sees it.
|
||||
# Full rationale in roles/k3s_server/defaults/main.yml, where k3s_oidc_issuer_url derives.
|
||||
#
|
||||
# WHY PROD MOVED (infra/PROD-PROVISIONING.md §10): both clusters were authored to serve the
|
||||
# SAME hostnames from the SAME Cloudflare zone, and on 2026-09-04 prod's external-dns
|
||||
# deleted every one of dev's public records. Dev stays on home.keithjackson.dev (it is LIVE);
|
||||
# prod takes prod.home.keithjackson.dev. Consequence here: prod's Keycloak issuer is
|
||||
# https://keycloak.prod.home.keithjackson.dev/realms/homelab, so the `kubernetes` OIDC client
|
||||
# in prod's OWN Keycloak realm is what prod's kubectl SSO authenticates against.
|
||||
k3s_base_domain: "prod.home.keithjackson.dev"
|
||||
# WHY THE SUFFIXES SPLIT (infra/PROD-PROVISIONING.md §10): both clusters were authored to serve
|
||||
# the SAME hostnames from the SAME Cloudflare zone, and on 2026-09-04 prod's external-dns
|
||||
# deleted every one of dev's public records.
|
||||
#
|
||||
# WHICH WAY ROUND — INVERTED 2026-09-04 by operator decision, superseding the interim split
|
||||
# committed earlier the same day (prod on `prod.home.` behind apex CNAMEs). The simpler target:
|
||||
# prod -> home.keithjackson.dev (THE APEX; this variable — no `prod.` label, no CNAMEs)
|
||||
# dev -> dev.home.keithjackson.dev (roles/k3s_server/defaults/main.yml)
|
||||
# So prod's Keycloak issuer is https://keycloak.home.keithjackson.dev/realms/homelab, and the
|
||||
# `kubernetes` OIDC client in PROD's OWN Keycloak realm is what prod's kubectl SSO authenticates
|
||||
# against.
|
||||
#
|
||||
# ORDERING HAZARD SPECIFIC TO THIS VARIABLE. keycloak.home.keithjackson.dev is the name the LIVE
|
||||
# dev cluster answers today, and prod's nodes resolve the issuer through PUBLIC upstreams (see
|
||||
# node_baseline_dns_search above). So until dev has vacated the apex and PROD's external-dns has
|
||||
# published that name at PROD's Gateway VIP (192.168.2.20), a prod node resolving its own issuer
|
||||
# gets DEV's VIP (192.168.1.53) — the wrong cluster, on the wrong LAN. kubectl OIDC SSO on prod
|
||||
# therefore does not work until §10's dev-vacate steps are complete; the cert-based admin
|
||||
# kubeconfig is the break-glass path and always works. Ordering, not a design gap.
|
||||
k3s_base_domain: "home.keithjackson.dev"
|
||||
|
||||
# --- Flux bootstrap: WHICH overlay this environment reconciles from (R-ENV-02) -----
|
||||
# roles/flux_bootstrap defaults flux_bootstrap_env_path to `clusters/dev` (dev is LIVE
|
||||
@@ -357,29 +356,3 @@ k3s_base_domain: "prod.home.keithjackson.dev"
|
||||
# from the prod nodes — it is outside every cluster and can simply be powered off;
|
||||
# - the `sops-age` Secret is installed out of band on prod (bootstrap/sops).
|
||||
flux_bootstrap_env_path: "clusters/prod"
|
||||
|
||||
# --- shared etcd/Longhorn device I/O arbitration (2026-09-07 incident) --------------
|
||||
# TURNS ON roles/etcd_io_priority FOR PROD ONLY. Role default is false, and dev must stay
|
||||
# that way: dev has a DEDICATED etcd device (/dev/vdc mounted at server/db,
|
||||
# docs/plans/ssd-etcd-isolation.md) so it has nothing to arbitrate, and pulling a LIVE
|
||||
# cluster onto a different I/O scheduler as a side effect of sharing a role would be a real
|
||||
# change for no benefit. Same seam as node_baseline_backports_kernel above.
|
||||
#
|
||||
# WHY PROD NEEDS IT. The §2 ruling put /data/k3s (etcd WAL) and /data/longhorn (every
|
||||
# replica) on ONE partition per node and recorded the contention as "accepted to be
|
||||
# MEASURED, not assumed". It has now been measured, the hard way: 2026-09-07 06:05:21Z
|
||||
# k3s-prod-1 went NotReady with the host still up, because etcd could not finish a WAL
|
||||
# fdatasync (3.4-6.7 s against a 1 s expectation) while Longhorn wrote to the same disk.
|
||||
# /proc/pressure/io FULL avg300 was 11.36 there against 0.75 (node 2, NVMe) and 1.29
|
||||
# (node 3). See PROD-PROVISIONING.md §2 "THE MEASUREMENT CAME BACK".
|
||||
#
|
||||
# WHAT ENABLING THIS DOES *NOT* DO: nothing, until the operator runs
|
||||
# playbooks/etcd-io-priority.yml. It is not applied by any other play except through
|
||||
# site.yml, and site.yml is not run against prod casually. Apply to k3s-prod-3 (the WD Blue,
|
||||
# IO PSI 1.29) FIRST and watch NodeIOPressureHigh for a day — the role's own README explains
|
||||
# why, what it cannot fix, and how to roll it back in three commands.
|
||||
#
|
||||
# k3s-prod-2 is NVMe-backed; the role detects that and deliberately installs nothing there
|
||||
# rather than leaving an inert drop-in behind, so this group-wide `true` is still correct
|
||||
# for it.
|
||||
etcd_io_priority_enabled: true
|
||||
@@ -31,17 +31,10 @@ k3s_secondary
|
||||
[hypervisor]
|
||||
localhost ansible_connection=local
|
||||
|
||||
# VPN egress-gateway node — runs the node-level split-tunnel OpenVPN + kill-switch
|
||||
# (ADR-0015, playbooks/vpn.egress.gateway.yml). EXACTLY ONE host belongs here.
|
||||
#
|
||||
# THIS GROUP IS SINGLETON BY DESIGN — a second host does NOT buy failover. Cilium OSS has no
|
||||
# egress-gateway failover: the CRD's own `egressGateway` description says "In case multiple nodes
|
||||
# are a match for the given set of labels, the first node in lexical ordering based on their name
|
||||
# will be selected". Two labelled nodes therefore means one silently wins by NAME ORDER (a labelled
|
||||
# `k3s-prod-1` beats a labelled `k3s-prod-3`) and the other carries no traffic. See
|
||||
# platform/vpn-egress/README.md "Resilience — there is no gateway failover" for the full statement
|
||||
# and the ADR-0016 escalation. Members are already defined above with their ansible_host under
|
||||
# k3s_secondary; this group only tags the egress role target.
|
||||
# VPN egress-gateway node(s) — run the node-level split-tunnel OpenVPN + kill-switch
|
||||
# (ADR-0015, playbooks/vpn.egress.gateway.yml). Initially just k3s-dev-3 (single gateway;
|
||||
# add a second host here for Cilium Egress Gateway HA). Members are already defined above
|
||||
# with their ansible_host under k3s_secondary; this group only tags the egress role target.
|
||||
[vpn_egress_gateway]
|
||||
k3s-dev-3
|
||||
|
||||
@@ -92,17 +85,9 @@ proxmox ansible_host=192.168.2.102 ansible_port=22 ansible_user=root ansible_ssh
|
||||
# =============================================================================
|
||||
|
||||
[k3s_prod_primary]
|
||||
# FOUNDER MOVED TO NODE 2 (2026-09-07). Node 1 founded this cluster with --cluster-init,
|
||||
# then its SSD was replaced and Debian reinstalled, so it is now a JOINING node like any
|
||||
# other. This group decides two things at once: which play sets k3s_server_mode: primary
|
||||
# (-> --cluster-init, which FOUNDS A NEW CLUSTER) and what k3s_api_host resolves to (the
|
||||
# --server join target). Leaving node 1 here would have pointed a rebuilt, empty node at
|
||||
# ITSELF and founded a second cluster beside the live one. Node 2 is the senior surviving
|
||||
# server; its own --cluster-init is inert because /etc/rancher/k3s/k3s.yaml already exists.
|
||||
k3s-prod-2 ansible_host=192.168.2.11 ansible_port=22 k3s_node_ip=192.168.2.11
|
||||
k3s-prod-1 ansible_host=192.168.2.10 ansible_port=22 k3s_node_ip=192.168.2.10
|
||||
|
||||
[k3s_prod_secondary]
|
||||
k3s-prod-1 ansible_host=192.168.2.10 ansible_port=22 k3s_node_ip=192.168.2.10
|
||||
# Uncomment a node only once it is actually on the bench and answering SSH, so no play can
|
||||
# target a host that isn't there. k3s-prod-2 came online 2026-09-01 (imaged identically to
|
||||
# node 1: same HP EliteDesk 800 G5 Mini, same BIOS R21 02.26.00, same i7-9700T, same
|
||||
@@ -118,6 +103,7 @@ k3s-prod-1 ansible_host=192.168.2.10 ansible_port=22 k3s_node_ip=192.168.2.10
|
||||
# three-way, not a node-1 outlier.
|
||||
# NOTE: node 3 also has wlp1s0 up on 192.168.2.195 (wifi). `ansible_host` is deliberately
|
||||
# the WIRED eno1 address — the same directly-routable 192.168.2.x LAN as nodes 1 and 2.
|
||||
k3s-prod-2 ansible_host=192.168.2.11 ansible_port=22 k3s_node_ip=192.168.2.11
|
||||
k3s-prod-3 ansible_host=192.168.2.12 ansible_port=22 k3s_node_ip=192.168.2.12
|
||||
|
||||
# Umbrella group group_vars/k3s_prod/ attaches to (mirrors dev's [k3s_servers:children]
|
||||
|
||||
@@ -1,38 +0,0 @@
|
||||
---
|
||||
# Give the k3s server process — and therefore the EMBEDDED ETCD inside it — a larger share of
|
||||
# the block device it shares with Longhorn. Mitigates the 2026-09-07 k3s-prod-1 outage, where
|
||||
# etcd's WAL fdatasync took 3.4-6.7 s (expects 1 s) and the node went NotReady while the host
|
||||
# stayed up. See roles/etcd_io_priority/README.md for the mechanism, the measurements, and the
|
||||
# list of things in this space that do NOT work on these nodes.
|
||||
#
|
||||
# NO-OP BY DEFAULT. The role's master switch (etcd_io_priority_enabled) is false in
|
||||
# defaults/main.yml and true only in inventory/group_vars/k3s_prod/vars.yml, so running this
|
||||
# with no extra vars against dev changes nothing on any node — dev already has a dedicated
|
||||
# etcd device and has nothing to arbitrate.
|
||||
#
|
||||
# TARGET (R-ENV-02): same `k3s_hosts` seam every other whole-cluster play uses.
|
||||
# ansible-playbook playbooks/etcd-io-priority.yml # dev, a no-op
|
||||
# ansible-playbook playbooks/etcd-io-priority.yml -e k3s_hosts=k3s_prod # prod
|
||||
#
|
||||
# RUN IT ON ONE NODE FIRST, AND CONFIRM THE TARGET BEFORE YOU RUN IT AT ALL. A prior mistake in
|
||||
# this repo silently targeted dev's three nodes; --list-hosts is how that is caught:
|
||||
# ansible-playbook playbooks/etcd-io-priority.yml -e k3s_hosts=k3s_prod \
|
||||
# --limit k3s-prod-3 --list-hosts
|
||||
# k3s-prod-3 first, deliberately: it is the WD Blue SATA node (IO PSI 1.29 during the incident),
|
||||
# so it exercises the identical code path as the failing node 1 (11.36) at a fraction of the
|
||||
# stakes. Watch NodeIOPressureHigh / the "Nodes — CPU / Memory / I-O Pressure" dashboard for a
|
||||
# day before touching node 1.
|
||||
#
|
||||
# ROLLBACK is immediate and needs no playbook — see the header of
|
||||
# /usr/local/sbin/k3s-etcd-io-priority on the node.
|
||||
|
||||
- name: I/O priority for the shared etcd + Longhorn device
|
||||
hosts: "{{ k3s_hosts | default('k3s_servers') }}"
|
||||
become: true
|
||||
# serial: 1 — one node at a time. Nothing here restarts k3s, but changing a live node's I/O
|
||||
# scheduler is a real change to a cluster member, and the whole point is that a bad outcome
|
||||
# on one node must never be simultaneous across all three.
|
||||
serial: 1
|
||||
gather_facts: true
|
||||
roles:
|
||||
- role: etcd_io_priority
|
||||
@@ -3,13 +3,6 @@
|
||||
# Control-plane ONLY (svc_enable=false) — Cilium owns Service LoadBalancing.
|
||||
# Writes the manifest into each server's k3s auto-deploy dir; k3s applies it.
|
||||
#
|
||||
# PART OF THE ONE-COMMAND BRING-UP SINCE 2026-09-06, not a separate operator step:
|
||||
# playbooks/site.yml imports this play BETWEEN k3s.yml and cilium.yml, because Cilium's
|
||||
# k8sServiceHost is now this VIP (k3s_apiserver_endpoint, group_vars/all.yml) and the CNI
|
||||
# cannot be installed against an endpoint nothing announces yet. The role's final task waits
|
||||
# for the VIP to answer on 6443, so a failure to float it stops the run here with a named
|
||||
# cause instead of surfacing later as unreachable cilium-agents.
|
||||
#
|
||||
# TARGET (R-ENV-02): `k3s_hosts` defaults to dev's `k3s_servers`; prod is
|
||||
# ansible-playbook playbooks/kube-vip.yml -e k3s_hosts=k3s_prod
|
||||
# (the VIP itself comes from kube_vip_address, already scoped per environment in
|
||||
|
||||
@@ -1,10 +1,7 @@
|
||||
---
|
||||
# Full dev/test cluster bring-up, in order (docs/02 sequencing; run AFTER
|
||||
# `tofu apply` has created and cloud-init has baselined the VMs):
|
||||
# k3s HA install -> kube-vip control-plane VIP -> Cilium CNI -> verify readiness
|
||||
# -> Flux + SOPS hand-off.
|
||||
# (kube-vip joined the chain 2026-09-06 — see the note at its import below; it is a
|
||||
# prerequisite of the CNI now, not a separate operator step.)
|
||||
# k3s HA install -> Cilium CNI -> verify readiness -> Flux + SOPS hand-off.
|
||||
#
|
||||
# TARGET (R-ENV-02): every imported play takes its host pattern from a var that
|
||||
# defaults to the dev group, so `ansible-playbook playbooks/site.yml` with no extra
|
||||
@@ -23,40 +20,6 @@
|
||||
- name: Import - install k3s HA control plane
|
||||
ansible.builtin.import_playbook: k3s.yml
|
||||
|
||||
# ADDED 2026-09-07, after the k3s-prod-1 etcd I/O-starvation outage. It must come AFTER
|
||||
# k3s.yml because it writes an IOWeight drop-in for k3s.service and verifies the value in
|
||||
# that unit's LIVE cgroup — on a cold build the cgroup does not exist until k3s runs, and
|
||||
# the role reports that rather than asserting on it. It restarts nothing.
|
||||
#
|
||||
# A NO-OP FOR DEV, by design: roles/etcd_io_priority defaults its master switch to false and
|
||||
# only inventory/group_vars/k3s_prod/vars.yml sets it true. Dev has a dedicated etcd device
|
||||
# and must not be dragged onto a different I/O scheduler by a shared role.
|
||||
- name: Import - I/O arbitration for the shared etcd + Longhorn device (prod only)
|
||||
ansible.builtin.import_playbook: etcd-io-priority.yml
|
||||
|
||||
# ADDED 2026-09-06, AND ITS POSITION IS LOAD-BEARING: kube-vip MUST come up between k3s and
|
||||
# Cilium. Cilium's k8sServiceHost is now the control-plane VIP (k3s_apiserver_endpoint =
|
||||
# kube_vip_address) rather than the founder's own node IP, so installing Cilium before the
|
||||
# VIP is announced would point every agent at an address nothing answers on — the exact
|
||||
# shape of the 2026-09-01 incident, where agents that could not reach the API server
|
||||
# deadlocked on a cold cluster.
|
||||
#
|
||||
# IT CANNOT COME EARLIER. kube-vip ships as a k3s auto-deploy manifest
|
||||
# (<data-dir>/server/manifests), so a cluster must exist before it can be applied — hence
|
||||
# after k3s.yml, not before.
|
||||
#
|
||||
# IT DOES NOT NEED THE CNI, which is what makes this ordering possible at all: the kube-vip
|
||||
# DaemonSet is hostNetwork with KUBERNETES_SERVICE_HOST=127.0.0.1 (roles/kube_vip's template
|
||||
# explains why), so it reaches the node-local apiserver with no CNI and no kube-proxy, and
|
||||
# DaemonSet pods carry the default not-ready/network-unavailable tolerations that let them
|
||||
# schedule onto a still-NotReady node. roles/kube_vip blocks until the VIP actually answers
|
||||
# on 6443, so cilium.yml below cannot start against a VIP that is not up.
|
||||
#
|
||||
# Re-running this against the LIVE dev cluster is a no-op for this play: dev already runs
|
||||
# kube-vip with the same rendered manifest, and the wait finds the VIP already answering.
|
||||
- name: Import - float the control-plane HA VIP (kube-vip) BEFORE the CNI dials it
|
||||
ansible.builtin.import_playbook: kube-vip.yml
|
||||
|
||||
- name: Import - install Cilium CNI
|
||||
ansible.builtin.import_playbook: cilium.yml
|
||||
|
||||
|
||||
@@ -2,17 +2,7 @@
|
||||
# cilium role defaults (ADR-0002, docs/02 Cilium-ownership contract).
|
||||
# This role performs the INITIAL Cilium install; the Flux platform/cilium
|
||||
# HelmRelease adopts config afterward. The parity-critical values
|
||||
# (kubeProxyReplacement, k8sServiceHost/Port) are templated from
|
||||
# k3s_apiserver_endpoint / k3s_api_port (group_vars/all.yml).
|
||||
#
|
||||
# k8sServiceHost IS THE HA VIP, NOT THE FOUNDER (changed 2026-09-06). It reads
|
||||
# k3s_apiserver_endpoint — which derives from kube_vip_address, so it is dev 192.168.1.49 /
|
||||
# prod 192.168.2.9 — and deliberately NOT k3s_api_host, the founder's own node IP. Those two
|
||||
# were one variable until 2026-09-06, and the conflation is what left all three prod
|
||||
# cilium-agents dialing k3s-prod-1 on an HA cluster. Do not re-merge them.
|
||||
#
|
||||
# THIS ROLE THEREFORE DEPENDS ON THE VIP EXISTING. playbooks/site.yml runs kube-vip.yml
|
||||
# before cilium.yml and roles/kube_vip blocks until the VIP answers on 6443.
|
||||
# (kubeProxyReplacement, k8sServiceHost/Port) are templated from k3s_api_host.
|
||||
|
||||
# Exact pin — MUST equal the platform/cilium HelmRelease chart.spec.version (1.20.1, an
|
||||
# exact pin, not a range). Renovate bumps both together.
|
||||
|
||||
@@ -89,7 +89,7 @@
|
||||
run_once: true
|
||||
changed_when: false
|
||||
|
||||
- name: Render the Cilium values file (parity-critical settings from k3s_apiserver_endpoint)
|
||||
- name: Render the Cilium values file (parity-critical settings from k3s_api_host)
|
||||
ansible.builtin.template:
|
||||
src: cilium-values.yaml.j2
|
||||
dest: "{{ cilium_values_path }}"
|
||||
|
||||
@@ -4,24 +4,9 @@
|
||||
---
|
||||
# [parity] k3s kube-proxy is disabled → Cilium replaces it (ADR-0001/0002).
|
||||
kubeProxyReplacement: true
|
||||
# [parity] = k3s_apiserver_endpoint, THE HA CONTROL-PLANE VIP (kube_vip_address) — dev
|
||||
# 192.168.1.49, prod 192.168.2.9. NOT k3s_api_host, which is the FOUNDER'S OWN node IP:
|
||||
# pointing the CNI's kube-proxy replacement at one machine on a 3-server HA cluster means
|
||||
# losing that machine stops every service/policy update reaching the datapath, silently
|
||||
# (the existing datapath keeps working from Cilium's cache, so nothing looks broken). That
|
||||
# was the live state of prod until 2026-09-06 — all three cilium-agents carried
|
||||
# KUBERNETES_SERVICE_HOST=192.168.2.10 while kube-vip held .9. See group_vars/all.yml.
|
||||
#
|
||||
# [parity] The Flux side is platform/cilium/helmrelease.yaml's k8sServiceHost:
|
||||
# "${K3S_API_HOST}", whose per-environment value comes from clusters/<env>/kustomization.yaml.
|
||||
# THAT VALUE MUST EQUAL k3s_apiserver_endpoint for the SAME environment. If it does not,
|
||||
# Flux and this bootstrap render different control-plane paths and fight over the CNI.
|
||||
#
|
||||
# THE VIP MUST BE UP BEFORE THIS ROLE RUNS. playbooks/site.yml imports kube-vip.yml between
|
||||
# k3s.yml and cilium.yml for exactly this reason, and roles/kube_vip blocks until the VIP
|
||||
# answers. Rendering this against a VIP that nothing announces yet reproduces the
|
||||
# 2026-09-01 shape: agents that cannot reach the API server on a cold cluster.
|
||||
k8sServiceHost: "{{ k3s_apiserver_endpoint }}"
|
||||
# [parity] = the k3s API endpoint (primary node IP). The Flux HelmRelease's
|
||||
# k8sServiceHost "REPLACE-ME" must be set to this same value.
|
||||
k8sServiceHost: "{{ k3s_api_host }}"
|
||||
k8sServicePort: {{ k3s_api_port }}
|
||||
|
||||
# Ingress via Gateway API — no Traefik (docs/09, ADR-0002).
|
||||
|
||||
@@ -1,142 +0,0 @@
|
||||
# roles/etcd_io_priority
|
||||
|
||||
Give the k3s server process — and therefore the **embedded etcd** it hosts — a larger share of
|
||||
the block device it shares with Longhorn.
|
||||
|
||||
## The incident
|
||||
|
||||
2026-09-07 06:05:21Z, `k3s-prod-1` went NotReady (`Kubelet stopped posting node status`) while
|
||||
the host stayed up, pingable and SSH-able. From the k3s journal:
|
||||
|
||||
```
|
||||
slow fdatasync took=3.56s / 6.71s / 4.95s / 3.40s (etcd expects 1s)
|
||||
apply request took too long took=8.30s (expects 100ms)
|
||||
etcdserver: too many requests
|
||||
Failed to check local etcd status for learner management: context deadline exceeded
|
||||
```
|
||||
|
||||
No OOM, no MCE, no hung task. All three prod nodes put `/data/k3s` (etcd's WAL **and** the
|
||||
containerd image store) and `/data/longhorn` (every replica) in two directories on **one
|
||||
partition**:
|
||||
|
||||
| node | device | drive | `/proc/pressure/io` full avg300 |
|
||||
|---|---|---|---|
|
||||
| k3s-prod-1 | `/dev/sda5` | C400-MTFDDAC512MAM — 2011 consumer SATA SSD, no PLP, 61,440 reallocated sectors | **11.36** |
|
||||
| k3s-prod-2 | `/dev/nvme0n1p5` | Samsung PM9A1 NVMe | 0.75 |
|
||||
| k3s-prod-3 | `/dev/sda5` | WDC WDS500G2B0A (WD Blue SATA) | 1.29 |
|
||||
|
||||
Recovery took a manual `systemctl restart k3s`; 18 Longhorn volumes then rebuilt onto the same
|
||||
disk etcd was trying to fsync on. The failure is self-reinforcing.
|
||||
|
||||
## What this role does
|
||||
|
||||
Two things, both instantly reversible, neither requiring a k3s restart:
|
||||
|
||||
1. **`k3s-etcd-io-priority.service`** (oneshot, ordered `Before=k3s.service`) switches the
|
||||
shared device to the **BFQ** I/O scheduler and lowers `nr_requests` from 64 to 32.
|
||||
2. **`IOWeight=10000` drop-ins** on `system.slice` and `k3s.service`, which BFQ turns into
|
||||
`io.bfq.weight=1000` — a 10:1 share against the default-weighted `kubepods.slice` where
|
||||
Longhorn's instance-managers live.
|
||||
|
||||
Then it **reads every one of those values back out of the kernel and asserts on them**. If the
|
||||
scheduler did not switch, or the weight did not land, the play fails. A mitigation that looks
|
||||
applied and is not is worse than none.
|
||||
|
||||
## Why not the obvious things — all checked on the live nodes, all inert
|
||||
|
||||
**`IOSchedulingClass=realtime` on `k3s.service`.** The scheduler here (`mq-deadline`, and BFQ
|
||||
too) *does* honour the I/O priority class, so this looks like the answer. It is not: I/O
|
||||
priority is inherited across `fork` and preserved across `exec` — `block/blk-ioc.c`'s
|
||||
`__copy_io()` copies `ioc->ioprio` into the child — and **containerd is a child of the k3s
|
||||
server process**. Verified on k3s-prod-1: `ps -eo pid,ppid,comm` shows containerd's PPID is the
|
||||
`k3s-server` PID (84553 → 84333), and
|
||||
|
||||
```
|
||||
$ ionice -c2 -n0 -- sh -c 'ionice -p $$; sh -c "ionice -p \$\$"'
|
||||
best-effort: prio 0
|
||||
best-effort: prio 0
|
||||
```
|
||||
|
||||
confirms inheritance through two levels of fork+exec on that node. Putting k3s in the realtime
|
||||
class therefore puts **every container on the node** in the realtime class, Longhorn included —
|
||||
no relative gain, and it demotes the rest of `system.slice` (sshd, journald) for nothing.
|
||||
Getting this to work would need containerd out of k3s's process tree, i.e. an external
|
||||
containerd via `--container-runtime-endpoint`. That is an architecture change, not a tweak.
|
||||
|
||||
**`IOSchedulingPriority` (the level within a class).** `mq-deadline` buckets requests by
|
||||
priority **class** only — `IOPRIO_CLASS_RT/BE/IDLE` map to three `dd_per_prio` queues and the
|
||||
level inside a class is not read. So best-effort 0 versus best-effort 4 is a no-op here.
|
||||
|
||||
**`io.latency` (blk-iolatency).** The ideal mechanism for this problem — scheduler-independent,
|
||||
protects one cgroup's latency target by throttling its siblings. **Not available:** Debian
|
||||
trixie's kernel does not build it. `grep CONFIG_BLK_CGROUP /boot/config-7.1.8+deb13-amd64`
|
||||
returns `CONFIG_BLK_CGROUP_IOCOST=y` and no `IOLATENCY`, and there is no `io.latency` file
|
||||
anywhere under `/sys/fs/cgroup`.
|
||||
|
||||
**`IOWeight=` on the stock configuration.** systemd writes `IOWeight=` to both `io.weight`
|
||||
(blk-iocost) and `io.bfq.weight` (BFQ). On these nodes as shipped, **neither reader exists**:
|
||||
the scheduler is `mq-deadline`, which has no cgroup awareness, and blk-iocost is compiled in but
|
||||
disabled (`/sys/fs/cgroup/io.cost.qos` is empty). Adding a drop-in without switching the
|
||||
scheduler would be pure decoration. That is why the scheduler change and the weights are one
|
||||
role and one verification, not two independent knobs.
|
||||
|
||||
**`IOWeight=1000` instead of `10000`.** systemd rescales `IOWeight`'s 1..10000 range onto
|
||||
`io.bfq.weight`'s 1..1000, so 1000 maps to roughly 101 — BFQ's default for an unweighted cgroup
|
||||
is 100. Only the top of the range separates anything. The role asserts the resulting
|
||||
`io.bfq.weight` is ≥ `etcd_io_priority_min_bfq_weight` (500) rather than trusting that mapping.
|
||||
|
||||
**A udev rule instead of a unit.** `ATTR{queue/scheduler}="bfq"` on the *boot* disk fires its
|
||||
`add` event in the initramfs, where `/etc/udev/rules.d` is absent and `bfq.ko` is not loaded —
|
||||
it silently does nothing and the disk comes up on `mq-deadline` with no error anywhere.
|
||||
|
||||
**A drop-in on `kubepods.slice`** (to lower Longhorn instead of raising etcd). `kubepods.slice`
|
||||
is a **transient** unit the kubelet creates through the systemd API (`systemctl cat
|
||||
kubepods.slice` shows `/run/systemd/transient/kubepods.slice`), and whether systemd merges an
|
||||
`/etc` drop-in into a transient unit was not verified on these nodes. Raising `system.slice`
|
||||
gets the same ratio through a unit systemd definitely owns.
|
||||
|
||||
## What this role does NOT fix
|
||||
|
||||
- **It does not separate etcd from Longhorn.** Both still write to one ext4 filesystem on one
|
||||
partition. Weights arbitrate the *device queue*; they cannot arbitrate the **shared ext4
|
||||
journal**. `fdatasync` on the WAL waits on a jbd2 commit, jbd2 is a kernel thread in the root
|
||||
cgroup, and Longhorn's replica writes dirty the same journal. That serialization point is
|
||||
invisible to every cgroup control in this file and is a hard limit of co-tenancy on one
|
||||
filesystem.
|
||||
- **It is unproven on this hardware.** BFQ is the right mechanism on paper and it is what
|
||||
systemd's `IOWeight` was built for, but nothing here has been measured on a C400 under a real
|
||||
Longhorn rebuild. The role fails loudly if the settings do not *land*; it cannot tell you they
|
||||
*helped*. Apply to `k3s-prod-3` first and watch the `NodeIOPressureHigh` alert and the
|
||||
"Nodes — CPU / Memory / I-O Pressure" dashboard before touching node 1.
|
||||
- **BFQ has costs.** It disables writeback throttling (`wbt_lat_usec` is 2000 on sda today),
|
||||
spends more CPU per request, and gives up some peak throughput. On a device that is already
|
||||
the bottleneck this is the right trade; it is still a trade.
|
||||
- **k3s-prod-2 gets nothing.** Its data dir is on NVMe, BFQ is a poor fit for NVMe, and its IO
|
||||
PSI was 0.75 during the incident. The role detects this and installs nothing rather than
|
||||
leaving inert files behind.
|
||||
- **The real fix is hardware and is the operator's**: replace the C400, or give etcd its own
|
||||
device and flip `node_baseline_etcd_source` back to `device` — the path
|
||||
`roles/node_baseline` still supports unchanged, and the exit condition already written into
|
||||
`infra/PROD-PROVISIONING.md` §2.
|
||||
|
||||
## Running it
|
||||
|
||||
```bash
|
||||
cd infra/ansible
|
||||
|
||||
# ALWAYS confirm the target first — a prior mistake in this repo silently hit dev.
|
||||
ansible-playbook playbooks/etcd-io-priority.yml -e k3s_hosts=k3s_prod \
|
||||
--limit k3s-prod-3 --list-hosts
|
||||
|
||||
ansible-playbook playbooks/etcd-io-priority.yml -e k3s_hosts=k3s_prod --limit k3s-prod-3
|
||||
```
|
||||
|
||||
Rollback, per node, immediate:
|
||||
|
||||
```bash
|
||||
systemctl disable --now k3s-etcd-io-priority.service
|
||||
printf 'mq-deadline\n' > /sys/block/sda/queue/scheduler
|
||||
printf '64\n' > /sys/block/sda/queue/nr_requests
|
||||
rm -f /etc/systemd/system/{system.slice,k3s.service}.d/10-etcd-io-weight.conf
|
||||
systemctl daemon-reload
|
||||
```
|
||||
@@ -1,84 +0,0 @@
|
||||
---
|
||||
# etcd_io_priority — give the k3s process (and therefore the embedded etcd it hosts) a
|
||||
# larger share of the block device it SHARES with Longhorn, on nodes where those two
|
||||
# tenants sit on one partition.
|
||||
#
|
||||
# READ roles/etcd_io_priority/README.md BEFORE CHANGING ANY OF THESE. Two of the obvious
|
||||
# knobs in this space are INERT on these nodes and are deliberately absent:
|
||||
# * IOSchedulingClass= / IOSchedulingPriority= on k3s.service. I/O priority is inherited
|
||||
# across fork and preserved across exec (block/blk-ioc.c __copy_io copies
|
||||
# ioc->ioprio), and containerd is a CHILD of the k3s server process — verified live on
|
||||
# k3s-prod-1: `ps -eo pid,ppid,comm` shows containerd's PPID is the k3s-server PID, and
|
||||
# `ionice -c2 -n0 -- sh -c 'ionice -p $$; sh -c "ionice -p \$\$"'` reports best-effort
|
||||
# prio 0 at BOTH levels. So raising k3s to the realtime I/O class raises every container
|
||||
# on the node with it, Longhorn's instance-managers included, and buys nothing against
|
||||
# the thing we are trying to beat.
|
||||
# * io.latency (blk-iolatency), which WOULD be the ideal scheduler-independent protection.
|
||||
# Not available: CONFIG_BLK_CGROUP_IOLATENCY is absent from Debian trixie's kernel
|
||||
# (`grep CONFIG_BLK_CGROUP /boot/config-7.1.8+deb13-amd64` returns IOCOST but not
|
||||
# IOLATENCY), and there is no /sys/fs/cgroup/**/io.latency on any prod node.
|
||||
# What is left, and what this role does, is cgroup-proportional weighting — which requires
|
||||
# a scheduler that implements it.
|
||||
|
||||
# MASTER SWITCH. Default FALSE so the LIVE dev cluster is untouched by a shared role:
|
||||
# dev already has a DEDICATED etcd device (/dev/vdc mounted at server/db,
|
||||
# docs/plans/ssd-etcd-isolation.md), so it has nothing to arbitrate and no reason to take a
|
||||
# scheduler change. Set true for the [k3s_prod] group in
|
||||
# inventory/group_vars/k3s_prod/vars.yml — the same seam node_baseline_backports_kernel,
|
||||
# kube_vip_address and k3s_data_dir use.
|
||||
etcd_io_priority_enabled: false
|
||||
|
||||
# THE I/O SCHEDULER, AND WHY IT HAS TO CHANGE FOR ANY OF THIS TO WORK.
|
||||
# Measured on k3s-prod-1: `cat /sys/block/sda/queue/scheduler` -> `none [mq-deadline]`.
|
||||
# mq-deadline honours the I/O priority CLASS (RT/BE/IDLE) but knows nothing about cgroups,
|
||||
# and cgroup v2's io.weight is implemented by blk-iocost — which is compiled in
|
||||
# (CONFIG_BLK_CGROUP_IOCOST=y) but DISABLED: /sys/fs/cgroup/io.cost.qos is empty. So on the
|
||||
# nodes as they stand today, systemd's IOWeight= writes a number nothing reads.
|
||||
# BFQ is the one scheduler here that implements cgroup weights (CONFIG_IOSCHED_BFQ=m,
|
||||
# CONFIG_BFQ_GROUP_IOSCHED=y — module present at
|
||||
# /lib/modules/7.1.8+deb13-amd64/kernel/block/bfq.ko.xz). Under BFQ, systemd's IOWeight=
|
||||
# lands in io.bfq.weight and becomes real.
|
||||
#
|
||||
# THE COSTS, STATED: BFQ turns OFF writeback throttling (wbt_lat_usec currently 2000 on
|
||||
# sda), costs some peak throughput and some CPU, and is a poor fit for NVMe — which is why
|
||||
# the role SKIPS the whole scheduler+weight path on an NVMe-backed node (k3s-prod-2), rather
|
||||
# than pretending to help there. Set to "" to install nothing but still run the report.
|
||||
etcd_io_priority_scheduler: bfq
|
||||
|
||||
# Block-layer queue depth for the shared device. sda's current value is 64 (blk-mq derives
|
||||
# it from the SATA NCQ depth). A shallower queue bounds how many Longhorn writes can already
|
||||
# be sitting in front of an etcd WAL flush at the DEVICE, below any scheduler's reach — the
|
||||
# one lever in this file that does not depend on cgroups or on the scheduler honouring
|
||||
# anything. Costs sequential throughput. Set to "" to leave nr_requests alone.
|
||||
etcd_io_priority_nr_requests: 32
|
||||
|
||||
# systemd IOWeight= for system.slice (which contains k3s.service, hence etcd) and for
|
||||
# k3s.service itself. THE VALUE IS 10000 — systemd's maximum — AND THAT IS NOT COSMETIC.
|
||||
# systemd rescales IOWeight (1..10000) onto io.bfq.weight (1..1000) linearly, so IOWeight=1000
|
||||
# maps to roughly 101 — indistinguishable from BFQ's own default of 100 for an unweighted
|
||||
# cgroup. Only near the top of the range does the weight actually separate: 10000 -> 1000,
|
||||
# a 10:1 share against the default-weighted kubepods.slice. A "reasonable-looking" 500 or
|
||||
# 1000 here would be a change that reads as applied and does nothing.
|
||||
etcd_io_priority_weight: 10000
|
||||
|
||||
# FAIL-LOUD FLOOR for the verification step. The role reads io.bfq.weight back out of the
|
||||
# live cgroups and asserts it is at least this. It exists because the rescaling above is
|
||||
# systemd's business, not ours: if a systemd version maps IOWeight differently, this role
|
||||
# must FAIL rather than leave a drop-in that looks applied. 500 = 5x an unweighted sibling.
|
||||
etcd_io_priority_min_bfq_weight: 500
|
||||
|
||||
# The k3s data dir whose backing device is the one to protect. Defers to the CLUSTER-WIDE
|
||||
# k3s_data_dir (group_vars/k3s_prod sets /data/k3s; the roles/k3s_server default restates
|
||||
# k3s's own /var/lib/rancher/k3s for dev) — NOT a second independent variable, for the same
|
||||
# reason node_baseline_k3s_data_dir defers to it.
|
||||
etcd_io_priority_k3s_data_dir: "{{ k3s_data_dir | default('/var/lib/rancher/k3s') }}"
|
||||
|
||||
# The systemd units that get an IOWeight drop-in, and the section each one takes.
|
||||
# BOTH ENTRIES ARE LOAD-BEARING — see tasks/main.yml "THE CGROUP WEIGHTS" for why weighting
|
||||
# k3s.service alone would arbitrate k3s against sshd and not against Longhorn at all.
|
||||
# system.slice must come first so the parent is weighted before its child.
|
||||
etcd_io_priority_weighted_units:
|
||||
- unit: system.slice
|
||||
section: Slice
|
||||
- unit: k3s.service
|
||||
section: Service
|
||||
@@ -1,18 +0,0 @@
|
||||
---
|
||||
# Re-run the oneshot so a changed script/unit takes effect NOW rather than at the next boot.
|
||||
# It only writes two sysfs files; there is nothing to drain and nothing to restart.
|
||||
- name: Apply k3s etcd I/O settings
|
||||
ansible.builtin.systemd_service:
|
||||
name: k3s-etcd-io-priority.service
|
||||
state: restarted
|
||||
daemon_reload: true
|
||||
|
||||
# daemon-reload re-realizes cgroup attributes for units that are already running, which is how
|
||||
# a changed IOWeight= reaches a LIVE k3s.service without restarting it. It is deliberately NOT
|
||||
# a `systemctl restart k3s`: restarting the k3s server to apply a performance tweak is exactly
|
||||
# the disruption this role exists to prevent, and on the primary it bounces etcd. If the reload
|
||||
# turns out not to be enough, the role's verification step FAILS and tells the operator to do a
|
||||
# rolling restart deliberately — it never quietly leaves the weight unapplied.
|
||||
- name: Reload systemd for I/O weights
|
||||
ansible.builtin.systemd_service:
|
||||
daemon_reload: true
|
||||
@@ -1,238 +0,0 @@
|
||||
---
|
||||
# etcd_io_priority — see README.md in this directory for the incident, the mechanism, and the
|
||||
# honest list of what this does NOT achieve. Everything runs under one `when` on the master
|
||||
# switch, so a run against dev (default false) touches nothing at all.
|
||||
#
|
||||
# TAG: `io-priority`. Every task carries it, so this role can be run alone against a live
|
||||
# cluster without dragging in any other role's work:
|
||||
# ansible-playbook playbooks/etcd-io-priority.yml -e k3s_hosts=k3s_prod --limit k3s-prod-3
|
||||
#
|
||||
# ALWAYS --limit AND ALWAYS CHECK --list-hosts FIRST. This role changes the I/O scheduler of a
|
||||
# live node's boot disk; it is safe and instantly reversible, but it is a real change and the
|
||||
# blast radius must be one node at a time (playbooks/etcd-io-priority.yml sets serial: 1 too).
|
||||
|
||||
- name: I/O priority for the shared etcd device
|
||||
when: etcd_io_priority_enabled | bool
|
||||
tags: [io-priority]
|
||||
block:
|
||||
|
||||
# -------------------------------------------------------------------------------------
|
||||
# DISCOVER THE DEVICE. Never hardcoded, and never taken from inventory: k3s-prod-1 and -3
|
||||
# boot from SATA (/dev/sda5) while k3s-prod-2 boots from NVMe (/dev/nvme0n1p5), and a
|
||||
# per-host var freezes today's hardware into inventory where it goes stale silently the
|
||||
# first time a disk is swapped. Inventory declares the INTENT ("the disk under k3s's data
|
||||
# dir"); the node answers what that is. Same reasoning as roles/node_baseline's disk
|
||||
# discovery.
|
||||
# -------------------------------------------------------------------------------------
|
||||
- name: Find the filesystem source backing the k3s data dir
|
||||
ansible.builtin.command:
|
||||
cmd: "findmnt --noheadings --output SOURCE --target {{ etcd_io_priority_k3s_data_dir }}"
|
||||
register: _eio_findmnt
|
||||
changed_when: false
|
||||
check_mode: false
|
||||
|
||||
- name: Find the whole-disk parent of that filesystem source
|
||||
ansible.builtin.command:
|
||||
cmd: "lsblk --noheadings --output PKNAME {{ _eio_findmnt.stdout | trim }}"
|
||||
register: _eio_pkname
|
||||
changed_when: false
|
||||
check_mode: false
|
||||
|
||||
# PKNAME is empty when the source IS already a whole disk (no partition), so fall back to
|
||||
# its basename rather than producing an empty device name.
|
||||
- name: Resolve the block device to tune
|
||||
ansible.builtin.set_fact:
|
||||
_eio_device: >-
|
||||
{{ (_eio_pkname.stdout_lines | map('trim') | reject('equalto', '') | list
|
||||
+ [_eio_findmnt.stdout | trim | basename]) | first }}
|
||||
|
||||
# -------------------------------------------------------------------------------------
|
||||
# NVMe IS DELIBERATELY LEFT ALONE, and this is the whole reason the role discovers the
|
||||
# device instead of assuming it. BFQ is a bad fit for NVMe (single software queue, real
|
||||
# throughput cost) and k3s-prod-2 — the NVMe node — measured /proc/pressure/io full avg300
|
||||
# of 0.75 during the 2026-09-07 incident against k3s-prod-1's 11.36. It does not have this
|
||||
# problem. Installing an inert weight drop-in there would be exactly the false confidence
|
||||
# this role exists to avoid, so nothing is written on such a node.
|
||||
# -------------------------------------------------------------------------------------
|
||||
- name: Decide whether a scheduler change applies to this device
|
||||
ansible.builtin.set_fact:
|
||||
_eio_scheduler: "{{ '' if _eio_device is match('nvme') else etcd_io_priority_scheduler }}"
|
||||
|
||||
- name: Report that this node is intentionally skipped (NVMe-backed)
|
||||
ansible.builtin.debug:
|
||||
msg: >-
|
||||
{{ inventory_hostname }}: k3s data dir {{ etcd_io_priority_k3s_data_dir }} is on
|
||||
/dev/{{ _eio_device }} (NVMe). BFQ and its cgroup weights are not applied to NVMe —
|
||||
nothing installed here. This is intentional, not a failure.
|
||||
when: not _eio_scheduler
|
||||
|
||||
# -------------------------------------------------------------------------------------
|
||||
# THE BLOCK-LAYER SETTINGS (scheduler + queue depth).
|
||||
# -------------------------------------------------------------------------------------
|
||||
- name: Install the block-layer settings script
|
||||
ansible.builtin.template:
|
||||
src: k3s-etcd-io-priority.sh.j2
|
||||
dest: /usr/local/sbin/k3s-etcd-io-priority
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0755"
|
||||
when: _eio_scheduler | length > 0
|
||||
notify: Apply k3s etcd I/O settings
|
||||
|
||||
- name: Install the boot-time unit that applies them
|
||||
ansible.builtin.template:
|
||||
src: k3s-etcd-io-priority.service.j2
|
||||
dest: /etc/systemd/system/k3s-etcd-io-priority.service
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
when: _eio_scheduler | length > 0
|
||||
notify: Apply k3s etcd I/O settings
|
||||
|
||||
- name: Enable and start the block-layer settings unit
|
||||
ansible.builtin.systemd_service:
|
||||
name: k3s-etcd-io-priority.service
|
||||
enabled: true
|
||||
state: started
|
||||
daemon_reload: true
|
||||
when: _eio_scheduler | length > 0
|
||||
|
||||
# -------------------------------------------------------------------------------------
|
||||
# THE CGROUP WEIGHTS.
|
||||
#
|
||||
# BOTH LEVELS ARE REQUIRED AND NEITHER IS SUFFICIENT. cgroup v2 weights are hierarchical:
|
||||
# k3s.service's weight only decides its share WITHIN system.slice, and system.slice's
|
||||
# weight is what competes with kubepods.slice (where Longhorn's instance-managers live).
|
||||
# Weighting only k3s.service — the intuitive thing to do — would arbitrate k3s against
|
||||
# sshd and journald and would not touch Longhorn at all.
|
||||
#
|
||||
# kubepods.slice is deliberately NOT given a drop-in of its own. It is a TRANSIENT unit
|
||||
# created by the kubelet through the systemd API (`systemctl cat kubepods.slice` on
|
||||
# k3s-prod-1 shows /run/systemd/transient/kubepods.slice), and whether systemd merges an
|
||||
# /etc drop-in into a transient unit is not something this change verified on these nodes.
|
||||
# Raising system.slice achieves the same ratio through a unit systemd definitely owns.
|
||||
# -------------------------------------------------------------------------------------
|
||||
- name: Ensure the systemd drop-in directories exist
|
||||
ansible.builtin.file:
|
||||
path: "/etc/systemd/system/{{ item.unit }}.d"
|
||||
state: directory
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0755"
|
||||
loop: "{{ etcd_io_priority_weighted_units }}"
|
||||
loop_control:
|
||||
label: "{{ item.unit }}"
|
||||
when: _eio_scheduler | length > 0
|
||||
|
||||
- name: Write the IOWeight drop-ins
|
||||
ansible.builtin.template:
|
||||
src: io-weight.conf.j2
|
||||
dest: "/etc/systemd/system/{{ item.unit }}.d/10-etcd-io-weight.conf"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
vars:
|
||||
_eio_dropin_unit: "{{ item.unit }}"
|
||||
_eio_dropin_section: "{{ item.section }}"
|
||||
loop: "{{ etcd_io_priority_weighted_units }}"
|
||||
loop_control:
|
||||
label: "{{ item.unit }}"
|
||||
when: _eio_scheduler | length > 0
|
||||
notify: Reload systemd for I/O weights
|
||||
|
||||
# Handlers must run BEFORE the verification below, or the verification reads the state from
|
||||
# before this run's changes and reports a stale pass.
|
||||
- name: Apply pending I/O changes before verifying them
|
||||
ansible.builtin.meta: flush_handlers
|
||||
|
||||
# -------------------------------------------------------------------------------------
|
||||
# VERIFY — the point of the role, not a formality.
|
||||
#
|
||||
# This repo has repeatedly shipped configuration that looked applied and was not (the
|
||||
# Keycloak reconciler that could not read its own Secret; Longhorn's orphanAutoDeletion,
|
||||
# silently dropped by the chart for two minor versions). Every mechanism here has a
|
||||
# plausible silent-failure mode: the scheduler module may not load; a systemd version may
|
||||
# rescale IOWeight differently; daemon-reload may not re-realize a running unit's cgroup.
|
||||
# So each is READ BACK OUT OF THE KERNEL and asserted. A failure here means the mitigation
|
||||
# is not in place — which is strictly better than believing that it is.
|
||||
# -------------------------------------------------------------------------------------
|
||||
- name: Read back the live block-layer state
|
||||
ansible.builtin.command:
|
||||
cmd: "cat /sys/block/{{ _eio_device }}/queue/scheduler /sys/block/{{ _eio_device }}/queue/nr_requests"
|
||||
register: _eio_queue_state
|
||||
changed_when: false
|
||||
check_mode: false
|
||||
when: _eio_scheduler | length > 0
|
||||
|
||||
- name: VERIFY the scheduler and queue depth actually took effect
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- "'[' ~ _eio_scheduler ~ ']' in _eio_queue_state.stdout_lines[0]"
|
||||
- "etcd_io_priority_nr_requests | string | length == 0
|
||||
or _eio_queue_state.stdout_lines[1] | trim | int == etcd_io_priority_nr_requests | int"
|
||||
fail_msg: >-
|
||||
/dev/{{ _eio_device }} did not take the requested settings. Wanted scheduler
|
||||
'{{ _eio_scheduler }}' and nr_requests '{{ etcd_io_priority_nr_requests }}'; the kernel
|
||||
reports {{ _eio_queue_state.stdout_lines | join(' / ') }}. The cgroup weights below are
|
||||
MEANINGLESS without the scheduler, so this is a hard failure, not a warning. Check
|
||||
`journalctl -u k3s-etcd-io-priority` and that bfq.ko exists for this kernel.
|
||||
success_msg: >-
|
||||
/dev/{{ _eio_device }}: scheduler {{ _eio_scheduler }}, nr_requests
|
||||
{{ _eio_queue_state.stdout_lines[1] | trim }}.
|
||||
when: _eio_scheduler | length > 0
|
||||
|
||||
# k3s.service's cgroup does not exist until k3s is running, so a cold-build run legitimately
|
||||
# has nothing to read there. Reported, not asserted away.
|
||||
- name: Check whether k3s.service has a live cgroup to verify
|
||||
ansible.builtin.stat:
|
||||
path: /sys/fs/cgroup/system.slice/k3s.service
|
||||
register: _eio_k3s_cgroup
|
||||
when: _eio_scheduler | length > 0
|
||||
|
||||
- name: Report that k3s is not running, so its own cgroup weight cannot be verified yet
|
||||
ansible.builtin.debug:
|
||||
msg: >-
|
||||
{{ inventory_hostname }}: /sys/fs/cgroup/system.slice/k3s.service does not exist — k3s
|
||||
is not running on this node, so its drop-in is written but UNVERIFIED. Re-run
|
||||
playbooks/etcd-io-priority.yml once k3s is up.
|
||||
when:
|
||||
- _eio_scheduler | length > 0
|
||||
- not (_eio_k3s_cgroup.stat.exists | default(false))
|
||||
|
||||
- name: Read back the live cgroup I/O weights
|
||||
ansible.builtin.command:
|
||||
cmd: "cat /sys/fs/cgroup/{{ item }}/io.bfq.weight"
|
||||
register: _eio_bfq_weights
|
||||
changed_when: false
|
||||
check_mode: false
|
||||
failed_when: false
|
||||
# `| default(false)`: Ansible templates `loop:` BEFORE evaluating `when:`, so on an NVMe
|
||||
# node — where the stat task above is skipped and _eio_k3s_cgroup has no `.stat` at all —
|
||||
# a bare `_eio_k3s_cgroup.stat.exists` raises even though this task will not run.
|
||||
loop: >-
|
||||
{{ ['system.slice']
|
||||
+ (['system.slice/k3s.service']
|
||||
if (_eio_k3s_cgroup.stat.exists | default(false)) else []) }}
|
||||
when: _eio_scheduler | length > 0
|
||||
|
||||
- name: VERIFY every live cgroup weight actually carries the boost
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- item.rc == 0
|
||||
- item.stdout | trim | int >= etcd_io_priority_min_bfq_weight | int
|
||||
fail_msg: >-
|
||||
/sys/fs/cgroup/{{ item.item }}/io.bfq.weight reads
|
||||
'{{ item.stdout | default('<unreadable>') | trim }}' (rc={{ item.rc }}), below the
|
||||
required floor of {{ etcd_io_priority_min_bfq_weight }}. IOWeight={{ etcd_io_priority_weight }}
|
||||
did NOT reach the kernel. Most likely causes, in order: (1) systemd rescaled the value
|
||||
differently from what defaults/main.yml assumes — compare
|
||||
`systemctl show {{ item.item | basename }} -p IOWeight` against the file; (2) the cgroup
|
||||
was not re-realized by daemon-reload, in which case a ROLLING `systemctl restart k3s`
|
||||
(one node at a time — etcd quorum is 2/3) applies it; (3) BFQ is not active on the
|
||||
device, which the previous assert should already have caught. The drop-in on disk is
|
||||
NOT the mitigation — this file is.
|
||||
success_msg: "{{ item.item }} io.bfq.weight = {{ item.stdout | default('') | trim }}"
|
||||
loop: "{{ _eio_bfq_weights.results | default([]) }}"
|
||||
loop_control:
|
||||
label: "{{ item.item | default('') }}"
|
||||
when: _eio_scheduler | length > 0
|
||||
@@ -1,21 +0,0 @@
|
||||
# {{ ansible_managed }}
|
||||
#
|
||||
# Proportional share of {{ _eio_device }} for {{ _eio_dropin_unit }}, which is where the k3s server
|
||||
# process — and therefore the EMBEDDED ETCD it hosts — does its WAL writes. Its competitor is
|
||||
# kubepods.slice (Longhorn's instance-managers), which stays at the default weight.
|
||||
#
|
||||
# THIS ONLY WORKS BECAUSE THE SCHEDULER IS {{ _eio_scheduler | upper }}. systemd writes IOWeight= to both
|
||||
# io.weight (blk-iocost, which is compiled in but DISABLED here — /sys/fs/cgroup/io.cost.qos is
|
||||
# empty) and io.bfq.weight (BFQ). Under the stock mq-deadline this file is a no-op; the sibling
|
||||
# k3s-etcd-io-priority.service is what makes it mean something. roles/etcd_io_priority verifies
|
||||
# io.bfq.weight after writing this and FAILS the play if the value did not land.
|
||||
#
|
||||
# NOT IOSchedulingClass=. I/O priority is inherited across fork and kept across exec, and
|
||||
# containerd is a child of the k3s server process — so realtime here would be realtime for every
|
||||
# container on the node, Longhorn included. See roles/etcd_io_priority/README.md.
|
||||
#
|
||||
# {{ etcd_io_priority_weight }} is systemd's MAXIMUM and that is deliberate: systemd rescales
|
||||
# 1..10000 onto io.bfq.weight's 1..1000, so IOWeight=1000 would map to ~101 — BFQ's unweighted
|
||||
# default is 100. Only the top of the range actually separates this cgroup from its siblings.
|
||||
[{{ _eio_dropin_section }}]
|
||||
IOWeight={{ etcd_io_priority_weight }}
|
||||
@@ -1,24 +0,0 @@
|
||||
# {{ ansible_managed }}
|
||||
#
|
||||
# WHY A ONESHOT UNIT AND NOT A UDEV RULE. The obvious form of this is
|
||||
# `ACTION=="add|change", KERNEL=="{{ _eio_device }}", ATTR{queue/scheduler}="{{ _eio_scheduler }}"`,
|
||||
# and it is unreliable for the ONE device that matters: {{ _eio_device }} is the boot disk, so its
|
||||
# `add` event fires in the initramfs, where /etc/udev/rules.d is not present and bfq.ko is not
|
||||
# loaded — the ATTR assignment silently does nothing and the disk comes up on mq-deadline with
|
||||
# no error anywhere. A unit ordered Before=k3s.service runs after modules are loadable, fails
|
||||
# LOUDLY if the scheduler is unavailable, and is visible in `systemctl status`.
|
||||
[Unit]
|
||||
Description=Block-layer I/O settings for the shared etcd + Longhorn device ({{ _eio_device }})
|
||||
Documentation=file:///etc/systemd/system/k3s-etcd-io-priority.service
|
||||
DefaultDependencies=no
|
||||
After=systemd-modules-load.service
|
||||
Before=k3s.service shutdown.target
|
||||
Conflicts=shutdown.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
ExecStart=/usr/local/sbin/k3s-etcd-io-priority
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,38 +0,0 @@
|
||||
#!/bin/sh
|
||||
# {{ ansible_managed }}
|
||||
#
|
||||
# Block-layer settings for the device that carries BOTH etcd's WAL ({{ etcd_io_priority_k3s_data_dir }})
|
||||
# and every Longhorn replica on this node. Run at boot by k3s-etcd-io-priority.service, and
|
||||
# safe to run by hand at any time — it is idempotent and takes effect immediately.
|
||||
#
|
||||
# Rollback (immediate, no reboot):
|
||||
# printf 'mq-deadline\n' > /sys/block/{{ _eio_device }}/queue/scheduler
|
||||
# printf '64\n' > /sys/block/{{ _eio_device }}/queue/nr_requests
|
||||
# and `systemctl disable --now k3s-etcd-io-priority.service` so it does not come back.
|
||||
set -eu
|
||||
|
||||
DEV="{{ _eio_device }}"
|
||||
QUEUE="/sys/block/${DEV}/queue"
|
||||
|
||||
if [ ! -d "${QUEUE}" ]; then
|
||||
echo "k3s-etcd-io-priority: /sys/block/${DEV} does not exist — device renamed?" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
{% if _eio_scheduler %}
|
||||
# BFQ is a module here (CONFIG_IOSCHED_BFQ=m); loading it is what makes it selectable at all.
|
||||
modprobe -q {{ _eio_scheduler }} 2>/dev/null || true
|
||||
|
||||
if ! grep -qw '{{ _eio_scheduler }}' "${QUEUE}/scheduler"; then
|
||||
echo "k3s-etcd-io-priority: scheduler '{{ _eio_scheduler }}' not available for ${DEV}: $(cat "${QUEUE}/scheduler")" >&2
|
||||
exit 1
|
||||
fi
|
||||
printf '%s\n' '{{ _eio_scheduler }}' > "${QUEUE}/scheduler"
|
||||
{% endif %}
|
||||
|
||||
{% if etcd_io_priority_nr_requests %}
|
||||
# ORDER MATTERS: switching the elevator resets nr_requests, so this must come second.
|
||||
printf '%s\n' '{{ etcd_io_priority_nr_requests }}' > "${QUEUE}/nr_requests"
|
||||
{% endif %}
|
||||
|
||||
exit 0
|
||||
@@ -59,17 +59,6 @@ k3s_server_mode: "secondary"
|
||||
# serving cert on restart when this set changes. Deduplicated (the primary IP is
|
||||
# also k3s_api_host).
|
||||
#
|
||||
# THIS LIST COVERS BOTH API ADDRESSES, WHICH IS WHY THE 2026-09-06 SPLIT NEEDED NO CERT
|
||||
# CHANGE AND NO k3s RESTART. group_vars/all.yml now distinguishes k3s_api_host (the
|
||||
# founder, for the cold join) from k3s_apiserver_endpoint (= kube_vip_address, what Cilium
|
||||
# and every steady-state client dial). Both are already in this set, so repointing a
|
||||
# consumer from one to the other cannot produce a TLS failure. VERIFIED LIVE 2026-09-06,
|
||||
# not assumed: prod's apiserver answered through https://192.168.2.9:6443 with full CA
|
||||
# verification and tls-server-name=192.168.2.9. Do not remove either entry — dropping
|
||||
# kube_vip_address breaks every client of the VIP; dropping k3s_api_host breaks nothing
|
||||
# today only because it is also in the k3s_servers_group node-IP list below, and that
|
||||
# coincidence is not a contract.
|
||||
#
|
||||
# k3s_servers_group — NOT a literal group name — is what makes this role environment-
|
||||
# agnostic (R-ENV-02): group_vars/all.yml sets it to `k3s_servers` (dev) and
|
||||
# group_vars/k3s_prod/vars.yml to `k3s_prod`, so each environment SANs its own servers.
|
||||
@@ -95,15 +84,26 @@ k3s_tls_sans: >-
|
||||
# while the cert-based admin kubeconfig keeps working — so it looks like "SSO is broken",
|
||||
# not like a config skew. The break-glass cert kubeconfig is why this is recoverable.
|
||||
#
|
||||
# THE DEFAULT IS THE DEV VALUE, restated explicitly rather than left implicit — the same
|
||||
# convention as k3s_data_dir above and cluster-vars' GATEWAY_VIP / LONGHORN_DATA_PATH: the
|
||||
# base default is the known-good LIVE value, so dev's rendered config.yaml is byte-for-byte
|
||||
# what it already is and the Restart handler does NOT fire on a re-run.
|
||||
# THE DEFAULT IS THE DEV VALUE — the same convention as k3s_data_dir above and cluster-vars'
|
||||
# GATEWAY_VIP / LONGHORN_DATA_PATH: the role default is dev's value and prod overrides it in
|
||||
# inventory/group_vars/k3s_prod/vars.yml (the same inventory seam k3s_data_dir / k3s_api_host /
|
||||
# kube_vip_address use, so a FOURTH prod server inherits it from group membership rather than
|
||||
# from anyone remembering).
|
||||
#
|
||||
# PROD OVERRIDES IT in inventory/group_vars/k3s_prod/vars.yml — the same inventory seam
|
||||
# k3s_data_dir / k3s_api_host / kube_vip_address use, so a FOURTH prod server inherits it
|
||||
# from group membership rather than from anyone remembering.
|
||||
k3s_base_domain: "home.keithjackson.dev"
|
||||
# *** WHICH DEV VALUE — UPDATED 2026-09-04, AND IT IS AHEAD OF THE LIVE DEV CLUSTER. ***
|
||||
# The operator inverted the suffix split: PROD is now the apex `home.keithjackson.dev` and DEV
|
||||
# carries the label `dev.home.keithjackson.dev`. This default therefore moved from the apex to
|
||||
# dev's subdomain, mirroring BASE_DOMAIN in clusters/dev/kustomization.yaml.
|
||||
#
|
||||
# CONSEQUENCE — this default no longer matches what the live dev nodes are running, so unlike
|
||||
# every previous edit to this file A DEV RE-RUN IS NOT A NO-OP. Rendering config.yaml with the
|
||||
# new issuer CHANGES the file, FIRES the Restart handler, and rolls k3s on all three dev
|
||||
# servers, which also invalidates every user's OIDC kubeconfig (SSO must be re-issued). That is
|
||||
# a deliberate step of the dev DNS migration, NOT a side effect to discover during unrelated
|
||||
# work: DO NOT run the k3s_server role against dev until step D6 of
|
||||
# infra/PROD-PROVISIONING.md §10, and never while the operator is away. The break-glass
|
||||
# cert-based admin kubeconfig keeps working throughout, which is what makes the step safe.
|
||||
k3s_base_domain: "dev.home.keithjackson.dev"
|
||||
|
||||
# Derived, so the realm path is stated once. Override k3s_base_domain, not this.
|
||||
k3s_oidc_issuer_url: "https://keycloak.{{ k3s_base_domain }}/realms/homelab"
|
||||
@@ -155,18 +155,6 @@
|
||||
| default(k3s_primary_token_raw.content | b64decode | trim, true) }}
|
||||
when: k3s_server_mode == 'secondary'
|
||||
|
||||
# THE JOIN TARGET IS k3s_api_host — THE FOUNDER — AND MUST NOT BECOME THE VIP.
|
||||
# group_vars/all.yml splits the two API addresses (k3s_api_host = the founder's own node IP,
|
||||
# k3s_apiserver_endpoint = the kube-vip HA VIP) and this is the canonical founder-only
|
||||
# consumer. On a COLD build the VIP does not exist at this point in the sequence: kube-vip
|
||||
# is a k3s auto-deploy manifest, so it cannot be applied until a cluster exists, and
|
||||
# playbooks/site.yml only imports kube-vip.yml AFTER k3s.yml. A `--server` pointed at the
|
||||
# VIP would therefore dial an address nothing announces and the cold build would be
|
||||
# unjoinable — with no diagnostic beyond a connection timeout.
|
||||
# On a WARM single-node rejoin (`--limit k3s-dev-N`) the founder is up and answering too, so
|
||||
# the founder works in both cases and the VIP works in only one. Cilium's k8sServiceHost is
|
||||
# the opposite trade (steady state, must survive losing the founder) and uses the endpoint
|
||||
# variable instead.
|
||||
- name: Install k3s secondary (join as server)
|
||||
ansible.builtin.command:
|
||||
cmd: >-
|
||||
|
||||
@@ -24,16 +24,6 @@ kube_vip_port: 6443
|
||||
# only if detection is wrong.
|
||||
kube_vip_interface: ""
|
||||
|
||||
# How long to wait for the VIP to actually ANSWER on kube_vip_port after the manifest is
|
||||
# written (seconds). This is a real gate, not cosmetics: since 2026-09-06 Cilium's
|
||||
# k8sServiceHost is this VIP (k3s_apiserver_endpoint), and playbooks/site.yml runs this role
|
||||
# immediately before cilium.yml — so if the VIP never comes up, the run must stop HERE with
|
||||
# a clear message rather than proceed to install a CNI pointed at an address nothing
|
||||
# answers. On a cold cluster the budget covers k3s's deploy controller applying the
|
||||
# manifest, the ghcr.io image pull and leader election; on an already-running cluster the
|
||||
# check returns immediately.
|
||||
kube_vip_wait_timeout: 300
|
||||
|
||||
# Leader-election tuning (seconds) — a snappy failover without thrashing.
|
||||
kube_vip_lease_duration: 15
|
||||
kube_vip_renew_deadline: 10
|
||||
|
||||
@@ -21,14 +21,6 @@
|
||||
executable: /bin/bash
|
||||
register: kube_vip_iface_detect
|
||||
changed_when: false
|
||||
# RUN THIS EVEN UNDER --check. It is `ip -o -4 addr show`: it reads and changes nothing.
|
||||
# Without check_mode: false the shell module is SKIPPED in check mode, kube_vip_iface_detect
|
||||
# has no stdout, and the assert two tasks down aborts the play with "Could not detect the
|
||||
# interface ... set kube_vip_interface explicitly" — a misleading failure that says nothing
|
||||
# about the node. That mattered less while this play was an out-of-band operator step; it
|
||||
# matters now that playbooks/site.yml imports it, because `--check` on the bring-up before a
|
||||
# cold prod build is exactly the rehearsal an operator should be able to run.
|
||||
check_mode: false
|
||||
|
||||
- name: Resolve the kube-vip VIP interface (override wins, else detected)
|
||||
ansible.builtin.set_fact:
|
||||
@@ -60,34 +52,3 @@
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0600"
|
||||
|
||||
# PROVE THE VIP IS ACTUALLY UP BEFORE THIS PLAY RETURNS (added 2026-09-06).
|
||||
#
|
||||
# Templating the manifest is not the same as having a floating endpoint: k3s's deploy
|
||||
# controller still has to apply it, the image still has to pull, and leader election still
|
||||
# has to pick an owner. Nothing used to check, which was tolerable while the VIP was only an
|
||||
# operator convenience. It is not tolerable now — Cilium's k8sServiceHost IS this VIP
|
||||
# (k3s_apiserver_endpoint) and playbooks/site.yml runs cilium.yml straight after this play,
|
||||
# so an unannounced VIP would mean installing a CNI that cannot reach the API server. That
|
||||
# failure is the 2026-09-01 shape and it is very expensive to diagnose from the far side;
|
||||
# failing here instead names the actual cause.
|
||||
#
|
||||
# run_once: the VIP is ONE address for the whole environment, so one probe answers for the
|
||||
# environment. Probing from a server host (not the controller) is deliberate: the servers
|
||||
# are on the VIP's own LAN in both environments, whereas the dev controller reaches dev's
|
||||
# API only through a pfSense NAT — a controller-side probe would have been a property of
|
||||
# where the play was launched from. READ-ONLY: it opens a TCP connection and changes nothing.
|
||||
- name: Wait for the control-plane VIP to answer (the CNI's endpoint depends on it)
|
||||
ansible.builtin.wait_for:
|
||||
host: "{{ kube_vip_address }}"
|
||||
port: "{{ kube_vip_port }}"
|
||||
timeout: "{{ kube_vip_wait_timeout }}"
|
||||
run_once: true
|
||||
changed_when: false
|
||||
vars:
|
||||
ansible_become: false
|
||||
# DELIBERATELY NOT check_mode: false, unlike the interface probe above — so a `--check`
|
||||
# rehearsal SKIPS this and does not sit on a socket for kube_vip_wait_timeout. wait_for
|
||||
# declares no check-mode support, so the skip is the module's own behaviour and this comment
|
||||
# is here only so the skip does not read as an oversight. CONSEQUENCE, stated plainly: a
|
||||
# `--check` run does NOT verify the VIP is up. Only a real run gates the CNI on it.
|
||||
@@ -11,13 +11,3 @@
|
||||
ansible.builtin.systemd:
|
||||
name: systemd-resolved
|
||||
state: restarted
|
||||
|
||||
# Masking a unit writes a symlink to /dev/null into /etc/systemd/system; the RUNNING
|
||||
# systemd keeps its old view of the unit until it re-reads unit files. The mask is read
|
||||
# fresh at the next boot either way, so this only exists so the running manager does not
|
||||
# disagree with on-disk state in between. `daemon_reload` re-parses unit files and
|
||||
# re-runs generators — it starts and stops nothing, so it cannot disturb k3s or Longhorn.
|
||||
# Fires only when a mask actually changed, so a re-run of the role is silent.
|
||||
- name: Reload systemd unit files
|
||||
ansible.builtin.systemd_service:
|
||||
daemon_reload: true
|
||||
@@ -331,15 +331,6 @@
|
||||
# The devices this role must never, under any circumstances, hand to e2label. Built from
|
||||
# gathered facts (`gather_facts: true` in the play) plus the live swap list, so it is the
|
||||
# NODE's account of what is load-bearing, not a hardcoded list of partition names.
|
||||
#
|
||||
# EXPECT THIS LIST TO BE EMPTY. Since the swap tasks in tasks/main.yml mask swap.target,
|
||||
# a baselined node has no active swap for `swapon` to report, so the swap half of
|
||||
# node_baseline_lh_protected_devices below contributes nothing. That is not a weakened
|
||||
# guard: the claim CANDIDATE is derived from whatever is mounted at
|
||||
# node_baseline_longhorn_claim_mountpoint, and a swap partition is neither mounted nor an
|
||||
# ext4 filesystem, so it can never be the candidate in the first place. The task stays
|
||||
# because the list must be the node's own account of what is load-bearing — including on
|
||||
# a node reached before baseline has run, or one an operator has deliberately swapped on.
|
||||
- name: "Longhorn (claim): list the devices currently backing active swap"
|
||||
ansible.builtin.command: swapon --noheadings --show=NAME
|
||||
register: node_baseline_lh_swapdevs
|
||||
|
||||
Loaded 100 of 147 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user