Revert "feat(dns): INVERT to operator's simpler layout — dev to subdomain, prod to apex"

This reverts commit 9137bbd0c3.
feat(ha): the CNI dials the control-plane VIP — kube-vip becomes a CNI prerequisite
2026-09-06 20:27:41 -05:00 · 2026-09-06 20:24:51 -05:00 · 2026-09-06 20:24:22 -05:00 · 2026-09-06 19:35:18 -05:00 · 2026-09-06 18:51:00 -05:00 · 2026-09-06 08:47:08 -05:00
133 changed files with 3649 additions and 11507 deletions
@@ -44,3 +44,6 @@ infra/tofu/.r2.env

 # Vendored ansible collections (installed by ansible-galaxy for node re-provision; not tracked or linted)
 infra/ansible/collections/
+
+# standup.sh DNS snapshots — run artifacts, not authored state
+.standup-dns/
@@ -1,57 +0,0 @@
-A|*.keithjackson.dev|136.51.115.252
-A|auth.home.keithjackson.dev|192.168.1.53
-A|brickbreak.keithjackson.dev|20.236.242.14
-A|crafty-mc.home.keithjackson.dev|192.168.1.55
-A|crafty.home.keithjackson.dev|192.168.1.53
-A|dev.brickbreak.keithjackson.dev|20.236.242.14
-A|gitea.home.keithjackson.dev|192.168.1.53
-A|grafana.home.keithjackson.dev|192.168.1.53
-A|headlamp.home.keithjackson.dev|192.168.1.53
-A|home.keithjackson.dev|192.168.1.53
-A|jackett.home.keithjackson.dev|192.168.1.53
-A|keycloak.home.keithjackson.dev|192.168.1.53
-A|nextcloud.home.keithjackson.dev|192.168.1.53
-A|pihole.home.keithjackson.dev|192.168.1.53
-A|plex.home.keithjackson.dev|192.168.1.53
-A|qbittorrent.home.keithjackson.dev|192.168.1.53
-A|radarr.home.keithjackson.dev|192.168.1.53
-A|seerr.home.keithjackson.dev|192.168.1.53
-A|semaphore.home.keithjackson.dev|192.168.1.53
-A|sonarr.home.keithjackson.dev|192.168.1.53
-A|unifi.home.keithjackson.dev|192.168.1.54
-TXT|a-auth.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/oauth2-proxy/oauth2-proxy
-TXT|a-crafty-mc.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/crafty/crafty-l4-tcp
-TXT|a-crafty.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/crafty/crafty
-TXT|a-gitea.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/gitea/gitea-3000
-TXT|a-grafana.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/monitoring/grafana
-TXT|a-headlamp.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/headlamp/headlamp
-TXT|a-home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/homepage/homepage
-TXT|a-jackett.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/jackett/jackett-9117
-TXT|a-keycloak.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/keycloak/keycloak
-TXT|a-nextcloud.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/nextcloud/nextcloud-80
-TXT|a-pihole.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/pihole/pihole-80
-TXT|a-plex.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/plex/plex-32400
-TXT|a-qbittorrent.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/qbittorrent/qbittorrent-8080
-TXT|a-radarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/radarr/radarr-7878
-TXT|a-seerr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/seerr/seerr-5055
-TXT|a-semaphore.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/semaphore/semaphore-3000
-TXT|a-sonarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/sonarr/sonarr-8989
-TXT|a-unifi.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/unifi/unifi-l4-tcp
-TXT|auth.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/oauth2-proxy/oauth2-proxy
-TXT|crafty-mc.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/crafty/crafty-l4-tcp
-TXT|crafty.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/crafty/crafty
-TXT|gitea.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/gitea/gitea-3000
-TXT|grafana.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/monitoring/grafana
-TXT|headlamp.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/headlamp/headlamp
-TXT|home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/homepage/homepage
-TXT|jackett.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/jackett/jackett-9117
-TXT|keycloak.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/keycloak/keycloak
-TXT|nextcloud.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/nextcloud/nextcloud-80
-TXT|pihole.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/pihole/pihole-80
-TXT|plex.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/plex/plex-32400
-TXT|qbittorrent.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/qbittorrent/qbittorrent-8080
-TXT|radarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/radarr/radarr-7878
-TXT|seerr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/seerr/seerr-5055
-TXT|semaphore.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/semaphore/semaphore-3000
-TXT|sonarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/sonarr/sonarr-8989
-TXT|unifi.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/unifi/unifi-l4-tcp
@@ -1,95 +0,0 @@
-A|*.keithjackson.dev|136.51.115.252
-A|auth.home.keithjackson.dev|192.168.1.53
-A|auth.prod.home.keithjackson.dev|192.168.2.20
-A|brickbreak.keithjackson.dev|20.236.242.14
-A|conformance-dns.home.keithjackson.dev|192.168.1.56
-A|crafty-mc.home.keithjackson.dev|192.168.1.55
-A|crafty-mc.prod.home.keithjackson.dev|192.168.2.22
-A|crafty.home.keithjackson.dev|192.168.1.53
-A|crafty.prod.home.keithjackson.dev|192.168.2.20
-A|dev.brickbreak.keithjackson.dev|20.236.242.14
-A|gitea.home.keithjackson.dev|192.168.1.53
-A|gitea.prod.home.keithjackson.dev|192.168.2.20
-A|grafana.home.keithjackson.dev|192.168.1.53
-A|grafana.prod.home.keithjackson.dev|192.168.2.20
-A|headlamp.home.keithjackson.dev|192.168.1.53
-A|headlamp.prod.home.keithjackson.dev|192.168.2.20
-A|home.keithjackson.dev|192.168.1.53
-A|jackett.home.keithjackson.dev|192.168.1.53
-A|jackett.prod.home.keithjackson.dev|192.168.2.20
-A|keycloak.home.keithjackson.dev|192.168.1.53
-A|keycloak.prod.home.keithjackson.dev|192.168.2.20
-A|nextcloud.home.keithjackson.dev|192.168.1.53
-A|nextcloud.prod.home.keithjackson.dev|192.168.2.20
-A|pihole.home.keithjackson.dev|192.168.1.53
-A|pihole.prod.home.keithjackson.dev|192.168.2.20
-A|plex.home.keithjackson.dev|192.168.1.53
-A|plex.prod.home.keithjackson.dev|192.168.2.20
-A|prod.home.keithjackson.dev|192.168.2.20
-A|qbittorrent.home.keithjackson.dev|192.168.1.53
-A|qbittorrent.prod.home.keithjackson.dev|192.168.2.20
-A|radarr.home.keithjackson.dev|192.168.1.53
-A|radarr.prod.home.keithjackson.dev|192.168.2.20
-A|seerr.home.keithjackson.dev|192.168.1.53
-A|seerr.prod.home.keithjackson.dev|192.168.2.20
-A|semaphore.home.keithjackson.dev|192.168.1.53
-A|semaphore.prod.home.keithjackson.dev|192.168.2.20
-A|sonarr.home.keithjackson.dev|192.168.1.53
-A|sonarr.prod.home.keithjackson.dev|192.168.2.20
-A|unifi.home.keithjackson.dev|192.168.1.54
-A|unifi.prod.home.keithjackson.dev|192.168.2.21
-TXT|a-auth.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/oauth2-proxy/oauth2-proxy
-TXT|a-auth.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/oauth2-proxy/oauth2-proxy
-TXT|a-conformance-dns.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/chainsaw-growing-grouse/dns-probe-svc
-TXT|a-crafty-mc.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/crafty/crafty-l4-tcp
-TXT|a-crafty-mc.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=service/crafty/crafty-l4-tcp
-TXT|a-crafty.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/crafty/crafty
-TXT|a-crafty.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/crafty/crafty
-TXT|a-gitea.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/gitea/gitea-3000
-TXT|a-gitea.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/gitea/gitea-3000
-TXT|a-grafana.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/monitoring/grafana
-TXT|a-grafana.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/monitoring/grafana
-TXT|a-headlamp.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/headlamp/headlamp
-TXT|a-headlamp.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/headlamp/headlamp
-TXT|a-home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/homepage/homepage
-TXT|a-jackett.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/jackett/jackett-9117
-TXT|a-jackett.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/jackett/jackett-9117
-TXT|a-keycloak.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/keycloak/keycloak
-TXT|a-keycloak.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/keycloak/keycloak
-TXT|a-nextcloud.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/nextcloud/nextcloud-80
-TXT|a-nextcloud.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/nextcloud/nextcloud-80
-TXT|a-pihole.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/pihole/pihole-80
-TXT|a-pihole.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/pihole/pihole-80
-TXT|a-plex.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/plex/plex-32400
-TXT|a-plex.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/plex/plex-32400
-TXT|a-prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/homepage/homepage
-TXT|a-qbittorrent.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/qbittorrent/qbittorrent-8080
-TXT|a-qbittorrent.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/qbittorrent/qbittorrent-8080
-TXT|a-radarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/radarr/radarr-7878
-TXT|a-radarr.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/radarr/radarr-7878
-TXT|a-seerr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/seerr/seerr-5055
-TXT|a-seerr.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/seerr/seerr-5055
-TXT|a-semaphore.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/semaphore/semaphore-3000
-TXT|a-semaphore.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/semaphore/semaphore-3000
-TXT|a-sonarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/sonarr/sonarr-8989
-TXT|a-sonarr.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/sonarr/sonarr-8989
-TXT|a-unifi.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/unifi/unifi-l4-tcp
-TXT|a-unifi.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=service/unifi/unifi-l4-tcp
-TXT|auth.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/oauth2-proxy/oauth2-proxy
-TXT|crafty-mc.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/crafty/crafty-l4-tcp
-TXT|crafty.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/crafty/crafty
-TXT|gitea.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/gitea/gitea-3000
-TXT|grafana.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/monitoring/grafana
-TXT|headlamp.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/headlamp/headlamp
-TXT|home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/homepage/homepage
-TXT|jackett.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/jackett/jackett-9117
-TXT|keycloak.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/keycloak/keycloak
-TXT|nextcloud.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/nextcloud/nextcloud-80
-TXT|pihole.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/pihole/pihole-80
-TXT|plex.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/plex/plex-32400
-TXT|qbittorrent.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/qbittorrent/qbittorrent-8080
-TXT|radarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/radarr/radarr-7878
-TXT|seerr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/seerr/seerr-5055
-TXT|semaphore.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/semaphore/semaphore-3000
-TXT|sonarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/sonarr/sonarr-8989
-TXT|unifi.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/unifi/unifi-l4-tcp
@@ -1,93 +0,0 @@
-A|*.keithjackson.dev|136.51.115.252
-A|auth.home.keithjackson.dev|192.168.1.53
-A|auth.prod.home.keithjackson.dev|192.168.2.20
-A|brickbreak.keithjackson.dev|20.236.242.14
-A|crafty-mc.home.keithjackson.dev|192.168.1.55
-A|crafty-mc.prod.home.keithjackson.dev|192.168.2.22
-A|crafty.home.keithjackson.dev|192.168.1.53
-A|crafty.prod.home.keithjackson.dev|192.168.2.20
-A|dev.brickbreak.keithjackson.dev|20.236.242.14
-A|gitea.home.keithjackson.dev|192.168.1.53
-A|gitea.prod.home.keithjackson.dev|192.168.2.20
-A|grafana.home.keithjackson.dev|192.168.1.53
-A|grafana.prod.home.keithjackson.dev|192.168.2.20
-A|headlamp.home.keithjackson.dev|192.168.1.53
-A|headlamp.prod.home.keithjackson.dev|192.168.2.20
-A|home.keithjackson.dev|192.168.1.53
-A|jackett.home.keithjackson.dev|192.168.1.53
-A|jackett.prod.home.keithjackson.dev|192.168.2.20
-A|keycloak.home.keithjackson.dev|192.168.1.53
-A|keycloak.prod.home.keithjackson.dev|192.168.2.20
-A|nextcloud.home.keithjackson.dev|192.168.1.53
-A|nextcloud.prod.home.keithjackson.dev|192.168.2.20
-A|pihole.home.keithjackson.dev|192.168.1.53
-A|pihole.prod.home.keithjackson.dev|192.168.2.20
-A|plex.home.keithjackson.dev|192.168.1.53
-A|plex.prod.home.keithjackson.dev|192.168.2.20
-A|prod.home.keithjackson.dev|192.168.2.20
-A|qbittorrent.home.keithjackson.dev|192.168.1.53
-A|qbittorrent.prod.home.keithjackson.dev|192.168.2.20
-A|radarr.home.keithjackson.dev|192.168.1.53
-A|radarr.prod.home.keithjackson.dev|192.168.2.20
-A|seerr.home.keithjackson.dev|192.168.1.53
-A|seerr.prod.home.keithjackson.dev|192.168.2.20
-A|semaphore.home.keithjackson.dev|192.168.1.53
-A|semaphore.prod.home.keithjackson.dev|192.168.2.20
-A|sonarr.home.keithjackson.dev|192.168.1.53
-A|sonarr.prod.home.keithjackson.dev|192.168.2.20
-A|unifi.home.keithjackson.dev|192.168.1.54
-A|unifi.prod.home.keithjackson.dev|192.168.2.21
-TXT|a-auth.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/oauth2-proxy/oauth2-proxy
-TXT|a-auth.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/oauth2-proxy/oauth2-proxy
-TXT|a-crafty-mc.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/crafty/crafty-l4-tcp
-TXT|a-crafty-mc.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=service/crafty/crafty-l4-tcp
-TXT|a-crafty.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/crafty/crafty
-TXT|a-crafty.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/crafty/crafty
-TXT|a-gitea.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/gitea/gitea-3000
-TXT|a-gitea.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/gitea/gitea-3000
-TXT|a-grafana.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/monitoring/grafana
-TXT|a-grafana.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/monitoring/grafana
-TXT|a-headlamp.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/headlamp/headlamp
-TXT|a-headlamp.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/headlamp/headlamp
-TXT|a-home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/homepage/homepage
-TXT|a-jackett.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/jackett/jackett-9117
-TXT|a-jackett.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/jackett/jackett-9117
-TXT|a-keycloak.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/keycloak/keycloak
-TXT|a-keycloak.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/keycloak/keycloak
-TXT|a-nextcloud.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/nextcloud/nextcloud-80
-TXT|a-nextcloud.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/nextcloud/nextcloud-80
-TXT|a-pihole.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/pihole/pihole-80
-TXT|a-pihole.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/pihole/pihole-80
-TXT|a-plex.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/plex/plex-32400
-TXT|a-plex.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/plex/plex-32400
-TXT|a-prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/homepage/homepage
-TXT|a-qbittorrent.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/qbittorrent/qbittorrent-8080
-TXT|a-qbittorrent.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/qbittorrent/qbittorrent-8080
-TXT|a-radarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/radarr/radarr-7878
-TXT|a-radarr.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/radarr/radarr-7878
-TXT|a-seerr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/seerr/seerr-5055
-TXT|a-seerr.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/seerr/seerr-5055
-TXT|a-semaphore.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/semaphore/semaphore-3000
-TXT|a-semaphore.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/semaphore/semaphore-3000
-TXT|a-sonarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/sonarr/sonarr-8989
-TXT|a-sonarr.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/sonarr/sonarr-8989
-TXT|a-unifi.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/unifi/unifi-l4-tcp
-TXT|a-unifi.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=service/unifi/unifi-l4-tcp
-TXT|auth.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/oauth2-proxy/oauth2-proxy
-TXT|crafty-mc.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/crafty/crafty-l4-tcp
-TXT|crafty.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/crafty/crafty
-TXT|gitea.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/gitea/gitea-3000
-TXT|grafana.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/monitoring/grafana
-TXT|headlamp.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/headlamp/headlamp
-TXT|home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/homepage/homepage
-TXT|jackett.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/jackett/jackett-9117
-TXT|keycloak.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/keycloak/keycloak
-TXT|nextcloud.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/nextcloud/nextcloud-80
-TXT|pihole.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/pihole/pihole-80
-TXT|plex.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/plex/plex-32400
-TXT|qbittorrent.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/qbittorrent/qbittorrent-8080
-TXT|radarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/radarr/radarr-7878
-TXT|seerr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/seerr/seerr-5055
-TXT|semaphore.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/semaphore/semaphore-3000
-TXT|sonarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/sonarr/sonarr-8989
-TXT|unifi.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/unifi/unifi-l4-tcp
@@ -69,6 +69,17 @@ infra/devcontainer/dev.sh run make -C ci gate   # the same gate CI runs
 infra/devcontainer/dev.sh                  # interactive shell (sops, flux, kubectl, tofu, ansible)
 ```

+`kubectl`/`flux`/`helm` in there talk to the **dev** cluster by default. Prefix `CLUSTER=prod` to
+target prod instead:
+
+```bash
+infra/devcontainer/dev.sh run kubectl get nodes                # dev (default)
+CLUSTER=prod infra/devcontainer/dev.sh run kubectl get nodes   # prod
+```
+
+Both clusters are always present in the container's `KUBECONFIG` as the contexts `k3s-dev` and
+`k3s-prod`; `CLUSTER` picks which one is current, and each run prints the cluster it is targeting.
+
 The container mounts the repo at `/work`, your age keyring, and the sibling spec repo read-only
 so the spec-immutability check can actually run. See
 [`infra/devcontainer/README.md`](infra/devcontainer/README.md).
@@ -170,27 +170,6 @@ spec:
      - port: '3000'
        protocol: TCP
 ---
-# CiliumNetworkPolicy in-cluster client ingress allow (R-NET-02): gitea-allow-flux-ingress
-apiVersion: cilium.io/v2
-kind: CiliumNetworkPolicy
-metadata:
-  name: gitea-allow-flux-ingress
-  namespace: gitea
-  labels:
-    app.kubernetes.io/name: gitea
-spec:
-  endpointSelector:
-    matchLabels:
-      app.kubernetes.io/name: gitea
-  ingress:
-  - fromEndpoints:
-    - matchLabels:
-        k8s:io.kubernetes.pod.namespace: flux-system
-    toPorts:
-    - ports:
-      - port: '3000'
-        protocol: TCP
---
 # CiliumNetworkPolicy world egress allow (egress class governs the path; R-NET-05): gitea-allow-internet-egress
 apiVersion: cilium.io/v2
 kind: CiliumNetworkPolicy
@@ -74,7 +74,7 @@ spec:
        homelab.keithjackson.dev/auth-mode: forward-auth
      annotations:
        homelab.keithjackson.dev/auth-mode: forward-auth
-        homelab.keithjackson.dev/auth-groups: media-users,platform-admins
+        homelab.keithjackson.dev/auth-groups: media-users
    spec:
      containers:
      - name: jackett
@@ -83,7 +83,7 @@ spec:
        - containerPort: 9117
        volumeMounts:
        - name: config
-          mountPath: /config
+          mountPath: /data/config
      - name: oauth2-proxy
        image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1
        imagePullPolicy: IfNotPresent
@@ -97,7 +97,6 @@ spec:
        - --scope=openid email profile
        - --oidc-groups-claim=groups
        - --allowed-group=media-users
-        - --allowed-group=platform-admins
        - --skip-provider-button=true
        - --redirect-url=https://jackett.${BASE_DOMAIN}/oauth2/callback
        - --cookie-domain=.${BASE_DOMAIN}
@@ -267,7 +266,7 @@ metadata:
    app.kubernetes.io/name: jackett
  annotations:
    homelab.keithjackson.dev/auth-mode: forward-auth
-    homelab.keithjackson.dev/auth-groups: media-users,platform-admins
+    homelab.keithjackson.dev/auth-groups: media-users
 spec:
  parentRefs:
  - name: shared
@@ -11,7 +11,6 @@ resources:
 - jackett-oauth2-proxy-credentials.sops.yaml
 - jackett.yaml
 - nextcloud-db-app.sops.yaml
- nextcloud-db-backup-credentials-prod.sops.yaml
 - nextcloud-db-backup-credentials.sops.yaml
 - nextcloud-db.yaml
 - pihole-oauth2-proxy-credentials.sops.yaml
@@ -20,6 +19,5 @@ resources:
 - qbittorrent-oauth2-proxy-credentials.sops.yaml
 - qbittorrent.yaml
 - semaphore-db-app.sops.yaml
- semaphore-db-backup-credentials-prod.sops.yaml
 - semaphore-db-backup-credentials.sops.yaml
 - semaphore-db.yaml
@@ -1,29 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-    name: nextcloud-db-backup-credentials-prod
-    namespace: nextcloud-db
-type: Opaque
-stringData:
-    ACCESS_KEY_ID: ENC[AES256_GCM,data:4lx6et4HcXDegulyIoa7jZR/lVgapGPEkukajfCKjvk=,iv:p6V7tizbikk55tOvpPOsPHTe/p52QLX5A5xrwuB14rA=,tag:oNJuipT/Lb6zynJ2K2Q71A==,type:str]
-    ACCESS_SECRET_KEY: ENC[AES256_GCM,data:V5bHPoPeBOVyLp1n1Jilyso4i5P2aPSH0hrryfweg2q/SctWZxRbLBQ/FLzOigbJsAhlwOGu069mQPma6xjfdw==,iv:8u4UOvI8dC/cm6mEWDlyN5E3SfRZmwKY9uZY/DjPEsQ=,tag:tw7tGs29TmUvMPBCtCszAA==,type:str]
-sops:
-    kms: []
-    gcp_kms: []
-    azure_kv: []
-    hc_vault: []
-    age:
-        - recipient: age175l8mdpdfu7hq0aml3exs3gykarwjmuvr23sq56s3wjejnyk44wsp4mfzg
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBiSUt5N2k2N2lDU2ZsMy9m
-            M1NFS0M2QjFsNFhnR0hCMzFyaTdiQWFZRFJZCmhBTWRXNmxLMGtsc3Rxd2FSMDZp
-            aktjcTE2c1hmU1EzM3RWNksyb3Q1YVUKLS0tIDBFS1I5aDBER2cvcWtLZ3BXTEp5
-            ZGI5ZmZIcStheUNQamxIVVVpc0ZMOHMKlE5aHewtHJpeU24F5sodc9CEm9KTjHQQ
-            8ThEQ/8vyazL9V4ojlICxSHU/SHkHpD1eBaTKcAQd/LZzujIcJ+QfA==
-            -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2026-09-07T06:36:33Z"
-    mac: ENC[AES256_GCM,data:8awz1Jh8t4zjDMyd93/v+6on9iiIaSi01293A7uYhxj9L1+g63zeO+DPmSn6+KbZvHdqU2A7up09H1NzfXAEdGuvVbB36cdGq0zKank3PUi8c7IT6qNuWydxRAcqLnbbD+c30Z6GB+gAuZHstNOe4Rgc4aJvMN6FuP6x6Z0870c=,iv:KGGKJq2QA5xztvoHs/SViXW72DYwT16i+W+dJBUeIJQ=,tag:3UpYqx2+RIK0QdMH/BSIpA==,type:str]
-    pgp: []
-    encrypted_regex: ^(data|stringData)$
-    version: 3.9.4
@@ -37,53 +37,16 @@ metadata:
  name: nextcloud-db
  namespace: nextcloud-db
 spec:
-  # THREE instances — one primary + two hot standbys (2026-09-07), same as keycloak-db; see
-  # platform/keycloak/postgres-cluster.yaml for the full rationale. Short version: Longhorn's 3-way
-  # replication saves the DATA on node loss, not the SERVICE, and with `instances: 1` all three CNPG
-  # primaries had drifted onto k3s-prod-2, so one node took Nextcloud down alongside SSO and
-  # Semaphore. A streaming standby lets CNPG promote in seconds and repoint the -rw Service.
-  # Gated on the Barman archive below actually working (ContinuousArchiving=True with a real
-  # firstRecoverabilityPoint) — standbys replicate corruption, they are not a backup.
-  #
-  # Scaling out is non-disruptive: CNPG creates -2 then -3 sequentially, each seeded by
-  # pg_basebackup from the running primary, which is neither restarted nor disconnected.
-  instances: 3
+  # Single Postgres instance on 3-replica Longhorn is acceptable at home scale (ADR-0014, same as
+  # keycloak-db). The storage is already 3-way replicated by Longhorn; raise to 3 for Postgres-level
+  # HA as a future hardening.
+  instances: 1
  imageName: ghcr.io/cloudnative-pg/postgresql:16.4
  # Stamp app.kubernetes.io/name=nextcloud-db onto every generated pod/service so the rendered
  # nextcloud egress CiliumNetworkPolicy's toEndpoints selector matches the primary pod (R-NET-02).
  inheritedMetadata:
    labels:
      app.kubernetes.io/name: nextcloud-db
-  # HARD anti-affinity — the three instances MUST land on three DIFFERENT nodes; and roll the
-  # primary by SWITCHOVER rather than CNPG's default in-place `restart`, which would delete and
-  # recreate the primary pod. Full rationale in platform/keycloak/postgres-cluster.yaml. Short
-  # version: CNPG's default `preferred` is a scheduler hint, so when CNPG recreates an instance
-  # after a node loss it may stack two instances on one node — which is how `instances: 3` decays
-  # back into the single-node failure domain this change removes. `required` makes it a constraint;
-  # the cost is that an instance whose node is gone stays Pending, which is correct (2 of 3 still
-  # serve). Satisfiable on both clusters: prod and dev each have 3 schedulable, untainted nodes.
-  #
-  # APPLYING THIS CHANGES THE POD SPEC, so CNPG rolls the instances: standbys first, then one
-  # switchover. Expect a single brief connection reset for Nextcloud.
-  affinity:
-    enablePodAntiAffinity: true
-    podAntiAffinityType: required
-  primaryUpdateMethod: switchover
-  # SYNCHRONOUS REPLICATION — quorum of ONE standby, PREFERRED durability (2026-09-07). Full
-  # rationale in platform/keycloak/postgres-cluster.yaml. Short version: async replication loses the
-  # last few transactions when a primary dies before its standbys catch up, and `method: any` +
-  # `number: 1` makes COMMIT wait for one standby to confirm. dataDurability MUST stay `preferred`,
-  # never `required` — `required` blocks writes outright when fewer than `number` healthy standbys
-  # are available, which on a 3-node cluster sharing nodes with etcd and the CNI turns one degraded
-  # standby into a total write outage. `preferred` relaxes to async instead.
-  #
-  # NON-DISRUPTIVE: compiles to `synchronous_standby_names`, a reload-only PostgreSQL parameter —
-  # no restart, no switchover, no client disconnect. Land it after the anti-affinity roll.
-  postgresql:
-    synchronous:
-      method: any
-      number: 1
-      dataDurability: preferred
  storage:
    storageClass: longhorn
    size: 5Gi
@@ -105,38 +68,29 @@ spec:
      memory: 256Mi
    limits:
      memory: 512Mi
-  # --- Off-cluster DR backup (R-STATE-03) — CNPG-native Barman → object storage ---
+  # --- Off-cluster DR backup (R-STATE-03) — CNPG-native Barman → the on-Proxmox MinIO `cnpg` bucket ---
  # Continuous WAL archiving + periodic base backups = proper Postgres PITR (realises the SD's
-  # backup.policy daily-offcluster). CNPG appends serverName (defaults to the Cluster name) to
-  # destinationPath, so objects land under ${CNPG_BACKUP_PATH}/nextcloud-db/ — dev
-  # s3://cnpg/nextcloud-db/ on the on-Proxmox MinIO, prod
-  # s3://k3s-prod-backup/barman/nextcloud-db/ on Cloudflare R2. Both endpointURL and
-  # destinationPath are cluster-vars knobs (R-ENV-02), substituted by `apps-wave-0`.
-  # Credentials: PER-ENVIRONMENT (2026-09-07). dev reads nextcloud-db-backup-credentials (the
-  # scoped MinIO key), prod reads nextcloud-db-backup-credentials-prod (the Cloudflare R2
-  # token). BOTH Secrets are applied to BOTH clusters by apps/wave-0/kustomization.yaml (a
-  # generated golden — render.py places the -prod sidecar in this DB's wave); the
-  # cluster-vars knob ${BACKUP_CREDENTIALS_SUFFIX} (dev "" | prod "-prod") chooses which
-  # one this Cluster reads. Both are in-namespace, which is all a pod can read. They were
-  # ONE shared Secret until the split, so giving prod its R2 token would have left dev
-  # signing MinIO requests with an R2 key id — 403 on every archive command, i.e. the
-  # disk-full clock below rather than a merely missing backup.
-  #
-  # archive_mode is ON, so a FAILING archive command is a disk-full clock, not a degraded backup:
-  # Postgres refuses to recycle a WAL segment it has not archived, and the pgdata PVC fills until
-  # the primary PANICs. Prod demonstrated this from bootstrap to 2026-09-07 with a placeholder
-  # endpoint. Treat ContinuousArchiving=False here as an availability incident.
+  # backup.policy daily-offcluster). destinationPath is the bucket ROOT; CNPG appends serverName
+  # (defaults to the Cluster name), so objects land under s3://cnpg/nextcloud-db/. Path-style HTTP.
+  # Credentials: the scoped key from the in-namespace SOPS Secret nextcloud-db-backup-credentials
+  # (same-ns only), kept in sync with platform/backup/minio-credentials.sops.yaml.
+  # endpointURL is ${S3_BACKUP_ENDPOINT} (cluster-vars, R-ENV-02; dev http://192.168.1.9:9000),
+  # substituted by the `apps-wave-0` Flux Kustomization. Prod supplies its own off-cluster target.
  backup:
    retentionPolicy: "30d"
    barmanObjectStore:
+      # PER-ENVIRONMENT PATH (cluster-vars). CNPG appends serverName, which defaults to the CLUSTER
+      # NAME — and both environments have a `nextcloud-db`. Sharing one MinIO with a shared path
+      # would interleave two Postgres servers' WAL into ONE Barman stream, corrupting the chain and
+      # destroying PITR for both. Dev keeps `s3://cnpg` (its existing, PROVEN backups); prod nests.
      destinationPath: ${CNPG_BACKUP_PATH}
      endpointURL: ${S3_BACKUP_ENDPOINT}
      s3Credentials:
        accessKeyId:
-          name: nextcloud-db-backup-credentials${BACKUP_CREDENTIALS_SUFFIX}
+          name: nextcloud-db-backup-credentials
          key: ACCESS_KEY_ID
        secretAccessKey:
-          name: nextcloud-db-backup-credentials${BACKUP_CREDENTIALS_SUFFIX}
+          name: nextcloud-db-backup-credentials
          key: ACCESS_SECRET_KEY
      wal:
        compression: gzip
@@ -35,34 +35,6 @@ spec:
    - protocol: TCP
      port: 53
 ---
-# PersistentVolume plex-media - static external NFS (bulk tier)
-apiVersion: v1
-kind: PersistentVolume
-metadata:
-  name: plex-media
-  labels:
-    app.kubernetes.io/name: plex
-spec:
-  capacity:
-    storage: 9Ti
-  accessModes:
-  - ReadOnlyMany
-  persistentVolumeReclaimPolicy: Retain
-  storageClassName: ''
-  mountOptions:
-  - nfsvers=4.1
-  - hard
-  - noatime
-  - timeo=600
-  - retrans=2
-  nfs:
-    server: ${BULK_NFS_SERVER}
-    path: /mnt/Storage/plex
-    readOnly: true
-  claimRef:
-    namespace: plex
-    name: plex-media
---
 # PVC plex-config (R-STATE)
 apiVersion: v1
 kind: PersistentVolumeClaim
@@ -79,23 +51,6 @@ spec:
    requests:
      storage: 8Gi
 ---
-# PVC plex-media (R-STATE)
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: plex-media
-  namespace: plex
-  labels:
-    app.kubernetes.io/name: plex
-spec:
-  accessModes:
-  - ReadOnlyMany
-  storageClassName: ''
-  volumeName: plex-media
-  resources:
-    requests:
-      storage: 9Ti
---
 # Deployment (R-PLAT-02)
 apiVersion: apps/v1
 kind: Deployment
@@ -135,16 +90,10 @@ spec:
        volumeMounts:
        - name: config
          mountPath: /config
-        - name: media
-          mountPath: /media
-          readOnly: true
      volumes:
      - name: config
        persistentVolumeClaim:
          claimName: plex-config
-      - name: media
-        persistentVolumeClaim:
-          claimName: plex-media
 ---
 # Service
 apiVersion: v1
@@ -74,7 +74,7 @@ spec:
        homelab.keithjackson.dev/auth-mode: forward-auth
      annotations:
        homelab.keithjackson.dev/auth-mode: forward-auth
-        homelab.keithjackson.dev/auth-groups: media-users,platform-admins
+        homelab.keithjackson.dev/auth-groups: media-users
    spec:
      containers:
      - name: qbittorrent
@@ -83,7 +83,7 @@ spec:
        - containerPort: 8080
        volumeMounts:
        - name: config
-          mountPath: /config
+          mountPath: /data/config
      - name: oauth2-proxy
        image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1
        imagePullPolicy: IfNotPresent
@@ -97,7 +97,6 @@ spec:
        - --scope=openid email profile
        - --oidc-groups-claim=groups
        - --allowed-group=media-users
-        - --allowed-group=platform-admins
        - --skip-provider-button=true
        - --redirect-url=https://qbittorrent.${BASE_DOMAIN}/oauth2/callback
        - --cookie-domain=.${BASE_DOMAIN}
@@ -267,7 +266,7 @@ metadata:
    app.kubernetes.io/name: qbittorrent
  annotations:
    homelab.keithjackson.dev/auth-mode: forward-auth
-    homelab.keithjackson.dev/auth-groups: media-users,platform-admins
+    homelab.keithjackson.dev/auth-groups: media-users
 spec:
  parentRefs:
  - name: shared
@@ -1,29 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-    name: semaphore-db-backup-credentials-prod
-    namespace: semaphore-db
-type: Opaque
-stringData:
-    ACCESS_KEY_ID: ENC[AES256_GCM,data:OhWEEYqrvmDB40NmiETgeUisBvaO9ocevCZAsurNijw=,iv:kEjOIg9NHbRYm+Ul1WrBAzb3gLbIv+ipiYCxFK0QoVc=,tag:KAB1JAIw0ATwbjDEKZVT9A==,type:str]
-    ACCESS_SECRET_KEY: ENC[AES256_GCM,data:C0PNKv07LCYpFONLqd55sz+ioyU/i2YJUkbZ8OX/dCErM/DilqchyUAT5EqnT/r3YJmdaA1pX85S3fxlVVHkbg==,iv:4ha28b+xIebvX2JPITmtTovIe4vdmj2XfI3cgkUHH4c=,tag:xbMyflcHojytPzgaoOYcXQ==,type:str]
-sops:
-    kms: []
-    gcp_kms: []
-    azure_kv: []
-    hc_vault: []
-    age:
-        - recipient: age175l8mdpdfu7hq0aml3exs3gykarwjmuvr23sq56s3wjejnyk44wsp4mfzg
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBZVTJ3cXR6cks3Y0RNS2hx
-            K3pjZEduYUxmL0V3SmFUcU9XbGVHSVFITWh3Cjl1eHlTLytoOW5PaVNFVkJoeTBo
-            T1FnRHNDVzJ6dWpuR09WaElvbWF4YU0KLS0tIEJoQ1VsM1cxRlg0R1dtNGdOenNn
-            aVNjMjVBTytLdHlFdmZCS2tPelo1Y2MKYt1G5t+sDiHgcg+Q9ECBzJn2gS/paSEC
-            7aoX60SBrPgHvnGls5lzc3Tin9nH4jWYl6Nu8DAvIE0YPXDFr5a0UA==
-            -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2026-09-07T06:36:33Z"
-    mac: ENC[AES256_GCM,data:mhVP7tjdZK2FR2imvpJbTyT7BigvcIKEpLSnBS1oSITUirB8xZC7aHBfyFuIoKOrpOJgEe40nHUdDYCHRJtb4dLeVja8+7meVaQWzxC8BWVHsksW+73hsqhjAzlHeobx6Irbk4PAMqIY0/jnf4MHryjnRn3dUuwu9U1CqL30JbE=,iv:57oCcp6FW9EAIVKnYGtkaTrnbijyv54T+vmZxMbMqnw=,tag:agrRP/JLypU60KOeN0s/5w==,type:str]
-    pgp: []
-    encrypted_regex: ^(data|stringData)$
-    version: 3.9.4
@@ -38,53 +38,16 @@ metadata:
  name: semaphore-db
  namespace: semaphore-db
 spec:
-  # THREE instances — one primary + two hot standbys (2026-09-07), same as keycloak-db /
-  # nextcloud-db; see platform/keycloak/postgres-cluster.yaml for the full rationale. Short version:
-  # Longhorn's 3-way replication saves the DATA on node loss, not the SERVICE, and with
-  # `instances: 1` all three CNPG primaries had drifted onto k3s-prod-2, so one node took Semaphore
-  # down alongside SSO and Nextcloud. A streaming standby lets CNPG promote in seconds and repoint
-  # the -rw Service. Gated on the Barman archive below actually working (ContinuousArchiving=True
-  # with a real firstRecoverabilityPoint) — standbys replicate corruption, they are not a backup.
-  #
-  # Scaling out is non-disruptive: CNPG creates -2 then -3 sequentially, each seeded by
-  # pg_basebackup from the running primary, which is neither restarted nor disconnected.
-  instances: 3
+  # Single Postgres instance on 3-replica Longhorn is acceptable at home scale (ADR-0014, same as
+  # keycloak-db / nextcloud-db). The storage is already 3-way replicated by Longhorn; raise to 3 for
+  # Postgres-level HA as a future hardening.
+  instances: 1
  imageName: ghcr.io/cloudnative-pg/postgresql:16.4
  # Stamp app.kubernetes.io/name=semaphore-db onto every generated pod/service so the rendered
  # semaphore egress CiliumNetworkPolicy's toEndpoints selector matches the primary pod (R-NET-02).
  inheritedMetadata:
    labels:
      app.kubernetes.io/name: semaphore-db
-  # HARD anti-affinity — the three instances MUST land on three DIFFERENT nodes; and roll the
-  # primary by SWITCHOVER rather than CNPG's default in-place `restart`, which would delete and
-  # recreate the primary pod. Full rationale in platform/keycloak/postgres-cluster.yaml. Short
-  # version: CNPG's default `preferred` is a scheduler hint, so when CNPG recreates an instance
-  # after a node loss it may stack two instances on one node — which is how `instances: 3` decays
-  # back into the single-node failure domain this change removes. `required` makes it a constraint;
-  # the cost is that an instance whose node is gone stays Pending, which is correct (2 of 3 still
-  # serve). Satisfiable on both clusters: prod and dev each have 3 schedulable, untainted nodes.
-  #
-  # APPLYING THIS CHANGES THE POD SPEC, so CNPG rolls the instances: standbys first, then one
-  # switchover. Expect a single brief connection reset for Semaphore.
-  affinity:
-    enablePodAntiAffinity: true
-    podAntiAffinityType: required
-  primaryUpdateMethod: switchover
-  # SYNCHRONOUS REPLICATION — quorum of ONE standby, PREFERRED durability (2026-09-07). Full
-  # rationale in platform/keycloak/postgres-cluster.yaml. Short version: async replication loses the
-  # last few transactions when a primary dies before its standbys catch up, and `method: any` +
-  # `number: 1` makes COMMIT wait for one standby to confirm. dataDurability MUST stay `preferred`,
-  # never `required` — `required` blocks writes outright when fewer than `number` healthy standbys
-  # are available, which on a 3-node cluster sharing nodes with etcd and the CNI turns one degraded
-  # standby into a total write outage. `preferred` relaxes to async instead.
-  #
-  # NON-DISRUPTIVE: compiles to `synchronous_standby_names`, a reload-only PostgreSQL parameter —
-  # no restart, no switchover, no client disconnect. Land it after the anti-affinity roll.
-  postgresql:
-    synchronous:
-      method: any
-      number: 1
-      dataDurability: preferred
  storage:
    storageClass: longhorn
    size: 5Gi
@@ -106,38 +69,29 @@ spec:
      memory: 256Mi
    limits:
      memory: 512Mi
-  # --- Off-cluster DR backup (R-STATE-03) — CNPG-native Barman → object storage ---
+  # --- Off-cluster DR backup (R-STATE-03) — CNPG-native Barman → the on-Proxmox MinIO `cnpg` bucket ---
  # Continuous WAL archiving + periodic base backups = proper Postgres PITR (realises the SD's
-  # backup.policy daily-offcluster). CNPG appends serverName (defaults to the Cluster name) to
-  # destinationPath, so objects land under ${CNPG_BACKUP_PATH}/semaphore-db/ — dev
-  # s3://cnpg/semaphore-db/ on the on-Proxmox MinIO, prod
-  # s3://k3s-prod-backup/barman/semaphore-db/ on Cloudflare R2. Both endpointURL and
-  # destinationPath are cluster-vars knobs (R-ENV-02), substituted by `apps-wave-0`.
-  # Credentials: PER-ENVIRONMENT (2026-09-07). dev reads semaphore-db-backup-credentials (the
-  # scoped MinIO key), prod reads semaphore-db-backup-credentials-prod (the Cloudflare R2
-  # token). BOTH Secrets are applied to BOTH clusters by apps/wave-0/kustomization.yaml (a
-  # generated golden — render.py places the -prod sidecar in this DB's wave); the
-  # cluster-vars knob ${BACKUP_CREDENTIALS_SUFFIX} (dev "" | prod "-prod") chooses which
-  # one this Cluster reads. Both are in-namespace, which is all a pod can read. They were
-  # ONE shared Secret until the split, so giving prod its R2 token would have left dev
-  # signing MinIO requests with an R2 key id — 403 on every archive command, i.e. the
-  # disk-full clock below rather than a merely missing backup.
-  #
-  # archive_mode is ON, so a FAILING archive command is a disk-full clock, not a degraded backup:
-  # Postgres refuses to recycle a WAL segment it has not archived, and the pgdata PVC fills until
-  # the primary PANICs. Prod demonstrated this from bootstrap to 2026-09-07 with a placeholder
-  # endpoint. Treat ContinuousArchiving=False here as an availability incident.
+  # backup.policy daily-offcluster). destinationPath is the bucket ROOT; CNPG appends serverName
+  # (defaults to the Cluster name), so objects land under s3://cnpg/semaphore-db/. Path-style HTTP.
+  # Credentials: the scoped key from the in-namespace SOPS Secret semaphore-db-backup-credentials
+  # (same-ns only), kept in sync with platform/backup/minio-credentials.sops.yaml.
+  # endpointURL is ${S3_BACKUP_ENDPOINT} (cluster-vars, R-ENV-02; dev http://192.168.1.9:9000),
+  # substituted by the `apps-wave-0` Flux Kustomization. Prod supplies its own off-cluster target.
  backup:
    retentionPolicy: "30d"
    barmanObjectStore:
+      # PER-ENVIRONMENT PATH (cluster-vars). CNPG appends serverName, which defaults to the CLUSTER
+      # NAME — and both environments have a `semaphore-db`. Sharing one MinIO with a shared path
+      # would interleave two Postgres servers' WAL into ONE Barman stream, corrupting the chain and
+      # destroying PITR for both. Dev keeps `s3://cnpg` (its existing, PROVEN backups); prod nests.
      destinationPath: ${CNPG_BACKUP_PATH}
      endpointURL: ${S3_BACKUP_ENDPOINT}
      s3Credentials:
        accessKeyId:
-          name: semaphore-db-backup-credentials${BACKUP_CREDENTIALS_SUFFIX}
+          name: semaphore-db-backup-credentials
          key: ACCESS_KEY_ID
        secretAccessKey:
-          name: semaphore-db-backup-credentials${BACKUP_CREDENTIALS_SUFFIX}
+          name: semaphore-db-backup-credentials
          key: ACCESS_SECRET_KEY
      wal:
        compression: gzip
@@ -1,159 +0,0 @@
---
-# Namespace (trust zone; default-deny + policies apply)
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: vm-demo
-  labels:
-    homelab.keithjackson.dev/app-namespace: 'true'
-  annotations:
-    homelab.keithjackson.dev/backup-policy: daily-offcluster
---
-# Default-deny baseline (R-NET-01): deny all, allow only DNS
-apiVersion: networking.k8s.io/v1
-kind: NetworkPolicy
-metadata:
-  name: default-deny
-  namespace: vm-demo
-  labels:
-    homelab.keithjackson.dev/baseline: default-deny
-spec:
-  podSelector: {}
-  policyTypes:
-  - Ingress
-  - Egress
-  ingress: []
-  egress:
-  - to:
-    - namespaceSelector:
-        matchLabels:
-          kubernetes.io/metadata.name: kube-system
-    ports:
-    - protocol: UDP
-      port: 53
-    - protocol: TCP
-      port: 53
---
-# CDI DataVolume (standalone; survives rebuild, R-STATE-01)
-apiVersion: cdi.kubevirt.io/v1beta1
-kind: DataVolume
-metadata:
-  name: vm-demo-rootdisk
-  namespace: vm-demo
-spec:
-  source:
-    registry:
-      url: docker://quay.io/containerdisks/fedora:41
-  storage:
-    accessModes:
-    - ReadWriteOnce
-    storageClassName: longhorn
-    resources:
-      requests:
-        storage: 6Gi
---
-# KubeVirt VirtualMachine (R-PLAT-03)
-apiVersion: kubevirt.io/v1
-kind: VirtualMachine
-metadata:
-  name: vm-demo
-  namespace: vm-demo
-  labels:
-    app.kubernetes.io/name: vm-demo
-spec:
-  running: false
-  template:
-    metadata:
-      labels:
-        kubevirt.io/vm: vm-demo
-        homelab.keithjackson.dev/egress-class: internet
-        homelab.keithjackson.dev/auth-mode: none
-    spec:
-      domain:
-        cpu:
-          cores: 1
-        memory:
-          guest: 1536Mi
-        devices:
-          disks:
-          - name: rootdisk
-            disk:
-              bus: virtio
-          - name: cloudinit
-            disk:
-              bus: virtio
-          interfaces:
-          - name: default
-            masquerade: {}
-        resources:
-          requests:
-            memory: 1536Mi
-      networks:
-      - name: default
-        pod: {}
-      volumes:
-      - name: rootdisk
-        dataVolume:
-          name: vm-demo-rootdisk
-      - name: cloudinit
-        cloudInitNoCloud:
-          userData: "#cloud-config\npackage_update: true\npackages:\n  - dnf-automatic\nwrite_files:\n  - path: /etc/dnf/automatic.conf\n    content: |\n      [commands]\n      upgrade_type = default\n      random_sleep = 0\n      download_updates = yes\n      apply_updates = yes\nruncmd:\n  - [ systemctl, enable, --now, dnf-automatic.timer ]\n"
---
-# CiliumNetworkPolicy world egress allow (egress class governs the path; R-NET-05): vm-demo-allow-internet-egress
-apiVersion: cilium.io/v2
-kind: CiliumNetworkPolicy
-metadata:
-  name: vm-demo-allow-internet-egress
-  namespace: vm-demo
-  labels:
-    app.kubernetes.io/name: vm-demo
-spec:
-  endpointSelector:
-    matchLabels:
-      app.kubernetes.io/name: vm-demo
-  egress:
-  - toEntities:
-    - world
---
-# ServiceMonitor (R-OBS-01)
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
-  name: vm-demo
-  namespace: vm-demo
-  labels:
-    release: kube-prometheus-stack
-    app.kubernetes.io/name: vm-demo
-spec:
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: vm-demo
-  namespaceSelector:
-    matchNames:
-    - vm-demo
-  endpoints:
-  - port: metrics
-    interval: 30s
---
-# PrometheusRule SLI alerts (R-OBS-01)
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
-  name: vm-demo-slis
-  namespace: vm-demo
-  labels:
-    release: kube-prometheus-stack
-    app.kubernetes.io/name: vm-demo
-spec:
-  groups:
-  - name: vm-demo.slis
-    rules:
-    - alert: vm-demo-down
-      expr: up{job="vm-demo"} == 0
-      for: 5m
-      labels:
-        severity: critical
-        service: vm-demo
-      annotations:
-        summary: vm-demo is down
-        description: 'vm-demo has been unreachable for 5m (SLI: up).'
@@ -1,228 +0,0 @@
---
-# Namespace (trust zone; default-deny + policies apply)
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: keycloak
-  labels:
-    homelab.keithjackson.dev/app-namespace: 'true'
-    gateway-access: shared
-  annotations:
-    homelab.keithjackson.dev/backup-policy: daily-offcluster
---
-# Default-deny baseline (R-NET-01): deny all, allow only DNS
-apiVersion: networking.k8s.io/v1
-kind: NetworkPolicy
-metadata:
-  name: default-deny
-  namespace: keycloak
-  labels:
-    homelab.keithjackson.dev/baseline: default-deny
-spec:
-  podSelector: {}
-  policyTypes:
-  - Ingress
-  - Egress
-  ingress: []
-  egress:
-  - to:
-    - namespaceSelector:
-        matchLabels:
-          kubernetes.io/metadata.name: kube-system
-    ports:
-    - protocol: UDP
-      port: 53
-    - protocol: TCP
-      port: 53
---
-# PVC keycloak-db (R-STATE)
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: keycloak-db
-  namespace: keycloak
-  labels:
-    app.kubernetes.io/name: keycloak
-spec:
-  accessModes:
-  - ReadWriteOnce
-  storageClassName: longhorn
-  resources:
-    requests:
-      storage: 10Gi
---
-# Deployment (R-PLAT-02)
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: keycloak
-  namespace: keycloak
-  labels:
-    app.kubernetes.io/name: keycloak
-spec:
-  replicas: 1
-  strategy:
-    type: Recreate
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: keycloak
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: keycloak
-        homelab.keithjackson.dev/egress-class: internet
-        homelab.keithjackson.dev/auth-mode: native
-    spec:
-      containers:
-      - name: keycloak
-        image: quay.io/keycloak/keycloak
-        ports:
-        - containerPort: 8080
-        volumeMounts:
-        - name: db
-          mountPath: /data/db
-      volumes:
-      - name: db
-        persistentVolumeClaim:
-          claimName: keycloak-db
---
-# Service
-apiVersion: v1
-kind: Service
-metadata:
-  name: keycloak
-  namespace: keycloak
-  labels:
-    app.kubernetes.io/name: keycloak
-spec:
-  selector:
-    app.kubernetes.io/name: keycloak
-  ports:
-  - name: port-8080
-    port: 8080
-    targetPort: 8080
---
-# CiliumNetworkPolicy egress allow (R-NET-02): keycloak-allow-keycloak-db-5432
-apiVersion: cilium.io/v2
-kind: CiliumNetworkPolicy
-metadata:
-  name: keycloak-allow-keycloak-db-5432
-  namespace: keycloak
-  labels:
-    app.kubernetes.io/name: keycloak
-spec:
-  endpointSelector:
-    matchLabels:
-      app.kubernetes.io/name: keycloak
-  egress:
-  - toEndpoints:
-    - matchLabels:
-        app.kubernetes.io/name: keycloak-db
-        k8s:io.kubernetes.pod.namespace: keycloak-db
-    toPorts:
-    - ports:
-      - port: '5432'
-        protocol: TCP
---
-# CiliumNetworkPolicy gateway-ingress allow (R-NET-01/02): keycloak-allow-gateway-ingress
-apiVersion: cilium.io/v2
-kind: CiliumNetworkPolicy
-metadata:
-  name: keycloak-allow-gateway-ingress
-  namespace: keycloak
-  labels:
-    app.kubernetes.io/name: keycloak
-spec:
-  endpointSelector:
-    matchLabels:
-      app.kubernetes.io/name: keycloak
-  ingress:
-  - fromEntities:
-    - ingress
-    toPorts:
-    - ports:
-      - port: '8080'
-        protocol: TCP
---
-# CiliumNetworkPolicy world egress allow (egress class governs the path; R-NET-05): keycloak-allow-internet-egress
-apiVersion: cilium.io/v2
-kind: CiliumNetworkPolicy
-metadata:
-  name: keycloak-allow-internet-egress
-  namespace: keycloak
-  labels:
-    app.kubernetes.io/name: keycloak
-spec:
-  endpointSelector:
-    matchLabels:
-      app.kubernetes.io/name: keycloak
-  egress:
-  - toEntities:
-    - world
---
-# HTTPRoute keycloak-8080 (R-TLS/R-DNS)
-apiVersion: gateway.networking.k8s.io/v1
-kind: HTTPRoute
-metadata:
-  name: keycloak-8080
-  namespace: keycloak
-  labels:
-    app.kubernetes.io/name: keycloak
-spec:
-  parentRefs:
-  - name: shared
-    namespace: gateway
-    sectionName: https
-  hostnames:
-  - keycloak.${BASE_DOMAIN}
-  rules:
-  - matches:
-    - path:
-        type: PathPrefix
-        value: /
-    backendRefs:
-    - name: keycloak
-      port: 8080
---
-# ServiceMonitor (R-OBS-01)
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
-  name: keycloak
-  namespace: keycloak
-  labels:
-    release: kube-prometheus-stack
-    app.kubernetes.io/name: keycloak
-spec:
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: keycloak
-  namespaceSelector:
-    matchNames:
-    - keycloak
-  endpoints:
-  - port: metrics
-    interval: 30s
---
-# PrometheusRule SLI alerts (R-OBS-01)
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
-  name: keycloak-slis
-  namespace: keycloak
-  labels:
-    release: kube-prometheus-stack
-    app.kubernetes.io/name: keycloak
-spec:
-  groups:
-  - name: keycloak.slis
-    rules:
-    - alert: keycloak-down
-      expr: up{job="keycloak"} == 0
-      for: 5m
-      labels:
-        severity: critical
-        service: keycloak
-      annotations:
-        summary: keycloak is down
-        description: 'keycloak has been unreachable for 5m (SLI: up).'
@@ -35,34 +35,6 @@ spec:
    - protocol: TCP
      port: 53
 ---
-# PersistentVolume nextcloud-legacy-archive - static external NFS (bulk tier)
-apiVersion: v1
-kind: PersistentVolume
-metadata:
-  name: nextcloud-legacy-archive
-  labels:
-    app.kubernetes.io/name: nextcloud
-spec:
-  capacity:
-    storage: 270Gi
-  accessModes:
-  - ReadOnlyMany
-  persistentVolumeReclaimPolicy: Retain
-  storageClassName: ''
-  mountOptions:
-  - nfsvers=4.1
-  - hard
-  - noatime
-  - timeo=600
-  - retrans=2
-  nfs:
-    server: ${BULK_NFS_SERVER}
-    path: /mnt/Storage/nextcloud
-    readOnly: true
-  claimRef:
-    namespace: nextcloud
-    name: nextcloud-legacy-archive
---
 # PVC nextcloud-data (R-STATE)
 apiVersion: v1
 kind: PersistentVolumeClaim
@@ -79,23 +51,6 @@ spec:
    requests:
      storage: 20Gi
 ---
-# PVC nextcloud-legacy-archive (R-STATE)
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: nextcloud-legacy-archive
-  namespace: nextcloud
-  labels:
-    app.kubernetes.io/name: nextcloud
-spec:
-  accessModes:
-  - ReadOnlyMany
-  storageClassName: ''
-  volumeName: nextcloud-legacy-archive
-  resources:
-    requests:
-      storage: 270Gi
---
 # Deployment (R-PLAT-02)
 apiVersion: apps/v1
 kind: Deployment
@@ -164,10 +119,6 @@ spec:
        volumeMounts:
        - name: data
          mountPath: /var/www/html
-        - name: legacy-archive
-          mountPath: /mnt/legacy-nextcloud
-          subPath: nextcloud
-          readOnly: true
        lifecycle:
          postStart:
            exec:
@@ -181,9 +132,6 @@ spec:
      - name: data
        persistentVolumeClaim:
          claimName: nextcloud-data
-      - name: legacy-archive
-        persistentVolumeClaim:
-          claimName: nextcloud-legacy-archive
 ---
 # Service
 apiVersion: v1
@@ -35,33 +35,6 @@ spec:
    - protocol: TCP
      port: 53
 ---
-# PersistentVolume radarr-media - static external NFS (bulk tier)
-apiVersion: v1
-kind: PersistentVolume
-metadata:
-  name: radarr-media
-  labels:
-    app.kubernetes.io/name: radarr
-spec:
-  capacity:
-    storage: 9Ti
-  accessModes:
-  - ReadWriteMany
-  persistentVolumeReclaimPolicy: Retain
-  storageClassName: ''
-  mountOptions:
-  - nfsvers=4.1
-  - hard
-  - noatime
-  - timeo=600
-  - retrans=2
-  nfs:
-    server: ${BULK_NFS_SERVER}
-    path: /mnt/Storage/plex
-  claimRef:
-    namespace: radarr
-    name: radarr-media
---
 # PVC radarr-config (R-STATE)
 apiVersion: v1
 kind: PersistentVolumeClaim
@@ -78,23 +51,6 @@ spec:
    requests:
      storage: 5Gi
 ---
-# PVC radarr-media (R-STATE)
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: radarr-media
-  namespace: radarr
-  labels:
-    app.kubernetes.io/name: radarr
-spec:
-  accessModes:
-  - ReadWriteMany
-  storageClassName: ''
-  volumeName: radarr-media
-  resources:
-    requests:
-      storage: 9Ti
---
 # Deployment (R-PLAT-02)
 apiVersion: apps/v1
 kind: Deployment
@@ -118,25 +74,16 @@ spec:
        homelab.keithjackson.dev/auth-mode: forward-auth
      annotations:
        homelab.keithjackson.dev/auth-mode: forward-auth
-        homelab.keithjackson.dev/auth-groups: media-users,platform-admins
+        homelab.keithjackson.dev/auth-groups: media-users
    spec:
      containers:
      - name: radarr
        image: lscr.io/linuxserver/radarr:5.21.1
        ports:
        - containerPort: 7878
-        env:
-        - name: PUID
-          value: '1000'
-        - name: PGID
-          value: '3000'
-        - name: TZ
-          value: Etc/UTC
        volumeMounts:
        - name: config
-          mountPath: /config
-        - name: media
-          mountPath: /media
+          mountPath: /data/config
      - name: oauth2-proxy
        image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1
        imagePullPolicy: IfNotPresent
@@ -150,7 +97,6 @@ spec:
        - --scope=openid email profile
        - --oidc-groups-claim=groups
        - --allowed-group=media-users
-        - --allowed-group=platform-admins
        - --skip-provider-button=true
        - --redirect-url=https://radarr.${BASE_DOMAIN}/oauth2/callback
        - --cookie-domain=.${BASE_DOMAIN}
@@ -202,15 +148,10 @@ spec:
            port: http
          initialDelaySeconds: 5
          periodSeconds: 15
-      securityContext:
-        fsGroup: 3000
      volumes:
      - name: config
        persistentVolumeClaim:
          claimName: radarr-config
-      - name: media
-        persistentVolumeClaim:
-          claimName: radarr-media
 ---
 # Service
 apiVersion: v1
@@ -347,7 +288,7 @@ metadata:
    app.kubernetes.io/name: radarr
  annotations:
    homelab.keithjackson.dev/auth-mode: forward-auth
-    homelab.keithjackson.dev/auth-groups: media-users,platform-admins
+    homelab.keithjackson.dev/auth-groups: media-users
 spec:
  parentRefs:
  - name: shared
@@ -5,15 +5,6 @@
 #   credential from its own namespace (a Pod cannot read a Secret in another namespace).
 # access-key-encryption: FIXED base64 32-byte AES key Semaphore uses to encrypt stored SSH keys /
 #   credentials; MUST stay stable across restarts or stored secrets become undecryptable.
-# oidc-client-secret: the Keycloak `semaphore` CONFIDENTIAL client secret, and ONLY that. It MIRRORS
-#   `semaphore-client-secret` in platform/keycloak/keycloak-realm-clients.sops.yaml (the realm import
-#   injects that copy into the client), same cross-namespace mirror pattern as db-password above.
-#   It replaced the former `oidc-providers-json` key (2026-09-07): that key held the WHOLE
-#   SEMAPHORE_OIDC_PROVIDERS blob, which buried the environment-specific realm-issuer and redirect
-#   URLs inside ciphertext where Flux's ${BASE_DOMAIN} postBuild substitution could not reach them,
-#   so prod ran with dev's URLs. Those URLs are PUBLIC and now live in plaintext in
-#   catalog/semaphore.yaml (rendered to apps/wave-1/semaphore.yaml); the kubelet splices THIS value
-#   back into the blob via $(SEMAPHORE_OIDC_CLIENT_SECRET). Only the secret half is encrypted.
 # Recover: infra/devcontainer/dev.sh run sops -d apps/semaphore-bootstrap-secret.sops.yaml
 apiVersion: v1
 kind: Secret
@@ -24,11 +15,11 @@ metadata:
        app.kubernetes.io/name: semaphore
 type: Opaque
 stringData:
-    admin-username: ENC[AES256_GCM,data:ju3wVdjNzRUP/w==,iv:d/5ZxgQZ5bympu3gT2ck3DJy9iUPPBAtZYsJvD3To7I=,tag:0UtBaDVqNSK4RCJrQicchw==,type:str]
-    admin-password: ENC[AES256_GCM,data:pxIwpgh3BoND2TI0mPTixOTJoictzT77dqtuRdMKgY/2WaW4wM1Ra6H6n/dxnyR2,iv:eYbFQGFYVJsnYGwn6cOI3BgAdzMcin9YggAhiuL+xqM=,tag:/sPWGv4gue0vjc2nMk7wWA==,type:str]
-    db-password: ENC[AES256_GCM,data:hBFpWXdpk1R5a23Pntup03t56NBRPS+SZU8CrHVb/ypIaIpgf7kjYtVy6nhTF6u0,iv:eBzoQgH5ZhWWpfKZ6AQmwaSal5QaiuN9gfexQ7GsVFs=,tag:j2k8EwhKDlj/Ud6O39ioVQ==,type:str]
-    access-key-encryption: ENC[AES256_GCM,data:SKtAV4HxJ9pNYDqUK/1uOdI1rtMji3pTfprI2rhQxFQyhc6dFlhqI+OUEcY=,iv:rKCP+FLidcOltOwmMMcO5M8tz4qiifIMI+W5soGO2M8=,tag:HaHzkX/JI+9wmcg2SD9GAQ==,type:str]
-    oidc-client-secret: ENC[AES256_GCM,data:uCo7TNzbM2QKw9ie+xzyUG9MFKco1OAlwr5qPn/AGpNvo1YyXSfbZyyM0CLJ1NS821hGg5Zr/m3R/xpy84s3FQ==,iv:O8a4haskufpmorPaxOkJ1Q99Y5GTlzmDFJAsHS+irLk=,tag:VojJpA9jNMbNQmIe1Qr44w==,type:str]
+    admin-username: ENC[AES256_GCM,data:uD/vO9A9MjerFg==,iv:i/NAqVD6rWg9oWWSGBrRkxzwcPp2IW1HLIKY0pdcPPg=,tag:r27tKLAJ+ydE7Ty3ccMNUw==,type:str]
+    admin-password: ENC[AES256_GCM,data:544nWY1UmBnlsdPJAQZJMXRumu4W9X7UXZvZ6vHN/ArjRgLXKTArOe5JkT/nNrNw,iv:AC1DDbZj2HfpU19EjzA7T+WpIaR4nZc3PNN61y8bAzU=,tag:r28LFi5w6ZeCCaRaX2jRdg==,type:str]
+    db-password: ENC[AES256_GCM,data:ySkdvaGRCW9fMQqChs5l8dY4zCG19i4HOEinRC48mgLuh9WOrjd16T6jwFgiwB55,iv:+1gztJ22HbwgQJiolZMMWOcKIsxrBJpnruJdtxdooOU=,tag:8CfJGwyPTjDGyr0CvalXMQ==,type:str]
+    access-key-encryption: ENC[AES256_GCM,data:1YAMLo1nVkpJWZc+KBZ7B4ia+Y8P+xMyOhHMcEGGJOXd3o+WC5FqirLBVh4=,iv:j28ozeaDhGWDjoaxFOhn2t9YbYE7L3ah/Mfi+i2DWwY=,tag:czG9Wxc/7Nsuh1W5iATOww==,type:str]
+    oidc-providers-json: ENC[AES256_GCM,data:90yt5mIVWLVI2KrlJWpxhL/BMDpEgRhv7+BL+mBRt79diRvU7pN9vtzhZprAKzcU1sEywlT4EwFlsvJpYyo5Qbkp+ZVknbyhAlGvHhJOBHukOPWgf4LignLerO2GzhXjOOzFqWfZ4EyUxNMWTbSe/xc0zIVHDDNQiYplGVRK+E/ukiI7sD/Tn2RrbrdvpC5GyK0NXWRsXn9ffhE3fCoiRHXagj9i/q+uUrQI5EKpPobrDUAGqvKoBXTl/wJ4phnWg9IqtCTE/c3yA7fLLz4j5dCtPZNSVPT4TKDx8Mdv8CfCgziVicjG2s+yHuWpsQn8lf6nvdJmpYPvIqTtLWdxFWuD8Vfo6Qbq2mqPbKR+KmDOGUnjl4gbZdL1SkE9JiqBUobz7UXeLXwYzxMIsinOfYZLcCk7alrD5Yndj9JDxDJZQ54sMMBfVbxDl94W1VHy03RxAR46KKAbPaxOP5wH4Q4z3SDceNwmuGCDRGChRXiHhN5X2X0GqYCbc8E6GdkIMcpmD80e26ydEmAjw7F89mCF4F/yMzJNiG82qaZIn8HeqgGA8urous9r9bYJNlitYTYQ,iv:jbfVdrrZ9fGiOxhQeoTJfFg1QwiOMkoQyaadiVfxX/w=,tag:JkepcRs34EuEgNb4buTTDQ==,type:str]
 sops:
    kms: []
    gcp_kms: []
@@ -38,14 +29,14 @@ sops:
        - recipient: age175l8mdpdfu7hq0aml3exs3gykarwjmuvr23sq56s3wjejnyk44wsp4mfzg
          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAyTzF6NTBtcFVWK1Vtc0I2
-            VGdySHBsRGgvc1h0VzgvQ01xYVBJV3g2Q1NRCjJyZHVYZGtuamo1MFJHWlBwWkxt
-            cHA4VkY1TkRpN3M4UTdMOWZTbFVGemMKLS0tIGVNdDBrWVJveU56VjVBU2JWdHF4
-            RGtaYStXYUpONTZjZCtmM0NyZ3Rka1UKIQpmQai8dz+uRAsXxyeuGaRlRwsFqIPQ
-            9AfrZJcslYwlaK3ptm1oqaCdrlxcMar2nI7thSqzbLe+e1nzA2z7vQ==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBuRDlqRFJ0dk14UHluSGRP
+            S1pESy9PWlpMakU0aTkzNVhqUGdtbWNEdTJjCm5RNG14azJHc28rdEJ4L0RzR0xO
+            MGRocGZ1V1JZTENWQkN3VGZzamp2eTgKLS0tIGt6QkNNaW5hdTBJVHY0SVo5d3JX
+            elp0SWxHcjg0RWF3aW9mWTJHZ0lhcUEKc0rU5VilAVDleCwqq4SOF+Y9sniu1Pka
+            w/MGDA9THSB0b+YgWRWiX/KiimFqp0ZFODOJ9cHVwFGY71+VadPwOg==
            -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2026-09-07T07:01:50Z"
-    mac: ENC[AES256_GCM,data:Xutm1rQt57J4RILElD8FkNOsQT1HK5jZlPtqqLyZ/VqOoDwo+IS1wp7hROprlUvQC0bUXy79tRRIl5Q0CCOD+fvAGIdmade+vB7qXlYN6Q1441dGhuSTPvckso6FaB2ddRjKhKGLc/Z4bJoz9VaRNNxJNpWXIrENTg+I+QhNhlo=,iv:klJOrEQ6QN5dTGkQfGnbMbgCWsgWwZPb5YXDMeWeE34=,tag:fuJr4t3Tt3dT9u3/0SFo8Q==,type:str]
+    lastmodified: "2026-08-27T22:12:32Z"
+    mac: ENC[AES256_GCM,data:nHftJTGnTr0O5h8yGXwgz3eflggXJfIXM3m8n3bzDJE/P4nvSpZzFeMb+sc2on+AL+t+SeKrIl8blzm33XsQ1lkaY85Zm+bc5cnED1g04BMDVmmK3brh/8w50Bp2wPEVzb4I01MDzhhyIbOYg7pFvZAH3/VuqvM1t6XmfF1NrOA=,iv:4KBa9PH/dfdGwLC16RenZIVB6BtSoGo6x3qt3h1R43I=,tag:KULiA1klP/RZMNigkloGDA==,type:str]
    pgp: []
    encrypted_regex: ^(data|stringData)$
    version: 3.9.4
@@ -118,13 +118,11 @@ spec:
              key: access-key-encryption
        - name: SEMAPHORE_PORT
          value: :3000
-        - name: SEMAPHORE_OIDC_CLIENT_SECRET
+        - name: SEMAPHORE_OIDC_PROVIDERS
          valueFrom:
            secretKeyRef:
              name: semaphore-bootstrap
-              key: oidc-client-secret
-        - name: SEMAPHORE_OIDC_PROVIDERS
-          value: '{"keycloak":{"display_name":"Keycloak","provider_url":"https://keycloak.${BASE_DOMAIN}/realms/homelab","client_id":"semaphore","client_secret":"$(SEMAPHORE_OIDC_CLIENT_SECRET)","redirect_url":"https://semaphore.${BASE_DOMAIN}/api/auth/oidc/keycloak/redirect","scopes":["openid","profile","email"],"username_claim":"preferred_username","email_claim":"email","name_claim":"name","order":1}}'
+              key: oidc-providers-json
        volumeMounts:
        - name: config
          mountPath: /etc/semaphore
@@ -35,33 +35,6 @@ spec:
    - protocol: TCP
      port: 53
 ---
-# PersistentVolume sonarr-media - static external NFS (bulk tier)
-apiVersion: v1
-kind: PersistentVolume
-metadata:
-  name: sonarr-media
-  labels:
-    app.kubernetes.io/name: sonarr
-spec:
-  capacity:
-    storage: 9Ti
-  accessModes:
-  - ReadWriteMany
-  persistentVolumeReclaimPolicy: Retain
-  storageClassName: ''
-  mountOptions:
-  - nfsvers=4.1
-  - hard
-  - noatime
-  - timeo=600
-  - retrans=2
-  nfs:
-    server: ${BULK_NFS_SERVER}
-    path: /mnt/Storage/plex
-  claimRef:
-    namespace: sonarr
-    name: sonarr-media
---
 # PVC sonarr-config (R-STATE)
 apiVersion: v1
 kind: PersistentVolumeClaim
@@ -78,23 +51,6 @@ spec:
    requests:
      storage: 5Gi
 ---
-# PVC sonarr-media (R-STATE)
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: sonarr-media
-  namespace: sonarr
-  labels:
-    app.kubernetes.io/name: sonarr
-spec:
-  accessModes:
-  - ReadWriteMany
-  storageClassName: ''
-  volumeName: sonarr-media
-  resources:
-    requests:
-      storage: 9Ti
---
 # Deployment (R-PLAT-02)
 apiVersion: apps/v1
 kind: Deployment
@@ -118,25 +74,16 @@ spec:
        homelab.keithjackson.dev/auth-mode: forward-auth
      annotations:
        homelab.keithjackson.dev/auth-mode: forward-auth
-        homelab.keithjackson.dev/auth-groups: media-users,platform-admins
+        homelab.keithjackson.dev/auth-groups: media-users
    spec:
      containers:
      - name: sonarr
        image: lscr.io/linuxserver/sonarr:4.0.15
        ports:
        - containerPort: 8989
-        env:
-        - name: PUID
-          value: '1000'
-        - name: PGID
-          value: '3000'
-        - name: TZ
-          value: Etc/UTC
        volumeMounts:
        - name: config
-          mountPath: /config
-        - name: media
-          mountPath: /media
+          mountPath: /data/config
      - name: oauth2-proxy
        image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1
        imagePullPolicy: IfNotPresent
@@ -150,7 +97,6 @@ spec:
        - --scope=openid email profile
        - --oidc-groups-claim=groups
        - --allowed-group=media-users
-        - --allowed-group=platform-admins
        - --skip-provider-button=true
        - --redirect-url=https://sonarr.${BASE_DOMAIN}/oauth2/callback
        - --cookie-domain=.${BASE_DOMAIN}
@@ -202,15 +148,10 @@ spec:
            port: http
          initialDelaySeconds: 5
          periodSeconds: 15
-      securityContext:
-        fsGroup: 3000
      volumes:
      - name: config
        persistentVolumeClaim:
          claimName: sonarr-config
-      - name: media
-        persistentVolumeClaim:
-          claimName: sonarr-media
 ---
 # Service
 apiVersion: v1
@@ -347,7 +288,7 @@ metadata:
    app.kubernetes.io/name: sonarr
  annotations:
    homelab.keithjackson.dev/auth-mode: forward-auth
-    homelab.keithjackson.dev/auth-groups: media-users,platform-admins
+    homelab.keithjackson.dev/auth-groups: media-users
 spec:
  parentRefs:
  - name: shared
@@ -11,7 +11,8 @@ Spec: [ADR-0004 (Flux)](../../../home-network-k3s/decisions/adr-0004-gitops-flux
 ## Files
 | File | Purpose |
 |---|---|
-| `bootstrap.sh` | **The single bootstrap entrypoint** the Infra worker's final Ansible play calls. render+apply the Flux controllers (`components/`) → verify `sops-age` Secret → `kubectl apply -k clusters/dev`. |
+| `bootstrap.sh` | **The single bootstrap entrypoint** the Infra worker's final Ansible play calls. render+apply the Flux controllers (`components/`) → **pre-apply chart CRDs (`preapply-crds.sh`)** → verify `sops-age` Secret → `kubectl apply -k ${FLUX_ENV_PATH}`. |
+| `preapply-crds.sh` | **Server-side pre-apply of the Kyverno CRD bundle** — the CRDs `platform/kyverno/admission/helmrelease.yaml` deliberately tells Helm *not* to install (`crds: {install: false}`, because the ~5.6 MiB bundle exceeds kubectl's 256 KiB last-applied annotation limit and stalls the release). Nothing else installs them, so without this step `platform` never goes Ready on a cold cluster. Chart name/version/repo are **derived from that HelmRelease** — no second pin. Idempotent; runnable standalone. |
 | `components/kustomization.yaml` | **Customization of the Flux CONTROLLERS themselves.** Overlays `flux install --export` output; currently enables `StrictPostBuildSubstitutions` on kustomize-controller. See "Customizing the controllers" below. |
 | `patches/platform-sops-decryption.yaml` | Strategic-merge patch enabling `decryption.provider: sops` + `secretRef: sops-age` on the `platform` Kustomization (realises the base TODO). |
 | `patches/apps-sops-decryption.yaml` | Same, for the app-wave Kustomizations. There is **no single `apps` Kustomization** any more — the apps tier is now `apps-wave-0/1/2` + `apps-vms`, and SOPS decryption is layered on **per wave** (each wave Kustomization is patched by name in `clusters/dev`). |
@@ -74,6 +75,49 @@ flux get kustomizations -A                                  # all 11 must return
 **Rollback** is the same command with `FLUX_INSTALL_PLAIN=1 bash bootstrap/flux/bootstrap.sh`, or
 simply `flux install --namespace=flux-system --components-extra=…` (drops the arg back off).

+## The CRD pre-apply step (`preapply-crds.sh`) — why bootstrap has to do this
+
+`platform/kyverno/admission/helmrelease.yaml` sets `crds: {install: false}`. That is correct and must
+stay: the Kyverno CRD bundle is **5,636,155 bytes across 22 CRDs** at chart 3.9.0 (measured with
+`helm template … --include-crds`, 2026-09-04), so Helm's client-side apply hits kubectl's 256 KiB
+`last-applied-configuration` annotation limit — `metadata.annotations: Too long` — which stalls the
+HelmRelease and blocks `platform` plus every tier that `dependsOn` it.
+
+The part that was never automated: **with CRD install off, nothing installs the CRDs.** Kyverno's
+Deployments start, its API types do not exist, `platform` never reaches Ready. Dev has them only
+because somebody once ran the command in a shell. `preapply-crds.sh` is that command, in git:
+
+```
+helm template <chart> --repo <url> --version <constraint> --include-crds --set crds.install=true
+  | yq 'select(.kind == "CustomResourceDefinition")'
+  | kubectl apply --server-side --force-conflicts
+```
+
+- **Version derivation, not a second pin.** `chart`, `version`, `namespace` and the repo `url` are
+  read out of the HelmRelease/HelmRepository pair with `yq`. `spec.chart.spec.version` is a semver
+  **constraint** (`3.9.x`), and `helm template --version` accepts the same constraint syntax
+  helm-controller resolves — so the rendered CRDs are the ones Flux will install, and a Renovate bump
+  of that one line carries this step with it. There is nothing here that can drift.
+- **CRDs only.** Piping the *whole* `helm template` output into kubectl (the by-hand command in the
+  HelmRelease comment) would create Deployments/webhooks outside Helm's ownership and then fight the
+  HelmRelease over field managers. The `yq` filter is load-bearing.
+- **Idempotent.** Server-side apply is an upsert; `--force-conflicts` adopts the fields dev's
+  hand-applied CRDs left owned by `kubectl-client-side-apply`. Re-running prints
+  `serverside-applied` and exits 0.
+- **Cannot degrade to a no-op.** Three guards: every derived coordinate must be non-empty and not
+  `null`; the rendered CRD count must be `> 0` (an empty stream would otherwise `apply` cleanly and
+  exit 0 — the exact silent-success bug this repo keeps hitting); and after the apply every CRD is
+  waited on for `condition=Established`. There is **no skip flag** on purpose.
+- **Environment-agnostic (R-ENV-02).** It names no environment and takes no overlay input — it reads
+  repo files and acts on `$KUBECONFIG`. Dev and prod run the identical step.
+
+Standalone use (an already-bootstrapped cluster whose `platform` is blocked on the missing CRDs):
+
+```bash
+# inside infra/devcontainer, KUBECONFIG pointed at the target cluster
+bash bootstrap/flux/preapply-crds.sh
+```
+
 ## Why `flux install` + apply, not `flux bootstrap git` (Phase 1)

 The reconciliation graph is **already authored** in `clusters/base` (read-only to this worker):
@@ -98,7 +142,7 @@ tofu apply → cloud-init → Ansible (k3s + Cilium)          [Infra worker, T-1
        ▼  (final Ansible play calls the entrypoints below)
 1. bootstrap/sops/age-keygen.sh          # operator generates age key OUTSIDE repo
 2. bootstrap/sops/age-key-secret.md      # operator installs private key as sops-age Secret
-3. bootstrap/flux/bootstrap.sh           # flux install → verify sops-age → apply -k clusters/dev
+3. bootstrap/flux/bootstrap.sh           # flux install → preapply-crds.sh → verify sops-age → apply -k ${FLUX_ENV_PATH}
        │
        ▼
 Flux reconciles: GitRepository(flux-system) → Kustomization(platform) → Kustomization(platform-config)
@@ -118,8 +162,10 @@ live in `infra/devcontainer/versions.env`).
   template). `bootstrap.sh` **fails fast** if this Secret is absent.
 3. `bootstrap/flux/bootstrap.sh` (this entrypoint).

-`bootstrap.sh` is idempotent-friendly (`flux install` and `kubectl apply -k` re-run safely) and
-never handles key material — it only *verifies* the Secret exists.
+`bootstrap.sh` is idempotent-friendly (`flux install`, `preapply-crds.sh`'s server-side apply, and
+`kubectl apply -k` all re-run safely) and never handles key material — it only *verifies* the Secret
+exists. Beyond `flux`/`kubectl` it now also needs **`helm` + `yq`** on PATH (step 2's CRD render;
+both are in `infra/devcontainer`) and outbound network to the Kyverno chart repo.

 ## Handoff — clusters owner / PM (files this worker may not edit)

@@ -33,10 +33,19 @@
 #                         the single apps.yaml was retired). Letting
 #                         bootstrap regenerate them would DUPLICATE/conflict. `flux install` +
 #                         applying the authored clusters/dev overlay is the reconciling path.
-#   2. sops-age Secret  — the operator installs the age PRIVATE key out-of-band BEFORE reconcile so
+#   2. CRD pre-apply   — ./preapply-crds.sh applies the Kyverno CRD bundle SERVER-SIDE, because
+#                         platform/kyverno/admission/helmrelease.yaml sets `crds: {install: false}`
+#                         (the ~5.6 MiB bundle blows kubectl's 256 KiB last-applied annotation limit,
+#                         which STALLS the HelmRelease and blocks the `platform` Kustomization and
+#                         every tier below it). With CRD install off, NOTHING else installs them, so
+#                         a cold cluster can never reconcile `platform` without this step. The chart
+#                         version is DERIVED from that same HelmRelease — no second pin. Dev only
+#                         ever had these CRDs because they were applied BY HAND; this is that step,
+#                         in git, idempotent and version-derived. See ./preapply-crds.sh.
+#   3. sops-age Secret  — the operator installs the age PRIVATE key out-of-band BEFORE reconcile so
 #                         decryption works on first pass (../sops/age-key-secret.md). This script
 #                         only VERIFIES it exists; it never handles key material.
-#   3. `kubectl apply -k ${FLUX_ENV_PATH}` — materialises the authored reconciliation graph:
+#   4. `kubectl apply -k ${FLUX_ENV_PATH}` — materialises the authored reconciliation graph:
 #                         GitRepository `flux-system` (source) -> Kustomization `platform` ->
 #                         `platform-config` -> the app-wave Kustomizations `apps-wave-0/1/2`
 #                         (chained dependsOn) + `apps-vms`, with cluster-vars overlay
@@ -71,7 +80,10 @@ repo_root="$(git -C "${script_dir}" rev-parse --show-toplevel 2>/dev/null || (cd
 echo "==> Flux + SOPS bootstrap — path=${FLUX_ENV_PATH}"

 # --- 0. tooling present (from infra/devcontainer; versions pinned there) ------
-for bin in flux kubectl; do
+# helm + yq are needed by step 2 (./preapply-crds.sh renders the CRD bundle from the authored
+# HelmRelease). Checked HERE as well as in that script so a missing binary stops the run BEFORE the
+# Flux controllers are installed, instead of half-way through.
+for bin in flux kubectl helm yq; do
  command -v "$bin" >/dev/null 2>&1 || { echo "ERROR: '$bin' not found (use infra/devcontainer)." >&2; exit 1; }
 done

@@ -99,7 +111,7 @@ done
 #
 # ESCAPE HATCH: FLUX_INSTALL_PLAIN=1 falls back to a bare `flux install` (no customization, so NO
 # strict substitution). Use only to unblock a bring-up if the render path misbehaves, and say so.
-echo "==> [1/3] installing Flux controllers${FLUX_COMPONENTS_EXTRA:+ (+ extra: ${FLUX_COMPONENTS_EXTRA})}"
+echo "==> [1/4] installing Flux controllers${FLUX_COMPONENTS_EXTRA:+ (+ extra: ${FLUX_COMPONENTS_EXTRA})}"
 flux_install_args=(--namespace="${FLUX_NAMESPACE}")
 [ -n "${FLUX_COMPONENTS_EXTRA}" ] && flux_install_args+=(--components-extra="${FLUX_COMPONENTS_EXTRA}")

@@ -130,18 +142,29 @@ else
  kubectl apply -f "${render_dir}/rendered.yaml"
 fi

-# --- 2. verify the out-of-band age key Secret exists -------------------------
+# --- 2. pre-apply the CRDs Helm is not allowed to install ---------------------
+# platform/kyverno/admission/helmrelease.yaml sets `crds: {install: false}`, so the 22 Kyverno CRDs
+# must exist BEFORE the platform Kustomization reconciles that HelmRelease — otherwise Kyverno's
+# controllers come up with no API types, `platform` never goes Ready, and all of its dependents stall.
+# ./preapply-crds.sh derives the chart version from the HelmRelease itself (no second pin), applies
+# only the CRD documents SERVER-SIDE (no 256 KiB annotation limit), and is idempotent. It FAILS HARD
+# rather than skipping — there is deliberately NO opt-out flag: a skipped CRD apply produces a
+# cluster that looks bootstrapped and reconciles nothing.
+echo "==> [2/4] pre-applying chart CRDs (server-side)"
+bash "${script_dir}/preapply-crds.sh"
+
+# --- 3. verify the out-of-band age key Secret exists -------------------------
 # We DO NOT create it here (it holds the private key). The operator installs it per
 # ../sops/age-key-secret.md. Fail fast if missing so decryption can't silently no-op.
-echo "==> [2/3] verifying ${SOPS_SECRET_NAME} Secret exists in ${FLUX_NAMESPACE}"
+echo "==> [3/4] verifying ${SOPS_SECRET_NAME} Secret exists in ${FLUX_NAMESPACE}"
 if ! kubectl -n "${FLUX_NAMESPACE}" get secret "${SOPS_SECRET_NAME}" >/dev/null 2>&1; then
  echo "ERROR: Secret ${FLUX_NAMESPACE}/${SOPS_SECRET_NAME} not found." >&2
  echo "       Install the age private key first: bootstrap/sops/age-key-secret.md" >&2
  exit 1
 fi

-# --- 3. apply the authored environment entrypoint ----------------------------
-echo "==> [3/3] kubectl apply -k ${FLUX_ENV_PATH}"
+# --- 4. apply the authored environment entrypoint ----------------------------
+echo "==> [4/4] kubectl apply -k ${FLUX_ENV_PATH}"
 kubectl apply -k "${repo_root}/${FLUX_ENV_PATH}"

 echo
@@ -0,0 +1,169 @@
+#!/usr/bin/env bash
+#
+# preapply-crds.sh — server-side pre-apply of the CRDs that Helm is NOT allowed to install.
+# ENVIRONMENT-AGNOSTIC: it reads the chart coordinates out of the authored HelmRelease and talks to
+# whatever cluster $KUBECONFIG points at, so dev and prod run the SAME step with no fork (R-ENV-02).
+# Called by ./bootstrap.sh (step 2/4); also safe to run STANDALONE against an already-bootstrapped
+# cluster (that is how prod gets unblocked without re-running the whole bootstrap).
+#
+# R-GITOPS-01 (the graph must reconcile from git, not from someone's shell history) · R-ENV-02.
+# =============================================================================
+#   *** AUTHORED, NOT RUN. Agents must not execute it — it MUTATES a cluster. ***
+# =============================================================================
+#
+# WHY THIS EXISTS
+#
+#   platform/kyverno/admission/helmrelease.yaml sets `crds: {install: false}`. That is deliberate:
+#   the Kyverno CRD bundle is ~5.6 MiB, far past kubectl's 256 KiB last-applied-configuration
+#   annotation limit, so Helm's client-side apply fails with "metadata.annotations: Too long" and
+#   STALLS the HelmRelease — which blocks the `platform` Kustomization and every tier that dependsOn
+#   it (the kps-cascade failure mode; see the HelmRelease's own comment).
+#
+#   The consequence nobody automated: with CRD install off, NOTHING installs those CRDs. Kyverno's
+#   Deployments come up but its API types do not exist, so `platform` never goes Ready on a cold
+#   cluster. The dev cluster only has them because they were once applied BY HAND — exactly the
+#   undocumented manual step this repo exists to eliminate. This script is that step, in git,
+#   version-derived and idempotent.
+#
+# HOW THE VERSION IS DERIVED (there is deliberately NO second copy of it)
+#
+#   Everything comes out of the HelmRelease/HelmRepository pair that Flux itself reconciles:
+#
+#     chart name      spec.chart.spec.chart            (HelmRelease)
+#     chart version   spec.chart.spec.version           (HelmRelease)  e.g. "3.9.x"
+#     chart repo url  spec.url                          (HelmRepository)
+#     namespace       metadata.namespace                (HelmRelease)
+#
+#   The version is a semver CONSTRAINT, not a fixed tag. `helm template --version` accepts the same
+#   constraint syntax Flux's helm-controller resolves (both use Masterminds/semver), so passing it
+#   through UNCHANGED renders the very chart version Flux will install. Renovate bumps one line in
+#   one file and this step follows automatically — it cannot drift, because there is nothing here to
+#   drift from.
+#
+# WHY ONLY THE CRD DOCUMENTS ARE APPLIED
+#
+#   The by-hand command recorded in the HelmRelease comment piped the WHOLE `helm template` output
+#   into kubectl. Do not do that: it would create Deployments/Services/webhooks outside Helm's
+#   ownership, which then fight the HelmRelease over field managers and leave orphans behind. We
+#   filter to `kind: CustomResourceDefinition` and apply nothing else. Values are the chart DEFAULTS
+#   (plus an explicit crds.install=true) — NOT the repo's values, which switch CRD rendering off.
+#
+# IDEMPOTENCY
+#
+#   `kubectl apply --server-side --force-conflicts` is a declarative upsert: the first run creates,
+#   every later run reports "serverside-applied" and exits 0, and field ownership is taken from
+#   whatever previously wrote those objects (that is what --force-conflicts is for — it resolves the
+#   hand-applied dev CRDs' `kubectl-client-side-apply` manager without a human deciding anything).
+#   Server-side apply does not write the last-applied annotation, which is the whole reason the
+#   5.6 MiB bundle can be applied at all.
+#
+# LOUD FAILURE (this script must never degrade to a successful no-op)
+#
+#   `set -euo pipefail`, plus three explicit guards, because a silent skip here reproduces the exact
+#   class of bug this repo has been bitten by more than once:
+#     1. every derived coordinate must be non-empty and not "null" — a moved YAML path fails here;
+#     2. the rendered CRD count must be > 0 — if a future chart moves/renames its CRD templates the
+#        filter would match nothing and `kubectl apply` on an empty stream EXITS 0. Guarded;
+#     3. after applying, every CRD is waited on for condition=Established — so "applied" is proven
+#        to mean "the API type exists", not merely "kubectl returned 0".
+#
+set -euo pipefail
+
+# --- config (env-overridable) ------------------------------------------------
+# The single source of truth for the chart coordinates. Repo-relative on purpose: this is the file
+# Flux reconciles, so it is the file this step must agree with.
+CRD_HELMRELEASE="${CRD_HELMRELEASE:-platform/kyverno/admission/helmrelease.yaml}"
+# How long to wait for the applied CRDs to become Established (apiextensions publishes the type).
+CRD_ESTABLISH_TIMEOUT="${CRD_ESTABLISH_TIMEOUT:-120s}"
+
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+repo_root="$(git -C "${script_dir}" rev-parse --show-toplevel 2>/dev/null || (cd "${script_dir}/../.." && pwd))"
+
+echo "==> CRD pre-apply — source of truth: ${CRD_HELMRELEASE}"
+
+# --- 0. tooling present (from infra/devcontainer; versions pinned there) ------
+# helm renders the bundle, yq selects the CRD documents, kubectl applies them server-side. All three
+# are baked into infra/devcontainer (HELM_VERSION / YQ_VERSION / KUBECTL_VERSION in versions.env).
+for bin in helm yq kubectl; do
+  command -v "$bin" >/dev/null 2>&1 || { echo "ERROR: '$bin' not found (use infra/devcontainer)." >&2; exit 1; }
+done
+
+helmrelease="${repo_root}/${CRD_HELMRELEASE}"
+[ -f "${helmrelease}" ] || {
+  echo "ERROR: ${helmrelease} not found — CRD_HELMRELEASE points nowhere." >&2
+  exit 1
+}
+
+# --- 1. derive the chart coordinates from the authored manifests --------------
+chart_name="$(yq eval-all --no-doc 'select(.kind == "HelmRelease") | .spec.chart.spec.chart' "${helmrelease}")"
+chart_version="$(yq eval-all --no-doc 'select(.kind == "HelmRelease") | .spec.chart.spec.version' "${helmrelease}")"
+chart_ns="$(yq eval-all --no-doc 'select(.kind == "HelmRelease") | .metadata.namespace' "${helmrelease}")"
+chart_repo="$(yq eval-all --no-doc 'select(.kind == "HelmRepository") | .spec.url' "${helmrelease}")"
+
+# Guard 1: a moved/renamed YAML path must STOP the bootstrap, not render a bogus chart. yq prints
+# "null" for a missing path and an empty string for a missing document, so both are rejected.
+for pair in "chart:${chart_name}" "version:${chart_version}" "namespace:${chart_ns}" "repo:${chart_repo}"; do
+  key="${pair%%:*}"; val="${pair#*:}"
+  case "${val}" in
+    ""|null|*$'\n'*)
+      echo "ERROR: could not derive a single '${key}' from ${CRD_HELMRELEASE} (got: '${val}')." >&2
+      echo "       The HelmRelease/HelmRepository shape changed — fix the yq paths in $(basename "${BASH_SOURCE[0]}")." >&2
+      exit 1
+      ;;
+  esac
+done
+
+echo "    chart=${chart_name} version=${chart_version} repo=${chart_repo} namespace=${chart_ns}"
+echo "    (version is the constraint Flux resolves — NOT a second copy pinned here)"
+
+# --- 2. render the chart and keep ONLY the CRD documents ----------------------
+render_dir="$(mktemp -d)"
+trap 'rm -rf "${render_dir}"' EXIT
+crd_file="${render_dir}/crds.yaml"
+
+# --include-crds covers charts that ship a native crds/ dir; crds.install=true covers Kyverno, whose
+# CRDs are subchart TEMPLATES that the repo's own values switch off. Chart defaults otherwise.
+echo "==> rendering ${chart_name} ${chart_version} (CRD documents only)"
+helm template "${chart_name}" "${chart_name}" \
+  --repo "${chart_repo}" \
+  --version "${chart_version}" \
+  --namespace "${chart_ns}" \
+  --include-crds \
+  --set crds.install=true \
+  | yq eval-all 'select(.kind == "CustomResourceDefinition")' - > "${crd_file}"
+
+# Blank/"null" lines are dropped on purpose: an EMPTY render makes `yq .metadata.name` emit one
+# empty line, which would otherwise count as a CRD and sail straight past guard 2 below.
+mapfile -t crd_names < <(yq eval-all --no-doc '.metadata.name' "${crd_file}" | grep -vx -e '' -e 'null' || true)
+crd_count="${#crd_names[@]}"
+
+# Guard 2: an empty stream applies cleanly and exits 0 — the silent-no-op trap. Refuse it.
+[ "${crd_count}" -gt 0 ] || {
+  echo "ERROR: rendered 0 CustomResourceDefinitions from ${chart_name} ${chart_version}." >&2
+  echo "       The chart's CRD layout or the crds.install value key changed. Refusing to continue:" >&2
+  echo "       applying nothing would look like success and leave 'platform' blocked forever." >&2
+  exit 1
+}
+echo "    ${crd_count} CRDs, $(wc -c < "${crd_file}") bytes (>256 KiB — this is why it must be server-side)"
+
+# --- 3. server-side apply (idempotent upsert) ---------------------------------
+# --server-side: no last-applied annotation, so the size limit does not apply.
+# --force-conflicts: adopt fields owned by an earlier manager (e.g. dev's hand-applied CRDs, or
+#   Helm's own manager if a past chart version did install them). Makes a re-run a plain no-op.
+echo "==> kubectl apply --server-side --force-conflicts (${crd_count} CRDs)"
+kubectl apply --server-side --force-conflicts -f "${crd_file}"
+
+# --- 4. prove the API types actually exist -----------------------------------
+# Without this, "kubectl returned 0" is the only evidence. Establishment is what Kyverno's
+# controllers and the platform Kustomization actually need.
+echo "==> waiting for condition=Established (timeout ${CRD_ESTABLISH_TIMEOUT})"
+crd_refs=()
+for n in "${crd_names[@]}"; do crd_refs+=("crd/${n}"); done
+kubectl wait --for=condition=Established --timeout="${CRD_ESTABLISH_TIMEOUT}" \
+  "${crd_refs[@]}" >/dev/null || {
+  echo "ERROR: not all ${crd_count} CRDs reached Established within ${CRD_ESTABLISH_TIMEOUT}." >&2
+  echo "       Inspect: kubectl get crd | grep ${chart_name}" >&2
+  exit 1
+}
+
+echo "==> ${crd_count} ${chart_name} CRDs applied and Established."
@@ -0,0 +1,988 @@
+#!/usr/bin/env bash
+#
+# standup.sh — bring a bootstrapped cluster's reconciliation graph all the way up.
+#
+# ENVIRONMENT-AGNOSTIC (R-ENV-02): it reads everything it needs from the cluster it is pointed at and
+# from the authored manifests. There is no dev/prod fork. What makes it safe to run against prod is
+# the MANDATORY --expect-server guard, not a name in the filename.
+#
+# WHY THIS EXISTS
+#   `bootstrap/flux/bootstrap.sh` installs Flux and applies the graph. It does NOT finish the job when
+#   tiers have been deliberately suspended (a DNS/TLS migration window, an incident recovery) or when
+#   a HelmRelease has wedged. Those last steps were living in shell history and in a human's head,
+#   which is the same failure the Kyverno CRD pre-apply had. This is that sequence, in git, ordered,
+#   idempotent, and gated.
+#
+# THE ORDERING IS LOAD-BEARING. Read infra/PROD-STANDUP.md before deviating from it.
+#
+# USAGE
+#   bootstrap/standup.sh --kubeconfig <path> --expect-server <url> [--confirm-destructive] <step...>
+#
+#   steps:  preflight  crds  unwedge  externaldns  s3check  resume  verify   (or: all)
+#
+#   --expect-server is REQUIRED and is compared against the live API server before ANY command runs.
+#   Pointing this at the wrong cluster is the one mistake with unbounded blast radius: dev is live and
+#   shares a Cloudflare zone with prod. The guard is not optional and there is no override flag.
+#
+#   --confirm-destructive is required by `unwedge` only. Everything else is additive or reversible.
+#
+#   --s3-endpoint <url> makes `s3check` probe a CANDIDATE backup endpoint instead of the live
+#   cluster-vars value, so a proposed target can be PROVEN before it is authored into cluster-vars.
+#   `s3check` writes and then deletes ONE throwaway object under `<prefix>/.s3check/`; it changes
+#   nothing else and applies nothing. See infra/BACKUP-R2.md.
+#
+set -euo pipefail
+
+# ---------------------------------------------------------------------------- args
+KUBECONFIG_ARG="" EXPECT_SERVER="" CONFIRM_DESTRUCTIVE=0 S3_ENDPOINT_OVERRIDE=""
+STEPS=()
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --kubeconfig)          KUBECONFIG_ARG="${2:?--kubeconfig needs a value}"; shift 2 ;;
+    --expect-server)       EXPECT_SERVER="${2:?--expect-server needs a value}"; shift 2 ;;
+    --confirm-destructive) CONFIRM_DESTRUCTIVE=1; shift ;;
+    --s3-endpoint)         S3_ENDPOINT_OVERRIDE="${2:?--s3-endpoint needs a value}"; shift 2 ;;
+    -h|--help)             sed -n '2,30p' "$0"; exit 0 ;;
+    -*)                    echo "ERROR: unknown flag '$1'" >&2; exit 2 ;;
+    *)                     STEPS+=("$1"); shift ;;
+  esac
+done
+
+[ -n "${KUBECONFIG_ARG}" ] || { echo "ERROR: --kubeconfig is required." >&2; exit 2; }
+[ -n "${EXPECT_SERVER}" ]  || { echo "ERROR: --expect-server is required (safety guard)." >&2; exit 2; }
+[ "${#STEPS[@]}" -gt 0 ]   || { echo "ERROR: no step given. One of: preflight crds unwedge externaldns s3check resume verify all" >&2; exit 2; }
+
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+repo_root="$(git -C "${script_dir}" rev-parse --show-toplevel 2>/dev/null || (cd "${script_dir}/.." && pwd))"
+cd "${repo_root}"
+
+export KUBECONFIG="${KUBECONFIG_ARG}"
+[ -f "${KUBECONFIG}" ] || { echo "ERROR: kubeconfig not found: ${KUBECONFIG}" >&2; exit 1; }
+
+# Where DNS snapshots go. This MUST be a path that survives between invocations: each `dev.sh run` is
+# a FRESH container, so a $TMPDIR default silently threw the baseline away and made `verify`'s
+# before/after comparison — the check that detects damage to the OTHER cluster's records — impossible
+# to run across steps. The repo is bind-mounted, so a repo-relative dir persists on the host.
+# Gitignored: these are run artifacts, not authored state.
+SNAP_DIR="${STANDUP_SNAP_DIR:-${repo_root}/.standup-dns}"
+mkdir -p "${SNAP_DIR}"
+
+say()  { printf '\n\033[1m==> %s\033[0m\n' "$*"; }
+info() { printf '    %s\n' "$*"; }
+die()  { printf '\033[31mERROR: %s\033[0m\n' "$*" >&2; exit 1; }
+warn() { printf '\033[33mWARN: %s\033[0m\n' "$*" >&2; }
+
+for bin in kubectl flux helm yq sops; do
+  command -v "$bin" >/dev/null 2>&1 || die "'$bin' not found — run this inside infra/devcontainer."
+done
+
+# ------------------------------------------------------------------ THE SAFETY GUARD
+# Checked once, before any step, and re-checked by every mutating helper. The failure mode this
+# prevents is running a prod bring-up against the LIVE dev cluster, whose public DNS records prod
+# has already destroyed once (2026-09-04).
+live_server="$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}')"
+[ "${live_server}" = "${EXPECT_SERVER}" ] || die \
+  "TARGET MISMATCH — refusing to run.
+       --expect-server : ${EXPECT_SERVER}
+       live API server : ${live_server}
+       The kubeconfig does not point where you think it does. Nothing has been changed."
+say "target confirmed: ${live_server}"
+
+# Dependency order. Flux enforces the edges itself via dependsOn; resuming in this order simply means
+# each wait is meaningful instead of blocking on a dependency that is still suspended.
+#   flux-system -> cilium -> platform -> platform-config -> platform-config-cdi
+#                                     -> {leaf tiers, apps-wave-0} -> wave-1 -> wave-2 -> apps-vms
+#
+# `platform-config-cdi` MUST stay in this list and MUST stay before apps-vms. It carries the CDI
+# StorageProfiles, which were split out of platform-config because StorageProfile's CRD does not exist
+# until cdi-operator has reconciled the CDI CR (clusters/base/platform-config-cdi.yaml). apps-vms
+# dependsOn it: apps/vms/unifi provisions on `longhorn-vm`, and without the profile applied CDI infers
+# ReadWriteOnce instead of the authored ReadWriteMany — permanently non-live-migratable for that disk.
+# Omitting it here would make `resume` return success without ever waiting on the tier that guarantees it.
+TIER_ORDER=(platform platform-config platform-config-cdi oauth2-proxy headlamp homepage crafty apps-wave-0 apps-wave-1 apps-wave-2 apps-vms)
+# velero is NOT in that list on purpose — see step `resume`: it consumes S3_BACKUP_ENDPOINT, which is
+# an intentionally-invalid placeholder until the operator supplies the real R2 endpoint.
+
+cvar() { kubectl -n flux-system get cm cluster-vars -o jsonpath="{.data.$1}" 2>/dev/null || true; }
+
+k_ready()  { kubectl -n flux-system get kustomization "$1" \
+               -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true; }
+k_msg()    { kubectl -n flux-system get kustomization "$1" \
+               -o jsonpath='{.status.conditions[?(@.type=="Ready")].message}' 2>/dev/null || true; }
+k_susp()   { kubectl -n flux-system get kustomization "$1" -o jsonpath='{.spec.suspend}' 2>/dev/null || true; }
+
+wait_ready() { # <kustomization> <timeout-seconds>
+  local n="$1" t="${2:-600}" i=0
+  while [ "$i" -lt "$t" ]; do
+    [ "$(k_ready "$n")" = "True" ] && { info "$n Ready (${i}s)"; return 0; }
+    sleep 10; i=$((i+10))
+  done
+  warn "$n did NOT reach Ready in ${t}s: $(k_msg "$n")"
+  return 1
+}
+
+# Snapshot the Cloudflare zone. This is how damage to the OTHER cluster's records becomes detectable.
+# Queried against the Cloudflare API, never a local resolver: the workstation's own dnsmasq wildcard
+# (address=/.home.keithjackson.dev/) answers the whole tree locally and produced a false
+# healthy reading during the 2026-09-04 incident and hid the damage for a while.
+dns_snapshot() { # <output-file>
+  local out="$1" tok zid page tp
+  tok="$(sops -d platform/externaldns/cloudflare-api-token.sops.yaml \
+         | yq '.stringData."api-token"' | tr -d '"')"
+  [ -n "${tok}" ] && [ "${tok}" != "null" ] || die "could not read the Cloudflare token (SOPS/age key available?)"
+  zid="$(curl -sS -H "Authorization: Bearer ${tok}" \
+        'https://api.cloudflare.com/client/v4/zones?name=keithjackson.dev' | yq '.result[0].id' | tr -d '"')"
+  [ -n "${zid}" ] && [ "${zid}" != "null" ] || die "Cloudflare zone lookup failed"
+  page=1; : > "${out}"
+  while :; do
+    local r; r="$(curl -sS -H "Authorization: Bearer ${tok}" \
+      "https://api.cloudflare.com/client/v4/zones/${zid}/dns_records?per_page=100&page=${page}")"
+    printf '%s' "${r}" | yq -o=json '.result' \
+      | yq -r '.[] | (.type + "|" + .name + "|" + .content)' | tr -d '"' >> "${out}"
+    tp="$(printf '%s' "${r}" | yq '.result_info.total_pages' | tr -d '"')"
+    [ "${page}" -ge "${tp:-1}" ] && break
+    page=$((page+1))
+  done
+  # Strip CRs, and sort under a FIXED collation. Both matter, and both were learned the hard way:
+  #   - a stray \r makes every line differ from a baseline captured without one;
+  #   - locale-dependent sort ordering puts `*.keithjackson.dev` in a different position on the host
+  #     than in the container, so a diff reports pairs of IDENTICAL lines as changed. That produced a
+  #     "RECORDS OUTSIDE ... CHANGED" alarm on a zone where nothing had changed at all.
+  # A regression check that cries wolf is worse than none — it trains you to skip the one alarm that
+  # matters. LC_ALL=C makes the ordering reproducible everywhere.
+  tr -d '\r' < "${out}" | grep -vE '^\s*$' | LC_ALL=C sort -o "${out}"
+  info "$(wc -l < "${out}") records -> ${out}"
+}
+
+dns_diff() { # <before> <after>  — returns non-zero if anything changed
+  if diff -u "$1" "$2" > "${SNAP_DIR}/dns.diff"; then
+    info "Cloudflare zone IDENTICAL — nothing created, modified or deleted."
+    return 0
+  fi
+  warn "Cloudflare zone CHANGED:"; cat "${SNAP_DIR}/dns.diff"
+  return 1
+}
+
+# ======================================================================= s3check plumbing
+# WHY THIS IS THIS BIG. The previous version of `s3check` curl'd /minio/health/live and reported
+# REACHABLE. That probe could not pass against Cloudflare R2 (no such path), and reachability is not
+# the property anyone cares about: a reachable endpoint with a wrong key, a missing bucket, or R2's
+# checksum rejection still yields a velero tier that reports Ready while NO backup exists. What must
+# be proven is the write path Velero and CNPG-barman actually use, with the credentials they actually
+# hold, from the network position they actually occupy.
+
+# Pinned. The AWS CLI is a deliberate choice over `minio/mc`:
+#   - it prints the S3 ERROR CODE (NoSuchBucket / AccessDenied / InvalidAccessKeyId / InvalidArgument),
+#     so a failure can be NAMED and turned into a fix instead of guessed at from an HTTP status;
+#   - it exposes the FLEXIBLE-CHECKSUM behaviour that is the R2 trap
+#     (AWS_REQUEST_CHECKSUM_CALCULATION), so the trap can be reproduced on purpose here instead of
+#     discovered by a silently-failing backup at 03:00;
+#   - its image carries bash + coreutils. `minio/mc` ships no sed/awk/diff — that minimalism has
+#     already broken one probe in this repo — and it cannot be made to send the checksum headers, so
+#     it cannot test for the trap at all.
+S3CHECK_IMAGE="amazon/aws-cli:2.36.40"
+S3CHECK_RUN_ID="$(date +%s)"
+
+# Throwaway probe pods, tracked so they are removed on EVERY exit path (including `die`). A probe
+# must never leave a pod behind on a live cluster.
+S3CHECK_PODS=()
+s3_cleanup_pods() {
+  local p
+  for p in ${S3CHECK_PODS[@]+"${S3CHECK_PODS[@]}"}; do
+    kubectl -n "${p%%/*}" delete pod "${p#*/}" --ignore-not-found --wait=false >/dev/null 2>&1 || true
+  done
+  S3CHECK_PODS=()
+}
+trap s3_cleanup_pods EXIT
+
+# TRUE when the endpoint is not a usable target. `TBD-*` alone was WRONG once the placeholder became
+# `https://TBD-R2-ACCOUNT-ID.r2.cloudflarestorage.com`: that does not START with TBD-, so the old
+# prefix pattern matched nothing and `resume` would have resumed velero straight onto a placeholder
+# hostname — the exact silent-no-backups failure the guard exists to prevent. Match anywhere.
+s3_is_placeholder() { # <endpoint>
+  case "$1" in
+    ""|*TBD*|*tbd-*|*REPLACE-ME*|*replace-me*|*CHANGE-ME*|*change-me*) return 0 ;;
+  esac
+  return 1
+}
+
+# Refuse any value that is not a plain URL/name fragment. This is BOTH a safety guard — these values
+# are interpolated into the probe pod's JSON overrides AND into a shell script that runs inside it, so
+# a quote, a backslash or a newline could rewrite either — and a correctness guard: a cluster-vars
+# value with a trailing space or a stray quote is a defect, not something to paper over.
+s3_safe_token() { # <label> <value>
+  case "$2" in
+    *[!A-Za-z0-9:/._-]*) die "refusing to probe: $1 contains a character outside [A-Za-z0-9:/._-].
+       That is either a corrupt cluster-vars value or an injection attempt. Value (quoted): $(printf '%q' "$2")" ;;
+  esac
+}
+
+# ---------------------------------------------------------------- the in-pod probe
+# Emitted to the pod base64-encoded, so nothing in it has to survive JSON or shell quoting.
+# CONTRACT WITH THE CALLER (this is what makes a silent no-op impossible):
+#   - every proven property prints exactly one `S3CHECK-OK[n] ...` line and increments n;
+#   - every failure prints `S3CHECK-FAIL ...` and is counted;
+#   - the LAST line is `S3CHECK-RESULT PASS|FAIL assertions=<n> failures=<n>`.
+# The caller independently knows how many assertions this environment must produce and re-counts the
+# OK lines itself, so neither a truncated log, a skipped bucket, an empty log, nor a lying result line
+# can be read as success. (A CRD guard in this repo once counted a blank line as "1 CRD" and sailed
+# past; that class of bug is designed out here rather than commented about.)
+s3_probe_script() {
+  cat <<'PROBE'
+set -u
+umask 077
+A=0; F=0
+ok()   { A=$((A+1)); printf 'S3CHECK-OK[%d] %s\n' "$A" "$*"; }
+bad()  { F=$((F+1)); printf 'S3CHECK-FAIL %s\n' "$*"; }
+note() { printf 'S3CHECK-NOTE %s\n' "$*"; }
+verdict() {
+  if [ "${F}" -eq 0 ] && [ "${A}" -gt 0 ]; then
+    printf 'S3CHECK-RESULT PASS assertions=%d failures=0\n' "${A}"; exit 0
+  fi
+  printf 'S3CHECK-RESULT FAIL assertions=%d failures=%d\n' "${A}" "${F}"; exit 1
+}
+
+for v in S3EP S3REGION S3TARGETS S3KEY CRED_MODE; do
+  eval "x=\${${v}-}"
+  [ -n "${x}" ] || { bad "required env ${v} is empty — the probe was launched wrong"; verdict; }
+done
+
+export AWS_EC2_METADATA_DISABLED=true
+export AWS_PAGER=""
+export AWS_DEFAULT_REGION="${S3REGION}"
+export AWS_REGION="${S3REGION}"
+export AWS_MAX_ATTEMPTS=2
+export AWS_RETRY_MODE=standard
+export AWS_CONFIG_FILE=/tmp/s3check-aws-config
+# Path-style addressing, because both targets need it: MinIO on a bare IP has no virtual-host names,
+# and R2's S3 API is documented path-style. Checksums OFF, mirroring `checksumAlgorithm: ""` on the
+# Velero BSL — this run must exercise the same wire shape the real backup will use. Written to the
+# CONFIG FILE as well as the environment so the setting holds even if one channel is ignored by a
+# future CLI version.
+{
+  echo '[default]'
+  echo 'request_checksum_calculation = when_required'
+  echo 'response_checksum_validation = when_required'
+  echo 's3 ='
+  echo '    addressing_style = path'
+} > "${AWS_CONFIG_FILE}"
+
+# ---- (1) the credential loads at all -------------------------------------------------
+# NEVER print a credential value, in either `key: value` or `key=value` shape. Only booleans,
+# byte counts and lengths leave this pod.
+case "${CRED_MODE}" in
+  file)
+    cf="${AWS_SHARED_CREDENTIALS_FILE-}"
+    if [ -z "${cf}" ] || [ ! -s "${cf}" ]; then
+      bad "credential file '${cf:-<unset>}' is missing or empty — the Secret key did not mount"
+    elif ! grep -Eq '^[[:space:]]*\[default\][[:space:]]*$' "${cf}"; then
+      bad "credential file has no [default] profile. velero-plugin-for-aws reads an AWS INI file from
+        the 'cloud' key of velero-cloud-credentials; without [default] every backup fails at RUNTIME"
+    elif ! grep -Eq '^[[:space:]]*aws_access_key_id[[:space:]]*=[[:space:]]*[^[:space:]]' "${cf}" \
+      || ! grep -Eq '^[[:space:]]*aws_secret_access_key[[:space:]]*=[[:space:]]*[^[:space:]]' "${cf}"; then
+      bad "credential file [default] lacks a non-empty aws_access_key_id / aws_secret_access_key"
+    else
+      ok "credential LOADS: AWS INI with [default] + both keys ($(wc -c < "${cf}") bytes; values never printed)"
+    fi
+    ;;
+  env)
+    if [ -z "${AWS_ACCESS_KEY_ID-}" ] || [ -z "${AWS_SECRET_ACCESS_KEY-}" ]; then
+      bad "AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY did not arrive from the Secret (missing ACCESS_KEY_ID / ACCESS_SECRET_KEY key?)"
+    else
+      ok "credential LOADS: access key id present (${#AWS_ACCESS_KEY_ID} chars; values never printed)"
+    fi
+    ;;
+  *) bad "unknown CRED_MODE '${CRED_MODE}'" ;;
+esac
+[ "${F}" -eq 0 ] || verdict     # no point talking to S3 with no credential
+
+OUT=""; RC=0
+awsx() { OUT="$(aws --endpoint-url "${S3EP}" "$@" 2>&1)"; RC=$?; return 0; }
+
+detail() { printf '%s\n' "${OUT}" | head -n 12 | while IFS= read -r l; do printf 'S3CHECK-DETAIL %s\n' "${l}"; done; }
+
+# Every failure is NAMED and given the fix. An unclassified failure is still a failure.
+diagnose() { # <operation label>
+  local op="$1"
+  case "${OUT}" in
+    *InvalidArgument*|*checksum*|*Checksum*)
+      bad "${op}: 400 InvalidArgument — THE R2 CHECKSUM TRAP. velero-plugin-for-aws v1.14+ bundles an
+        AWS SDK that sends x-amz-sdk-checksum-algorithm / x-amz-checksum-crc32 on every PUT, and
+        Cloudflare R2 REJECTS those headers with HTTP 400 InvalidArgument, so every Velero backup and
+        every kopia blob upload fails. FIX: checksumAlgorithm: \"\" must be present in the
+        backupStorageLocation config in platform/velero/helmrelease.yaml. This probe already disables
+        checksums, so seeing this here means the client sent one anyway — treat it as a hard blocker." ;;
+    *"Could not connect"*|*EndpointConnectionError*|*ConnectTimeoutError*|*"Failed to establish"*|\
+    *"Name or service not known"*|*"nodename nor servname"*|*"Temporary failure in name resolution"*|\
+    *"Connection reset"*|*"Read timeout"*)
+      bad "${op}: this POD cannot reach ${S3EP} (connect / DNS / timeout). Velero and CNPG would
+        install perfectly clean and then fail every backup at RUNTIME — a tier that looks healthy
+        while no backup exists. This is the reading that matters: the workstation's own reachability
+        is irrelevant (prod sits on the SOHO net, pfSense's WAN; the dev LAN's MinIO sits INSIDE
+        pfSense) and a host-side probe has already misled this project once. Fix pod egress/routing,
+        or give this environment its own endpoint." ;;
+    *NoSuchBucket*|*"(404)"*|*"Not Found"*)
+      bad "${op}: the bucket does not exist at this endpoint. Create it before enabling velero/CNPG
+        (R2: dashboard -> R2 -> Create bucket; see infra/BACKUP-R2.md)." ;;
+    *InvalidAccessKeyId*|*SignatureDoesNotMatch*|*"(401)"*|*Unauthorized*)
+      bad "${op}: the endpoint REJECTED these credentials. Wrong key, or a key belonging to a
+        different R2 account / MinIO. Re-encrypt the credential Secrets — infra/BACKUP-R2.md." ;;
+    *AccessDenied*|*"(403)"*|*Forbidden*)
+      bad "${op}: authenticated but FORBIDDEN. The token is not scoped to this bucket. An R2 API token
+        needs Object Read & Write on BOTH the velero and cnpg buckets." ;;
+    *) bad "${op}: unclassified failure (rc=${RC}). Treated as a FAILURE, never as unknown-so-fine." ;;
+  esac
+  detail
+}
+
+PAY=/tmp/s3check.payload
+GOT=/tmp/s3check.got
+head -c 4096 /dev/urandom | base64 > "${PAY}"
+[ -s "${PAY}" ] || { bad "could not generate a probe payload inside the pod"; verdict; }
+
+TRAP_DONE=0
+for t in ${S3TARGETS}; do
+  B="${t%%:*}"; P="${t#*:}"
+  [ "${P}" = "${B}" ] && P=""
+  if [ -n "${P}" ]; then K="${P}/.s3check/${S3KEY}"; else K=".s3check/${S3KEY}"; fi
+
+  # ---- (2) the bucket exists and is LISTABLE. This is the operation Velero's BSL sync and Barman's
+  #          WAL scan both perform; ListBuckets deliberately is NOT used, because neither consumer
+  #          needs it and a correctly bucket-scoped R2 token may not grant it.
+  args=(s3api list-objects-v2 --bucket "${B}" --max-keys 1)
+  [ -n "${P}" ] && args+=(--prefix "${P}/")
+  awsx "${args[@]}"
+  if [ "${RC}" -eq 0 ]; then
+    ok "${B}: bucket EXISTS, AUTHENTICATES and is LISTABLE (prefix '${P:-<bucket root>}')"
+  else
+    diagnose "LIST ${B}"
+    continue      # a short assertion count is how the caller learns this target was abandoned
+  fi
+
+  # ---- (3) PUT -> GET -> byte-compare -> DELETE. A real write, not a HEAD: HEAD on a bucket proves
+  #          nothing about whether an object can be STORED (wrong scope, checksum rejection and
+  #          read-only tokens all pass a HEAD and fail a backup).
+  awsx s3api put-object --bucket "${B}" --key "${K}" --body "${PAY}"
+  if [ "${RC}" -eq 0 ]; then ok "${B}: PUT ${K} accepted (genuine write path)"
+  else diagnose "PUT ${B}/${K}"; continue; fi
+
+  rm -f "${GOT}"
+  awsx s3api get-object --bucket "${B}" --key "${K}" "${GOT}"
+  if [ "${RC}" -eq 0 ]; then ok "${B}: GET ${K} returned an object"
+  else diagnose "GET ${B}/${K}"
+       aws --endpoint-url "${S3EP}" s3api delete-object --bucket "${B}" --key "${K}" >/dev/null 2>&1 || true
+       continue; fi
+
+  sp="$(wc -c < "${PAY}")"; sg="$(wc -c < "${GOT}" 2>/dev/null || echo 0)"
+  hp="$(sha256sum < "${PAY}")"; hp="${hp%% *}"
+  hg="$(sha256sum < "${GOT}")"; hg="${hg%% *}"
+  if [ "${sp}" = "${sg}" ] && [ -n "${hp}" ] && [ "${hp}" = "${hg}" ]; then
+    ok "${B}: round trip is BYTE-IDENTICAL (${sp} bytes, sha256 matches)"
+  else
+    bad "${B}: round trip CORRUPTED — wrote ${sp} bytes (sha256 ${hp}), read back ${sg} bytes (sha256 ${hg})"
+  fi
+
+  awsx s3api delete-object --bucket "${B}" --key "${K}"
+  if [ "${RC}" -eq 0 ]; then ok "${B}: DELETE ${K} accepted"
+  else diagnose "DELETE ${B}/${K}"; fi
+
+  awsx s3api head-object --bucket "${B}" --key "${K}"
+  if [ "${RC}" -ne 0 ]; then ok "${B}: scratch object is GONE — the probe left nothing behind"
+  else bad "${B}: scratch object ${K} STILL EXISTS after DELETE. Remove it by hand; the probe leaked."; fi
+
+  # ---- (4) the R2 checksum trap, probed ON PURPOSE. Informational by design: the authored BSL sets
+  #          checksumAlgorithm: "", so a rejection here does NOT mean backups are broken — it means
+  #          that line is LOAD-BEARING. Reported either way so the operator learns which world this
+  #          endpoint lives in before, not after, enabling velero.
+  if [ "${TRAP_DONE}" -eq 0 ]; then
+    TRAP_DONE=1
+    TK="${K}.checksum-trap-probe"
+    OUT="$(AWS_REQUEST_CHECKSUM_CALCULATION=when_supported \
+           aws --endpoint-url "${S3EP}" s3api put-object --bucket "${B}" --key "${TK}" --body "${PAY}" 2>&1)"; RC=$?
+    if [ "${RC}" -eq 0 ]; then
+      note "checksum probe: this endpoint ACCEPTS a checksum-bearing PUT — the R2 checksum trap is NOT
+        present here (expected for MinIO). checksumAlgorithm: \"\" stays harmless."
+      aws --endpoint-url "${S3EP}" s3api delete-object --bucket "${B}" --key "${TK}" >/dev/null 2>&1 \
+        || note "checksum probe: could NOT delete ${TK} — remove it by hand"
+    else
+      case "${OUT}" in
+        *InvalidArgument*)
+          note "checksum probe: R2 CHECKSUM TRAP CONFIRMED at this endpoint — a PUT carrying
+        x-amz-sdk-checksum-algorithm / x-amz-checksum-crc32 is rejected with 400 InvalidArgument.
+        checksumAlgorithm: \"\" in platform/velero/helmrelease.yaml is LOAD-BEARING: delete it, or bump
+        velero-plugin-for-aws without re-checking it, and EVERY backup breaks." ;;
+        *) note "checksum probe: inconclusive (rc=${RC}) — neither confirms nor rules out the trap." ;;
+      esac
+    fi
+  fi
+done
+
+verdict
+PROBE
+}
+
+# ---------------------------------------------------------------- run one probe pod
+# <ns> <file|env> <secret> <space-separated bucket:prefix targets>  <endpoint> <region>
+s3_probe() {
+  local ns="$1" mode="$2" sec="$3" targets="$4" ep="$5" region="$6"
+  # $$ alone is NOT unique here: inside the dev container this script is frequently PID 1, so two runs
+  # would reuse the same pod name AND the same scratch object key. Mix in the epoch second.
+  local pod="s3check-$$-${S3CHECK_RUN_ID}-${mode}"
+
+  # What this environment MUST produce: 1 credential assertion + 6 per bucket (list, put, get,
+  # byte-compare, delete, delete-verified). Computed HERE, by the caller, from the target list — the
+  # pod's own claim is never the only source of the expected count.
+  local t n=0
+  for t in ${targets}; do n=$((n+1)); done
+  local min=$((1 + 6 * n))
+
+  local volsjson="" mountsjson="" creds=""
+  case "${mode}" in
+    file)  # exactly how velero consumes it: the `cloud` key as an AWS INI credentials FILE
+      volsjson='"volumes":[{"name":"s3creds","secret":{"secretName":"'"${sec}"'","items":[{"key":"cloud","path":"credentials"}],"defaultMode":256}}],'
+      mountsjson='"volumeMounts":[{"name":"s3creds","mountPath":"/s3creds","readOnly":true}],'
+      creds='{"name":"AWS_SHARED_CREDENTIALS_FILE","value":"/s3creds/credentials"},{"name":"CRED_MODE","value":"file"}' ;;
+    env)   # exactly how CNPG/barman consumes it: discrete ACCESS_KEY_ID / ACCESS_SECRET_KEY keys
+      creds='{"name":"AWS_ACCESS_KEY_ID","valueFrom":{"secretKeyRef":{"name":"'"${sec}"'","key":"ACCESS_KEY_ID"}}},{"name":"AWS_SECRET_ACCESS_KEY","valueFrom":{"secretKeyRef":{"name":"'"${sec}"'","key":"ACCESS_SECRET_KEY"}}},{"name":"CRED_MODE","value":"env"}' ;;
+    *) die "internal: unknown probe mode '${mode}'" ;;
+  esac
+
+  local b64; b64="$(s3_probe_script | base64 | tr -d '\n')"
+  [ -n "${b64}" ] || die "internal: probe script encoded to nothing"
+
+  # egress-class label: harmless here (the velero namespace is not an app namespace) but required by
+  # the require-egress-class-label ClusterPolicy in any namespace labelled app-namespace=true, which
+  # the CNPG credential namespaces are. Without it the probe pod would be REJECTED by admission and
+  # the check would report "unknown" instead of a result.
+  local overrides
+  overrides='{"apiVersion":"v1","metadata":{"labels":{"app.kubernetes.io/name":"s3check","homelab.keithjackson.dev/egress-class":"internet"}},"spec":{"restartPolicy":"Never","automountServiceAccountToken":false,'"${volsjson}"'"containers":[{"name":"s3check","image":"'"${S3CHECK_IMAGE}"'","imagePullPolicy":"IfNotPresent","command":["/bin/bash","-c","printf %s \"$PROBE_B64\" | base64 -d > /tmp/probe.sh && exec bash /tmp/probe.sh"],'"${mountsjson}"'"env":['"${creds}"',{"name":"PROBE_B64","value":"'"${b64}"'"},{"name":"S3EP","value":"'"${ep}"'"},{"name":"S3REGION","value":"'"${region}"'"},{"name":"S3TARGETS","value":"'"${targets}"'"},{"name":"S3KEY","value":"'"${pod}"'.probe"},{"name":"AWS_REQUEST_CHECKSUM_CALCULATION","value":"when_required"},{"name":"AWS_RESPONSE_CHECKSUM_VALIDATION","value":"when_required"}],"resources":{"requests":{"cpu":"50m","memory":"64Mi"},"limits":{"memory":"256Mi"}}}]}}'
+
+  say "probe: ns=${ns} secret=${sec} (${mode}) targets='${targets}'  → ${min} assertions required"
+  kubectl -n "${ns}" delete pod "${pod}" --ignore-not-found >/dev/null 2>&1 || true
+  S3CHECK_PODS+=("${ns}/${pod}")
+  kubectl -n "${ns}" run "${pod}" --restart=Never --image="${S3CHECK_IMAGE}" \
+      --overrides="${overrides}" >/dev/null 2>&1 \
+    || die "could NOT create the probe pod ${ns}/${pod} (admission policy? RBAC? quota?).
+       That is a FAILURE, not an unknown: nothing about ${ep} has been proven."
+
+  local i=0 phase=""
+  while [ "${i}" -lt 300 ]; do
+    phase="$(kubectl -n "${ns}" get pod "${pod}" -o jsonpath='{.status.phase}' 2>/dev/null || true)"
+    case "${phase}" in Succeeded|Failed) break ;; esac
+    sleep 5; i=$((i+5))
+  done
+  case "${phase}" in
+    Succeeded|Failed) ;;
+    *) local reason
+       reason="$(kubectl -n "${ns}" get pod "${pod}" \
+                 -o jsonpath='{.status.containerStatuses[0].state.waiting.reason}' 2>/dev/null || true)"
+       die "probe pod ${ns}/${pod} did not finish in ${i}s (phase=${phase:-<none>}, waiting=${reason:-none}).
+       NOT a pass. If waiting=ErrImagePull/ImagePullBackOff this cluster cannot pull ${S3CHECK_IMAGE}." ;;
+  esac
+
+  local log; log="$(kubectl -n "${ns}" logs "${pod}" 2>/dev/null || true)"
+  [ -n "${log}" ] || die "probe pod ${ns}/${pod} produced NO output (phase=${phase}). Failing closed —
+       an empty log is the shape of a probe that never ran, not of a healthy backup target."
+  printf '%s\n' "${log}" | while IFS= read -r l; do printf '    %s\n' "${l}"; done
+
+  # ---- the four independent accounting checks that make a silent no-op impossible ----
+  local okcount failcount resline nres verd a f
+  okcount="$(printf '%s\n' "${log}" | grep -c '^S3CHECK-OK\[' || true)"
+  failcount="$(printf '%s\n' "${log}" | grep -c '^S3CHECK-FAIL' || true)"
+  nres="$(printf '%s\n' "${log}" | grep -c '^S3CHECK-RESULT ' || true)"
+  [ "${nres}" = "1" ] || die "expected exactly ONE S3CHECK-RESULT line from ${ns}/${pod}, found ${nres}.
+       A missing verdict line means the probe died mid-way; it is never a pass."
+  resline="$(printf '%s\n' "${log}" | grep '^S3CHECK-RESULT ')"
+  read -r _ verd a f <<<"${resline}"
+  a="${a#assertions=}"; f="${f#failures=}"
+  case "${a}" in ''|*[!0-9]*) die "malformed verdict from ${ns}/${pod}: '${resline}'" ;; esac
+  case "${f}" in ''|*[!0-9]*) die "malformed verdict from ${ns}/${pod}: '${resline}'" ;; esac
+  [ "${verd}" = "PASS" ] || die "S3 ROUND TRIP FAILED against ${ep} (${ns}/${pod}: ${f} failure(s)).
+       Read the S3CHECK-FAIL lines above — each one names the cause and the fix. Do NOT enable velero
+       or CNPG backups against this endpoint until they are all gone."
+  [ "${f}" = "0" ]          || die "${ns}/${pod} claimed PASS while reporting ${f} failures — refusing to believe it."
+  [ "${failcount}" = "0" ]  || die "${ns}/${pod} claimed PASS but the log carries ${failcount} S3CHECK-FAIL line(s)."
+  [ "${a}" -ge "${min}" ]   || die "${ns}/${pod} claimed PASS with only ${a} assertions; ${min} are required
+       (1 credential + 6 per bucket). A short count means steps were SKIPPED — exactly the
+       degrade-to-a-successful-no-op this check exists to prevent."
+  [ "${okcount}" = "${a}" ] || die "assertion accounting mismatch for ${ns}/${pod}: ${okcount} S3CHECK-OK
+       lines in the log vs a claimed ${a}. Refusing to pass on a self-report that does not add up."
+  [ "${phase}" = "Succeeded" ] || die "${ns}/${pod} printed PASS but the pod ended ${phase}."
+  info "PROVEN: ${a} assertions, 0 failures, pod ${phase}, no object left behind."
+}
+
+# ============================================================================ steps
+step_preflight() {
+  say "PREFLIGHT (read-only)"
+  info "nodes Ready: $(kubectl get nodes --no-headers | awk '$2=="Ready"' | wc -l)/$(kubectl get nodes --no-headers | wc -l)"
+  info "flux controllers Running: $(kubectl -n flux-system get pods --no-headers | awk '$3=="Running"' | wc -l)"
+  # Count BOTH groups the Kyverno chart ships. Grepping only 'kyverno.io' undercounts by 2 — the
+  # policy-report types are in the wgpolicyk8s.io group — which reads as "CRDs are missing" when the
+  # pre-apply in fact succeeded. Expect 22 for kyverno 3.9.x.
+  info "kyverno CRDs present: $(kubectl get crd --no-headers 2>/dev/null | grep -cE 'kyverno\.io|wgpolicyk8s\.io' || true) (expect 22 for kyverno 3.9.x)"
+
+  local base owner filter s3 acme
+  base="$(cvar BASE_DOMAIN)"; owner="$(cvar EXTERNALDNS_TXT_OWNER_ID)"
+  filter="$(cvar EXTERNALDNS_DOMAIN_FILTER)"; s3="$(cvar S3_BACKUP_ENDPOINT)"; acme="$(cvar ACME_CLUSTER_ISSUER)"
+  say "cluster-vars guards"
+  info "BASE_DOMAIN                = ${base}"
+  info "EXTERNALDNS_TXT_OWNER_ID   = ${owner}"
+  info "EXTERNALDNS_DOMAIN_FILTER  = ${filter}"
+  info "ACME_CLUSTER_ISSUER        = ${acme}"
+  info "S3_BACKUP_ENDPOINT         = ${s3}"
+
+  # THE GUARD THAT MATTERS. Two clusters sharing one Cloudflare zone are isolated by txtOwnerId and
+  # by serving disjoint hostnames — NOT by domainFilters (finding F-DNS1: a subdomain filter drops the
+  # parent zone and publishes nothing). Both clusters inheriting the chart default `default` is the
+  # single root cause of the 2026-09-04 incident, in which prod deleted every one of dev's records.
+  # preflight is a DIAGNOSTIC: it must always finish and report. The only thing it refuses over is the
+  # target guard, because that one means you are pointed at the wrong cluster. Everything else is
+  # reported and warned about here, and REFUSED on the mutating path (step_externaldns) where acting
+  # on a bad value is what actually causes harm. Dying here instead hid the rest of dev's health
+  # report behind a condition that was, for dev, simply the long-standing status quo.
+  if [ -z "${owner}" ]; then
+    warn "EXTERNALDNS_TXT_OWNER_ID is EMPTY — external-dns falls back to the chart default 'default'.
+       Safe only if this is the sole cluster publishing into the zone. 'externaldns' will refuse."
+  elif [ "${owner}" = "unset-see-cluster-vars" ]; then
+    warn "EXTERNALDNS_TXT_OWNER_ID is still the base sentinel — the overlay did not apply.
+       'externaldns' will refuse until a real per-environment value is set."
+  elif [ "${owner}" = "default" ]; then
+    # NOT fatal here, because `default` is DEV's legitimate, established owner id — its live records
+    # are registered under it and cannot be renamed without orphaning them. Dying on it would make
+    # this script refuse to run against dev at all, which would be an environment fork (R-ENV-02) in
+    # a script whose whole claim is that it has none. preflight only READS, so it warns; the
+    # MUTATING path (step_externaldns) still refuses outright.
+    warn "EXTERNALDNS_TXT_OWNER_ID is 'default' — the Helm chart's default value.
+       That is correct for DEV (its records are registered under it) and WRONG for any second cluster
+       sharing this zone: two clusters both defaulting to 'default' is the exact root cause of the
+       2026-09-04 incident, in which prod treated dev's TXT registry entries as its own and
+       policy: sync deleted the records behind them."
+  else
+    info "owner id is distinct and non-default: OK"
+  fi
+
+  if s3_is_placeholder "${s3}"; then
+    warn "S3_BACKUP_ENDPOINT is unset/placeholder ('${s3}'). velero AND apps-wave-0 consume it.
+       velero will be SKIPPED by the resume step. apps-wave-0 will still apply (the token IS defined,
+       so strict substitution succeeds) but anything writing to that endpoint fails at RUNTIME, not at
+       apply time — so a green apps-wave-0 does NOT mean backups work. Operator decision required."
+  else
+    info "S3 backup endpoint set: OK"
+  fi
+
+  # COLD-START CRD ORDERING. Flux server-side dry-runs EVERY object in a Kustomization before applying
+  # any of them, so a single unknown `kind` aborts the whole tier. platform-config applies the CDI CR
+  # *and* CDI StorageProfiles in the same pass — but the StorageProfile CRD does not exist until the
+  # cdi-operator has reconciled that CR. On an established cluster the CRDs are long since there and
+  # this is invisible; on a COLD one it is a deadlock, and it is the same class of bug as Kyverno's
+  # 5.7 MiB CRD bundle. Report it here so `resume` does not burn 15 minutes discovering it.
+  say "CRDs that later tiers apply resources against"
+  local crd missing=0
+  for crd in storageprofiles.cdi.kubevirt.io cdis.cdi.kubevirt.io kubevirts.kubevirt.io \
+             volumesnapshotclasses.snapshot.storage.k8s.io; do
+    if kubectl get crd "${crd}" >/dev/null 2>&1; then
+      printf '    %-52s present\n' "${crd}"
+    else
+      printf '    \033[33m%-52s MISSING\033[0m\n' "${crd}"; missing=1
+    fi
+  done
+  if [ "${missing}" = "1" ]; then
+    warn "a MISSING crd above will fail platform-config's dry-run and block every tier below it.
+       CDI's own CRDs (StorageProfile, DataVolume, ...) are created by cdi-operator only AFTER it
+       reconciles the CDI CR. Check that the CDI CR is applied and cdi-operator is Available; if the
+       CRD appeared only recently this can also be a stale discovery cache in kustomize-controller,
+       which a forced reconcile clears."
+    kubectl -n cdi get cdi,deploy 2>/dev/null | head -12 || info "(no cdi namespace yet)"
+  fi
+
+  say "tiers"
+  local n
+  for n in flux-system cilium "${TIER_ORDER[@]}" velero; do
+    printf '    %-16s suspend=%-6s ready=%-6s %s\n' "$n" "$(k_susp "$n")" "$(k_ready "$n")" "$(k_msg "$n" | head -c 90)"
+  done
+
+  say "HelmReleases not Ready"
+  kubectl get helmrelease -A \
+    -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name,SUSPEND:.spec.suspend,READY:.status.conditions[?\(@.type==\"Ready\"\)].status,MSG:.status.conditions[?\(@.type==\"Ready\"\)].message \
+    --no-headers 2>/dev/null | grep -v ' True ' || info "(all Ready)"
+
+  # SUSPENDED releases are invisible above: a suspended HelmRelease keeps whatever Ready condition it
+  # last had, so a STALE `Ready=True` hides the fact that Flux is not managing it at all. external-dns
+  # is deliberately in this state until `standup.sh externaldns` runs.
+  say "HelmReleases SUSPENDED (Flux is not reconciling these — a stale Ready=True means nothing)"
+  kubectl get helmrelease -A \
+    -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name,SUSPEND:.spec.suspend --no-headers 2>/dev/null \
+    | awk '$3=="true"' || true
+}
+
+step_crds() {
+  say "CRD PRE-APPLY (idempotent)"
+  # Charts whose CRD bundle exceeds kubectl's 256 KiB annotation limit set crds.install=false, so
+  # nothing installs them and `platform` can never go Ready. See bootstrap/flux/preapply-crds.sh.
+  bash "${script_dir}/flux/preapply-crds.sh"
+
+  # ---------------------------------------------------------------- the CDI cold-start deadlock
+  # SAME CLASS OF BUG, DIFFERENT OPERATOR. `platform-config` applies the CDI CR *and*
+  # platform/kubevirt/cdi-storageprofiles.yaml in ONE pass. But StorageProfile's CRD does not exist
+  # until cdi-operator has reconciled that CR — and Flux server-side dry-runs EVERY object before
+  # applying ANY of them, so the unknown kind aborts the whole tier, the CR is never applied, the CRD
+  # is never created, and platform-config can never converge. A true deadlock, invisible on an
+  # established cluster (the CRDs are long since there) and fatal on a cold one.
+  #
+  # Breaking it needs exactly one thing: get the CDI CR in. Then cdi-operator creates the remaining
+  # CDI CRDs and platform-config's next reconcile dry-runs clean. We apply ONLY the CR — the minimum
+  # — and leave every other object to Flux. The CR carries no ${} tokens, so no substitution is
+  # needed and applying it here cannot diverge from what Flux would render.
+  local cdi_cr="${repo_root}/platform/kubevirt/cdi-cr.yaml"
+  if kubectl get crd storageprofiles.cdi.kubevirt.io >/dev/null 2>&1; then
+    info "storageprofiles CRD already present — CDI is past its cold start"
+  elif [ ! -f "${cdi_cr}" ]; then
+    warn "expected ${cdi_cr} and it is missing — skipping the CDI unblock"
+  else
+    say "CDI cold-start unblock: applying the CDI CR so cdi-operator can create its CRDs"
+    kubectl get ns cdi >/dev/null 2>&1 || die "namespace 'cdi' absent — the platform tier has not applied the CDI operator yet. Run 'resume' far enough to make platform Ready first."
+    kubectl apply --server-side --force-conflicts -f "${cdi_cr}"
+    local i=0
+    while [ "$i" -lt 300 ]; do
+      kubectl get crd storageprofiles.cdi.kubevirt.io >/dev/null 2>&1 && break
+      sleep 10; i=$((i+10))
+    done
+    kubectl get crd storageprofiles.cdi.kubevirt.io >/dev/null 2>&1 \
+      || die "cdi-operator did not create storageprofiles.cdi.kubevirt.io within ${i}s.
+       Inspect: kubectl -n cdi get cdi,pods,deploy — platform-config stays blocked until it exists."
+    kubectl wait --for=condition=Established --timeout=60s crd/storageprofiles.cdi.kubevirt.io >/dev/null
+    info "storageprofiles.cdi.kubevirt.io created and Established (${i}s)"
+  fi
+}
+
+step_unwedge() {
+  say "UNWEDGE STUCK HELM RELEASES"
+  # A Helm release left in uninstalling/pending-* cannot be advanced by helm-controller: it reports
+  # StateError ("unable to determine state for release with status 'uninstalling'") forever, which
+  # keeps `platform` (wait: true) not-Ready and every tier below it blocked.
+  #
+  # How prod got here: Kyverno's install timed out because its pods were crash-looping with no CRDs;
+  # install.remediation.retries:3 then fired an uninstall, and that uninstall timed out too.
+  #
+  # Pre-applied CRDs live OUTSIDE the Helm release and SURVIVE this — that is precisely why the
+  # pre-apply is a separate step and must run before this one.
+  local found=0 ns rel st
+  while read -r ns rel st; do
+    [ -n "${ns:-}" ] || continue
+    found=1
+    warn "${ns}/${rel} is '${st}'"
+    [ "${CONFIRM_DESTRUCTIVE}" = "1" ] || {
+      info "DRY RUN. This DELETES the release's Kubernetes objects and lets Flux install cold."
+      info "Re-run with --confirm-destructive to act."
+      continue
+    }
+    say "clearing ${ns}/${rel}"
+    # --no-hooks and --wait=false: the WAIT is what hung the original remediation, so do not repeat it.
+    helm uninstall "${rel}" -n "${ns}" --no-hooks --wait=false || warn "helm uninstall returned non-zero; continuing to check state"
+    if helm list -n "${ns}" --all -q 2>/dev/null | grep -qx "${rel}"; then
+      warn "${ns}/${rel} still present after uninstall — inspect manually, NOT deleting release secrets by hand."
+    else
+      info "${ns}/${rel} cleared"
+      flux reconcile helmrelease "${rel}" -n "${ns}" --force --timeout=10m || warn "reconcile of ${ns}/${rel} did not complete in 10m"
+    fi
+  done < <(
+    for ns in $(kubectl get helmrelease -A -o jsonpath='{range .items[*]}{.metadata.namespace}{"\n"}{end}' | sort -u); do
+      helm list -n "${ns}" --all -o json 2>/dev/null \
+        | yq -r '.[] | select(.status | test("uninstalling|pending-install|pending-upgrade|pending-rollback")) | "'"${ns}"' " + .name + " " + .status' 2>/dev/null || true
+    done
+  )
+  [ "${found}" = "1" ] || info "no wedged releases found."
+}
+
+step_externaldns() {
+  say "EXTERNAL-DNS — SAFE UPGRADE (prove the owner id BEFORE the pod can run)"
+  # THE HAZARD: a Helm upgrade sets replicas back to the chart default (1) and applies the new args in
+  # ONE operation. There is no window in which to check the result first. If --txt-owner-id is missing
+  # the pod starts as owner `default`, claims the other cluster's TXT registry entries, and under
+  # `policy: sync` DELETES the records behind them. That is the 2026-09-04 incident.
+  #
+  # THE FIX IS TO VERIFY IN ADVANCE, NOT AFTER: render the chart with the LIVE HelmRelease values and
+  # assert the flag appears. Rendering is deterministic and touches nothing, so this converts a race
+  # into a precondition.
+  local ns=external-dns rel=external-dns
+  local owner chart ver repo vals
+  owner="$(cvar EXTERNALDNS_TXT_OWNER_ID)"
+  [ -n "${owner}" ] && [ "${owner}" != "default" ] && [ "${owner}" != "unset-see-cluster-vars" ] \
+    || die "refusing: EXTERNALDNS_TXT_OWNER_ID is '${owner}'"
+
+  chart="$(kubectl -n "${ns}" get helmrelease "${rel}" -o jsonpath='{.spec.chart.spec.chart}')"
+  ver="$(kubectl -n "${ns}" get helmrelease "${rel}" -o jsonpath='{.spec.chart.spec.version}')"
+  repo="$(yq eval-all --no-doc 'select(.kind == "HelmRepository") | .spec.url' platform/externaldns/helmrepository.yaml | tr -d '"')"
+  [ -n "${repo}" ] || die "could not derive the external-dns chart repo url"
+
+  vals="${SNAP_DIR}/ed-values.yaml"
+  kubectl -n "${ns}" get helmrelease "${rel}" -o jsonpath='{.spec.values}' | yq -P > "${vals}"
+  info "chart=${chart} version=${ver} repo=${repo}"
+  info "live spec.values.txtOwnerId = $(yq '.txtOwnerId' "${vals}")"
+
+  say "rendering the chart with the LIVE values to prove the flag"
+  local rendered="${SNAP_DIR}/ed-rendered.yaml"
+  helm template "${rel}" "${chart}" --repo "${repo}" --version "${ver}" -n "${ns}" -f "${vals}" > "${rendered}"
+  local args; args="$(yq -r 'select(.kind == "Deployment") | .spec.template.spec.containers[0].args[]' "${rendered}")"
+  printf '%s\n' "${args}" | sed 's/^/      /'
+  printf '%s\n' "${args}" | grep -qx -- "--txt-owner-id=${owner}" \
+    || die "the render does NOT contain --txt-owner-id=${owner}.
+       An upgrade would start external-dns as owner 'default' and delete the other cluster's records.
+       REFUSING. Fix the chart values key before retrying."
+  info "PROVEN: the upgrade will carry --txt-owner-id=${owner}"
+
+  say "snapshotting the Cloudflare zone (baseline for the post-check)"
+  dns_snapshot "${SNAP_DIR}/dns-before.txt"
+
+  say "resuming + forcing the upgrade"
+  kubectl -n "${ns}" get helmrelease "${rel}" -o jsonpath='{.spec.suspend}' | grep -q true \
+    && flux resume helmrelease "${rel}" -n "${ns}" || info "(HelmRelease was not suspended)"
+  flux reconcile helmrelease "${rel}" -n "${ns}" --force --timeout=10m || warn "reconcile did not complete in 10m"
+
+  say "verifying the LIVE deployment args"
+  local live; live="$(kubectl -n "${ns}" get deploy "${rel}" -o jsonpath='{.spec.template.spec.containers[0].args}' 2>/dev/null || true)"
+  printf '    %s\n' "${live}"
+  printf '%s' "${live}" | grep -q -- "--txt-owner-id=${owner}" \
+    || { kubectl -n "${ns}" scale deploy "${rel}" --replicas=0 || true
+         die "live args LACK --txt-owner-id=${owner}. Scaled to 0. Investigate before scaling up."; }
+  info "live deployment carries --txt-owner-id=${owner}"
+
+  say "post-check: did anything in the shared zone change?"
+  sleep 45   # external-dns --interval=1m; give it a full cycle to act before judging it safe
+  dns_snapshot "${SNAP_DIR}/dns-after.txt"
+  dns_diff "${SNAP_DIR}/dns-before.txt" "${SNAP_DIR}/dns-after.txt" \
+    || warn "records changed — expected ONLY additions under this cluster's own BASE_DOMAIN ($(cvar BASE_DOMAIN)).
+       Any change to a name outside it, or to a TXT entry owned by another cluster, is a REGRESSION."
+}
+
+step_s3check() {
+  say "S3 BACKUP TARGET — PROVE A REAL ROUND TRIP FROM INSIDE THE CLUSTER"
+  # Velero and CNPG run in-cluster, so pod egress is the only reachability that counts: prod sits on
+  # the SOHO network (pfSense's WAN) while the dev LAN's MinIO sits INSIDE pfSense, so host and pod
+  # egress differ — and a healthy host-side reading has already misled this project once.
+  #
+  # WHAT THIS PROVES (and it fails closed on every one of them):
+  #   1. the committed credential loads and AUTHENTICATES, in the exact shape each consumer reads it
+  #      (velero: an AWS INI file from the `cloud` key; CNPG: discrete ACCESS_KEY_ID/ACCESS_SECRET_KEY);
+  #   2. each required bucket EXISTS and is LISTABLE under this environment's own prefix;
+  #   3. PUT -> GET -> byte-compare -> DELETE of a throwaway object under that prefix — the genuine
+  #      write path, not a HEAD (a HEAD passes with a read-only token and with R2 rejecting writes);
+  #   4. whether the R2 CHECKSUM TRAP is present at this endpoint, named explicitly either way.
+  #
+  # WHAT IT STILL DOES NOT PROVE: that Velero's kopia uploader and CNPG's barman-cloud themselves work
+  # against this target. Both are UNPROVEN against R2 in this repo. See infra/BACKUP-R2.md.
+  local ep live region
+  live="$(cvar S3_BACKUP_ENDPOINT)"
+  info "live S3_BACKUP_ENDPOINT = ${live:-<unset>}"
+  if [ -n "${S3_ENDPOINT_OVERRIDE}" ]; then
+    # Probe a CANDIDATE endpoint before adopting it. The whole point is to find out whether a proposed
+    # backup target works from this cluster's pods BEFORE authoring it into cluster-vars — otherwise
+    # the first evidence is silently-missing backups.
+    ep="${S3_ENDPOINT_OVERRIDE}"
+    info "probing CANDIDATE endpoint from --s3-endpoint: ${ep}"
+  else
+    ep="${live}"
+  fi
+  if s3_is_placeholder "${ep}"; then
+    die "no usable endpoint to probe (endpoint='${ep:-<unset>}').
+       cluster-vars still carries the loud placeholder. Pass --s3-endpoint https://<account-id>.r2.cloudflarestorage.com
+       to test a CANDIDATE before authoring it. See infra/BACKUP-R2.md."
+  fi
+  s3_safe_token S3_BACKUP_ENDPOINT "${ep}"
+  case "${ep}" in
+    http://*|https://*) ;;
+    *) die "endpoint '${ep}' carries no http:// or https:// scheme — the AWS SDK needs one." ;;
+  esac
+
+  # Region is a knob, not a constant: MinIO ignores it, R2 requires exactly `auto`, and an EMPTY value
+  # in cluster-vars silently builds a BSL with no region.
+  region="$(cvar S3_BACKUP_REGION)"
+  case "${ep}" in
+    *.r2.cloudflarestorage.com*)
+      if [ "${region}" != "auto" ]; then
+        warn "endpoint is Cloudflare R2 but cluster-vars S3_BACKUP_REGION='${region:-<unset>}'. R2 requires
+       exactly 'auto'. Probing with 'auto' anyway; FIX clusters/base/cluster-vars.yaml before enabling
+       velero, or the BackupStorageLocation is built with the wrong region."
+      fi
+      region=auto ;;
+    *)
+      if [ -z "${region}" ]; then
+        region="us-east-1"
+        warn "cluster-vars S3_BACKUP_REGION is EMPTY on this cluster. Probing with 'us-east-1' (MinIO
+       ignores it) — but the BSL would be rendered with region: \"\". Set it explicitly."
+      fi ;;
+  esac
+  s3_safe_token S3_BACKUP_REGION "${region}"
+
+  # Buckets come from the AUTHORED manifests + cluster-vars, never from a hardcoded name here: the
+  # whole point is to probe what this environment will really write to (R-ENV-02, no dev/prod fork).
+  local vbucket vprefix cpath cbucket cprefix rest
+  vbucket="$(yq -r '.spec.values.configuration.backupStorageLocation[0].bucket' \
+             platform/velero/helmrelease.yaml 2>/dev/null | tr -d '"')"
+  [ -n "${vbucket}" ] && [ "${vbucket}" != "null" ] \
+    || die "could not read the Velero bucket name from platform/velero/helmrelease.yaml"
+  vprefix="$(cvar VELERO_BACKUP_PREFIX)"
+
+  cpath="$(cvar CNPG_BACKUP_PATH)"
+  if [ -z "${cpath}" ]; then
+    warn "cluster-vars CNPG_BACKUP_PATH is EMPTY on this cluster. barmanObjectStore.destinationPath would
+       substitute to the empty string and EVERY WAL archive would fail at runtime. Probing the historical
+       default s3://cnpg so the round trip is still proven — but SET THIS VALUE."
+    cpath="s3://cnpg"
+  fi
+  case "${cpath}" in s3://*) ;; *) die "CNPG_BACKUP_PATH='${cpath}' is not an s3:// URL." ;; esac
+  rest="${cpath#s3://}"; cbucket="${rest%%/*}"
+  if [ "${rest}" = "${cbucket}" ]; then cprefix=""; else cprefix="${rest#*/}"; fi
+  cprefix="${cprefix%/}"
+  [ -n "${cbucket}" ] || die "CNPG_BACKUP_PATH='${cpath}' names no bucket."
+
+  s3_safe_token velero-bucket "${vbucket}"; s3_safe_token VELERO_BACKUP_PREFIX "${vprefix}"
+  s3_safe_token cnpg-bucket   "${cbucket}"; s3_safe_token CNPG_BACKUP_PATH-prefix "${cprefix}"
+
+  say "what is about to be proven"
+  info "endpoint    ${ep}"
+  info "region      ${region}"
+  info "velero      bucket '${vbucket}'  prefix '${vprefix:-<bucket root>}'   (authored bucket + VELERO_BACKUP_PREFIX)"
+  info "cnpg        bucket '${cbucket}'  prefix '${cprefix:-<bucket root>}'   (from CNPG_BACKUP_PATH)"
+  info "scratch key <prefix>/.s3check/<pod>.probe — written, read back, byte-compared, DELETED"
+
+  local ran=0 keys
+  # (a) Velero's own credential, from its own namespace, mounted the way the aws plugin reads it.
+  if kubectl get ns velero >/dev/null 2>&1 \
+     && kubectl -n velero get secret velero-cloud-credentials >/dev/null 2>&1; then
+    # List KEY NAMES only. Never pipe or print a value (a careless redaction has leaked a password
+    # into a transcript on this project already). go-template, not jsonpath: kubectl's jsonpath
+    # cannot range over a map's keys (`range $k,$v :=` is rejected, U+002C) and returns EMPTY, which
+    # silently looked like "the key is missing".
+    keys="$(kubectl -n velero get secret velero-cloud-credentials \
+            -o go-template='{{range $k,$v := .data}}{{$k}}{{"\n"}}{{end}}' 2>/dev/null || true)"
+    printf '%s\n' "${keys}" | grep -qx 'cloud' \
+      || die "Secret velero/velero-cloud-credentials carries no 'cloud' key (keys: $(printf '%s' "${keys}" | tr '\n' ' ')).
+       velero-plugin-for-aws reads exactly that key as an AWS INI file; without it every backup fails
+       at RUNTIME while the HelmRelease reports Ready."
+    s3_probe velero file velero-cloud-credentials "${vbucket}:${vprefix}" "${ep}" "${region}"
+    ran=$((ran+1))
+  else
+    warn "no Secret velero/velero-cloud-credentials on this cluster — Velero's own credential could NOT
+       be proven. That is expected before the velero tier has ever reconciled here; it is not a pass."
+  fi
+
+  # (b) A CNPG/barman credential, discovered rather than hardcoded (R-ENV-02: no per-environment list
+  #     of app names in this script). Any one of them proves the key + the cnpg bucket + the discrete
+  #     ACCESS_KEY_ID/ACCESS_SECRET_KEY shape; they are all kept in sync by design.
+  local pick cns csec
+  pick="$(kubectl get secret -A -o jsonpath='{range .items[*]}{.metadata.namespace}{"/"}{.metadata.name}{"\n"}{end}' 2>/dev/null \
+          | grep -- '-db-backup-credentials$' | head -n 1 || true)"
+  if [ -n "${pick}" ]; then
+    cns="${pick%%/*}"; csec="${pick#*/}"
+    s3_probe "${cns}" env "${csec}" "${cbucket}:${cprefix}" "${ep}" "${region}"
+    ran=$((ran+1))
+  else
+    warn "no *-db-backup-credentials Secret found in any namespace — the CNPG/barman credential could
+       NOT be proven."
+  fi
+
+  [ "${ran}" -gt 0 ] || die "NO backup credential Secret exists on this cluster, so NOTHING could be
+       proven about ${ep}. That is a FAILURE, not a skip: a green s3check must never mean 'I found
+       nothing to check'."
+
+  say "S3 ROUND TRIP PROVEN from inside the cluster: ${ep} (${ran} credential path(s))"
+  info "STILL UNPROVEN by this step: Velero's kopia uploader and CNPG barman-cloud against this target,"
+  info "and a RESTORE. A backup that has never been restored is not a backup — see infra/BACKUP-R2.md."
+}
+
+step_resume() {
+  say "RESUME TIERS (dependency order)"
+  local n s3; s3="$(cvar S3_BACKUP_ENDPOINT)"
+  for n in "${TIER_ORDER[@]}"; do
+    if [ "$(k_susp "$n")" = "true" ]; then
+      info "resuming $n"; flux resume kustomization "$n" --timeout=1s >/dev/null 2>&1 || true
+    else
+      info "$n already unsuspended"
+    fi
+    wait_ready "$n" 900 || { warn "stopping at $n — a partial, understood bring-up beats a forced green."; return 1; }
+  done
+
+  # The placeholder test MUST be substring-based. It was `TBD-*` (a PREFIX match) while the placeholder
+  # was `TBD-PROD-…`; the R2 move changed it to `https://TBD-R2-ACCOUNT-ID.r2.cloudflarestorage.com`,
+  # which does not START with TBD- — so this case arm silently stopped matching and `resume` would have
+  # RESUMED velero onto a placeholder hostname on prod. That is the precise failure this guard exists
+  # to prevent, reintroduced by a value change in another file. s3_is_placeholder matches anywhere.
+  if s3_is_placeholder "${s3}"; then
+    warn "velero left SUSPENDED: S3_BACKUP_ENDPOINT='${s3}' is not a usable endpoint.
+       Resuming it would create a BackupStorageLocation that can never reach a bucket, i.e. a tier
+       that reports Ready-ish while backups silently do not exist. Set the real endpoint first, prove
+       it with \`s3check\`, and only then resume (infra/BACKUP-R2.md)."
+  else
+    info "resuming velero"; flux resume kustomization velero --timeout=1s >/dev/null 2>&1 || true
+    wait_ready velero 900 || warn "velero did not converge"
+  fi
+}
+
+step_verify() {
+  say "VERIFY"
+  local base; base="$(cvar BASE_DOMAIN)"
+  info "BASE_DOMAIN = ${base}"
+  say "kustomizations"
+  kubectl get kustomization -A \
+    -o custom-columns=NAME:.metadata.name,SUSPEND:.spec.suspend,READY:.status.conditions[?\(@.type==\"Ready\"\)].status --no-headers
+  say "certificates (must be Ready before any hostname is trusted)"
+  kubectl get certificate -A --no-headers 2>/dev/null || info "(none yet)"
+  say "pods not Running/Completed"
+  kubectl get pods -A --no-headers | awk '$4!="Running" && $4!="Completed"' || true
+  info "total pods: $(kubectl get pods -A --no-headers | wc -l)"
+  say "gateway / LoadBalancer VIPs"
+  kubectl get svc -A --field-selector spec.type=LoadBalancer --no-headers 2>/dev/null || true
+  say "published DNS for this cluster's own domain"
+  local owner; owner="$(cvar EXTERNALDNS_TXT_OWNER_ID)"
+  dns_snapshot "${SNAP_DIR}/dns-verify.txt"
+  info "records under ${base}: $(grep -c "${base}" "${SNAP_DIR}/dns-verify.txt" || true)"
+
+  # Every TXT registry entry this cluster publishes must carry OUR owner id. One reading another
+  # cluster's id means we have claimed its records and `policy: sync` will delete them next cycle.
+  local foreign
+  foreign="$(grep "${base}" "${SNAP_DIR}/dns-verify.txt" | grep 'heritage=external-dns' \
+             | grep -v "external-dns/owner=${owner}" || true)"
+  if [ -n "${foreign}" ]; then
+    warn "TXT entries under ${base} NOT owned by '${owner}' — this cluster has claimed records it does not own:"
+    printf '%s\n' "${foreign}" | sed 's/^/      /'
+  else
+    info "every TXT registry entry under ${base} reads owner=${owner}: OK"
+  fi
+
+  # THE REGRESSION CHECK. Additions under our own BASE_DOMAIN are expected. ANY change to a name
+  # outside it means we have touched the other cluster's zone — the 2026-09-04 incident.
+  if [ -f "${SNAP_DIR}/dns-before.txt" ]; then
+    say "regression check against the pre-standup baseline"
+    local outside
+    outside="$(diff "${SNAP_DIR}/dns-before.txt" "${SNAP_DIR}/dns-verify.txt" \
+               | grep -E '^[<>]' | grep -v "${base}" || true)"
+    if [ -n "${outside}" ]; then
+      warn "RECORDS OUTSIDE ${base} CHANGED — investigate before doing anything else:"
+      printf '%s\n' "${outside}" | sed 's/^/      /'
+    else
+      info "no record outside ${base} was added, changed or removed: OK"
+      info "added under ${base}: $(diff "${SNAP_DIR}/dns-before.txt" "${SNAP_DIR}/dns-verify.txt" | grep -c '^>' || true)"
+    fi
+  else
+    warn "no ${SNAP_DIR}/dns-before.txt baseline — cannot prove the other cluster's records are untouched.
+       Take one BEFORE the next standup so this check can run."
+  fi
+}
+
+# ============================================================================ dispatch
+for s in "${STEPS[@]}"; do
+  case "$s" in
+    preflight)   step_preflight ;;
+    s3check)     step_s3check ;;
+    crds)        step_crds ;;
+    unwedge)     step_unwedge ;;
+    externaldns) step_externaldns ;;
+    resume)      step_resume ;;
+    verify)      step_verify ;;
+    all)         step_preflight; step_crds; step_unwedge; step_externaldns; step_resume; step_verify ;;
+    *)           die "unknown step '$s' (preflight crds unwedge externaldns s3check resume verify all)" ;;
+  esac
+done
+
+say "done: ${STEPS[*]}"
@@ -180,34 +180,6 @@ dependencies: []                # default-deny: reaches only DNS; git clients re
 ingressRoutes:
  - host: gitea.${BASE_DOMAIN}
    port: 3000                  # Gitea HTTP web/API; TLS terminates at the shared Gateway (docs/08, docs/09)
-# THIS IS WHAT KEEPS PROD'S GITOPS OUT OF A DEADLOCK — it is not an incidental allowance. Remove it
-# and prod cannot reconcile AT ALL, in a way that is very hard to diagnose from cold.
-#
-# Prod's Flux source is THIS SERVICE, not a public hostname:
-#   clusters/prod/kustomization.yaml -> GitRepository.spec.url =
-#   http://gitea.gitea.svc.cluster.local:3000/Home-Network/home-network-k3s-impl.git
-# chosen deliberately so prod's GitOps depends on NEITHER public DNS, Cloudflare, prod's own
-# Gateway, NOR ${BASE_DOMAIN} — a future domain change cannot sever prod from its own source. That
-# file documents the other half of this pair at length; the two must be read together.
-#
-# The only other ingress this entry renders is gitea-allow-gateway-ingress, which admits
-# `fromEntities: [ingress]` — the shared Gateway's Envoy proxy — and nothing else. Flux's
-# source-controller dials the ClusterIP DIRECTLY, so under the namespace default-deny (R-NET-01) its
-# clone was DROPPED (`wget` from source-controller timed out; srcReady=False, "unable to list
-# remote"). The deadlock that made this urgent: the HTTPRoute that would have served the new public
-# name after the 2026-09-13 apex flip lives in a DOWNSTREAM tier, which cannot apply while the
-# source is down — source down -> routes never rename -> source stays down. It was broken on
-# 2026-09-13 with a live `kubectl apply` of a hand-written gitea-allow-flux-ingress, i.e. with a
-# cluster object that existed OUTSIDE git. This declaration renders that object, so one
-# `kubectl delete` or one cluster rebuild can no longer put prod back in the deadlock.
-inClusterClients:
-  # `name` is pinned to `flux` (rather than defaulting to the namespace) so the rendered policy name
-  # equals the live object's, `gitea-allow-flux-ingress`, and Flux ADOPTS it in place: no
-  # near-duplicate policy beside it, and no window in which prod's own source has no allow.
-  - name: flux
-    namespace: flux-system      # matched by NAMESPACE alone — the flux controllers' pod labels belong
-                                # to upstream's manifests, not to this platform's contract
-    port: 3000                  # ONLY the Gitea HTTP port — the same single port the Gateway allow opens
 auth:
  mode: oidc                    # native Keycloak OIDC client (docs/13 SSO row, docs/15 "Native OIDC" set; R-IDP-02)
  groups:
@@ -23,17 +23,6 @@ storage:
  - name: config
    size: 2Gi
    class: longhorn             # indexer definitions/API keys on 3-replica Longhorn (docs/04; R-STATE-02)
-    mountPath: /config          # THE APP WRITES /config, NOT the renderer default /data/config.
-                                # LSIO images run with `-data=/config` (Jackett/qBittorrent likewise use
-                                # /config), so without this the Longhorn PVC mounted at /data/config sat
-                                # holding nothing but lost+found while the app wrote its entire
-                                # configuration to the container's EPHEMERAL writable layer — destroyed on
-                                # every pod recreation: restart, reschedule, image bump. Measured in-cluster
-                                # 2026-09-12: /data/config was dev=2080 with [lost+found], /config was
-                                # dev=2053 on the node's own disk with [Sentry asp config.xml logs].
-                                # This is the SAME failure catalog/plex.yaml:54 already records fixing.
-                                # The live /config contents were copied onto each PVC before this landed,
-                                # so API keys and credentials other apps reference were preserved.
 dependencies: []                # default-deny: reaches only DNS; arr reach it north of its own egress (docs/03; R-NET-01/02)
 ingressRoutes:
  - host: jackett.${BASE_DOMAIN}
@@ -42,15 +31,6 @@ auth:
  mode: forward-auth            # oauth2-proxy -> Keycloak; bare UI, no usable native OIDC (docs/15; R-IDP-02)
  groups:
    - media-users               # authenticated AND in an authorized group (R-IDP-06)
-    # platform-admins is a SUPERSET, not a widening. --allowed-group is repeatable and OR-ed, so
-    # listing both means "media consumers OR platform operators". Without it the media UIs are
-    # reachable by NOBODY the moment media-users is empty: on prod the realm's only user is the
-    # operator, who is in platform-admins only, so jackett/qbittorrent/radarr/sonarr answered an
-    # immediate 403 at the oauth2-proxy callback (no login prompt, because the Keycloak SSO session
-    # already existed) while pihole/crafty — which already gate on platform-admins — worked.
-    # This grants no real privilege: platform-admins already maps to cluster-admin via
-    # k8s-oidc-rbac.yaml (oidc:platform-admins), so a member can already exec into these pods.
-    - platform-admins           # operators of these services must not be locked out of them
 update:
  channel: stable               # R-UPD-01
  range: ">=0.22.0"             # stay on the Jackett 0.22.x stable line
@@ -223,44 +223,6 @@ storage:
    mountPath: /var/www/html    # 3-replica tier; irreplaceable, DR-critical (docs/04; R-STATE-02).
                                # The whole html dir is persisted so config.php + user data survive a
                                # rebuild (the default /data/<name> would strand them on ephemeral rootfs).
-  # THE LEGACY NEXTCLOUD DATASET (Storage/nextcloud, 270 G) — MOUNTED READ-ONLY, DELIBERATELY, AND
-  # NOT AS THIS INSTANCE'S DATA DIRECTORY. Read this before changing either decision.
-  #
-  # WHAT IS ACTUALLY IN THERE (measured 2026-09-07, not assumed):
-  #     /mnt/Storage/nextcloud/nextcloud/config          103 M
-  #     /mnt/Storage/nextcloud/nextcloud/config_old      3.4 G
-  #     /mnt/Storage/nextcloud/nextcloud/config_backup   267 G   <- 99% of the dataset
-  # There is NO `data/` directory. This is the OLD, non-k3s Nextcloud's configuration plus a backup
-  # blob — an ARCHIVE, not a live user-data tree.
-  #
-  # WHY READ-ONLY. Two independent reasons, either sufficient:
-  #   1. The old instance is STILL LIVE on it. The export is currently shared rw to 192.168.1.6 and
-  #      192.168.1.5, and the operator has said the existing non-k3s prod must remain active. Two
-  #      Nextcloud instances writing one instance's config directory, each with its own database and
-  #      therefore its own oc_filecache, is a corruption scenario with no clean recovery.
-  #   2. Nothing here needs writing. An archive is read.
-  # "Do not grant more than needed" therefore lands on `ro`. If the operator's intent is in fact a
-  # MIGRATION (read this data in, then decommission the old instance), that is a different and
-  # larger change than a mount, and it should be a deliberate one — flip `readOnly` and update the
-  # export at that point, not before. Enforced by the PV's nfs.readOnly, so it is a real `ro` mount.
-  #
-  # WHY NOT /var/www/html/data. That path is the LONGHORN PVC above and is this instance's own
-  # state; mounting a foreign tree into it would have Nextcloud scan and index an archive it does
-  # not own. A neutral path keeps the two completely separate. To expose it to users, add it as a
-  # Nextcloud External Storage (Local) mount pointing at this path — an in-app, per-user decision.
-  #
-  # subPath skips the redundant single-child directory: the dataset root holds exactly one entry,
-  # `nextcloud/`, so mounting that subtree puts config/ config_old/ config_backup/ at the top.
-  #
-  # UID: nothing to do. The tree is 0755 owned 1000:3000 and the mount is read-only, so www-data
-  # (uid 33) can read and traverse it. (fsGroup would be a no-op on an NFS volume in any case.)
-  - name: legacy-archive
-    class: bulk
-    export: /mnt/Storage/nextcloud   # the ZFS dataset Storage/nextcloud, on ${BULK_NFS_SERVER}
-    subPath: nextcloud
-    mountPath: /mnt/legacy-nextcloud
-    readOnly: true
-    size: 270Gi                      # ADVISORY ONLY on a static NFS PV — the dataset's used size.
 dependencies:
  - to: nextcloud-db            # explicit allow: Nextcloud -> its PostgreSQL (docs/13 graph; R-NET-02).
    port: 5432                  # Renders the egress half; the DB namespace mirrors keycloak-db (no
@@ -4,11 +4,21 @@
 # auth: native (Plex accounts, NOT Keycloak — docs/13 + docs/15 documented exception).
 # Validates against schemas/servicedefinition.schema.json.
 #
-# BULK MEDIA — NOW ATTACHED (docs/04 bulk tier; the long-standing OQ-2 prerequisite, resolved for
-# the STATIC case). The library is 8.33 T of ZFS on the Proxmox host at /mnt/Storage/plex, and it
-# already exists: nothing is provisioned, an existing dataset is mounted. See
-# docs/BULK-NFS-STORAGE.md for the layout, the export change and the uid analysis. (The bulk-tier
-# requirement ID is deliberately still not typed here — see that document's "gate" note.)
+# BULK MEDIA (spec bulk-tier requirement, docs/04) — OPEN PREREQUISITE, intentionally NOT declared
+# here yet (the requirement ID is cited in the WS-APPS report, not as a token here: this change does
+# NOT exercise the bulk tier, so it must not be selected as a "touched" requirement whose in-cluster
+# conformance test is still a Phase-1 gap).
+#   The Plex library belongs on the `bulk` tier (docs/04 / docs/13 cross-cutting note: metadata DB
+#   on replicated Longhorn, large library on bulk). The `bulk` StorageClass exists
+#   (platform/longhorn/storageclasses.yaml) and the schema+renderer both support `class: bulk`, but
+#   that class is a documented STUB: OQ-2 (external NFS/SMB backing) is UNRESOLVED and the NFS CSI
+#   driver is not installed, so a `bulk` PVC will not bind and would leave the Plex pod Pending.
+#   To onboard the running Plex shell now (compute + config + public exposure), the media volume is
+#   omitted. Once OQ-2 is resolved and `bulk` provisions, attach the library with a one-line add:
+#     storage:
+#       - name: media
+#         class: bulk        # external mount; size omitted (sized outside the cluster)
+#   then re-render. See the WS-APPS report for the full bulk-media finding.
 name: plex
 kind: container
 source:
@@ -60,31 +70,6 @@ storage:
                              # UI sign-in, the operator adds a PLEX_CLAIM env from a fresh https://plex.tv/claim token
                              # (4-min TTL) — deliberately NOT wired here: the env secretKeyRef pattern has no `optional`,
                              # so a placeholder/absent claim Secret would crashloop the pod. See operator hand-off.
-  # THE LIBRARY (docs/04 bulk tier). One READ-ONLY mount of the export ROOT, giving /media/movies,
-  # /media/tv and /media/photos — the whole tree the layout actually has (`ls /mnt/Storage/plex`:
-  # downloads/ movies/ photos/ tv/).
-  #
-  # WHY THE ROOT AND NOT THREE subPath MOUNTS. Nothing is gained by narrowing: the mount is `ro`,
-  # so no subPath could remove a privilege that matters, while three mounts would triple the NFS
-  # mounts per pod and mean a manifest change every time a top-level library is added.
-  #
-  # WHY READ-ONLY, when the export is `rw` and the other consumers write. Plex is a SERVER: it
-  # indexes and streams. Every write on that tree belongs to Sonarr and Radarr. Marking it here
-  # costs nothing and removes Plex from the set of things that can damage 8.33 T. Enforced by the
-  # PV's nfs.readOnly (a real `ro` mount), not only by the volumeMount flag.
-  #
-  # UID: no change needed. The tree is 0755 owned 1000:3000 throughout, so it is world-readable and
-  # world-traversable, and PLEX_UID/PLEX_GID above already make the process uid 1000 in any case.
-  # (fsGroup would have been a no-op here regardless — the in-tree NFS plugin does no ownership
-  # management. That is the trap the arrs' PUID/PGID exists to avoid; Plex never hits it.)
-  - name: media
-    class: bulk
-    export: /mnt/Storage/plex   # the ZFS dataset Storage/plex, on ${BULK_NFS_SERVER}
-    mountPath: /media
-    readOnly: true
-    size: 9Ti                   # ADVISORY ONLY (nothing enforces a static NFS PV's capacity): the
-                                # dataset's 8.33 T used, rounded up, so `kubectl get pv` reports
-                                # something true. The real limit is the ZFS pool.
 dependencies: []              # default-deny: reaches only DNS; clients/Seerr reach IT (docs/03; R-NET-01/02)
 ingressRoutes:
  - host: plex.${BASE_DOMAIN}
@@ -38,17 +38,6 @@ storage:
  - name: config
    size: 5Gi
    class: longhorn             # settings/categories/session on 3-replica Longhorn; DR-critical (docs/04; R-STATE-02)
-    mountPath: /config          # THE APP WRITES /config, NOT the renderer default /data/config.
-                                # LSIO images run with `-data=/config` (Jackett/qBittorrent likewise use
-                                # /config), so without this the Longhorn PVC mounted at /data/config sat
-                                # holding nothing but lost+found while the app wrote its entire
-                                # configuration to the container's EPHEMERAL writable layer — destroyed on
-                                # every pod recreation: restart, reschedule, image bump. Measured in-cluster
-                                # 2026-09-12: /data/config was dev=2080 with [lost+found], /config was
-                                # dev=2053 on the node's own disk with [Sentry asp config.xml logs].
-                                # This is the SAME failure catalog/plex.yaml:54 already records fixing.
-                                # The live /config contents were copied onto each PVC before this landed,
-                                # so API keys and credentials other apps reference were preserved.
 dependencies: []                # default-deny: reaches only DNS; arr reach it north of its own egress (docs/03; R-NET-01/02)
 ingressRoutes:
  - host: qbittorrent.${BASE_DOMAIN}
@@ -57,11 +46,6 @@ auth:
  mode: forward-auth            # oauth2-proxy -> Keycloak; bare UI, no usable native OIDC (docs/15; R-IDP-02)
  groups:
    - media-users               # authenticated AND in an authorized group (R-IDP-06)
-    # See catalog/jackett.yaml for the full rationale: --allowed-group is repeatable and OR-ed, so
-    # platform-admins here is a superset, not a widening (that group already carries cluster-admin
-    # via k8s-oidc-rbac.yaml). Without it, an empty media-users locks every operator out with an
-    # immediate 403 at the oauth2-proxy callback — observed on prod 2026-09-12.
-    - platform-admins           # operators of these services must not be locked out of them
 update:
  channel: stable               # R-UPD-01
  range: ">=5.0.0 <6.0.0"       # stay on the qBittorrent 5.x stable line
@@ -9,23 +9,6 @@ name: radarr
 kind: container
 source:
  image: lscr.io/linuxserver/radarr:5.21.1   # pinned stable v5 tag (R-UPD-01); update.range governs advancement
-# LinuxServer images run their service as the user `abc`, whose uid/gid these two set. They are
-# LOAD-BEARING for the bulk mount, not cosmetic: see the storage note below. The default is 911,
-# which cannot write a 1000:3000-owned 0755 tree.
-env:
-  - name: PUID
-    value: "1000"               # owner uid of every directory under /mnt/Storage/plex
-  - name: PGID
-    value: "3000"               # dominant group there; keeps files this writes readable to the
-                                # EXISTING non-k3s consumers of the same export, which stay live
-  - name: TZ
-    value: Etc/UTC              # same safe default plex uses; the arrs schedule by local time
-podSecurityContext:
-  # For the LONGHORN config PVC only. fsGroup does nothing on the NFS volume (the in-tree plugin
-  # does no ownership management), but on a Longhorn filesystem the kubelet applies it, which is
-  # what makes /config writable by the now-uid-1000 process WITHOUT relying on the image chowning
-  # /config for itself when PUID changes. Matches PGID so the primary group owns the config too.
-  fsGroup: 3000
 exposure: internal              # LAN-only web UI at radarr.${BASE_DOMAIN} (docs/13 row 3; R-DNS/R-NET)
 egress: vpn                     # indexer scrape + downloader handoff steered via the shared VPN egress gateway,
                                # kill-switch enforced OUTSIDE the workload (docs/03; ADR-0015; R-NET-05/06)
@@ -33,66 +16,11 @@ storage:
  - name: config
    size: 5Gi
    class: longhorn             # app DB / config on the 3-replica tier; DR-critical (docs/04; R-STATE-02)
-    mountPath: /config          # THE APP WRITES /config, NOT the renderer default /data/config.
-                                # LSIO images run with `-data=/config` (Jackett/qBittorrent likewise use
-                                # /config), so without this the Longhorn PVC mounted at /data/config sat
-                                # holding nothing but lost+found while the app wrote its entire
-                                # configuration to the container's EPHEMERAL writable layer — destroyed on
-                                # every pod recreation: restart, reschedule, image bump. Measured in-cluster
-                                # 2026-09-12: /data/config was dev=2080 with [lost+found], /config was
-                                # dev=2053 on the node's own disk with [Sentry asp config.xml logs].
-                                # This is the SAME failure catalog/plex.yaml:54 already records fixing.
-                                # The live /config contents were copied onto each PVC before this landed,
-                                # so API keys and credentials other apps reference were preserved.
-  # THE MEDIA TREE (docs/04 bulk tier) — ONE read-WRITE mount of the export ROOT at /media, giving
-  # /media/movies (the library) and /media/downloads (the import source) UNDER THE SAME MOUNT.
-  #
-  # *** THE ROOT MOUNT IS THE POINT, NOT A CONVENIENCE. *** Radarr imports by HARDLINKING or
-  # atomically renaming a finished download into the library. Both are same-FILESYSTEM operations,
-  # and to the kernel "same filesystem" means SAME MOUNT. The layout on the host is:
-  #     /mnt/Storage/plex/movies/                      the library
-  #     /mnt/Storage/plex/downloads/complete/radarr/  where finished releases land
-  # — siblings under one dataset, so a hardlink between them is legal on the server. But mounting
-  # them as two subPath volumes (subPath: movies at /media/movies, subPath: downloads elsewhere) creates
-  # TWO mounts, and link() across mounts fails EXDEV no matter what the server thinks. Radarr then
-  # silently degrades to copy-then-delete: every import rewrites the whole file over NFS and the
-  # seeding copy is destroyed. Their smallest common ancestor IS the export root, so the root mount
-  # is the MINIMAL mount that preserves hardlinks — least privilege bounded below by a hard
-  # requirement, not a shortcut. (The cost is stated plainly: Radarr can also write /media/tv,
-  # which is sonarr's. Narrowing that would cost hardlinks, which is a worse trade.)
-  #
-  # *** /media, NOT /data — AND THAT IS NOT A STYLE CHOICE. *** The config PVC above renders at the
-  # renderer default /data/config. A media mount at /data would be mounted OVER it and MASK the
-  # config volume entirely; the pod would come up with its app database shadowed by an 8 T media
-  # tree. /media collides with nothing in a LinuxServer image and matches the path plex uses for
-  # the same export, so when qbittorrent is eventually given this mount too, the arrs and the
-  # download client will already agree on absolute paths — which is the other precondition for
-  # hardlinked imports (otherwise every import needs a Remote Path Mapping).
-  #
-  # SEPARATE LATENT BUG, NOT FIXED HERE (deliberately — it changes live on-disk state): /data/config
-  # is almost certainly the wrong place for this PVC in the first place. LinuxServer's arr images run
-  # with `-data=/config`, so the app writes /config — the container's EPHEMERAL layer — while the
-  # Longhorn PVC at /data/config sits unused. That is the EXACT failure plex's own mountPath comment
-  # documents having already been bitten by. Fixing it means starting from an empty config volume, so
-  # it belongs in its own change with the operator present.
-  #
-  # UID/GID — THE FAILURE THIS AVOIDS IS "MOUNTS FINE, CANNOT WRITE". The export is no_root_squash
-  # + sec=sys, so the client's numeric uid is passed through verbatim and the server's ownership is
-  # the whole story. The tree is 0755 owned 1000:3000 (some dirs 1000:1000), so ONLY uid 1000 can
-  # write it. LinuxServer images default to PUID/PGID 911 — which would mount cleanly and then fail
-  # every import with EACCES. The PUID/PGID set above make the process uid 1000, matching the owner.
-  #
-  # AND NOTE fsGroup CANNOT SUBSTITUTE FOR THAT. The in-tree NFS volume plugin performs no
-  # ownership management, so podSecurityContext.fsGroup is a NO-OP on this volume — it is not the
-  # lever here, and reaching for it is how this failure normally gets misdiagnosed. The fsGroup
-  # that IS set above exists for the LONGHORN config PVC, so that changing PUID does not depend on
-  # the image re-chowning /config for itself.
-  - name: media
-    class: bulk
-    export: /mnt/Storage/plex   # the ZFS dataset Storage/plex, on ${BULK_NFS_SERVER}
-    mountPath: /media           # => /media/movies (Root Folder) and /media/downloads (import source)
-    size: 9Ti                   # ADVISORY ONLY on a static NFS PV: the dataset's 8.33 T used,
-                                # rounded up, so `kubectl get pv` reports something true.
+  # NOTE (bulk-media PREREQUISITE): docs/13 row 3 places the media library / downloads volume on the
+  # `bulk` tier (docs/04 bulk storage). The `bulk` StorageClass exists (platform/longhorn/storageclasses.yaml)
+  # but is a NON-provisioning STUB pending OQ-2 (NFS/SMB backing = REPLACE-ME). Adding a `class: bulk`
+  # volume now would leave the PVC (and therefore the pod) Pending. The media/downloads volume is
+  # therefore DEFERRED until OQ-2 is resolved; config-only on longhorn keeps Radarr deployable today.
 dependencies:
  - to: qbittorrent             # arr → download client (docs/13 dependency graph; R-NET-02 explicit allow)
    port: 8080
@@ -107,11 +35,6 @@ auth:
  mode: forward-auth            # bare UI, no usable OIDC: oauth2-proxy → Keycloak at the Gateway (docs/15; R-IDP-02)
  groups:
    - media-users               # authenticated AND in the media group (R-IDP-06)
-    # See catalog/jackett.yaml for the full rationale: --allowed-group is repeatable and OR-ed, so
-    # platform-admins here is a superset, not a widening (that group already carries cluster-admin
-    # via k8s-oidc-rbac.yaml). Without it, an empty media-users locks every operator out with an
-    # immediate 403 at the oauth2-proxy callback — observed on prod 2026-09-12.
-    - platform-admins           # operators of these services must not be locked out of them
 update:
  channel: stable               # R-UPD-01
  range: ">=5.0.0 <6.0.0"       # stay on the Radarr v5 stable line
@@ -4,8 +4,7 @@
 Consumes ONE ServiceDefinition (schemas/servicedefinition.schema.json, catalog/*.yaml)
 and emits the concrete manifest set the renderer contract specifies
 (schemas/renderer-contract.md): Namespace + default-deny baseline, Deployment/Service or
-KubeVirt VirtualMachine + CDI DataVolume, PVCs, per-dependency and per-in-cluster-client
-CiliumNetworkPolicy allows,
+KubeVirt VirtualMachine + CDI DataVolume, PVCs, per-dependency CiliumNetworkPolicy allows,
 egress-class steering label, HTTPRoute(s), ServiceMonitor/alerts, Flux image-automation,
 and the auth-mode / backup wiring hints.

@@ -54,30 +53,6 @@ LBL = "homelab.keithjackson.dev"
 # Hardcoding the literal here is what let prod be authored to serve dev's public hostnames,
 # and prod's external-dns then deleted dev's records (infra/PROD-PROVISIONING.md §10).
 ZONE = "${BASE_DOMAIN}"
-# The bulk (external NFS) tier's SERVER, emitted as a Flux post-build TOKEN for exactly the same
-# reason as ZONE: dev and prod reach the same Proxmox ZFS host by DIFFERENT addresses (dev's nodes
-# are on the host's own LAN and dial it directly; prod's nodes are on the far side of pfSense and
-# must be given whichever address the still-pending routing decision produces). Hardcoding either
-# one authors a manifest that is silently wrong in the other environment. Declared in
-# clusters/base/cluster-vars.yaml and patched by BOTH overlays; written BARE (it is a real knob,
-# not a shell expansion), so StrictPostBuildSubstitutions fails the tier if an overlay forgets it.
-BULK_NFS_SERVER = "${BULK_NFS_SERVER}"
-# Mount options for every bulk NFS volume. These are a PLATFORM decision, not a per-service one,
-# so they live here rather than on the ServiceDefinition surface:
-#   nfsvers=4.1  NFSv4 carries mount+lock IN the protocol, so it needs ONE port (tcp/2049) and no
-#                rpcbind/111, mountd or statd. That is what makes the prod path a single pfSense
-#                rule instead of four, and it is why v4 is chosen over v3. The server advertises
-#                +3 +4 +4.1 +4.2; 4.1 is the conservative floor every node kernel already speaks.
-#   hard         A `soft` mount turns a server hiccup into a short read / EIO handed to the
-#                application, which for a media library and an archive is silent corruption.
-#                `hard` blocks instead. The cost is explicit: an NFS outage HANGS I/O rather than
-#                failing it, so an outage shows up as stuck pods, not as damaged files. Chosen
-#                deliberately -- hung is recoverable, truncated is not.
-#   noatime      The datasets are relatime, so every read would otherwise generate a metadata
-#                WRITE back to the pool. Plex walking an 8 T library must not do that.
-#   timeo=600/retrans=2  the TCP defaults, stated explicitly so a future reader knows the retry
-#                budget was considered rather than inherited.
-BULK_NFS_MOUNT_OPTIONS = ["nfsvers=4.1", "hard", "noatime", "timeo=600", "retrans=2"]
 GATEWAY_NAME = "shared"
 GATEWAY_NS = "gateway"
 KP_RELEASE = "kube-prometheus-stack"  # ServiceMonitor/PrometheusRule discovery label
@@ -117,7 +92,6 @@ def apply_defaults(sd: dict) -> dict:
    s.setdefault("storage", [])
    s.setdefault("dependencies", [])
    s.setdefault("ingressRoutes", [])
-    s.setdefault("inClusterClients", [])

    # auth.mode default posture: forward-auth for any exposed UI, else none (internal, no
    # UI). An explicit auth block always wins (R-IDP-02).
@@ -137,19 +111,10 @@ def apply_defaults(sd: dict) -> dict:
    obs.setdefault("slis", [])
    s["observability"] = obs

-    # storage entry class default is longhorn (replicated tier); readOnly default is False
-    # (grant-nothing-extra is expressed by SETTING it, so the default stays back-compatible and
-    # every pre-existing golden renders byte-identical).
-    s["storage"] = [{**{"class": "longhorn", "readOnly": False}, **v} for v in s["storage"]]
+    # storage entry class default is longhorn (replicated tier).
+    s["storage"] = [{**{"class": "longhorn"}, **v} for v in s["storage"]]
    # dependency protocol default is tcp.
    s["dependencies"] = [{**{"protocol": "tcp"}, **d} for d in s["dependencies"]]
-    # in-cluster client protocol default is tcp (as for a dependency); its `name` — which only
-    # ever appears in the rendered POLICY name — defaults to the namespace it admits, so the
-    # common case declares just `namespace` + `port` and reads like a dependency edge.
-    s["inClusterClients"] = [
-        {**{"protocol": "tcp", "name": c.get("namespace", "")}, **c}
-        for c in s["inClusterClients"]
-    ]
    return s


@@ -235,139 +200,18 @@ def _service_ports(s: dict) -> list:
    return [{"name": f"port-{p}", "port": p, "targetPort": p} for p in ports]


-def _bulk_access_mode(vol: dict) -> str:
-    """A bulk volume's access mode follows its read/write posture.
-
-    Every bulk volume is `...Many`: the SAME export is consumed by several namespaces at once
-    (plex, sonarr and radarr all sit on /mnt/Storage/plex) and by several nodes at once, which is
-    precisely what ReadWriteOnce cannot express. WHICH `Many` it is says whether this consumer
-    writes -- so `kubectl get pv` reports the intent, not merely the topology.
-    """
-    return "ReadOnlyMany" if vol.get("readOnly") else "ReadWriteMany"
-
-
-def r_bulk_pvs(s: dict) -> list:
-    """One STATIC PersistentVolume per `class: bulk` storage entry (docs/04 bulk tier).
-
-    WHY A PV AT ALL, RATHER THAN AN INLINE `nfs:` POD VOLUME
-    -------------------------------------------------------
-    An inline volume is fewer objects and it would work. It is rejected for four reasons, in
-    descending order of what they would cost to discover later:
-
-      1. RECLAIM POLICY. `persistentVolumeReclaimPolicy: Retain` is a declared, reviewable
-         property of a PV. There is no equivalent for an inline volume -- and the data behind
-         these mounts is 8.33 TiB of media and a 267 GiB archive that ALSO has live non-k3s
-         consumers. A statement in git that nothing in this cluster may reclaim it is worth an
-         object.
-      2. READ-ONLY LIVES IN ONE PLACE. `spec.nfs.readOnly: true` becomes a real `ro` mount option
-         for every pod that ever binds this volume. The same intent expressed on an inline volume
-         is one Deployment edit away from being dropped silently, with no diff anywhere near the
-         storage. (It is still a CLIENT-side guarantee -- the export cannot distinguish plex from
-         sonarr, since both may run on any node -- but it is one that lives in one reviewable
-         place.)
-      3. VISIBILITY. `kubectl get pv,pvc` is where an operator looks for "what external storage is
-         attached, and who holds it". An inline volume is invisible there, and invisible to
-         anything that reasons over PVCs.
-      4. IT IS THE CONTRACT THE REPO ALREADY HAS. Every other storage entry is a PVC; the
-         renderer, the Kyverno ServiceDefinition policy and the backup wiring all key on that. An
-         inline volume would be a second, parallel storage model inside the same renderer.
-
-    CROSS-NAMESPACE. A PV binds 1:1 to a PVC, so "plex, sonarr and radarr share one export" is NOT
-    one shared PV -- it is three PVs, one per namespace, each naming the same server+path. That is
-    the correct model rather than a workaround: the sharing happens on the NFS server, which is
-    built for concurrent clients, and each namespace keeps its own independently reviewable access
-    mode (plex ReadOnlyMany, the arrs ReadWriteMany). One shared PV would have forced a single
-    posture on all three.
-
-    BINDING IS PINNED FROM BOTH ENDS -- `claimRef` here and `volumeName` on the PVC. Either alone
-    leaves a race in which some other PVC of matching size and mode grabs the volume first;
-    together the binding is deterministic. `storageClassName: ""` (explicitly empty, NOT the `bulk`
-    StorageClass) is what tells the binder this volume is statically provisioned: it suppresses the
-    default-StorageClass annotation and guarantees no provisioner is ever asked to create anything.
-    It also sidesteps a trap -- a StorageClass's `provisioner` is IMMUTABLE, so re-pointing the
-    existing `bulk` class (whose stub names an NFS CSI driver that is not installed) at this data
-    would make Flux fail forever on that one object.
-    """
-    out = []
-    for vol in s["storage"]:
-        if vol.get("class") != "bulk":
-            continue
-        name = f"{s['name']}-{vol['name']}"
-        # `export` and `size` are both structurally required of a bulk entry, and the JSON
-        # Schema only enforces the first. A PersistentVolume with no `capacity` is rejected by
-        # the API server, so a missing size must fail HERE with a sentence a human can act on
-        # rather than as a bare KeyError three frames down.
-        for field, why in (
-            ("export", "the absolute NFS export path this volume attaches"),
-            ("size", "the PV capacity (advisory, but the API requires the field)"),
-        ):
-            if not vol.get(field):
-                raise ValueError(
-                    f"{s['name']}: storage entry {vol['name']!r} has class 'bulk' but no "
-                    f"'{field}' — {why}. A bulk volume attaches storage that already exists "
-                    f"outside the cluster, so it cannot be rendered without it."
-                )
-        nfs = {"server": BULK_NFS_SERVER, "path": vol["export"]}
-        if vol.get("readOnly"):
-            nfs["readOnly"] = True
-        spec = {
-            "capacity": {"storage": vol["size"]},
-            "accessModes": [_bulk_access_mode(vol)],
-            # Retain: deleting the PVC must NEVER be able to reach the data. There is nothing to
-            # "reclaim" here in any case -- the volume was not provisioned by the cluster.
-            "persistentVolumeReclaimPolicy": "Retain",
-            "storageClassName": "",
-            "mountOptions": list(BULK_NFS_MOUNT_OPTIONS),
-            "nfs": nfs,
-            "claimRef": {"namespace": s["name"], "name": name},
-        }
-        out.append(
-            {
-                "apiVersion": "v1",
-                "kind": "PersistentVolume",
-                "metadata": {
-                    "name": name,
-                    "labels": {"app.kubernetes.io/name": s["name"]},
-                },
-                "spec": spec,
-            }
-        )
-    return out
-
-
 def r_pvcs(s: dict) -> list:
    """One PVC per storage entry (R-STATE-01/02/04).

-    longhorn -> 3-replica replicated tier, dynamically provisioned, ReadWriteOnce.
-    bulk     -> the STATIC external NFS PV rendered by r_bulk_pvs, pre-bound by volumeName.
+    longhorn -> 3-replica replicated tier; bulk -> external media tier (not 3x replicated).
    """
    out = []
    for vol in s["storage"]:
        cls = vol.get("class", "longhorn")
-        if cls == "bulk":
-            # Mirror image of the PV: the same explicitly-empty class (so nothing dynamically
-            # provisions), the same access mode, and volumeName completing the two-sided binding.
-            out.append(
-                {
-                    "apiVersion": "v1",
-                    "kind": "PersistentVolumeClaim",
-                    "metadata": {
-                        "name": f"{s['name']}-{vol['name']}",
-                        "namespace": s["name"],
-                        "labels": {"app.kubernetes.io/name": s["name"]},
-                    },
-                    "spec": {
-                        "accessModes": [_bulk_access_mode(vol)],
-                        "storageClassName": "",
-                        "volumeName": f"{s['name']}-{vol['name']}",
-                        "resources": {"requests": {"storage": vol["size"]}},
-                    },
-                }
-            )
-            continue
+        storage_class = "longhorn" if cls == "longhorn" else "bulk"
        spec = {
            "accessModes": ["ReadWriteOnce"],
-            "storageClassName": "longhorn",
+            "storageClassName": storage_class,
        }
        if vol.get("size"):
            spec["resources"] = {"requests": {"storage": vol["size"]}}
@@ -413,17 +257,7 @@ def r_deployment(s: dict) -> dict:
        # it verbatim for apps that need a fixed on-disk layout (e.g. Gitea keeps all state
        # directly under /data). Omitting it preserves the /data/<name> default (back-compat).
        mount_path = vol.get("mountPath", f"/data/{vol['name']}")
-        mount = {"name": vol["name"], "mountPath": mount_path}
-        # subPath mounts a subtree of the volume instead of its root. Emitted only when declared,
-        # so every pre-existing golden stays byte-identical.
-        if vol.get("subPath"):
-            mount["subPath"] = vol["subPath"]
-        # readOnly on the mount as well as on the PV's nfs source. Belt and braces on purpose: the
-        # PV field is what makes the kernel mount `ro`; this one is what a reviewer reading the
-        # Deployment sees. An edit that drops one still leaves the other.
-        if vol.get("readOnly"):
-            mount["readOnly"] = True
-        volume_mounts.append(mount)
+        volume_mounts.append({"name": vol["name"], "mountPath": mount_path})
        volumes.append(
            {
                "name": vol["name"],
@@ -709,82 +543,6 @@ def r_gateway_ingress_policy(s: dict):
    }


-def r_in_cluster_client_policies(s: dict) -> list:
-    """East-west allow: let a declared IN-CLUSTER client reach this service DIRECTLY (R-NET-02).
-
-    `ingressRoutes` opens the NORTH-SOUTH path only — r_gateway_ingress_policy admits the
-    reserved `ingress` identity (the Gateway proxy) and nothing else — and `dependencies`
-    compiles edges only BETWEEN CATALOG SERVICES (both halves are rendered from a catalog
-    entry's own declaration). A PLATFORM CONTROLLER that dials this service's ClusterIP
-    directly therefore had no authorable form at all, and under default-deny (R-NET-01) it is
-    DROPPED.
-
-    That gap is not theoretical: prod's `flux-system` source-controller clones the platform's
-    own repo from the in-cluster Gitea Service (clusters/prod/kustomization.yaml pins
-    `url: http://gitea.gitea.svc.cluster.local:3000/...` so prod's GitOps depends on neither
-    public DNS, Cloudflare, its own Gateway nor ${BASE_DOMAIN}). gitea-allow-gateway-ingress
-    admits `fromEntities: [ingress]` only, so the clone was denied, the source could not
-    reconcile — and the HTTPRoute that would have served a public name instead lives in a
-    DOWNSTREAM tier that cannot apply while the source is down. Source down -> routes never
-    rename -> source stays down. It had to be broken with a live `kubectl apply`, i.e. with a
-    cluster object outside git, which one `kubectl delete` or one rebuild would take away.
-
-    Each entry renders ONE least-privilege ingress CiliumNetworkPolicy, shaped exactly like the
-    reciprocal-dependency half (r_reverse_dependency_policies) but selecting the client by
-    NAMESPACE alone, because a platform controller's pod labels are the upstream chart's to
-    change and are not part of this platform's contract: `fromEndpoints` carrying ONLY the
-    reserved `k8s:io.kubernetes.pod.namespace` label, `toPorts` carrying ONLY the declared
-    port/protocol. Never `fromEntities: all`, and never the whole service. Additive: the
-    namespace default-deny baseline keeps denying every undeclared source and port.
-
-    The policy NAME is `<service>-allow-<name>-ingress`, `name` defaulting to the namespace —
-    it is operator-facing (`kubectl get cnp`, a Hubble drop), and letting an author pin it is
-    what allows an allowance that already had to be applied live to be authored under its
-    EXISTING name, so Flux adopts that object in place rather than creating a near-duplicate
-    beside it and leaving the live one unmanaged.
-    """
-    out = []
-    for client in s["inClusterClients"]:
-        proto = client.get("protocol", "tcp").upper()
-        out.append(
-            {
-                "apiVersion": "cilium.io/v2",
-                "kind": "CiliumNetworkPolicy",
-                "metadata": {
-                    "name": f"{s['name']}-allow-{client['name']}-ingress",
-                    "namespace": s["name"],
-                    "labels": {"app.kubernetes.io/name": s["name"]},
-                },
-                "spec": {
-                    "endpointSelector": {
-                        "matchLabels": {"app.kubernetes.io/name": s["name"]}
-                    },
-                    "ingress": [
-                        {
-                            "fromEndpoints": [
-                                {
-                                    "matchLabels": {
-                                        "k8s:io.kubernetes.pod.namespace": client[
-                                            "namespace"
-                                        ]
-                                    }
-                                }
-                            ],
-                            "toPorts": [
-                                {
-                                    "ports": [
-                                        {"port": str(client["port"]), "protocol": proto}
-                                    ]
-                                }
-                            ],
-                        }
-                    ],
-                },
-            }
-        )
-    return out
-
-
 def _route_meta(s: dict, route: dict) -> dict:
    meta = {
        "name": f"{s['name']}-{route['port']}",
@@ -1337,14 +1095,6 @@ def render(sd: dict, image_automation: bool = True, inbound: list | None = None)
    if s["kind"] == "container":
        # Container PVCs. (For a vm, the CDI DataVolume owns/creates the disk PVC, so we
        # never render a plain PVC for the VM root disk — it would collide with CDI.)
-        # The static bulk PV is rendered BEFORE the PVC that names it. Document order inside one
-        # kustomize build is not an apply-ordering guarantee, but a PVC whose PV does not exist
-        # yet merely stays Pending and binds on the next reconcile, so this is a readability
-        # convention rather than a correctness requirement.
-        for pv in r_bulk_pvs(s):
-            docs.append(
-                (f"PersistentVolume {pv['metadata']['name']} - static external NFS (bulk tier)", pv)
-            )
        for pvc in r_pvcs(s):
            docs.append((f"PVC {pvc['metadata']['name']} (R-STATE)", pvc))
        docs.append(("Deployment (R-PLAT-02)", r_deployment(s)))
@@ -1377,15 +1127,6 @@ def render(sd: dict, image_automation: bool = True, inbound: list | None = None)
            )
        )

-    for cnp in r_in_cluster_client_policies(s):
-        docs.append(
-            (
-                "CiliumNetworkPolicy in-cluster client ingress allow (R-NET-02): "
-                f"{cnp['metadata']['name']}",
-                cnp,
-            )
-        )
-
    ie = r_internet_egress(s)
    docs.append(
        (
@@ -1580,11 +1321,6 @@ def plan_apps_waves(sds: list, present_files: set) -> dict:
      - `<svc>-db.yaml`                     -> wave(svc-db) (a strictly-lower wave; CNPG DB)
      - `<svc>-db-app.sops.yaml`            -> wave(svc-db) (CNPG initdb owner secret)
      - `<svc>-db-backup-credentials.sops.yaml` -> wave(svc-db) (CNPG Barman S3 backup secret)
-      - `<svc>-db-backup-credentials-prod.sops.yaml` -> wave(svc-db) (the PROD half of that
-        secret: the two environments carry DIFFERENT object-store credentials, selected in
-        the Cluster CR by `${BACKUP_CREDENTIALS_SUFFIX}`. Both objects ship to both clusters,
-        so both must be placed in the DB wave — the Secret has to exist before the Cluster
-        that names it.)
    Non-catalog units (EXTRA_UNITS, e.g. gitea-actions) place by their declared anchor+offset.
    Only files that actually exist in `present_files` are placed, so the plan is robust to a
    service having no bootstrap secret or no DB. Deterministic (sorted).
@@ -1612,9 +1348,6 @@ def plan_apps_waves(sds: list, present_files: set) -> dict:
            add(wdb, f"{db}.yaml")
            add(wdb, f"{db}-app.sops.yaml")
            add(wdb, f"{db}-backup-credentials.sops.yaml")   # CNPG Barman S3 creds (R-STATE-03)
-            # PROD half of the same credential (2026-09-07 per-env split). Placed only when
-            # the file exists, so a DB that has not been split is unaffected.
-            add(wdb, f"{db}-backup-credentials-prod.sops.yaml")

    for unit, cfg in EXTRA_UNITS.items():
        anchor = cfg["anchor"]
@@ -1902,101 +1635,6 @@ def _selftest() -> int:
        "a non-exposed service must not render a gateway-ingress allow (least privilege)",
    )

-    # 4c) inClusterClients -> one ADDITIVE, least-privilege ingress allow per entry that admits a
-    #     PLATFORM controller's NAMESPACE (not the reserved `ingress` identity, not `all`) to ONE
-    #     declared port. This is what keeps a cluster whose GitOps source is an in-cluster Service
-    #     reconcilable: fromEntities[ingress] does not cover a controller dialling the ClusterIP.
-    icc_docs = render(
-        {
-            "name": "g",
-            "source": {"image": "a:1"},
-            "ingressRoutes": [{"host": "g.home.keithjackson.dev", "port": 3000}],
-            "auth": {"mode": "none"},
-            "inClusterClients": [
-                {"name": "flux", "namespace": "flux-system", "port": 3000}
-            ],
-        }
-    )
-    icc = [
-        d[1]
-        for d in icc_docs
-        if d[1]["kind"] == "CiliumNetworkPolicy"
-        and d[1]["metadata"]["name"] == "g-allow-flux-ingress"
-    ]
-    check(
-        len(icc) == 1,
-        "an inClusterClients entry must render one ingress CNP named "
-        "<service>-allow-<name>-ingress",
-    )
-    if icc:
-        p = icc[0]
-        check(
-            p["metadata"]["namespace"] == "g"
-            and p["spec"]["endpointSelector"]["matchLabels"]
-            == {"app.kubernetes.io/name": "g"},
-            "in-cluster client allow must live in the service's namespace and select the service",
-        )
-        ing = p["spec"]["ingress"]
-        check(
-            ing[0]["fromEndpoints"]
-            == [{"matchLabels": {"k8s:io.kubernetes.pod.namespace": "flux-system"}}],
-            "in-cluster client allow must match the client NAMESPACE only (reserved k8s: label)",
-        )
-        check(
-            "fromEntities" not in ing[0],
-            "in-cluster client allow must not widen to an entity (never fromEntities: all)",
-        )
-        icc_ports = {
-            (pt["port"], pt["protocol"])
-            for r in ing
-            for tp in r["toPorts"]
-            for pt in tp["ports"]
-        }
-        check(
-            icc_ports == {("3000", "TCP")},
-            "in-cluster client allow must be least-privilege (only the declared port/protocol)",
-        )
-    # ADDITIVE: default-deny ingress stays [] and the gateway allow is untouched.
-    icc_dd = next(d[1] for d in icc_docs if d[1]["kind"] == "NetworkPolicy")
-    check(
-        icc_dd["spec"]["ingress"] == [],
-        "in-cluster client allow must be ADDITIVE — default-deny ingress stays [] (R-NET-01)",
-    )
-    check(
-        any(
-            d[1]["metadata"].get("name") == "g-allow-gateway-ingress"
-            for d in icc_docs
-            if d[1]["kind"] == "CiliumNetworkPolicy"
-        ),
-        "in-cluster client allow must not displace the north-south gateway allow",
-    )
-    # `name` is OPTIONAL and defaults to the namespace, so the common case is namespace + port.
-    icc_default = render(
-        {
-            "name": "g",
-            "source": {"image": "a:1"},
-            "inClusterClients": [{"namespace": "monitoring", "port": 9090}],
-        }
-    )
-    check(
-        any(
-            d[1]["metadata"].get("name") == "g-allow-monitoring-ingress"
-            for d in icc_default
-            if d[1]["kind"] == "CiliumNetworkPolicy"
-        ),
-        "inClusterClients.name must default to the admitted namespace",
-    )
-    # A service that declares none must render NO such policy (grant-nothing-extra).
-    check(
-        not any(
-            d[1]["kind"] == "CiliumNetworkPolicy"
-            and d[1]["metadata"]["name"].endswith("-ingress")
-            and "fromEndpoints" in d[1]["spec"].get("ingress", [{}])[0]
-            for d in render({"name": "q", "source": {"image": "a:1"}})
-        ),
-        "a service declaring no inClusterClients must render no in-cluster client allow",
-    )
-
    # 5) vm kind -> VirtualMachine + standalone DataVolume.
    vm_docs = render(
        {
@@ -2192,7 +1830,6 @@ def _selftest() -> int:
    present = {
        "a.yaml", "svc.yaml", "svc-bootstrap-secret.sops.yaml",
        "svc-db.yaml", "svc-db-app.sops.yaml", "svc-db-backup-credentials.sops.yaml",
-        "svc-db-backup-credentials-prod.sops.yaml",
        "gitea.yaml",
        "gitea-actions.yaml", "gitea-actions-runner-secret.sops.yaml",
    }
@@ -2202,10 +1839,8 @@ def _selftest() -> int:
    flat = {f: wv for wv, fs in plan.items() for f in fs}
    check(flat.get("a.yaml") == 0, "leaf app must land in wave 0")
    check(flat.get("svc-db.yaml") == 0 and flat.get("svc-db-app.sops.yaml") == 0
-          and flat.get("svc-db-backup-credentials.sops.yaml") == 0
-          and flat.get("svc-db-backup-credentials-prod.sops.yaml") == 0,
-          "CNPG DB manifest + its owner secret + BOTH halves of its per-env Barman backup secret "
-          "must be in the DB's (lower) wave")
+          and flat.get("svc-db-backup-credentials.sops.yaml") == 0,
+          "CNPG DB manifest + its owner secret + its Barman backup secret must be in the DB's (lower) wave")
    check(flat.get("svc.yaml") == 1 and flat.get("svc-bootstrap-secret.sops.yaml") == 1,
          "a DB-backed app + its bootstrap secret must be in the consumer wave, above the DB")
    check(flat.get("gitea.yaml") == 0, "gitea (leaf) must be wave 0")
@@ -78,34 +78,16 @@ env:
  #     (util ConfigType `oidc_providers`, env-tagged SEMAPHORE_OIDC_PROVIDERS). The value is a single
  #     JSON blob for the `keycloak` provider — provider_url is the homelab realm issuer (auto-discovery),
  #     client_id=semaphore, redirect_url=/api/auth/oidc/keycloak/redirect, claims map to the realm's
-  #     token. The name is NOT a kubelet service-link var (those are SEMAPHORE_SERVICE_HOST /
-  #     SEMAPHORE_PORT_*), so no collision. The LOCAL SEMAPHORE_ADMIN above stays break-glass (docs/15).
-  #
-  #     PER-ENVIRONMENT URLS (2026-09-07 fix). The blob USED to live whole inside the SOPS-encrypted
-  #     semaphore-bootstrap Secret (key `oidc-providers-json`) because it CONTAINS the confidential
-  #     client secret. That hid two environment-specific, NON-SECRET URLs — the realm issuer and the
-  #     redirect URI — inside ciphertext, where Flux's ${BASE_DOMAIN} postBuild substitution CANNOT
-  #     reach them. Both were frozen at the apex `home.keithjackson.dev`, so PROD (BASE_DOMAIN =
-  #     prod.home.keithjackson.dev) pointed its OIDC at DEV's Keycloak — unroutable from prod's
-  #     192.168.2.0/24 (the identical root cause as the Headlamp login timeout; see
-  #     platform/headlamp/README.md). Dev was only ever "right" because dev IS the apex.
-  #
-  #     THE SPLIT: the URLs are public, so the blob is now a PLAINTEXT literal here and gets
-  #     ${BASE_DOMAIN}-substituted per environment like every other hostname. Only the confidential
-  #     client secret stays SOPS-encrypted, in its own key (`oidc-client-secret`), and is spliced back
-  #     in by the KUBELET's dependent-env-var expansion: an env `value` may reference an EARLIER env
-  #     var of the same container as $(NAME), and secretKeyRef-sourced vars are eligible. So
-  #     SEMAPHORE_OIDC_CLIENT_SECRET MUST STAY IMMEDIATELY ABOVE — the kubelet resolves env in list
-  #     order, and a reference to a not-yet-defined name is left as the LITERAL text $(...). It is
-  #     fail-loud in the way that matters: if the Secret key is missing the Pod never starts
-  #     (CreateContainerConfigError), rather than silently authenticating against nothing. ---
-  - name: SEMAPHORE_OIDC_CLIENT_SECRET
+  #     token. It CONTAINS the confidential client secret, so the whole blob is held SOPS-encrypted in the
+  #     semaphore-bootstrap Secret (key oidc-providers-json; same secret value SOPS-held in
+  #     keycloak-realm-clients) and injected here by reference — never in cleartext. The name is NOT a
+  #     kubelet service-link var (those are SEMAPHORE_SERVICE_HOST / SEMAPHORE_PORT_*), so no collision.
+  #     The LOCAL SEMAPHORE_ADMIN above stays as the break-glass path (docs/15). ---
+  - name: SEMAPHORE_OIDC_PROVIDERS
    valueFrom:
      secretKeyRef:
        name: semaphore-bootstrap
-        key: oidc-client-secret
-  - name: SEMAPHORE_OIDC_PROVIDERS
-    value: '{"keycloak":{"display_name":"Keycloak","provider_url":"https://keycloak.${BASE_DOMAIN}/realms/homelab","client_id":"semaphore","client_secret":"$(SEMAPHORE_OIDC_CLIENT_SECRET)","redirect_url":"https://semaphore.${BASE_DOMAIN}/api/auth/oidc/keycloak/redirect","scopes":["openid","profile","email"],"username_claim":"preferred_username","email_claim":"email","name_claim":"name","order":1}}'
+        key: oidc-providers-json
 # The semaphoreui/semaphore image runs as the non-root user UID 1001 / GID 0 (tini -> server-wrapper),
 # and on boot the wrapper writes its generated config.json into the config dir. A mounted Longhorn PVC
 # is root:root, so UID 1001 cannot write it -> the pod panics ("permission denied"). fsGroup=1001 makes
@@ -135,12 +117,10 @@ auth:
  groups:
    - platform-admins           # access scoped to an authorized group (R-IDP-06)
  # WIRED (native OIDC): the Keycloak `semaphore` confidential client (platform/keycloak/keycloak-realm-
-  # import.yaml + live kcadm) is consumed via the SEMAPHORE_OIDC_PROVIDERS env — a PLAINTEXT JSON blob
-  # (see `env` above) whose per-environment URLs are ${BASE_DOMAIN}-substituted, with the confidential
-  # client secret spliced in at runtime from the semaphore-bootstrap Secret key `oidc-client-secret`.
-  # The `groups` claim is exposed for authorization (R-IDP-06). Semaphore KEEPS its LOCAL admin as the
-  # break-glass path so an IdP outage never locks the operator out (docs/15 "Availability caveat").
-  # No secret is embedded here.
+  # import.yaml + live kcadm) is consumed via the SEMAPHORE_OIDC_PROVIDERS env (JSON blob held encrypted
+  # in the semaphore-bootstrap Secret, oidc-providers-json). The `groups` claim is exposed for
+  # authorization (R-IDP-06). Semaphore KEEPS its LOCAL admin as the break-glass path so an IdP outage
+  # never locks the operator out (docs/15 "Availability caveat"). No secret is embedded here.
 update:
  channel: stable               # R-UPD-01
  range: ">=2.19.0 <3.0.0"      # stay on the 2.x stable line
@@ -9,23 +9,6 @@ name: sonarr
 kind: container
 source:
  image: lscr.io/linuxserver/sonarr:4.0.15   # pinned stable v4 tag (R-UPD-01); update.range governs advancement
-# LinuxServer images run their service as the user `abc`, whose uid/gid these two set. They are
-# LOAD-BEARING for the bulk mount, not cosmetic: see the storage note below. The default is 911,
-# which cannot write a 1000:3000-owned 0755 tree.
-env:
-  - name: PUID
-    value: "1000"               # owner uid of every directory under /mnt/Storage/plex
-  - name: PGID
-    value: "3000"               # dominant group there; keeps files this writes readable to the
-                                # EXISTING non-k3s consumers of the same export, which stay live
-  - name: TZ
-    value: Etc/UTC              # same safe default plex uses; the arrs schedule by local time
-podSecurityContext:
-  # For the LONGHORN config PVC only. fsGroup does nothing on the NFS volume (the in-tree plugin
-  # does no ownership management), but on a Longhorn filesystem the kubelet applies it, which is
-  # what makes /config writable by the now-uid-1000 process WITHOUT relying on the image chowning
-  # /config for itself when PUID changes. Matches PGID so the primary group owns the config too.
-  fsGroup: 3000
 exposure: internal              # LAN-only web UI at sonarr.${BASE_DOMAIN} (docs/13 row 2; R-DNS/R-NET)
 egress: vpn                     # indexer scrape + downloader handoff steered via the shared VPN egress gateway,
                                # kill-switch enforced OUTSIDE the workload (docs/03; ADR-0015; R-NET-05/06)
@@ -33,66 +16,11 @@ storage:
  - name: config
    size: 5Gi
    class: longhorn             # app DB / config on the 3-replica tier; DR-critical (docs/04; R-STATE-02)
-    mountPath: /config          # THE APP WRITES /config, NOT the renderer default /data/config.
-                                # LSIO images run with `-data=/config` (Jackett/qBittorrent likewise use
-                                # /config), so without this the Longhorn PVC mounted at /data/config sat
-                                # holding nothing but lost+found while the app wrote its entire
-                                # configuration to the container's EPHEMERAL writable layer — destroyed on
-                                # every pod recreation: restart, reschedule, image bump. Measured in-cluster
-                                # 2026-09-12: /data/config was dev=2080 with [lost+found], /config was
-                                # dev=2053 on the node's own disk with [Sentry asp config.xml logs].
-                                # This is the SAME failure catalog/plex.yaml:54 already records fixing.
-                                # The live /config contents were copied onto each PVC before this landed,
-                                # so API keys and credentials other apps reference were preserved.
-  # THE MEDIA TREE (docs/04 bulk tier) — ONE read-WRITE mount of the export ROOT at /media, giving
-  # /media/tv (the library) and /media/downloads (the import source) UNDER THE SAME MOUNT.
-  #
-  # *** THE ROOT MOUNT IS THE POINT, NOT A CONVENIENCE. *** Sonarr imports by HARDLINKING or
-  # atomically renaming a finished download into the library. Both are same-FILESYSTEM operations,
-  # and to the kernel "same filesystem" means SAME MOUNT. The layout on the host is:
-  #     /mnt/Storage/plex/tv/                      the library
-  #     /mnt/Storage/plex/downloads/complete/tv-sonarr/  where finished releases land
-  # — siblings under one dataset, so a hardlink between them is legal on the server. But mounting
-  # them as two subPath volumes (subPath: tv at /media/tv, subPath: downloads elsewhere) creates
-  # TWO mounts, and link() across mounts fails EXDEV no matter what the server thinks. Sonarr then
-  # silently degrades to copy-then-delete: every import rewrites the whole file over NFS and the
-  # seeding copy is destroyed. Their smallest common ancestor IS the export root, so the root mount
-  # is the MINIMAL mount that preserves hardlinks — least privilege bounded below by a hard
-  # requirement, not a shortcut. (The cost is stated plainly: Sonarr can also write /media/movies,
-  # which is radarr's. Narrowing that would cost hardlinks, which is a worse trade.)
-  #
-  # *** /media, NOT /data — AND THAT IS NOT A STYLE CHOICE. *** The config PVC above renders at the
-  # renderer default /data/config. A media mount at /data would be mounted OVER it and MASK the
-  # config volume entirely; the pod would come up with its app database shadowed by an 8 T media
-  # tree. /media collides with nothing in a LinuxServer image and matches the path plex uses for
-  # the same export, so when qbittorrent is eventually given this mount too, the arrs and the
-  # download client will already agree on absolute paths — which is the other precondition for
-  # hardlinked imports (otherwise every import needs a Remote Path Mapping).
-  #
-  # SEPARATE LATENT BUG, NOT FIXED HERE (deliberately — it changes live on-disk state): /data/config
-  # is almost certainly the wrong place for this PVC in the first place. LinuxServer's arr images run
-  # with `-data=/config`, so the app writes /config — the container's EPHEMERAL layer — while the
-  # Longhorn PVC at /data/config sits unused. That is the EXACT failure plex's own mountPath comment
-  # documents having already been bitten by. Fixing it means starting from an empty config volume, so
-  # it belongs in its own change with the operator present.
-  #
-  # UID/GID — THE FAILURE THIS AVOIDS IS "MOUNTS FINE, CANNOT WRITE". The export is no_root_squash
-  # + sec=sys, so the client's numeric uid is passed through verbatim and the server's ownership is
-  # the whole story. The tree is 0755 owned 1000:3000 (some dirs 1000:1000), so ONLY uid 1000 can
-  # write it. LinuxServer images default to PUID/PGID 911 — which would mount cleanly and then fail
-  # every import with EACCES. The PUID/PGID set above make the process uid 1000, matching the owner.
-  #
-  # AND NOTE fsGroup CANNOT SUBSTITUTE FOR THAT. The in-tree NFS volume plugin performs no
-  # ownership management, so podSecurityContext.fsGroup is a NO-OP on this volume — it is not the
-  # lever here, and reaching for it is how this failure normally gets misdiagnosed. The fsGroup
-  # that IS set above exists for the LONGHORN config PVC, so that changing PUID does not depend on
-  # the image re-chowning /config for itself.
-  - name: media
-    class: bulk
-    export: /mnt/Storage/plex   # the ZFS dataset Storage/plex, on ${BULK_NFS_SERVER}
-    mountPath: /media           # => /media/tv (Root Folder) and /media/downloads (import source)
-    size: 9Ti                   # ADVISORY ONLY on a static NFS PV: the dataset's 8.33 T used,
-                                # rounded up, so `kubectl get pv` reports something true.
+  # NOTE (bulk-media PREREQUISITE): docs/13 row 2 places the media library / downloads volume on the
+  # `bulk` tier (docs/04 bulk storage). The `bulk` StorageClass exists (platform/longhorn/storageclasses.yaml)
+  # but is a NON-provisioning STUB pending OQ-2 (NFS/SMB backing = REPLACE-ME). Adding a `class: bulk`
+  # volume now would leave the PVC (and therefore the pod) Pending. The media/downloads volume is
+  # therefore DEFERRED until OQ-2 is resolved; config-only on longhorn keeps Sonarr deployable today.
 dependencies:
  - to: qbittorrent             # arr → download client (docs/13 dependency graph; R-NET-02 explicit allow)
    port: 8080
@@ -107,11 +35,6 @@ auth:
  mode: forward-auth            # bare UI, no usable OIDC: oauth2-proxy → Keycloak at the Gateway (docs/15; R-IDP-02)
  groups:
    - media-users               # authenticated AND in the media group (R-IDP-06)
-    # See catalog/jackett.yaml for the full rationale: --allowed-group is repeatable and OR-ed, so
-    # platform-admins here is a superset, not a widening (that group already carries cluster-admin
-    # via k8s-oidc-rbac.yaml). Without it, an empty media-users locks every operator out with an
-    # immediate 403 at the oauth2-proxy callback — observed on prod 2026-09-12.
-    - platform-admins           # operators of these services must not be locked out of them
 update:
  channel: stable               # R-UPD-01
  range: ">=4.0.0 <5.0.0"       # stay on the Sonarr v4 stable line
@@ -137,82 +137,6 @@ else
  esac
 fi

-# ---- Stage 3d: Keycloak client-secret declaration + reconciler (UNCONDITIONAL) ----
-# The Keycloak realm import resolves each OIDC client's secret from an `(env:NAME)`
-# placeholder. When the operator cannot resolve one it stores the LITERAL placeholder as
-# the client secret AND THE IMPORT JOB STILL REPORTS SUCCESS, then rests at Done=True and
-# never re-runs. On 2026-09-06 that had all 12 prod clients unusable ("Invalid client or
-# Invalid client credentials") behind an entirely green cluster; dev had one. The check
-# asserts the declaration is self-consistent (every placeholder defined, every Secret key
-# present, names matching the reconciler's contract), that the Flux ordering is a REAL
-# dependsOn edge rather than a comment, and — by extracting the shipped script from its
-# ConfigMap and running its --selftest — that the reconciler is compare-then-write,
-# idempotent, non-logging and blast-radius-bounded.
-#
-# WHY ITS OWN UNCONDITIONAL STAGE, same reasoning as 3c: Stage 3b scopes the conformance
-# run to the TOUCHED requirements (R-TEST-03). This check's invariant spans platform/,
-# platform-config/ and clusters/, so a change that breaks it need not attribute R-IDP-03/04
-# and the requirement-scoped selection would filter it out of exactly the change that
-# introduces the bug. It is a sub-second offline scan plus a loopback selftest, with no
-# side effects, so running it twice when 3b also selects it is not worth avoiding.
-KEYCLOAK_SECRET_CHECK="${KEYCLOAK_SECRET_CHECK:-${REPO_ROOT}/conformance/tests/l0-static/keycloak-client-secret-reconciler/check.sh}"
-log "Stage: Keycloak client-secret reconciler safety (${KEYCLOAK_SECRET_CHECK})"
-if [ ! -f "$KEYCLOAK_SECRET_CHECK" ]; then
-  skip "keycloak client-secret check not present at '${KEYCLOAK_SECRET_CHECK}' — UNWIRED, not green"
-  summary "- keycloak-client-secrets: **SKIPPED** (check missing)"
-  if [ "$rc" -eq 0 ]; then rc=3; fi
-else
-  kc_rc=0
-  bash "$KEYCLOAK_SECRET_CHECK" || kc_rc=$?
-  case "$kc_rc" in
-    0) ok "Keycloak client-secret declaration + reconciler passed"
-       summary "- keycloak-client-secrets: **passed**" ;;
-    3) skip "keycloak client-secret check SKIPPED (platform/keycloak absent) — not a pass"
-       summary "- keycloak-client-secrets: **SKIPPED** (nothing to audit)"
-       if [ "$rc" -eq 0 ]; then rc=3; fi ;;
-    *) err "Keycloak client-secret declaration + reconciler FAILED"
-       summary "- keycloak-client-secrets: **FAILED**"; rc=1 ;;
-  esac
-fi
-
-# ---- Stage 3e: bulk tier distinct from the replicated tier (UNCONDITIONAL) ----
-# The `bulk` StorageClass sat as a REPLACE-ME stub for six roadmap phases, and the only
-# thing keeping a multi-terabyte media library off the 3-replica `longhorn` class was four
-# hand-written comments in ServiceDefinitions. A bulk volume that lands on the replicated
-# tier does not fail loudly — Longhorn accepts the PVC and starts replicating 3x of data
-# three mini-PCs cannot hold. The check asserts the tier separation, that a bulk volume is
-# statically pre-bound from BOTH ends (so nothing can dynamically provision an EMPTY
-# stand-in that mounts and Binds perfectly while the real data is never attached), Retain,
-# hard/NFSv4 mount options, the read-only coupling between PV and volumeMount, that no
-# workload mounts one export twice (which breaks Sonarr/Radarr hardlinks with EXDEV), and
-# that the server is a per-environment knob every overlay supplies.
-#
-# WHY ITS OWN UNCONDITIONAL STAGE, same reasoning as 3c/3d: Stage 3b scopes the conformance
-# run to the TOUCHED requirements (R-TEST-03), and ci/touched-requirements.sh's path map
-# keys NEITHER apps/* nor platform/longhorn/* to the bulk requirement. So the change most
-# likely to break this — moving a bulk volume onto the replicated tier, or narrowing one
-# export into two mounts — could attribute no requirement at all and be gated by nothing.
-# The invariant spans catalog/, apps/ and clusters/, so it runs on EVERY gate invocation.
-BULK_TIER_CHECK="${BULK_TIER_CHECK:-${REPO_ROOT}/conformance/tests/l0-static/bulk-tier-distinct-from-replicated/check.sh}"
-log "Stage: bulk tier distinct from replicated tier (${BULK_TIER_CHECK})"
-if [ ! -f "$BULK_TIER_CHECK" ]; then
-  skip "bulk tier check not present at '${BULK_TIER_CHECK}' — UNWIRED, not green"
-  summary "- bulk-tier: **SKIPPED** (check missing)"
-  if [ "$rc" -eq 0 ]; then rc=3; fi
-else
-  bulk_rc=0
-  bash "$BULK_TIER_CHECK" || bulk_rc=$?
-  case "$bulk_rc" in
-    0) ok "Bulk tier distinctness + static binding passed"
-       summary "- bulk-tier: **passed**" ;;
-    3) skip "bulk tier check SKIPPED (catalog/ or apps/ absent) — not a pass"
-       summary "- bulk-tier: **SKIPPED** (nothing to audit)"
-       if [ "$rc" -eq 0 ]; then rc=3; fi ;;
-    *) err "Bulk tier distinctness + static binding FAILED"
-       summary "- bulk-tier: **FAILED**"; rc=1 ;;
-  esac
-fi
-
 case "$rc" in
  0) ok "L0 stage complete" ;;
  3) skip "L0 stage incomplete (unwired seam / missing tool) — not a pass" ;;
@@ -1,214 +0,0 @@
-#!/usr/bin/env python3
-# =============================================================================
-# ci/render-keycloak-client-urls.py
-#
-# Render platform/keycloak/client-urls.yaml FROM platform/keycloak/keycloak-realm-import.yaml.
-#
-# WHY THIS EXISTS
-# ---------------
-# The Keycloak realm import is a ONE-SHOT that fails OPEN: once it rests at
-# Done=True / Started=False it never re-runs, so a redirect URI changed in git is
-# never applied to the realm. The client-secret half of that failure class is
-# already healed by a level-triggered CronJob (platform/keycloak/client-secret-
-# reconciler.yaml). This renderer supplies the DESIRED STATE that the same CronJob
-# needs to heal the URL half.
-#
-# THE CONSTRAINT THAT SHAPES IT: the reconciler pod runs with
-# automountServiceAccountToken: false. It has NO Kubernetes API access at all, by
-# design (R-SEC-02) — so it cannot read the KeycloakRealmImport CR to learn what
-# the realm is supposed to look like. Its inputs must be MOUNTED. Hence a
-# ConfigMap.
-#
-# A hand-authored ConfigMap would be a SECOND SOURCE OF TRUTH for every public
-# hostname in the realm, free to disagree with the realm import — and "two
-# declarations that can disagree" is the same shape as the bug this whole effort
-# is closing. So the ConfigMap is GENERATED from the realm import and the
-# generation is GATED: conformance/tests/l0-static/keycloak-client-secret-
-# reconciler/check.sh runs this script with --check on every merge, and the merge
-# fails if the committed file is not what this renderer produces. That is the same
-# render==golden idiom catalog/render.py + ci/render-golden.sh already use for
-# apps/.
-#
-# STRINGS ARE COPIED VERBATIM, NEVER RE-DERIVED. If a URL in the realm import is
-# written as a Flux substitution token (https://grafana.${BASE_DOMAIN}/...), the
-# token is carried through into the ConfigMap CHARACTER FOR CHARACTER. Both files
-# are rendered by the SAME Flux Kustomization (./platform-config, whose
-# postBuild.substituteFrom reads cluster-vars), so both are substituted in the
-# same pass with the same values and cannot disagree about what BASE_DOMAIN is.
-# This renderer never parses, splits, or reconstructs a hostname, and never reads
-# cluster-vars — so there is no second place for the domain to be decided.
-#
-# USAGE
-#   python3 ci/render-keycloak-client-urls.py            # print the rendered file
-#   python3 ci/render-keycloak-client-urls.py --write     # write it (regenerate)
-#   python3 ci/render-keycloak-client-urls.py --check      # exit 1 if it has drifted
-# =============================================================================
-from __future__ import annotations
-
-import argparse
-import difflib
-import json
-import sys
-from pathlib import Path
-
-try:
-    import yaml
-except ImportError:  # pragma: no cover
-    print("FATAL: PyYAML is required (same dependency as catalog/render.py)",
-          file=sys.stderr)
-    raise SystemExit(1)
-
-ROOT = Path(__file__).resolve().parent.parent
-IMPORT_FILE = "platform/keycloak/keycloak-realm-import.yaml"
-OUT_FILE = "platform/keycloak/client-urls.yaml"
-DATA_KEY = "client-urls.json"
-CONFIGMAP_NAME = "keycloak-client-urls"
-NAMESPACE = "keycloak"
-
-# The client fields this renderer exports, and therefore the ONLY fields the
-# reconciler may write. Every one of them is an environment-derived URL: it changes
-# when BASE_DOMAIN changes, and a stale value breaks OIDC login (a wrong
-# redirectUri) or silently widens trust (a stale wildcard left behind).
-#
-# DELIBERATELY ABSENT: `secret` (healed by the secret pass from the SOPS Secret,
-# and it must never appear in a ConfigMap), and every non-URL field —
-# protocolMappers, defaultClientScopes, attributes, flags. Those are not
-# environment-derived, so nothing re-renders them on a domain flip; keeping them
-# out keeps the reconciler's write surface as narrow as the problem.
-LIST_FIELDS = ("redirectUris", "webOrigins")
-STR_FIELDS = ("rootUrl", "baseUrl", "adminUrl")
-
-
-def render(root: Path) -> str:
-    """Return the full text of client-urls.yaml as it should be committed."""
-    docs = list(yaml.safe_load_all((root / IMPORT_FILE).read_text()))
-    imp = next((d for d in docs if d and d.get("kind") == "KeycloakRealmImport"), None)
-    if imp is None:
-        raise SystemExit("FATAL: no KeycloakRealmImport document in %s" % IMPORT_FILE)
-    clients = (imp.get("spec") or {}).get("realm", {}).get("clients") or []
-    if not clients:
-        # Fail closed. Emitting an empty desired state would make the reconciler
-        # believe every client is unmanaged and silently stop reconciling URLs.
-        raise SystemExit("FATAL: %s declares zero clients — refusing to render an "
-                         "empty desired state" % IMPORT_FILE)
-
-    want: dict[str, dict] = {}
-    for c in clients:
-        cid = c.get("clientId")
-        if not isinstance(cid, str) or not cid:
-            raise SystemExit("FATAL: %s has a client with no clientId" % IMPORT_FILE)
-        entry: dict[str, object] = {}
-        for f in LIST_FIELDS:
-            if f not in c:
-                continue                 # not declared => NOT MANAGED (never forced empty)
-            v = c[f]
-            if not isinstance(v, list) or not all(isinstance(x, str) for x in v):
-                raise SystemExit("FATAL: %s client %r field %s is not a list of strings"
-                                 % (IMPORT_FILE, cid, f))
-            entry[f] = list(v)           # verbatim, order as authored
-        for f in STR_FIELDS:
-            if f not in c:
-                continue
-            v = c[f]
-            if not isinstance(v, str):
-                raise SystemExit("FATAL: %s client %r field %s is not a string"
-                                 % (IMPORT_FILE, cid, f))
-            entry[f] = v
-        if entry:
-            want[cid] = entry
-    if not want:
-        raise SystemExit("FATAL: no client in %s declares any of %s — refusing to render "
-                         "an empty desired state"
-                         % (IMPORT_FILE, ", ".join(LIST_FIELDS + STR_FIELDS)))
-
-    body = json.dumps(want, indent=2, sort_keys=True, ensure_ascii=False)
-    indented = "\n".join(("    " + ln) if ln else "" for ln in body.splitlines())
-    return HEADER + indented + "\n"
-
-
-HEADER = """---
-# =============================================================================
-# GENERATED FILE — DO NOT EDIT BY HAND.
-#
-#   source:      platform/keycloak/keycloak-realm-import.yaml (spec.realm.clients)
-#   regenerate:  python3 ci/render-keycloak-client-urls.py --write
-#   gated by:    conformance/tests/l0-static/keycloak-client-secret-reconciler
-#                (runs this renderer with --check on every merge)
-#
-# WHAT THIS IS: the DESIRED STATE for every environment-derived client URL in the
-# `homelab` realm — redirectUris, webOrigins, rootUrl, baseUrl, adminUrl. The
-# client-secret reconciler CronJob mounts it and makes the live realm match it.
-#
-# WHY IT IS A SEPARATE OBJECT: the realm import is a ONE-SHOT that never re-runs
-# once Done=True, so a redirect URI edited in git never reaches the realm. Prod's
-# stored redirect URIs were measured still pointing at the pre-migration hostname
-# while git declared the new one, and every OIDC login would have failed with a
-# redirect_uri mismatch. The reconciler is the level-triggered loop that closes
-# that; this file is what it reconciles TOWARDS.
-#
-# WHY IT IS GENERATED AND NOT AUTHORED: a hand-written copy of every public
-# hostname would be a second source of truth, free to disagree with the realm
-# import. It is rendered from the import and the render is gated, so the two
-# cannot drift. URL strings are copied VERBATIM — if the source uses a
-# ${BASE_DOMAIN} token it appears here unchanged, and both files are substituted
-# by the SAME Flux Kustomization (./platform-config), so the domain is decided in
-# exactly one place.
-#
-# A field that is NOT declared on a client in the realm import is simply ABSENT
-# here, and the reconciler leaves it alone. Absence means UNMANAGED, never
-# "force to empty".
-# =============================================================================
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: %s
-  namespace: %s
-  labels:
-    app.kubernetes.io/name: %s
-    app.kubernetes.io/part-of: keycloak
-data:
-  %s: |
-""" % (CONFIGMAP_NAME, NAMESPACE, CONFIGMAP_NAME, DATA_KEY)
-
-
-def main() -> int:
-    ap = argparse.ArgumentParser(description=__doc__)
-    g = ap.add_mutually_exclusive_group()
-    g.add_argument("--write", action="store_true", help="write %s" % OUT_FILE)
-    g.add_argument("--check", action="store_true",
-                   help="exit 1 if the committed file has drifted from the source")
-    args = ap.parse_args()
-
-    want = render(ROOT)
-    target = ROOT / OUT_FILE
-
-    if args.write:
-        target.write_text(want, encoding="utf-8")
-        print("wrote %s" % OUT_FILE)
-        return 0
-
-    if args.check:
-        if not target.is_file():
-            print("CHECK FAIL: %s does not exist. Run: python3 "
-                  "ci/render-keycloak-client-urls.py --write" % OUT_FILE, file=sys.stderr)
-            return 1
-        have = target.read_text(encoding="utf-8")
-        if have == want:
-            print("client-urls.yaml is in sync with %s" % IMPORT_FILE)
-            return 0
-        print("CHECK FAIL: %s has DRIFTED from %s — the reconciler would converge the "
-              "realm on a desired state the realm import does not declare.\n"
-              "Regenerate with: python3 ci/render-keycloak-client-urls.py --write\n"
-              % (OUT_FILE, IMPORT_FILE), file=sys.stderr)
-        for line in difflib.unified_diff(have.splitlines(True), want.splitlines(True),
-                                         fromfile="committed " + OUT_FILE,
-                                         tofile="rendered from " + IMPORT_FILE):
-            sys.stderr.write(line)
-        return 1
-
-    sys.stdout.write(want)
-    return 0
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())
@@ -180,29 +180,6 @@ path_requirements() {
    platform/externaldns/*)    echo "R-DNS-02" ;;
    platform/velero/*)         echo "R-STATE-03" ;;
    platform/monitoring/*)     echo "R-OBS-01 R-OBS-02" ;;
-    # platform/cnpg was UNMAPPED until 2026-09-06. It is the PostgreSQL DR path for keycloak-db /
-    # nextcloud-db / semaphore-db (barmanObjectStore WAL archiving + ScheduledBackup), so a change
-    # there is an R-IDP-05 / R-STATE-03 change; it now also carries the platform-health alert rules
-    # for that path (R-OBS-04). Without this line an edit under platform/cnpg/ contributed ZERO
-    # requirements unless it happened to retype an R-ID on a non-comment line, and escaped the gate.
-    platform/cnpg/*)           echo "R-IDP-05 R-STATE-03 R-OBS-04" ;;
-    # vpn-egress was UNMAPPED until 2026-09-06. platform/vpn-egress/egress-steering.yaml IS the
-    # R-NET-04 datapath (the rule that selects between the two egress paths) and the R-NET-05/06/07
-    # steering policy — it even self-declares those IDs in its `homelab.keithjackson.dev/requirements`
-    # annotation. But the content scan only reads ADDED/REMOVED lines, so any edit that does not
-    # happen to retype an R-ID (a selector change, an excludedCIDRs change, a comment) contributed
-    # ZERO requirements and escaped the gate entirely. Mapped explicitly; the annotation still wins
-    # when a diff does carry the tags.
-    platform/vpn-egress/*)     echo "R-NET-04 R-NET-05 R-NET-06 R-NET-07" ;;
-    # Keycloak is the whole SSO surface: the realm import (declarative identity config),
-    # the SOPS client secrets, and the reconciler that heals them. Added 2026-09-06 after
-    # the client-secret incident, in which editing platform/keycloak/ attributed NO
-    # requirement, so the L2 keycloak tests were filtered out of the merge gate's touched
-    # subset — the change that could break every OIDC login was gated by nothing.
-    # R-IDP-05 is in scope because platform/keycloak/ also carries the CNPG DB Cluster and
-    # its off-cluster backup credential, not only the realm.
-    platform/keycloak/*)       echo "R-IDP-01 R-IDP-02 R-IDP-03 R-IDP-04 R-IDP-05 R-IDP-06" ;;
-    platform/oauth2-proxy/*)   echo "R-IDP-02 R-IDP-06" ;;
    platform/image-automation/*) echo "R-UPD-01 R-UPD-02" ;;
    conformance/*)             echo "R-TEST-01 R-TEST-02 R-TEST-03 R-TEST-04" ;;
    clusters/*)                echo "R-GITOPS-01 R-ENV-02" ;;
@@ -34,9 +34,6 @@ spec:
    substituteFrom:
      - kind: ConfigMap
        name: cluster-vars
-  # SOPS decryption for the `cloud` INI credentials — velero-cloud.sops.yaml (dev's scoped MinIO key)
-  # and velero-cloud-prod.sops.yaml (prod's Cloudflare R2 token) — is layered per environment by the
-  # overlay (clusters/dev and clusters/prod patch THIS Kustomization by name), identical to how
-  # headlamp/oauth2-proxy/crafty decrypt their own SOPS secrets. Both Secrets are applied on both
-  # clusters; ${BACKUP_CREDENTIALS_SUFFIX} (substituted from cluster-vars above) decides which one the
-  # velero Deployment actually mounts. See clusters/dev/kustomization.yaml.
+  # SOPS decryption for velero-cloud.sops.yaml (the scoped MinIO `cloud` credential) is layered per
+  # environment by the overlay (clusters/dev patches THIS Kustomization by name), identical to how
+  # headlamp/oauth2-proxy/crafty decrypt their own SOPS secrets. See clusters/dev/kustomization.yaml.
@@ -96,22 +96,9 @@ data:
  #
  # Substitution coverage (verify before adding a knob here): every path that consumes these declares
  # postBuild.substituteFrom cluster-vars — platform (K3S_API_HOST), platform-config
-  # (CILIUM_LB_POOL_*, and keycloak-db's S3_BACKUP_ENDPOINT + CNPG_BACKUP_PATH: the Cluster lives at
-  # platform/keycloak/postgres-cluster.yaml but is rendered by platform-config, which is the ONLY
-  # tier that lists it), crafty (CRAFTY_L4_VIP), apps-wave-0 (nextcloud-db + semaphore-db:
-  # S3_BACKUP_ENDPOINT, CNPG_BACKUP_PATH), velero (S3_BACKUP_ENDPOINT, S3_BACKUP_REGION,
-  # VELERO_BACKUP_BUCKET, VELERO_BACKUP_PREFIX), apps-vms (UNIFI_L4_VIP). A ${TOKEN} in an
-  # UNSUBSTITUTED path would ship the literal string into the cluster.
-  #
-  # ADDING A KNOB IS SAFE UNDER StrictPostBuildSubstitutions ONLY BECAUSE OF THE ROOT EDGE. An
-  # UNDEFINED variable is a HARD Kustomization failure, so a manifest that references a knob the
-  # live ConfigMap does not yet carry breaks its whole tier. `clusters/base/platform.yaml` gained
-  # `dependsOn: [flux-system]` for exactly this: dependsOn is REVISION-gated, so no substituting
-  # tier can reconcile revision N until the ROOT Kustomization has APPLIED revision N — i.e. until
-  # the new cluster-vars key EXISTS. Every substituting consumer chains transitively through
-  # `platform`, so one edge covers the graph, and a new knob plus its first reference may ship in
-  # ONE commit. (A hand-run `kubectl apply -k clusters/<env>` bypasses that ordering entirely: apply
-  # the overlay BEFORE the tier that references the knob, or the tier fails until it retries.)
+  # (CILIUM_LB_POOL_*, S3_BACKUP_ENDPOINT), crafty (CRAFTY_L4_VIP), apps-wave-0 (S3_BACKUP_ENDPOINT),
+  # velero (S3_BACKUP_ENDPOINT), apps-vms (UNIFI_L4_VIP). A ${TOKEN} in an UNSUBSTITUTED path would
+  # ship the literal string into the cluster.

  # k3s API endpoint Cilium's kube-proxy replacement dials (platform/cilium/helmrelease.yaml).
  #
@@ -160,45 +147,29 @@ data:
  UNIFI_L4_VIP: "192.168.1.54"               # UniFi controller VM: inform/STUN/adoption (apps/vms/unifi)
  CRAFTY_L4_VIP: "192.168.1.55"              # Crafty: Minecraft Java (TCP) + Bedrock (UDP) game ports

-  # --- Bulk (external NFS) storage host (docs/04 bulk tier) ---------------------------------------
-  # THE address of the Proxmox host whose ZFS datasets back every `class: bulk` volume: the media
-  # library (Storage/plex, 8.33 T — plex, sonarr, radarr) and the legacy Nextcloud archive
-  # (Storage/nextcloud, 270 G). The renderer emits it into each static NFS PersistentVolume's
-  # `spec.nfs.server` (catalog/render.py, BULK_NFS_SERVER); the export PATH is per-dataset and is
-  # declared in the ServiceDefinition, because the path is identical in every environment and only
-  # the ADDRESS is not.
+  # Off-cluster S3 backup target (R-STATE-03, R-IDP-05). Consumed by BOTH backup paths: every CNPG
+  # Cluster's barmanObjectStore.endpointURL (keycloak-db, nextcloud-db, semaphore-db) and Velero's
+  # backupStorageLocation s3Url. Full URL including scheme (and port, if not the scheme default).
  #
-  # WHY THIS IS A KNOB AND NOT A LITERAL — the two clusters are on OPPOSITE SIDES of pfSense.
-  # The storage host has exactly one interface, vmbr0 192.168.1.9/24, on pfSense's LAN.
-  #   dev  nodes are 192.168.1.40/.41/.42 — the SAME L2 segment. They reach it directly, and one
-  #        NFS export to them already exists on that host (/mnt/Storage/k8s-volumes in
-  #        /etc/exports), so this path is proven, not theoretical. Base default = the dev value,
-  #        per the convention of the LAN block above.
-  #   prod nodes are 192.168.2.10/.11/.12 — pfSense's WAN. tcp/2049 does NOT traverse that today;
-  #        22 and 8006 answer only because pfSense port-forwards them via 192.168.2.102. Opening
-  #        the storage path is a PENDING OPERATOR DECISION with two candidate outcomes that give
-  #        DIFFERENT values here: 192.168.2.102 (another port-forward) or 192.168.1.9 (a route).
-  # So prod holds a loud TBD-PROD-* placeholder — see clusters/prod/kustomization.yaml for the
-  # blast radius of shipping it unset, which is NOT the usual "fails to apply".
-  BULK_NFS_SERVER: "192.168.1.9"
+  # OPERATOR DECISION 2026-09-05: **Cloudflare R2**, replacing the on-Proxmox MinIO — "for the time
+  # being until a better on-prem backup solution is created". Two reasons beyond cost:
+  #   1. The k3s prod cluster is on the SOHO network (pfSense's WAN) and CANNOT reach the MinIO inside
+  #      pfSense's LAN — measured, from a pod: connection timed out. R2 needs no firewall change.
+  #   2. That MinIO runs on the SAME Proxmox host as dev's VMs, so dev's "off-cluster" backup shared a
+  #      physical failure domain with dev. One host loss took the cluster AND its backups. R2 is
+  #      genuinely off-site.
+  # Measured to justify the cost: dev's whole estate is 4.0 GiB Velero (6,264 objects, 7d retention) +
+  # 211 MiB CNPG (1,755 objects) — pennies per month, and R2 egress is free, so restore REHEARSALS
+  # stay free and R-STATE-03 has no cost disincentive to stay proven.
+  #
+  # The MinIO copy is NOT deleted — it stays as a historical/local copy (3-2-1). Restoring from it
+  # means pointing this knob back at http://192.168.1.9:9000 temporarily.
+  S3_BACKUP_ENDPOINT: "https://TBD-R2-ACCOUNT-ID.r2.cloudflarestorage.com"

-  # Off-cluster S3 backup target (R-STATE-03, R-IDP-05) — the on-Proxmox MinIO, plain HTTP, path-style.
-  # Consumed by BOTH backup paths: every CNPG Cluster's barmanObjectStore.endpointURL (keycloak-db,
-  # nextcloud-db, semaphore-db) and Velero's backupStorageLocation s3Url. Full URL incl. scheme+port.
-  # Prod needs its OWN target: a backup that lands on the dev LAN's MinIO is not off-cluster DR —
-  # and, measured 2026-09-05 from a pod INSIDE prod, it CANNOT reach it anyway (prod is on the SOHO
-  # network 192.168.2.0/24, which is pfSense's WAN, while this MinIO sits inside pfSense's LAN:
-  # `curl http://192.168.1.9:9000/minio/health/live` timed out after 15s). Prod uses Cloudflare R2.
-  S3_BACKUP_ENDPOINT: "http://192.168.1.9:9000"
-
-  # S3 region. Every S3 client REQUIRES one; MinIO ignores whatever it is; Cloudflare R2 demands
-  # exactly `auto` (its `head-bucket` reports the physical hint `WNAM`, which is NOT a usable region
-  # string). This was hardcoded `us-east-1` inside platform/velero/helmrelease.yaml — correct-by-
-  # accident for MinIO and wrong for anything else, which is why it is a knob now. The base default
-  # is dev's LIVE value, deliberately: dev's Velero has been signing requests with `us-east-1`
-  # against that MinIO since bootstrap, and this repo's rule is that base == the known-good dev
-  # value. Prod overrides to `auto`.
-  S3_BACKUP_REGION: "us-east-1"
+  # S3 region. Required by every S3 client, IGNORED by MinIO, and must be exactly `auto` for R2.
+  # This was hardcoded `us-east-1` inside platform/velero/helmrelease.yaml — correct-by-accident for
+  # MinIO and wrong for R2, which is why it is a knob now.
+  S3_BACKUP_REGION: "auto"

  # --- storage (ADR-0003) ---
  LONGHORN_DEFAULT_REPLICA_COUNT: "3"       # default StorageClass replica count (3 on 3 nodes)
@@ -252,98 +223,38 @@ data:
  VELERO_BACKUP_SCHEDULE: "0 3 * * *"        # cron; overlays may differ
  VELERO_RETENTION: "168h"                   # dev short | prod long

-  # ==========================================================================================
-  # OBJECT-STORE TENANCY — the three knobs that keep two clusters and two backup tools from
-  # writing into each other's data. THESE ARE SAFETY KNOBS, not preferences.
-  # ==========================================================================================
-  # Nothing about the object layout was per-environment before this: Velero's bucket was the
-  # hardcoded literal `velero` with no prefix, and every CNPG Cluster used the hardcoded
-  # `s3://cnpg` with barman's `serverName` defaulting to the CLUSTER NAME — and BOTH environments
-  # have a keycloak-db, a nextcloud-db and a semaphore-db. Point two clusters at one bucket under
-  # that layout and:
-  #   (a) each cluster's Velero ADOPTS the other's backups and TTL-DELETES them on retention; and
-  #   (b) two different Postgres servers stream WAL into ONE Barman prefix, which does not merely
-  #       mix files — it CORRUPTS THE CHAIN and destroys PITR for both. That is the worse half:
-  #       dev's DR restore is PROVEN (R-STATE-03) and this would silently un-prove it.
+  # BACKUP TENANCY — the two knobs that keep two clusters sharing ONE MinIO from destroying each
+  # other's backups. Added 2026-09-05 when the operator ruled prod uses the SAME endpoint as dev.
  #
-  # WHY THERE IS NO SINGLE `S3_BACKUP_BUCKET`. The two consumers take differently-SHAPED values:
-  # the Velero chart wants `bucket` and `prefix` as separate fields, while CNPG's
-  # barmanObjectStore.destinationPath is ONE full s3:// URL whose first path segment IS the bucket.
-  # CNPG's bucket therefore already lives inside CNPG_BACKUP_PATH; adding a second knob that had to
-  # hold the same bucket name a second time would be a rotation waiting to drift — the exact class
-  # of duplication this repo has been bitten by. Three knobs, each shaped for its own consumer, and
-  # no two of them can contradict each other.
+  # Sharing an endpoint is fine. Sharing a PATH is not, and nothing here was per-environment before:
  #
-  # BASE = DEV'S LIVE LAYOUT, BYTE-FOR-BYTE, AND MUST STAY THAT WAY. Dev's existing backups are at
-  # `s3://velero/` (bucket root, no prefix) and `s3://cnpg/<clusterName>/` on that MinIO. Giving dev
-  # a prefix, or renaming its buckets, would ORPHAN the very backups its proven restore depends on —
-  # the same reason dev's EXTERNALDNS_TXT_OWNER_ID stays `default` and its LONGHORN_DATA_PATH stays
-  # /var/lib/longhorn. Verified: with an EMPTY prefix the Velero chart omits `objectStorage.prefix`
-  # from the rendered BackupStorageLocation entirely (helm template, chart 12.1.0), so dev's live
-  # BSL is unchanged by this parameterization rather than merely equivalent to it.
+  #   VELERO  — bucket `velero`, no prefix, and BOTH clusters run schedules named `daily` and
+  #             `daily-resources-only`. Each cluster's Velero SYNCS backups it finds in the BSL into
+  #             itself as Backup CRs and GARBAGE-COLLECTS them when their TTL expires. Prod would
+  #             therefore adopt dev's backups and delete them on schedule.
+  #   CNPG    — destinationPath `s3://cnpg` with CNPG appending serverName, which defaults to the
+  #             CLUSTER NAME. Both environments have `keycloak-db`, `nextcloud-db`, `semaphore-db`,
+  #             so prod's Postgres would archive WAL into the exact prefix dev's does. Two servers
+  #             interleaving WAL into one Barman stream CORRUPTS THE CHAIN and destroys PITR — and
+  #             dev's DR restore is PROVEN (R-STATE-03). This is the worst of the two.
  #
-  # PROD, which has no history to preserve, gets ONE bucket with TWO disjoint top-level prefixes:
+  # Both environments are now NAMESPACED SYMMETRICALLY, which was not possible an hour ago and is
+  # worth explaining, because the earlier reasoning was the opposite:
  #
-  #     s3://k3s-prod-backup/velero/…                 Velero (kopia repo + backup metadata)
-  #     s3://k3s-prod-backup/barman/<clusterName>/…   CNPG WAL + base backups
+  #   While the target was the MinIO, dev's base value HAD to be the bucket root — its existing
+  #   backups lived there and giving dev a prefix would have ORPHANED the very backups its proven
+  #   restore depends on (the same reason dev's EXTERNALDNS_TXT_OWNER_ID stays `default`).
  #
-  # The operator created exactly one R2 bucket, so velero-vs-barman separation has to come from the
-  # prefix rather than the bucket. The BUCKET ROOT IS DELIBERATELY LEFT EMPTY: if a prefix ever
-  # renders wrong, its objects land somewhere obviously bogus instead of silently colliding with a
-  # real tenant's data. Dev↔prod isolation is doubly independent — different endpoint AND different
-  # bucket — so neither cluster can reach the other's objects even with a mis-substituted prefix.
-  VELERO_BACKUP_BUCKET: "velero"             # dev: the `velero` MinIO bucket | prod: k3s-prod-backup
-  VELERO_BACKUP_PREFIX: ""                   # dev: bucket root (live layout) | prod: "velero"
-  CNPG_BACKUP_PATH: "s3://cnpg"              # dev: the `cnpg` MinIO bucket root | prod: s3://k3s-prod-backup/barman
-
-  # ------------------------------------------------------------------------------------------------
-  # WHICH CREDENTIAL SECRET THE BACKUP CONSUMERS READ (added 2026-09-07 — the per-env credential split)
-  # ------------------------------------------------------------------------------------------------
-  # The three knobs above say WHERE the backups go. This one says WHICH KEY signs the requests, and
-  # it exists because the endpoint moved per-environment while the credential did not.
+  #   Moving to R2 starts BOTH clusters on an EMPTY bucket, so there is no history to orphan and no
+  #   reason to keep the asymmetry. `dev` and `prod` is clearer, greppable, and leaves the bucket root
+  #   free — so an unset/misrendered prefix lands somewhere obviously wrong instead of silently
+  #   colliding with a real environment's backups.
  #
-  # THE BUG IT FIXES. The four backup-credential Secrets — velero-cloud.sops.yaml and the three
-  # <db>-backup-credentials.sops.yaml — are SHARED BASE manifests: platform/ and apps/ are rendered
-  # by the same Flux tiers on both clusters, and cluster-vars substitution is the ONLY thing that
-  # differs (R-ENV-02). So writing prod's R2 token into those four files did not GIVE prod a
-  # credential, it REPLACED DEV'S: dev's Velero and all three dev CNPG Clusters would have started
-  # signing MinIO requests with an R2 key id. For Velero that is a broken BSL; for CNPG it is worse,
-  # because `archive_mode=on` means Postgres REFUSES TO RECYCLE a WAL segment it could not archive,
-  # so a 403 loop is a monotonically filling pgdata PVC ending in a PANIC — the same failure prod
-  # spent its whole bring-up in with a placeholder endpoint. One shared Secret also means rotating
-  # either environment's key silently breaks the other; that is the structural reason for the split,
-  # independent of dev's eventual decommission.
-  #
-  # THE MECHANISM. Each credential is now TWO Secret objects, both applied to BOTH clusters (they are
-  # base resources; nothing here forks a manifest per environment). The consumer names one:
-  #
-  #     velero:   credentials.existingSecret: velero-cloud-credentials${BACKUP_CREDENTIALS_SUFFIX}
-  #     CNPG:     s3Credentials.*.name:  <db>-backup-credentials${BACKUP_CREDENTIALS_SUFFIX}
-  #
-  # ONE knob, not four, ON PURPOSE. Four independent secret-name knobs could be updated three-
-  # quarters of the way, leaving one consumer pointed at the OTHER environment's key with nothing
-  # rendering wrong — and StrictPostBuildSubstitutions cannot catch that, because it only fails on an
-  # UNDEFINED variable, never on a defined-but-stale one. With a single suffix the four move together
-  # or not at all. The cost is that the rendered Secret name is not greppable from the consumer; the
-  # names are spelled out in full in each kustomization.yaml beside the resources.
-  #
-  # THE BASE DEFAULT IS THE EMPTY STRING, AND THAT IS LOAD-BEARING. It renders the UNSUFFIXED names,
-  # which are the objects that already exist and that the live dev cluster is running. A default
-  # naming a Secret that does not exist would be worse than the shared-Secret state it replaces:
-  # Velero's pod would sit in ContainerCreating forever on a missing `existingSecret` volume, and a
-  # CNPG Cluster whose s3Credentials Secret is absent stops archiving WAL — i.e. the disk-full clock
-  # again — so an unresolvable default breaks the referencing cluster HARDER than a wrong key does.
-  # Empty is also the value dev must carry, so base and dev agree, exactly like VELERO_BACKUP_PREFIX.
-  #
-  # An EMPTY VALUE IS SAFE UNDER StrictPostBuildSubstitutions: strict mode fails on a variable that is
-  # not DEFINED, not on one defined as "". VELERO_BACKUP_PREFIX above has shipped empty since the
-  # gate was enabled and proves it. Both overlays state this knob explicitly — dev "" and prod
-  # "-prod" — so neither inherits it silently.
-  #
-  # ADDING A NEW BACKUP CREDENTIAL: create BOTH halves (`x-credentials.sops.yaml` and
-  # `x-credentials-prod.sops.yaml`), list both in the kustomization, and reference
-  # `x-credentials${BACKUP_CREDENTIALS_SUFFIX}`. Never a single shared Secret again.
-  BACKUP_CREDENTIALS_SUFFIX: ""              # dev: "" (MinIO creds) | prod: "-prod" (R2 creds)
+  # *** THESE TWO VALUES AND S3_BACKUP_ENDPOINT MUST CHANGE TOGETHER. *** Setting a `dev` prefix while
+  # the endpoint is still the MinIO would orphan dev's existing MinIO backups for no benefit. They are
+  # one change; do not split the commit.
+  VELERO_BACKUP_PREFIX: "dev"                # dev: "dev" | prod: "prod"
+  CNPG_BACKUP_PATH: "s3://cnpg/dev"          # dev: s3://cnpg/dev | prod: s3://cnpg/prod

  # --- placement ---
  PLATFORM_NODE_SELECTOR_KEY: "kubernetes.io/os"   # base no-op selector; prod may pin roles
@@ -106,11 +106,8 @@ patches:
          secretRef:
            name: sops-age
  # velero (backup controller) has its own post-platform Flux Kustomization (clusters/base/backup.yaml).
-  # It carries BOTH halves of the per-env `cloud` INI credential — velero-cloud.sops.yaml (dev's scoped
-  # MinIO key) and velero-cloud-prod.sops.yaml (prod's R2 token) — so it needs its own SOPS decryption
-  # stanza (same as headlamp/oauth2-proxy/crafty). Which one the Deployment mounts is chosen by
-  # BACKUP_CREDENTIALS_SUFFIX in the cluster-vars patch below, not here. The `sops-age` Secret is
-  # out-of-band (name only).
+  # It carries velero-cloud.sops.yaml (the scoped MinIO `cloud` credential), so it needs its own SOPS
+  # decryption stanza (same as headlamp/oauth2-proxy/crafty). The `sops-age` Secret is out-of-band (name only).
  - patch: |-
      apiVersion: kustomize.toolkit.fluxcd.io/v1
      kind: Kustomization
@@ -225,67 +222,20 @@ patches:
      - op: replace
        path: /data/CRAFTY_L4_VIP
        value: "192.168.1.55"
-      # Bulk (external NFS) storage host: the Proxmox host itself, on the SAME LAN as dev's nodes
-      # (192.168.1.40/.41/.42), which already hold an NFS export from it (/mnt/Storage/k8s-volumes).
-      # Restated verbatim rather than inherited, per this overlay's "supply every knob" convention.
-      # Dev is therefore the environment in which the bulk manifests can actually be VALIDATED
-      # end-to-end before prod's network path exists — provided the additive `zfs set sharenfs=`
-      # change described in docs/BULK-NFS-STORAGE.md is applied for the dev node addresses.
-      - op: replace
-        path: /data/BULK_NFS_SERVER
-        value: "192.168.1.9"
-      # Off-cluster backup target: the on-Proxmox MinIO (plain HTTP, path-style) that every CNPG
-      # barmanObjectStore and Velero's backupStorageLocation write to.
+      # OFF-CLUSTER BACKUP TARGET — no op here any more, deliberately.
      #
-      # DEV IS DELIBERATELY STAYING ON THIS MinIO while prod goes to Cloudflare R2 (2026-09-07).
-      # Prod could not use it at all — it cannot even route to it — but dev's move is a separate,
-      # operator-gated decision, because R2 would start dev on an EMPTY prefix: Velero would write
-      # a fresh kopia repository, CNPG would take new base backups, and the 30-day Barman window
-      # would rebuild from zero. Dev's DR restore is the one PROVEN recovery path in this repo
-      # (R-STATE-03); resetting its PITR window is not a side effect to take while fixing prod.
-      - op: replace
-        path: /data/S3_BACKUP_ENDPOINT
-        value: "http://192.168.1.9:9000"
-      # DO NOT CHANGE THESE FOUR. They are dev's LIVE object layout, restated verbatim for the same
-      # reason as LONGHORN_DATA_PATH and EXTERNALDNS_TXT_OWNER_ID below/above: they equal the base
-      # defaults today, and a silent drift in base would ORPHAN dev's existing backups — the ones
-      # its proven restore depends on — without any manifest looking wrong.
+      # This used to pin the on-Proxmox MinIO (http://192.168.1.9:9000). As of 2026-09-05 the operator
+      # moved backups to Cloudflare R2 "for the time being until a better on-prem backup solution is
+      # created", and R2 is a single environment-INDEPENDENT endpoint: both clusters write to the same
+      # buckets and are isolated by VELERO_BACKUP_PREFIX / CNPG_BACKUP_PATH (dev vs prod), not by URL.
+      # Under R-ENV-02 a value that does not differ per environment belongs in base and must NOT be
+      # restated here — two copies of one endpoint is just a rotation waiting to drift.
      #
-      # `us-east-1` is what dev's Velero has signed every request with since bootstrap; MinIO
-      # ignores the value, but changing it is a needless risk against a live, working backup path.
-      # An EMPTY VELERO_BACKUP_PREFIX means the bucket root, which is where dev's backups already
-      # are — and the Velero chart omits `objectStorage.prefix` entirely when it is empty (verified
-      # against chart 12.1.0), so dev's live BackupStorageLocation is byte-unchanged by this.
-      - op: replace
-        path: /data/S3_BACKUP_REGION
-        value: "us-east-1"
-      - op: replace
-        path: /data/VELERO_BACKUP_BUCKET
-        value: "velero"
-      - op: replace
-        path: /data/VELERO_BACKUP_PREFIX
-        value: ""
-      - op: replace
-        path: /data/CNPG_BACKUP_PATH
-        value: "s3://cnpg"
-      # WHICH CREDENTIAL SIGNS THOSE REQUESTS. Empty suffix = the UNSUFFIXED Secrets
-      # (velero-cloud-credentials, <db>-backup-credentials) — the scoped MinIO keys dev's Velero and
-      # its three CNPG Clusters are running RIGHT NOW, byte-identical to what is on the cluster. Prod
-      # renders "-prod" and reads the Cloudflare R2 token from the parallel `-prod` Secrets instead.
-      #
-      # RESTATED EXPLICITLY even though it equals the base default, for the same reason as
-      # EXTERNALDNS_TXT_OWNER_ID and the four backup-layout knobs above: the four Secrets are SHARED
-      # base manifests, so before this split prod's R2 token landed in the objects DEV reads. Dev
-      # signing MinIO with an R2 key id is 403 on every barman-cloud-wal-archive, and archive_mode=on
-      # turns that into a filling pgdata PVC — the disk-full clock, on the one cluster whose DR
-      # restore is PROVEN (R-STATE-03). This line is what keeps dev on its own credential.
-      #
-      # DEV IS NOT BEING BACKED UP GOING FORWARD and is slated for decommission (operator,
-      # 2026-09-07), so this is not protecting a future backup — it is protecting the EXISTING
-      # backups and the WAL chain until dev is retired. Do not "simplify" it away while dev runs.
-      - op: replace
-        path: /data/BACKUP_CREDENTIALS_SUFFIX
-        value: ""
+      # *** MERGING THIS RESETS DEV'S PITR WINDOW. *** Dev's R2 prefix starts EMPTY, so Velero writes a
+      # fresh kopia repository and CNPG takes new base backups; the 30-day Barman window rebuilds from
+      # zero. The existing MinIO copy is NOT deleted and stays restorable — point S3_BACKUP_ENDPOINT
+      # back at http://192.168.1.9:9000 (and VELERO_BACKUP_PREFIX back to "") to reach it. Do not merge
+      # this while dev's proven restore is the only recovery path you are relying on.
      # dev mirrors prod's 3-node topology, so the replicated tier keeps 3 replicas.
      - op: replace
        path: /data/LONGHORN_DEFAULT_REPLICA_COUNT
@@ -185,56 +185,9 @@ patches:
        name: flux-system
        namespace: flux-system
      spec:
-        # CUTOVER DONE 2026-09-06 (operator direction: "move the repo to prod"). Prod now reconciles
-        # from PROD'S OWN in-cluster Gitea. The workstation bootstrap container
-        # (http://192.168.2.132:3000) is retired as prod's source — it was a podman container on the
-        # operator's workstation holding a DHCP lease on WIFI, i.e. production's source of truth was a
-        # laptop. It is kept only as a recovery path (bootstrap/gitea/bootstrap-gitea.sh).
-        #
-        # THIS LINE IS SELF-REFERENTIAL AND THAT IS THE WHOLE DIFFICULTY. The root Kustomization
-        # renders ./clusters/prod, which contains this patch — so a live `kubectl patch` of
-        # GitRepository.spec.url is REVERTED on the next reconcile unless the authored value moves
-        # too. The cutover therefore had to be: (1) push the repo to prod's Gitea, (2) fast-forward
-        # its `main` to the exact commit prod was already running so the source swap changed nothing
-        # else, (3) live-patch the url to open a window, (4) land THIS commit through that window so
-        # the authored value agrees. Dev hit the identical trap on 2026-09-05
-        # (infra/PROD-PROVISIONING.md §10a "D0/D1 as executed").
-        #
-        # Verified before the switch: a throwaway pod INSIDE prod resolved this host through
-        # in-cluster CoreDNS to the Gateway VIP and cloned over the public LE cert with no CA
-        # supplied. The workstation being able to reach it proves nothing about pod egress.
-        # MOVED WITH THE APEX FLIP (2026-09-13) — and it had to move in the SAME fetch.
-        # This URL is a hardcoded literal, NOT templated on ${BASE_DOMAIN}, so the flip did not
-        # carry it. But the flip DOES rename prod's own HTTPRoute hostnames, so:
-        #   - external-dns (policy=sync) deletes gitea.prod.home.keithjackson.dev, and
-        #   - platform/coredns/coredns-custom.yaml answers the whole ${BASE_DOMAIN} zone with
-        #     ${GATEWAY_VIP}, so in-cluster resolution moves from prod.home. to home.
-        # Leaving this on the old name would therefore have severed prod from its OWN GitOps
-        # source moments after the flip reconciled — self-reference, the same class of trap as
-        # patching this url live and having the root Kustomization revert it. The fix has to land
-        # THROUGH THE OLD SOURCE, which is why it is in the same push as the flip rather than
-        # after it.
-        # FOLLOW-UP worth doing: point this at the in-cluster Service
-        # (gitea-http.gitea.svc.cluster.local) so prod's GitOps stops depending on public DNS and
-        # on its own Gateway at all. Deferred: it needs the TLS/credential path rethought.
-        # IN-CLUSTER SERVICE, NOT A PUBLIC HOSTNAME (2026-09-13). This is the durable fix for a
-        # deadlock the apex flip walked straight into, and it is worth stating precisely.
-        #
-        # The url used to be a public name. The flip renamed prod's HTTPRoute hostnames, so the
-        # name this GitRepository pointed at stopped serving — but the HTTPRoute that WOULD have
-        # served the new name lives in a DOWNSTREAM tier, which cannot reconcile while the source
-        # is broken. Source down -> routes never rename -> source stays down. Observed live:
-        # `srcReady=False`, "unable to list remote for https://gitea.home.keithjackson.dev".
-        #
-        # Pointing at the Service breaks the cycle permanently: prod's GitOps no longer depends on
-        # public DNS, on Cloudflare, on its own Gateway, or on ${BASE_DOMAIN} at all. A future
-        # domain change cannot sever prod from its own source again.
-        #
-        # Plain http is deliberate and safe here: it is cluster-internal traffic to a ClusterIP,
-        # TLS terminates at the Gateway for external clients only, the GitRepository has no
-        # secretRef (the repo is cloned anonymously), and every secret in the tree is SOPS-encrypted
-        # at rest regardless of transport.
-        url: http://gitea.gitea.svc.cluster.local:3000/Home-Network/home-network-k3s-impl.git
+        # TEMPORARY bootstrap source — workstation container, NOT in any cluster. See the block
+        # above for the cutover to https://gitea.home.keithjackson.dev/... . Do not treat as final.
+        url: http://192.168.2.132:3000/Home-Network/home-network-k3s-impl.git
        ref:
          branch: main
  - target:
@@ -272,30 +225,20 @@ patches:
      # `<svc>.home.keithjackson.dev` point at prod are PHASE 2, deliberately not done here — they
      # would take traffic off the live dev cluster. infra/PROD-PROVISIONING.md §10.
      #
+      # THE APEX INVERSION IS DELIBERATELY NOT ON THIS BRANCH. Moving prod to the bare apex (and dev
+      # to dev.home.) is a LIVE MIGRATION of the dev cluster, not a variable change — see
+      # infra/PROD-PROVISIONING.md §10a. It is kept as a single commit on branch
+      # `migration/dns-apex-inversion` (9137bbd) and is applied as step D3 of that runbook, with the
+      # operator present. Everything else on this branch is independent of it, which is the whole
+      # reason it was split out.
+      #
      # [parity] MUST equal k3s_base_domain in infra/ansible/inventory/group_vars/k3s_prod/vars.yml.
      # The kube-apiserver's oidc-issuer-url is an INSTALL-TIME k3s flag, not a manifest, so it is
      # set Ansible-side from that variable and cannot read this one. If they disagree, kubectl SSO
      # trusts a different Keycloak than the one prod actually runs.
      - op: replace
        path: /data/BASE_DOMAIN
-        # APEX AS OF 2026-09-13. Prod now serves home.keithjackson.dev directly; dev has been
-        # decommissioned and released the name. Prerequisites completed and VERIFIED before this
-        # flip, in this order:
-        #   1. dev's 18 A records released by its own external-dns (policy=sync, sources deleted)
-        #   2. dev's 18 LEGACY UNPREFIXED TXT registry entries deleted from Cloudflare by hand —
-        #      external-dns v0.21.0 removed only the `a-` prefixed form and then reported "All
-        #      records are already up to date", so it would NEVER have cleared them. They carried
-        #      `owner=default`, and prod runs `owner=prod`, so prod could not have claimed these
-        #      names while they existed. Confirmed gone at the AUTHORITY (betty.ns.cloudflare.com);
-        #      1.1.1.1 served stale cache for minutes afterwards and must not be trusted here.
-        #   3. dev's `*.home.keithjackson.dev` + apex Certificate deleted, so it stops renewing.
-        #      It was issued ~18d ago, so prod's request is #1 this week against LE's duplicate
-        #      limit of 5 per identical name set per week.
-        # NOTE the zone has wildcards `*.keithjackson.dev` AND `*.home.keithjackson.dev` pointing
-        # at the public IP 136.51.115.252 (legacy non-k3s prod). Dev's specific records were
-        # SHADOWING that wildcard; prod's new specific records shadow it again. The legacy stack
-        # lives on `*.keithjackson.dev` and is untouched by any of this.
-        value: "home.keithjackson.dev"
+        value: "prod.home.keithjackson.dev"
      # SAFETY NET #2 (independent of the owner id below): narrow ExternalDNS's name scope to prod's
      # OWN subtree, so prod cannot create, update or delete any record outside it no matter what the
      # TXT registry says. VERIFY ON FIRST RECONCILE — the Cloudflare provider uses this value for
@@ -441,145 +384,37 @@ patches:
        path: /data/CRAFTY_L4_VIP
        value: "192.168.2.22"
      # ------------------------------------------------------------------------------------------
-      # BULK (EXTERNAL NFS) STORAGE HOST — *** NOT YET DECIDED. THIS PLACEHOLDER IS LOAD-BEARING. ***
+      # OFF-CLUSTER DR TARGET — Cloudflare R2 (operator, 2026-09-05).
+      #
+      # THERE IS DELIBERATELY NO `S3_BACKUP_ENDPOINT` OP HERE, and that is a reversal of the rule
+      # stated at the top of this overlay ("an OMITTED op means prod silently inherits a dev
+      # address"). That rule exists because base defaults are dev's LAN addresses. R2 is not one: it
+      # is a single shared, environment-independent endpoint, so under R-ENV-02 it belongs in base and
+      # overriding it with an identical value would only create two places to change on a rotation —
+      # the exact drift this repo has been bitten by before. Isolation does NOT come from the
+      # endpoint; it comes from the two path knobs below.
+      #
+      # WHY R2 AND NOT THE MinIO the operator first named: prod is on the SOHO network
+      # 192.168.2.0/24, which is pfSense's WAN, while the MinIO sits inside pfSense's LAN
+      # 192.168.1.0/24. Measured from a pod INSIDE prod (the workstation proves nothing about pod
+      # egress): `curl http://192.168.1.9:9000/minio/health/live` → connection timed out after 15s.
+      # Reaching it would need a WAN→LAN rule that does not exist. R2 needs no firewall change, and
+      # unlike that MinIO — which runs on the same Proxmox host as dev's VMs — it is genuinely
+      # off-site. Verify prod can reach the new endpoint before resuming velero:
+      #   bootstrap/standup.sh ... --s3-endpoint <the R2 endpoint> s3check
+      #
+      # THE PATH KNOBS ARE WHAT MAKE ONE SHARED BUCKET SAFE. Nothing about the paths was
+      # per-environment before 2026-09-05: Velero used one unprefixed bucket, and every CNPG Cluster
+      # used `s3://cnpg` with serverName defaulting to the CLUSTER NAME — and both environments have
+      # keycloak-db / nextcloud-db / semaphore-db. Without these knobs prod would (a) adopt and
+      # TTL-delete dev's Velero backups and (b) interleave two Postgres servers' WAL into ONE Barman
+      # stream, corrupting the chain and destroying PITR for both. See clusters/base/cluster-vars.yaml.
      # ------------------------------------------------------------------------------------------
-      # Backs every `class: bulk` volume: the 8.33 T media library (Storage/plex — plex, sonarr,
-      # radarr) and the 270 G legacy Nextcloud archive (Storage/nextcloud).
-      #
-      # THE STORAGE HOST IS NOT REACHABLE FROM PROD AT ALL RIGHT NOW. It has one interface,
-      # vmbr0 192.168.1.9/24, on pfSense's LAN; prod's nodes are on 192.168.2.0/24, pfSense's WAN.
-      # 22 and 8006 answer from prod only because pfSense port-forwards them via 192.168.2.102;
-      # 2049 and 111 do not. Opening that path is an OPERATOR decision, and the two candidate
-      # outcomes give DIFFERENT values here:
-      #     port-forward 192.168.2.102:2049 -> 192.168.1.9:2049   =>  "192.168.2.102"
-      #     a WAN->LAN route to 192.168.1.0/24                    =>  "192.168.1.9"
-      # Guessing either one authors a prod cluster that mounts nothing, or — worse, if the guess
-      # ever became reachable — mounts something unintended. So it stays a placeholder.
-      #
-      # WHICH PORTS THE DECISION MUST OPEN: exactly ONE, tcp/2049. The PVs mount nfsvers=4.1, and
-      # NFSv4 carries mount and lock in-protocol — no rpcbind/111, no mountd, no statd. Choosing v4
-      # over v3 was in part this decision's cost: v3 would have needed four ports and three of them
-      # on ephemeral numbers.
-      #
-      # WHAT THE EXPORT MUST ALLOW IS ALREADY SETTLED, and it is the same under BOTH outcomes.
-      # Measured 2026-09-07 from this workstation (192.168.2.132) through the existing port-forward:
-      # the storage host logged the connection as coming FROM 192.168.2.132, not from a pfSense LAN
-      # address (`ss -tn` peer 192.168.2.132, and the host's only route off-LAN is its default via
-      # 192.168.1.1). pfSense DNATs WITHOUT source NAT, so the NFS server will see prod's REAL node
-      # addresses either way. The additive `zfs set sharenfs=` values are in docs/BULK-NFS-STORAGE.md.
-      #
-      # *** BLAST RADIUS — READ BEFORE MERGING THIS TO A BRANCH PROD RECONCILES. ***
-      # A PersistentVolume's spec.nfs.server is an unvalidated string: the API server ACCEPTS this
-      # placeholder, the PV is created and the PVC BINDS to it (binding is pure control-plane). The
-      # failure surfaces only when a pod mounts — MountVolume.SetUp "Failed to resolve server
-      # TBD-PROD-BULK-NFS-SERVER" — so plex, sonarr, radarr and nextcloud would sit in
-      # ContainerCreating on prod. Unlike the LAN knobs above, this placeholder does NOT fail at
-      # apply time and there is no CRD validation to lean on. Set this value BEFORE prod reconciles
-      # a revision containing the bulk volumes; the dev overlay renders a working value, which is
-      # where these manifests are meant to be proven first.
-      - op: replace
-        path: /data/BULK_NFS_SERVER
-        # 192.168.2.102 is pfSense's WAN address, which DNATs tcp/2049 through to the Proxmox
-        # host's REAL address 192.168.1.9 on pfSense's LAN. Prod sits on the WAN side and cannot
-        # reach 192.168.1.9 directly, so the cluster mounts the NAT address. Dev is on the LAN and
-        # keeps the base value 192.168.1.9.
-        #
-        # PROVEN from k3s-prod-1 on 2026-09-12, not inferred:
-        #   - mount of 192.168.2.102:/mnt/Storage/plex negotiated nfs4 vers=4.1 proto=tcp
-        #   - a write as uid 1000 succeeded. Tested as 1000 and NOT root deliberately: the export
-        #     carries no_root_squash, so a root write would have passed and proved nothing, while
-        #     LSIO images default to PUID 911 and would have gone Ready then failed every import
-        #     with EACCES. The tree is 0755 1000:3000, so uid 1000 is the only writer.
-        #   - ln() across downloads/ -> tv/ returned link count 2 (same inode), so the arrs get
-        #     real atomic moves instead of silently degrading to copy-then-delete on EXDEV.
-        # pfSense DNATs WITHOUT source NAT (the host logs prod's own node addresses as the peer),
-        # so the ZFS export ACL names 192.168.2.10/.11/.12 directly rather than one translated IP.
-        value: "192.168.2.102"
-      # ==========================================================================================
-      # OFF-CLUSTER DR TARGET — Cloudflare R2 (operator, 2026-09-05; bucket + token 2026-09-07).
-      # ==========================================================================================
-      # Dev's versitygw/MinIO (192.168.1.9:9000) sits on the DEV side of the network and is NOT an
-      # option for prod, for two independent reasons:
-      #   1. UNREACHABLE. Prod is on the SOHO network 192.168.2.0/24 — pfSense's WAN — while that
-      #      MinIO is inside pfSense's LAN. Measured from a pod INSIDE prod (the workstation proves
-      #      nothing about pod egress): `curl http://192.168.1.9:9000/minio/health/live` → connection
-      #      timed out after 15s. Reaching it would need a WAN→LAN rule that does not exist.
-      #   2. NOT OFF-CLUSTER DR. It runs on the same Proxmox host as dev's VMs, so it already shares
-      #      a physical failure domain with dev. R2 is genuinely off-site, and its free egress means
-      #      restore REHEARSALS cost nothing — R-STATE-03 has no financial disincentive to stay proven.
-      #
-      # THIS UNBLOCKS AN ACTIVE INCIDENT, not just a roadmap item. All three prod CNPG Clusters ship
-      # `archive_mode=on` with `endpointURL: TBD-PROD-S3-BACKUP-ENDPOINT`, so barman-cloud-wal-archive
-      # exits 4 on every segment and Postgres — correctly — REFUSES TO RECYCLE WAL. Verified live
-      # 2026-09-07: keycloak-db / nextcloud-db / semaphore-db all report
-      # ContinuousArchiving=False/ContinuousArchivingFailing, and keycloak-db's 5 GiB PVC is at
-      # 579 MiB used / 4360 MiB free and climbing monotonically. A disk-full PANIC there takes the
-      # cluster's SSO database down, and with it kubectl OIDC, Grafana, Headlamp, Gitea and
-      # forward-auth. Fixing the endpoint lets Postgres drain the whole backlog.
-      #
-      # ENDPOINT IS ACCOUNT-SCOPED, so it stays a prod-overlay value even though it is not a LAN
-      # address: this is the operator's Cloudflare account id, confirmed from inside a prod pod
-      # (`head-bucket` on k3s-prod-backup → BucketRegion WNAM; PUT and DELETE both succeed).
-      # It is NOT put in base, because base is dev's live MinIO and dev is deliberately STAYING
-      # there — moving dev to R2 would reset its PITR window and orphan the backups its PROVEN
-      # restore depends on. Two environments, two targets, isolated by URL *and* by bucket.
-      - op: replace
-        path: /data/S3_BACKUP_ENDPOINT
-        value: "https://7ce452a1b3d6c86daf2c020edcb82d37.r2.cloudflarestorage.com"
-      # R2 requires the region string to be exactly `auto`. (`head-bucket` answers
-      # `{"BucketRegion":"WNAM"}` — that is a PHYSICAL LOCATION HINT, not a signing region; using it
-      # here produces AuthorizationHeaderMalformed.) Dev keeps the base `us-east-1` its live MinIO
-      # has always been signed with.
-      - op: replace
-        path: /data/S3_BACKUP_REGION
-        value: "auto"
-      # ------------------------------------------------------------------------------------------
-      # OBJECT TENANCY INSIDE THE ONE BUCKET. The operator created a SINGLE R2 bucket, so velero and
-      # barman cannot be separated by bucket the way they are on dev's MinIO (`velero` and `cnpg`).
-      # They are separated by disjoint TOP-LEVEL PREFIXES instead:
-      #
-      #     s3://k3s-prod-backup/velero/…                 Velero: kopia repo + backup metadata
-      #     s3://k3s-prod-backup/barman/<clusterName>/…   CNPG: WAL + base backups
-      #
-      # The bucket ROOT is left empty on purpose — a misrendered prefix then lands somewhere
-      # obviously wrong instead of silently colliding with a live tenant. Velero's retention TTL
-      # deletes objects under its own prefix only, so it can never reap a WAL segment; and barman's
-      # per-cluster serverName subdirectory keeps the three Postgres servers' chains apart, which
-      # matters because interleaving two servers' WAL into one prefix corrupts the chain outright.
-      # Dev cannot be reached from any of these: different endpoint AND different bucket names.
-      # ------------------------------------------------------------------------------------------
-      - op: replace
-        path: /data/VELERO_BACKUP_BUCKET
-        value: "k3s-prod-backup"
      - op: replace
        path: /data/VELERO_BACKUP_PREFIX
-        value: "velero"
+        value: "prod"
+      # Barman objects land at s3://cnpg/prod/<clusterName>/; dev's at s3://cnpg/dev/<clusterName>/.
      - op: replace
        path: /data/CNPG_BACKUP_PATH
-        value: "s3://k3s-prod-backup/barman"
-      # ------------------------------------------------------------------------------------------
-      # WHICH CREDENTIAL SIGNS THE REQUESTS TO THAT BUCKET (per-env credential split, 2026-09-07).
-      # ------------------------------------------------------------------------------------------
-      # The knobs above moved the ENDPOINT and the OBJECT LAYOUT per environment. The credential did
-      # not move with them, and it could not: the four backup-credential Secrets are SHARED BASE
-      # manifests, applied to both clusters by the same Flux tiers. Writing the R2 token into them
-      # gave prod a working key by TAKING DEV'S AWAY — dev would have signed MinIO requests with an
-      # R2 key id, and under archive_mode=on that is a filling pgdata PVC, not a missing backup.
-      #
-      # So each credential is now two objects, and this suffix selects prod's:
-      #     velero-cloud-credentials-prod           the R2 token in the AWS INI `cloud` shape
-      #     <db>-backup-credentials-prod            the same token as ACCESS_KEY_ID / ACCESS_SECRET_KEY
-      # for keycloak-db, nextcloud-db and semaphore-db. Both halves ship to both clusters (they are
-      # base resources); ONLY this value decides which one each consumer reads. It is a SUFFIX rather
-      # than four secret-name knobs deliberately — four could be updated three-quarters of the way,
-      # leaving one consumer on dev's MinIO key with nothing rendering wrong, and strict-mode
-      # substitution cannot catch a defined-but-stale variable.
-      #
-      # THE R2 TOKEN IS THE ONE PROVEN FROM INSIDE PROD (chore/r2-credentials 0a6a862): head-bucket on
-      # k3s-prod-backup returns BucketRegion WNAM, and PUT and DELETE both succeed. Rotating it now
-      # touches ONLY the four `-prod` files; dev's MinIO keys are untouched objects. That isolation is
-      # the structural reason for the split, independent of dev's eventual decommission.
-      - op: replace
-        path: /data/BACKUP_CREDENTIALS_SUFFIX
-        value: "-prod"
+        value: "s3://cnpg/prod"
      # TODO(WS-INFRA, Phase 7): pin prod node placement once the bare-metal role labels exist.
@@ -467,14 +467,7 @@ def git_sha(repo: Path) -> str:
                             capture_output=True, text=True, check=True).stdout.strip()
    except (subprocess.CalledProcessError, OSError):
        return "unknown"
-    # EXCLUDE conformance/runs/. It is TRACKED, so `--record` writing ledger.jsonl dirtied the
-    # tree and every LATER run in the same session was stamped `-dirty` — recording one test
-    # poisoned the next, and check.sh refuses to honour a dirty proven row. That cost a second
-    # live fault-injection run on prod to work around. The docstring above already states the
-    # intent for the untracked runs/<id>/ half of this same artifact; the ledger is the tracked
-    # half. Mirrors status.py's IMPL_EXCLUDE_PREFIXES, which lists this path for the same reason.
-    dirty = subprocess.run(["git", "-C", str(repo), "diff", "--quiet", "HEAD", "--",
-                            ".", ":(exclude)conformance/runs/"],
+    dirty = subprocess.run(["git", "-C", str(repo), "diff", "--quiet", "HEAD", "--"],
                           capture_output=True).returncode != 0
    return f"{sha}-dirty" if dirty else sha

@@ -1,12 +0,0 @@
-<?xml version="1.0" ?>
-<testsuites name="home-k3s-conformance" tests="1" failures="0" errors="0" skipped="0">
-  <testsuite name="home-k3s-conformance L0/L1 (offline)" tests="1" failures="0" errors="0" skipped="0">
-    <testcase classname="L1:R-NET-05,R-NET-06,R-NET-07" name="egress-class-render">
-      <properties>
-        <property name="requirements" value="R-NET-05,R-NET-06,R-NET-07"/>
-        <property name="layer" value="L1"/>
-        <property name="path" value="/work/.claude/conf-clone/conformance/tests/l1-rendering/egress-class-render"/>
-      </properties>
-    </testcase>
-  </testsuite>
-</testsuites>
@@ -1,7 +0,0 @@
-[1m[run.sh][0m Coverage report first (always)
-[coverage] 49/51 requirements have a scaffolded test (96.1%); 2 gap(s), 0 safety-critical gap(s).
-[coverage] wrote /work/.claude/conf-clone/conformance/_artifacts/coverage/coverage.md and /work/.claude/conf-clone/conformance/_artifacts/coverage/coverage.json
-[1m[run.sh][0m Running 1 offline L0/L1 check(s) (no cluster; docs/14).
-[offline] L1  PASS egress-class-render (R-NET-05,R-NET-06,R-NET-07)
-[offline] wrote JUnit (1 case(s), 0 failure(s)) -> /work/.claude/conf-clone/conformance/_artifacts/report.junit.xml
-[1m[run.sh][0m Done (offline l1). JUnit: /work/.claude/conf-clone/conformance/_artifacts/report.junit.xml | Coverage: /work/.claude/conf-clone/conformance/_artifacts/coverage/coverage.md
@@ -1,4 +0,0 @@
-<testsuites name="chainsaw-report" tests="1" failures="0" errors="0" time="0.000000"><testsuite name="/work/.claude/conf-clone/conformance/tests/l3-fault-injection/vpn-killswitch" tests="1" failures="0" errors="0" id="0" time="">
-    <testcase name="vpn-killswitch" classname="" time="174.632779" />
-  </testsuite>
-</testsuites>
@@ -1,237 +0,0 @@
-[1m[run.sh][0m Coverage report first (always)
-[coverage] 49/51 requirements have a scaffolded test (96.1%); 2 gap(s), 0 safety-critical gap(s).
-[coverage] wrote /work/.claude/conf-clone/conformance/_artifacts/coverage/coverage.md and /work/.claude/conf-clone/conformance/_artifacts/coverage/coverage.json
-[1m[run.sh][0m No L0/L1 offline checks match the selection (layer=l3 reqs='R-NET-06').
-[1m[run.sh][0m LIGHT pass: 1 test(s), parallel=2 (pinned v0.2.12)
-Version: 0.2.12
-Loading config (/work/.claude/conf-clone/conformance/config/chainsaw-config.yaml)...
- Using test file: chainsaw-test
- TestDirs [/work/.claude/conf-clone/conformance/tests/l3-fault-injection/vpn-killswitch]
- SkipDelete false
- FailFast false
- ReportFormat 'JUNIT-TEST'
- ReportName 'chainsaw-report-light'
- ReportPath '/work/.claude/conf-clone/conformance/_artifacts'
- Namespace ''
- FullName false
- IncludeTestRegex ''
- ExcludeTestRegex ''
- ApplyTimeout 1m30s
- AssertTimeout 3m0s
- CleanupTimeout 2m0s
- DeleteTimeout 2m0s
- ErrorTimeout 1m0s
- ExecTimeout 3m0s
- DeletionPropagationPolicy Background
- Parallel 2
- Template true
- NoCluster false
- PauseOnFailure false
-Loading tests...
- vpn-killswitch (/work/.claude/conf-clone/conformance/tests/l3-fault-injection/vpn-killswitch)
-Loading values...
-Running tests...
-=== RUN   chainsaw
-=== PAUSE chainsaw
-=== CONT  chainsaw
-=== RUN   chainsaw/vpn-killswitch
-=== PAUSE chainsaw/vpn-killswitch
-=== CONT  chainsaw/vpn-killswitch
-    l.go:52: | 01:44:38 | [32;1mvpn-killswitch[0;22m | [32;1m@chainsaw                                [0;22m | [32;1mCREATE[0;22m    | [32;1mOK[0;22m    | v1/Namespace @ chainsaw-credible-bird
-    l.go:52: | 01:44:38 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads   [0;22m | [36;1mTRY[0;22m       | [36;1mBEGIN[0;22m |
-    l.go:52: | 01:44:38 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads   [0;22m | [36;1mCMD[0;22m       | [36;1mRUN[0;22m   |
-        === COMMAND
-        /usr/bin/sh -c set -eu
-        kubectl label namespace "$NAMESPACE" \
-          pod-security.kubernetes.io/enforce=privileged \
-          pod-security.kubernetes.io/warn=privileged \
-          pod-security.kubernetes.io/audit=privileged \
-          --overwrite
-        echo "NS_PRIVILEGED_OK"
-    l.go:52: | 01:44:39 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads   [0;22m | [36;1mSCRIPT[0;22m    | [36;1mLOG[0;22m   |
-        === STDOUT
-        namespace/chainsaw-credible-bird labeled
-        NS_PRIVILEGED_OK
-    l.go:52: | 01:44:39 | [32;1mvpn-killswitch[0;22m | [32;1mgiven-healthy-tunnel-and-two-workloads   [0;22m | [32;1mSCRIPT[0;22m    | [32;1mDONE[0;22m  |
-    l.go:52: | 01:44:39 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads   [0;22m | [36;1mAPPLY[0;22m     | [36;1mRUN[0;22m   | v1/Pod @ chainsaw-credible-bird/egress-internet
-    l.go:52: | 01:44:39 | [32;1mvpn-killswitch[0;22m | [32;1mgiven-healthy-tunnel-and-two-workloads   [0;22m | [32;1mCREATE[0;22m    | [32;1mOK[0;22m    | v1/Pod @ chainsaw-credible-bird/egress-internet
-    l.go:52: | 01:44:39 | [32;1mvpn-killswitch[0;22m | [32;1mgiven-healthy-tunnel-and-two-workloads   [0;22m | [32;1mAPPLY[0;22m     | [32;1mDONE[0;22m  | v1/Pod @ chainsaw-credible-bird/egress-internet
-    l.go:52: | 01:44:39 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads   [0;22m | [36;1mAPPLY[0;22m     | [36;1mRUN[0;22m   | v1/Pod @ chainsaw-credible-bird/egress-vpn
-    l.go:52: | 01:44:39 | [32;1mvpn-killswitch[0;22m | [32;1mgiven-healthy-tunnel-and-two-workloads   [0;22m | [32;1mCREATE[0;22m    | [32;1mOK[0;22m    | v1/Pod @ chainsaw-credible-bird/egress-vpn
-    l.go:52: | 01:44:39 | [32;1mvpn-killswitch[0;22m | [32;1mgiven-healthy-tunnel-and-two-workloads   [0;22m | [32;1mAPPLY[0;22m     | [32;1mDONE[0;22m  | v1/Pod @ chainsaw-credible-bird/egress-vpn
-    l.go:52: | 01:44:39 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads   [0;22m | [36;1mASSERT[0;22m    | [36;1mRUN[0;22m   | v1/Pod @ chainsaw-credible-bird/egress-internet
-    l.go:52: | 01:44:41 | [32;1mvpn-killswitch[0;22m | [32;1mgiven-healthy-tunnel-and-two-workloads   [0;22m | [32;1mASSERT[0;22m    | [32;1mDONE[0;22m  | v1/Pod @ chainsaw-credible-bird/egress-internet
-    l.go:52: | 01:44:41 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads   [0;22m | [36;1mASSERT[0;22m    | [36;1mRUN[0;22m   | v1/Pod @ chainsaw-credible-bird/egress-vpn
-    l.go:52: | 01:44:41 | [32;1mvpn-killswitch[0;22m | [32;1mgiven-healthy-tunnel-and-two-workloads   [0;22m | [32;1mASSERT[0;22m    | [32;1mDONE[0;22m  | v1/Pod @ chainsaw-credible-bird/egress-vpn
-    l.go:52: | 01:44:41 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads   [0;22m | [36;1mCMD[0;22m       | [36;1mRUN[0;22m   |
-        === COMMAND
-        /usr/bin/sh -c set -eu
-        ns="$NAMESPACE"
-        get_ip() {
-          kubectl -n "$ns" exec "$1" -c curl -- \
-            curl -fsS --max-time 10 https://api.ipify.org 2>/dev/null || true
-        }
-        wan="" vpn=""
-        for i in $(seq 1 12); do
-          [ -n "$wan" ] || wan="$(get_ip egress-internet)"
-          [ -n "$vpn" ] || vpn="$(get_ip egress-vpn)"
-          if [ -n "$wan" ] && [ -n "$vpn" ] && [ "$wan" != "$vpn" ]; then break; fi
-          [ "$wan" = "$vpn" ] && vpn=""   # not steered yet; re-poll
-          sleep 5
-        done
-        echo "wan=$wan vpn=$vpn"
-        [ -n "$wan" ] || { echo "FAIL: no WAN IP from internet-class pod" >&2; exit 1; }
-        [ -n "$vpn" ] || { echo "FAIL: vpn-class egress not working at baseline" >&2; exit 1; }
-        [ "$wan" != "$vpn" ] || { echo "FAIL: vpn-class == WAN at baseline (not tunneled)" >&2; exit 1; }
-        echo "TUNNEL_HEALTHY_OK"
-    l.go:52: | 01:44:43 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads   [0;22m | [36;1mSCRIPT[0;22m    | [36;1mLOG[0;22m   |
-        === STDOUT
-        wan=136.51.115.252 vpn=158.173.241.11
-        TUNNEL_HEALTHY_OK
-    l.go:52: | 01:44:43 | [32;1mvpn-killswitch[0;22m | [32;1mgiven-healthy-tunnel-and-two-workloads   [0;22m | [32;1mSCRIPT[0;22m    | [32;1mDONE[0;22m  |
-    l.go:52: | 01:44:43 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads   [0;22m | [36;1mTRY[0;22m       | [36;1mEND[0;22m   |
-    l.go:52: | 01:44:43 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak      [0;22m | [36;1mTRY[0;22m       | [36;1mBEGIN[0;22m |
-    l.go:52: | 01:44:43 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak      [0;22m | [36;1mAPPLY[0;22m     | [36;1mRUN[0;22m   | batch/v1/Job @ chainsaw-credible-bird/tunnel-stop
-    l.go:52: | 01:44:43 | [32;1mvpn-killswitch[0;22m | [32;1mwhen-force-tunnel-down-then-no-leak      [0;22m | [32;1mCREATE[0;22m    | [32;1mOK[0;22m    | batch/v1/Job @ chainsaw-credible-bird/tunnel-stop
-    l.go:52: | 01:44:43 | [32;1mvpn-killswitch[0;22m | [32;1mwhen-force-tunnel-down-then-no-leak      [0;22m | [32;1mAPPLY[0;22m     | [32;1mDONE[0;22m  | batch/v1/Job @ chainsaw-credible-bird/tunnel-stop
-    l.go:52: | 01:44:43 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak      [0;22m | [36;1mASSERT[0;22m    | [36;1mRUN[0;22m   | batch/v1/Job @ chainsaw-credible-bird/tunnel-stop
-    l.go:52: | 01:44:48 | [32;1mvpn-killswitch[0;22m | [32;1mwhen-force-tunnel-down-then-no-leak      [0;22m | [32;1mASSERT[0;22m    | [32;1mDONE[0;22m  | batch/v1/Job @ chainsaw-credible-bird/tunnel-stop
-    l.go:52: | 01:44:48 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak      [0;22m | [36;1mCMD[0;22m       | [36;1mRUN[0;22m   |
-        === COMMAND
-        /usr/bin/sh -c set -u
-        ns="$NAMESPACE"
-        wan="$(kubectl -n "$ns" exec egress-internet -c curl -- \
-                 curl -fsS --max-time 10 https://api.ipify.org 2>/dev/null || true)"
-        for i in $(seq 1 8); do
-          out="$(kubectl -n "$ns" exec egress-vpn -c curl -- \
-                   curl -fsS --max-time 8 https://api.ipify.org 2>/dev/null || true)"
-          if [ -n "$out" ]; then
-            if [ -n "$wan" ] && [ "$out" = "$wan" ]; then
-              echo "LEAK: vpn-class reached the WAN IP ($out) with the tunnel DOWN" >&2; exit 1
-            fi
-            echo "LEAK: vpn-class reached the internet ($out) with the tunnel DOWN" >&2; exit 1
-          fi
-          echo "attempt $i: vpn-class blackholed (no response), good"
-          sleep 3
-        done
-        echo "NO_LEAK_OK"
-    l.go:52: | 01:46:18 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak      [0;22m | [36;1mSCRIPT[0;22m    | [36;1mLOG[0;22m   |
-        === STDOUT
-        attempt 1: vpn-class blackholed (no response), good
-        attempt 2: vpn-class blackholed (no response), good
-        attempt 3: vpn-class blackholed (no response), good
-        attempt 4: vpn-class blackholed (no response), good
-        attempt 5: vpn-class blackholed (no response), good
-        attempt 6: vpn-class blackholed (no response), good
-        attempt 7: vpn-class blackholed (no response), good
-        attempt 8: vpn-class blackholed (no response), good
-        NO_LEAK_OK
-    l.go:52: | 01:46:18 | [32;1mvpn-killswitch[0;22m | [32;1mwhen-force-tunnel-down-then-no-leak      [0;22m | [32;1mSCRIPT[0;22m    | [32;1mDONE[0;22m  |
-    l.go:52: | 01:46:18 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak      [0;22m | [36;1mCMD[0;22m       | [36;1mRUN[0;22m   |
-        === COMMAND
-        /usr/bin/sh -c set -eu
-        ns="$NAMESPACE"
-        out="$(kubectl -n "$ns" exec egress-internet -c curl -- \
-                 curl -fsS --max-time 10 https://api.ipify.org 2>/dev/null || true)"
-        echo "internet_ip=$out"
-        [ -n "$out" ] || { echo "FAIL: internet-class egress broke while tunnel down (must be unaffected)" >&2; exit 1; }
-        echo "INTERNET_UNAFFECTED_OK"
-    l.go:52: | 01:46:18 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak      [0;22m | [36;1mSCRIPT[0;22m    | [36;1mLOG[0;22m   |
-        === STDOUT
-        internet_ip=136.51.115.252
-        INTERNET_UNAFFECTED_OK
-    l.go:52: | 01:46:18 | [32;1mvpn-killswitch[0;22m | [32;1mwhen-force-tunnel-down-then-no-leak      [0;22m | [32;1mSCRIPT[0;22m    | [32;1mDONE[0;22m  |
-    l.go:52: | 01:46:18 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak      [0;22m | [36;1mTRY[0;22m       | [36;1mEND[0;22m   |
-    l.go:52: | 01:46:18 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak      [0;22m | [36;1mFINALLY[0;22m   | [36;1mBEGIN[0;22m |
-    l.go:52: | 01:46:18 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak      [0;22m | [36;1mCMD[0;22m       | [36;1mRUN[0;22m   |
-        === COMMAND
-        /usr/bin/sh -c set -eu
-        ns="$NAMESPACE"
-        # Distinct Job name (tunnel-restart) so it never collides with the
-        # tunnel-stop Job. Left-aligned manifest so the block scalar yields
-        # valid YAML at column 0 at runtime.
-        cat <<'JOB' | kubectl -n "$ns" apply -f -
-        apiVersion: batch/v1
-        kind: Job
-        metadata:
-          name: tunnel-restart
-          labels: { app: vpn-killswitch-fault }
-        spec:
-          backoffLimit: 3
-          activeDeadlineSeconds: 90
-          ttlSecondsAfterFinished: 600
-          template:
-            metadata:
-              labels: { app: vpn-killswitch-fault }
-            spec:
-              restartPolicy: Never
-              hostPID: true
-              nodeSelector:
-                homelab.keithjackson.dev/egress-gateway: "true"
-              tolerations:
-                - operator: Exists
-              containers:
-                - name: nsenter
-                  image: busybox:1.37.0
-                  securityContext:
-                    privileged: true
-                  command: ["nsenter","-t","1","-m","-u","-i","-n","-p","--","systemctl","start","vpn-egress-tunnel"]
-        JOB
-        # Block teardown until the restart actually completed on the host — a
-        # leaked-down tunnel is unacceptable. `wait` exits non-zero on failure
-        # or timeout, which (set -e) fails the step and surfaces the problem.
-        kubectl -n "$ns" wait --for=condition=complete --timeout=110s job/tunnel-restart
-        echo "TUNNEL_SELF_HEAL_OK"
-    l.go:52: | 01:46:21 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak      [0;22m | [36;1mSCRIPT[0;22m    | [36;1mLOG[0;22m   |
-        === STDOUT
-        job.batch/tunnel-restart created
-        job.batch/tunnel-restart condition met
-        TUNNEL_SELF_HEAL_OK
-    l.go:52: | 01:46:21 | [32;1mvpn-killswitch[0;22m | [32;1mwhen-force-tunnel-down-then-no-leak      [0;22m | [32;1mSCRIPT[0;22m    | [32;1mDONE[0;22m  |
-    l.go:52: | 01:46:21 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak      [0;22m | [36;1mFINALLY[0;22m   | [36;1mEND[0;22m   |
-    l.go:52: | 01:46:21 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-tunnel-recovers-connectivity-returns[0;22m | [36;1mTRY[0;22m       | [36;1mBEGIN[0;22m |
-    l.go:52: | 01:46:21 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-tunnel-recovers-connectivity-returns[0;22m | [36;1mCMD[0;22m       | [36;1mRUN[0;22m   |
-        === COMMAND
-        /usr/bin/sh -c set -eu
-        ns="$NAMESPACE"
-        wan="$(kubectl -n "$ns" exec egress-internet -c curl -- \
-                 curl -fsS --max-time 10 https://api.ipify.org 2>/dev/null || true)"
-        vpn=""
-        for i in $(seq 1 24); do
-          vpn="$(kubectl -n "$ns" exec egress-vpn -c curl -- \
-                   curl -fsS --max-time 8 https://api.ipify.org 2>/dev/null || true)"
-          if [ -n "$vpn" ] && [ "$vpn" != "$wan" ]; then break; fi
-          vpn=""
-          sleep 5
-        done
-        echo "recovered_vpn=$vpn wan=$wan"
-        [ -n "$vpn" ] || { echo "FAIL: vpn-class egress did not recover after tunnel restart" >&2; exit 1; }
-        [ "$vpn" != "$wan" ] || { echo "FAIL: vpn-class egress recovered onto the WAN (not tunneled)" >&2; exit 1; }
-        echo "TUNNEL_RECOVERED_OK"
-    l.go:52: | 01:46:25 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-tunnel-recovers-connectivity-returns[0;22m | [36;1mSCRIPT[0;22m    | [36;1mLOG[0;22m   |
-        === STDOUT
-        recovered_vpn=158.173.241.11 wan=136.51.115.252
-        TUNNEL_RECOVERED_OK
-    l.go:52: | 01:46:25 | [32;1mvpn-killswitch[0;22m | [32;1mwhen-tunnel-recovers-connectivity-returns[0;22m | [32;1mSCRIPT[0;22m    | [32;1mDONE[0;22m  |
-    l.go:52: | 01:46:25 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-tunnel-recovers-connectivity-returns[0;22m | [36;1mTRY[0;22m       | [36;1mEND[0;22m   |
-    l.go:52: | 01:46:25 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak      [0;22m | [36;1mCLEANUP[0;22m   | [36;1mBEGIN[0;22m |
-    l.go:52: | 01:46:25 | [32;1mvpn-killswitch[0;22m | [32;1mwhen-force-tunnel-down-then-no-leak      [0;22m | [32;1mDELETE[0;22m    | [32;1mOK[0;22m    | batch/v1/Job @ chainsaw-credible-bird/tunnel-stop
-    l.go:52: | 01:46:25 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak      [0;22m | [36;1mCLEANUP[0;22m   | [36;1mEND[0;22m   |
-    l.go:52: | 01:46:25 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads   [0;22m | [36;1mCLEANUP[0;22m   | [36;1mBEGIN[0;22m |
-    l.go:52: | 01:46:25 | [32;1mvpn-killswitch[0;22m | [32;1mgiven-healthy-tunnel-and-two-workloads   [0;22m | [32;1mDELETE[0;22m    | [32;1mOK[0;22m    | v1/Pod @ chainsaw-credible-bird/egress-vpn
-    l.go:52: | 01:46:56 | [32;1mvpn-killswitch[0;22m | [32;1mgiven-healthy-tunnel-and-two-workloads   [0;22m | [32;1mDELETE[0;22m    | [32;1mOK[0;22m    | v1/Pod @ chainsaw-credible-bird/egress-internet
-    l.go:52: | 01:47:26 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads   [0;22m | [36;1mCLEANUP[0;22m   | [36;1mEND[0;22m   |
-    l.go:52: | 01:47:26 | [36;1mvpn-killswitch[0;22m | [36;1m@chainsaw                                [0;22m | [36;1mCLEANUP[0;22m   | [36;1mBEGIN[0;22m |
-    l.go:52: | 01:47:26 | [32;1mvpn-killswitch[0;22m | [32;1m@chainsaw                                [0;22m | [32;1mDELETE[0;22m    | [32;1mOK[0;22m    | v1/Namespace @ chainsaw-credible-bird
-    l.go:52: | 01:47:33 | [36;1mvpn-killswitch[0;22m | [36;1m@chainsaw                                [0;22m | [36;1mCLEANUP[0;22m   | [36;1mEND[0;22m   |
--- PASS: chainsaw (0.00s)
-    --- PASS: chainsaw/vpn-killswitch (174.63s)
-PASS
-Tests Summary...
- Passed  tests 1
- Failed  tests 0
- Skipped tests 0
-Done.
-[1m[run.sh][0m HEAVY pass: no heavy tests selected.
-merge_junit: merged 1 report(s) -> /work/.claude/conf-clone/conformance/_artifacts/report.junit.xml (tests=1 failures=0 errors=0 skipped=0)
-[1m[run.sh][0m Done. JUnit: /work/.claude/conf-clone/conformance/_artifacts/report.junit.xml | Coverage: /work/.claude/conf-clone/conformance/_artifacts/coverage/coverage.md
@@ -1,4 +0,0 @@
-<testsuites name="chainsaw-report" tests="1" failures="0" errors="0" time="0.000000"><testsuite name="/work/.claude/conf-clone/conformance/tests/l3-fault-injection/vpn-killswitch" tests="1" failures="0" errors="0" id="0" time="">
-    <testcase name="vpn-killswitch" classname="" time="173.473422" />
-  </testsuite>
-</testsuites>
@@ -1,237 +0,0 @@
-[1m[run.sh][0m Coverage report first (always)
-[coverage] 49/51 requirements have a scaffolded test (96.1%); 2 gap(s), 0 safety-critical gap(s).
-[coverage] wrote /work/.claude/conf-clone/conformance/_artifacts/coverage/coverage.md and /work/.claude/conf-clone/conformance/_artifacts/coverage/coverage.json
-[1m[run.sh][0m No L0/L1 offline checks match the selection (layer=l3 reqs='R-NET-06').
-[1m[run.sh][0m LIGHT pass: 1 test(s), parallel=2 (pinned v0.2.12)
-Version: 0.2.12
-Loading config (/work/.claude/conf-clone/conformance/config/chainsaw-config.yaml)...
- Using test file: chainsaw-test
- TestDirs [/work/.claude/conf-clone/conformance/tests/l3-fault-injection/vpn-killswitch]
- SkipDelete false
- FailFast false
- ReportFormat 'JUNIT-TEST'
- ReportName 'chainsaw-report-light'
- ReportPath '/work/.claude/conf-clone/conformance/_artifacts'
- Namespace ''
- FullName false
- IncludeTestRegex ''
- ExcludeTestRegex ''
- ApplyTimeout 1m30s
- AssertTimeout 3m0s
- CleanupTimeout 2m0s
- DeleteTimeout 2m0s
- ErrorTimeout 1m0s
- ExecTimeout 3m0s
- DeletionPropagationPolicy Background
- Parallel 2
- Template true
- NoCluster false
- PauseOnFailure false
-Loading tests...
- vpn-killswitch (/work/.claude/conf-clone/conformance/tests/l3-fault-injection/vpn-killswitch)
-Loading values...
-Running tests...
-=== RUN   chainsaw
-=== PAUSE chainsaw
-=== CONT  chainsaw
-=== RUN   chainsaw/vpn-killswitch
-=== PAUSE chainsaw/vpn-killswitch
-=== CONT  chainsaw/vpn-killswitch
-    l.go:52: | 01:49:24 | [32;1mvpn-killswitch[0;22m | [32;1m@chainsaw                                [0;22m | [32;1mCREATE[0;22m    | [32;1mOK[0;22m    | v1/Namespace @ chainsaw-beloved-toad
-    l.go:52: | 01:49:24 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads   [0;22m | [36;1mTRY[0;22m       | [36;1mBEGIN[0;22m |
-    l.go:52: | 01:49:24 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads   [0;22m | [36;1mCMD[0;22m       | [36;1mRUN[0;22m   |
-        === COMMAND
-        /usr/bin/sh -c set -eu
-        kubectl label namespace "$NAMESPACE" \
-          pod-security.kubernetes.io/enforce=privileged \
-          pod-security.kubernetes.io/warn=privileged \
-          pod-security.kubernetes.io/audit=privileged \
-          --overwrite
-        echo "NS_PRIVILEGED_OK"
-    l.go:52: | 01:49:24 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads   [0;22m | [36;1mSCRIPT[0;22m    | [36;1mLOG[0;22m   |
-        === STDOUT
-        namespace/chainsaw-beloved-toad labeled
-        NS_PRIVILEGED_OK
-    l.go:52: | 01:49:24 | [32;1mvpn-killswitch[0;22m | [32;1mgiven-healthy-tunnel-and-two-workloads   [0;22m | [32;1mSCRIPT[0;22m    | [32;1mDONE[0;22m  |
-    l.go:52: | 01:49:24 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads   [0;22m | [36;1mAPPLY[0;22m     | [36;1mRUN[0;22m   | v1/Pod @ chainsaw-beloved-toad/egress-internet
-    l.go:52: | 01:49:24 | [32;1mvpn-killswitch[0;22m | [32;1mgiven-healthy-tunnel-and-two-workloads   [0;22m | [32;1mCREATE[0;22m    | [32;1mOK[0;22m    | v1/Pod @ chainsaw-beloved-toad/egress-internet
-    l.go:52: | 01:49:24 | [32;1mvpn-killswitch[0;22m | [32;1mgiven-healthy-tunnel-and-two-workloads   [0;22m | [32;1mAPPLY[0;22m     | [32;1mDONE[0;22m  | v1/Pod @ chainsaw-beloved-toad/egress-internet
-    l.go:52: | 01:49:24 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads   [0;22m | [36;1mAPPLY[0;22m     | [36;1mRUN[0;22m   | v1/Pod @ chainsaw-beloved-toad/egress-vpn
-    l.go:52: | 01:49:25 | [32;1mvpn-killswitch[0;22m | [32;1mgiven-healthy-tunnel-and-two-workloads   [0;22m | [32;1mCREATE[0;22m    | [32;1mOK[0;22m    | v1/Pod @ chainsaw-beloved-toad/egress-vpn
-    l.go:52: | 01:49:25 | [32;1mvpn-killswitch[0;22m | [32;1mgiven-healthy-tunnel-and-two-workloads   [0;22m | [32;1mAPPLY[0;22m     | [32;1mDONE[0;22m  | v1/Pod @ chainsaw-beloved-toad/egress-vpn
-    l.go:52: | 01:49:25 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads   [0;22m | [36;1mASSERT[0;22m    | [36;1mRUN[0;22m   | v1/Pod @ chainsaw-beloved-toad/egress-internet
-    l.go:52: | 01:49:26 | [32;1mvpn-killswitch[0;22m | [32;1mgiven-healthy-tunnel-and-two-workloads   [0;22m | [32;1mASSERT[0;22m    | [32;1mDONE[0;22m  | v1/Pod @ chainsaw-beloved-toad/egress-internet
-    l.go:52: | 01:49:26 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads   [0;22m | [36;1mASSERT[0;22m    | [36;1mRUN[0;22m   | v1/Pod @ chainsaw-beloved-toad/egress-vpn
-    l.go:52: | 01:49:26 | [32;1mvpn-killswitch[0;22m | [32;1mgiven-healthy-tunnel-and-two-workloads   [0;22m | [32;1mASSERT[0;22m    | [32;1mDONE[0;22m  | v1/Pod @ chainsaw-beloved-toad/egress-vpn
-    l.go:52: | 01:49:26 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads   [0;22m | [36;1mCMD[0;22m       | [36;1mRUN[0;22m   |
-        === COMMAND
-        /usr/bin/sh -c set -eu
-        ns="$NAMESPACE"
-        get_ip() {
-          kubectl -n "$ns" exec "$1" -c curl -- \
-            curl -fsS --max-time 10 https://api.ipify.org 2>/dev/null || true
-        }
-        wan="" vpn=""
-        for i in $(seq 1 12); do
-          [ -n "$wan" ] || wan="$(get_ip egress-internet)"
-          [ -n "$vpn" ] || vpn="$(get_ip egress-vpn)"
-          if [ -n "$wan" ] && [ -n "$vpn" ] && [ "$wan" != "$vpn" ]; then break; fi
-          [ "$wan" = "$vpn" ] && vpn=""   # not steered yet; re-poll
-          sleep 5
-        done
-        echo "wan=$wan vpn=$vpn"
-        [ -n "$wan" ] || { echo "FAIL: no WAN IP from internet-class pod" >&2; exit 1; }
-        [ -n "$vpn" ] || { echo "FAIL: vpn-class egress not working at baseline" >&2; exit 1; }
-        [ "$wan" != "$vpn" ] || { echo "FAIL: vpn-class == WAN at baseline (not tunneled)" >&2; exit 1; }
-        echo "TUNNEL_HEALTHY_OK"
-    l.go:52: | 01:49:27 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads   [0;22m | [36;1mSCRIPT[0;22m    | [36;1mLOG[0;22m   |
-        === STDOUT
-        wan=136.51.115.252 vpn=158.173.241.11
-        TUNNEL_HEALTHY_OK
-    l.go:52: | 01:49:27 | [32;1mvpn-killswitch[0;22m | [32;1mgiven-healthy-tunnel-and-two-workloads   [0;22m | [32;1mSCRIPT[0;22m    | [32;1mDONE[0;22m  |
-    l.go:52: | 01:49:27 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads   [0;22m | [36;1mTRY[0;22m       | [36;1mEND[0;22m   |
-    l.go:52: | 01:49:27 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak      [0;22m | [36;1mTRY[0;22m       | [36;1mBEGIN[0;22m |
-    l.go:52: | 01:49:27 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak      [0;22m | [36;1mAPPLY[0;22m     | [36;1mRUN[0;22m   | batch/v1/Job @ chainsaw-beloved-toad/tunnel-stop
-    l.go:52: | 01:49:27 | [32;1mvpn-killswitch[0;22m | [32;1mwhen-force-tunnel-down-then-no-leak      [0;22m | [32;1mCREATE[0;22m    | [32;1mOK[0;22m    | batch/v1/Job @ chainsaw-beloved-toad/tunnel-stop
-    l.go:52: | 01:49:27 | [32;1mvpn-killswitch[0;22m | [32;1mwhen-force-tunnel-down-then-no-leak      [0;22m | [32;1mAPPLY[0;22m     | [32;1mDONE[0;22m  | batch/v1/Job @ chainsaw-beloved-toad/tunnel-stop
-    l.go:52: | 01:49:27 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak      [0;22m | [36;1mASSERT[0;22m    | [36;1mRUN[0;22m   | batch/v1/Job @ chainsaw-beloved-toad/tunnel-stop
-    l.go:52: | 01:49:31 | [32;1mvpn-killswitch[0;22m | [32;1mwhen-force-tunnel-down-then-no-leak      [0;22m | [32;1mASSERT[0;22m    | [32;1mDONE[0;22m  | batch/v1/Job @ chainsaw-beloved-toad/tunnel-stop
-    l.go:52: | 01:49:31 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak      [0;22m | [36;1mCMD[0;22m       | [36;1mRUN[0;22m   |
-        === COMMAND
-        /usr/bin/sh -c set -u
-        ns="$NAMESPACE"
-        wan="$(kubectl -n "$ns" exec egress-internet -c curl -- \
-                 curl -fsS --max-time 10 https://api.ipify.org 2>/dev/null || true)"
-        for i in $(seq 1 8); do
-          out="$(kubectl -n "$ns" exec egress-vpn -c curl -- \
-                   curl -fsS --max-time 8 https://api.ipify.org 2>/dev/null || true)"
-          if [ -n "$out" ]; then
-            if [ -n "$wan" ] && [ "$out" = "$wan" ]; then
-              echo "LEAK: vpn-class reached the WAN IP ($out) with the tunnel DOWN" >&2; exit 1
-            fi
-            echo "LEAK: vpn-class reached the internet ($out) with the tunnel DOWN" >&2; exit 1
-          fi
-          echo "attempt $i: vpn-class blackholed (no response), good"
-          sleep 3
-        done
-        echo "NO_LEAK_OK"
-    l.go:52: | 01:51:01 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak      [0;22m | [36;1mSCRIPT[0;22m    | [36;1mLOG[0;22m   |
-        === STDOUT
-        attempt 1: vpn-class blackholed (no response), good
-        attempt 2: vpn-class blackholed (no response), good
-        attempt 3: vpn-class blackholed (no response), good
-        attempt 4: vpn-class blackholed (no response), good
-        attempt 5: vpn-class blackholed (no response), good
-        attempt 6: vpn-class blackholed (no response), good
-        attempt 7: vpn-class blackholed (no response), good
-        attempt 8: vpn-class blackholed (no response), good
-        NO_LEAK_OK
-    l.go:52: | 01:51:01 | [32;1mvpn-killswitch[0;22m | [32;1mwhen-force-tunnel-down-then-no-leak      [0;22m | [32;1mSCRIPT[0;22m    | [32;1mDONE[0;22m  |
-    l.go:52: | 01:51:01 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak      [0;22m | [36;1mCMD[0;22m       | [36;1mRUN[0;22m   |
-        === COMMAND
-        /usr/bin/sh -c set -eu
-        ns="$NAMESPACE"
-        out="$(kubectl -n "$ns" exec egress-internet -c curl -- \
-                 curl -fsS --max-time 10 https://api.ipify.org 2>/dev/null || true)"
-        echo "internet_ip=$out"
-        [ -n "$out" ] || { echo "FAIL: internet-class egress broke while tunnel down (must be unaffected)" >&2; exit 1; }
-        echo "INTERNET_UNAFFECTED_OK"
-    l.go:52: | 01:51:02 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak      [0;22m | [36;1mSCRIPT[0;22m    | [36;1mLOG[0;22m   |
-        === STDOUT
-        internet_ip=136.51.115.252
-        INTERNET_UNAFFECTED_OK
-    l.go:52: | 01:51:02 | [32;1mvpn-killswitch[0;22m | [32;1mwhen-force-tunnel-down-then-no-leak      [0;22m | [32;1mSCRIPT[0;22m    | [32;1mDONE[0;22m  |
-    l.go:52: | 01:51:02 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak      [0;22m | [36;1mTRY[0;22m       | [36;1mEND[0;22m   |
-    l.go:52: | 01:51:02 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak      [0;22m | [36;1mFINALLY[0;22m   | [36;1mBEGIN[0;22m |
-    l.go:52: | 01:51:02 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak      [0;22m | [36;1mCMD[0;22m       | [36;1mRUN[0;22m   |
-        === COMMAND
-        /usr/bin/sh -c set -eu
-        ns="$NAMESPACE"
-        # Distinct Job name (tunnel-restart) so it never collides with the
-        # tunnel-stop Job. Left-aligned manifest so the block scalar yields
-        # valid YAML at column 0 at runtime.
-        cat <<'JOB' | kubectl -n "$ns" apply -f -
-        apiVersion: batch/v1
-        kind: Job
-        metadata:
-          name: tunnel-restart
-          labels: { app: vpn-killswitch-fault }
-        spec:
-          backoffLimit: 3
-          activeDeadlineSeconds: 90
-          ttlSecondsAfterFinished: 600
-          template:
-            metadata:
-              labels: { app: vpn-killswitch-fault }
-            spec:
-              restartPolicy: Never
-              hostPID: true
-              nodeSelector:
-                homelab.keithjackson.dev/egress-gateway: "true"
-              tolerations:
-                - operator: Exists
-              containers:
-                - name: nsenter
-                  image: busybox:1.37.0
-                  securityContext:
-                    privileged: true
-                  command: ["nsenter","-t","1","-m","-u","-i","-n","-p","--","systemctl","start","vpn-egress-tunnel"]
-        JOB
-        # Block teardown until the restart actually completed on the host — a
-        # leaked-down tunnel is unacceptable. `wait` exits non-zero on failure
-        # or timeout, which (set -e) fails the step and surfaces the problem.
-        kubectl -n "$ns" wait --for=condition=complete --timeout=110s job/tunnel-restart
-        echo "TUNNEL_SELF_HEAL_OK"
-    l.go:52: | 01:51:05 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak      [0;22m | [36;1mSCRIPT[0;22m    | [36;1mLOG[0;22m   |
-        === STDOUT
-        job.batch/tunnel-restart created
-        job.batch/tunnel-restart condition met
-        TUNNEL_SELF_HEAL_OK
-    l.go:52: | 01:51:05 | [32;1mvpn-killswitch[0;22m | [32;1mwhen-force-tunnel-down-then-no-leak      [0;22m | [32;1mSCRIPT[0;22m    | [32;1mDONE[0;22m  |
-    l.go:52: | 01:51:05 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak      [0;22m | [36;1mFINALLY[0;22m   | [36;1mEND[0;22m   |
-    l.go:52: | 01:51:05 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-tunnel-recovers-connectivity-returns[0;22m | [36;1mTRY[0;22m       | [36;1mBEGIN[0;22m |
-    l.go:52: | 01:51:05 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-tunnel-recovers-connectivity-returns[0;22m | [36;1mCMD[0;22m       | [36;1mRUN[0;22m   |
-        === COMMAND
-        /usr/bin/sh -c set -eu
-        ns="$NAMESPACE"
-        wan="$(kubectl -n "$ns" exec egress-internet -c curl -- \
-                 curl -fsS --max-time 10 https://api.ipify.org 2>/dev/null || true)"
-        vpn=""
-        for i in $(seq 1 24); do
-          vpn="$(kubectl -n "$ns" exec egress-vpn -c curl -- \
-                   curl -fsS --max-time 8 https://api.ipify.org 2>/dev/null || true)"
-          if [ -n "$vpn" ] && [ "$vpn" != "$wan" ]; then break; fi
-          vpn=""
-          sleep 5
-        done
-        echo "recovered_vpn=$vpn wan=$wan"
-        [ -n "$vpn" ] || { echo "FAIL: vpn-class egress did not recover after tunnel restart" >&2; exit 1; }
-        [ "$vpn" != "$wan" ] || { echo "FAIL: vpn-class egress recovered onto the WAN (not tunneled)" >&2; exit 1; }
-        echo "TUNNEL_RECOVERED_OK"
-    l.go:52: | 01:51:08 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-tunnel-recovers-connectivity-returns[0;22m | [36;1mSCRIPT[0;22m    | [36;1mLOG[0;22m   |
-        === STDOUT
-        recovered_vpn=158.173.241.11 wan=136.51.115.252
-        TUNNEL_RECOVERED_OK
-    l.go:52: | 01:51:08 | [32;1mvpn-killswitch[0;22m | [32;1mwhen-tunnel-recovers-connectivity-returns[0;22m | [32;1mSCRIPT[0;22m    | [32;1mDONE[0;22m  |
-    l.go:52: | 01:51:08 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-tunnel-recovers-connectivity-returns[0;22m | [36;1mTRY[0;22m       | [36;1mEND[0;22m   |
-    l.go:52: | 01:51:08 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak      [0;22m | [36;1mCLEANUP[0;22m   | [36;1mBEGIN[0;22m |
-    l.go:52: | 01:51:08 | [32;1mvpn-killswitch[0;22m | [32;1mwhen-force-tunnel-down-then-no-leak      [0;22m | [32;1mDELETE[0;22m    | [32;1mOK[0;22m    | batch/v1/Job @ chainsaw-beloved-toad/tunnel-stop
-    l.go:52: | 01:51:08 | [36;1mvpn-killswitch[0;22m | [36;1mwhen-force-tunnel-down-then-no-leak      [0;22m | [36;1mCLEANUP[0;22m   | [36;1mEND[0;22m   |
-    l.go:52: | 01:51:08 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads   [0;22m | [36;1mCLEANUP[0;22m   | [36;1mBEGIN[0;22m |
-    l.go:52: | 01:51:08 | [32;1mvpn-killswitch[0;22m | [32;1mgiven-healthy-tunnel-and-two-workloads   [0;22m | [32;1mDELETE[0;22m    | [32;1mOK[0;22m    | v1/Pod @ chainsaw-beloved-toad/egress-vpn
-    l.go:52: | 01:51:39 | [32;1mvpn-killswitch[0;22m | [32;1mgiven-healthy-tunnel-and-two-workloads   [0;22m | [32;1mDELETE[0;22m    | [32;1mOK[0;22m    | v1/Pod @ chainsaw-beloved-toad/egress-internet
-    l.go:52: | 01:52:10 | [36;1mvpn-killswitch[0;22m | [36;1mgiven-healthy-tunnel-and-two-workloads   [0;22m | [36;1mCLEANUP[0;22m   | [36;1mEND[0;22m   |
-    l.go:52: | 01:52:10 | [36;1mvpn-killswitch[0;22m | [36;1m@chainsaw                                [0;22m | [36;1mCLEANUP[0;22m   | [36;1mBEGIN[0;22m |
-    l.go:52: | 01:52:10 | [32;1mvpn-killswitch[0;22m | [32;1m@chainsaw                                [0;22m | [32;1mDELETE[0;22m    | [32;1mOK[0;22m    | v1/Namespace @ chainsaw-beloved-toad
-    l.go:52: | 01:52:18 | [36;1mvpn-killswitch[0;22m | [36;1m@chainsaw                                [0;22m | [36;1mCLEANUP[0;22m   | [36;1mEND[0;22m   |
--- PASS: chainsaw (0.00s)
-    --- PASS: chainsaw/vpn-killswitch (173.47s)
-PASS
-Tests Summary...
- Passed  tests 1
- Failed  tests 0
- Skipped tests 0
-Done.
-[1m[run.sh][0m HEAVY pass: no heavy tests selected.
-merge_junit: merged 1 report(s) -> /work/.claude/conf-clone/conformance/_artifacts/report.junit.xml (tests=1 failures=0 errors=0 skipped=0)
-[1m[run.sh][0m Done. JUnit: /work/.claude/conf-clone/conformance/_artifacts/report.junit.xml | Coverage: /work/.claude/conf-clone/conformance/_artifacts/coverage/coverage.md
@@ -1,5 +1,2 @@
 {"chainsaw_version": "v0.2.12", "env": "dev", "git_sha": "57d4df8c4f992732a49f3e2daddcace866778ab0", "junit_result": "passed", "kube_version": "v1.36.4+k3s1", "layer": "L3", "note": "run log OUTPUT carries every declared pass token (NODE_LOSS_VOLUME_AVAILABLE_OK) and no skip token [log scoped to this testcase; tokens read from real-output sections only]", "proof": "proven", "requirements": ["R-STATE-02"], "run_id": "20260901T235117Z-e9a4b66", "sentinel": "NODE_LOSS_VOLUME_AVAILABLE_OK", "test": "node-loss", "ts_utc": "2026-09-01T23:51:17Z"}
 {"chainsaw_version": "v0.2.12", "env": "dev", "git_sha": "fdc3265585f4b775c0a6ae3fb33c837b0c643655", "junit_result": "passed", "kube_version": "v1.36.4+k3s1", "layer": "L3", "note": "run log OUTPUT carries every declared pass token (OFF_CLUSTER_TARGET_OK, BACKUPS_ARE_SCHEDULED_OK, DR_RESTORED_FROM_OFF_CLUSTER_BACKUP_OK) and no skip token [log scoped to this testcase; tokens read from real-output sections only]", "proof": "proven", "requirements": ["R-STATE-03"], "run_id": "20260902T001907Z-fdc3265", "sentinel": "DR_RESTORED_FROM_OFF_CLUSTER_BACKUP_OK", "test": "dr-restore", "ts_utc": "2026-09-02T00:19:07Z"}
-{"chainsaw_version": "v0.2.12", "env": "prod", "git_sha": "19fc8bcb27eab42a0007b28a831519d3e325c7b6", "junit_result": "passed", "kube_version": "v1.36.4+k3s1", "layer": "L1", "note": "L1 offline check exited 0; run_l1_offline.py renders its skips as JUnit <skipped/>, so a green JUnit is truthful without a sentinel", "proof": "proven", "requirements": ["R-NET-05", "R-NET-06", "R-NET-07"], "run_id": "20260913T014321Z-19fc8bc", "sentinel": null, "test": "egress-class-render", "ts_utc": "2026-09-13T01:43:21Z"}
-{"chainsaw_version": "v0.2.12", "env": "prod", "git_sha": "19fc8bcb27eab42a0007b28a831519d3e325c7b6-dirty", "junit_result": "passed", "kube_version": "v1.36.4+k3s1", "layer": "L3", "note": "run log OUTPUT carries every declared pass token (NS_PRIVILEGED_OK, TUNNEL_HEALTHY_OK, NO_LEAK_OK, INTERNET_UNAFFECTED_OK, TUNNEL_SELF_HEAL_OK, TUNNEL_RECOVERED_OK) and no skip token [log scoped to this testcase; tokens read from real-output sections only]", "proof": "proven", "requirements": ["R-NET-06"], "run_id": "20260913T014437Z-19fc8bc", "sentinel": "TUNNEL_RECOVERED_OK", "test": "vpn-killswitch", "ts_utc": "2026-09-13T01:44:37Z"}
-{"chainsaw_version": "v0.2.12", "env": "prod", "git_sha": "9d0814ae61e6a0ba64d3789a71f1cd9c985efbf0", "junit_result": "passed", "kube_version": "v1.36.4+k3s1", "layer": "L3", "note": "run log OUTPUT carries every declared pass token (NS_PRIVILEGED_OK, TUNNEL_HEALTHY_OK, NO_LEAK_OK, INTERNET_UNAFFECTED_OK, TUNNEL_SELF_HEAL_OK, TUNNEL_RECOVERED_OK) and no skip token [log scoped to this testcase; tokens read from real-output sections only]", "proof": "proven", "requirements": ["R-NET-06"], "run_id": "20260913T014923Z-9d0814a", "sentinel": "TUNNEL_RECOVERED_OK", "test": "vpn-killswitch", "ts_utc": "2026-09-13T01:49:23Z"}
@@ -250,11 +250,6 @@ try:
    sys.dont_write_bytecode = True
    import record_run as rr                                   # noqa: E402

-    # status.py owns the latest-wins rule; import it rather than reimplementing it, so the two
-    # cannot disagree about which rows are live evidence.
-    import status as _st                                        # noqa: E402
-    rr_latest_by_test = _st.latest_by_test
-
    smap = rr.sentinel_map(CONF / "tests")
    check(len(smap) >= 10,
          f"record_run.py's sentinel map contains {len(smap)} test(s) — the parser lost its "
@@ -391,15 +386,6 @@ try:
        check(not bad,
              f"conformance/runs/ledger.jsonl has unparsable line(s) {bad} — the audit record "
              "is not machine-readable, so no proof can be derived from it")
-        # Which rows are HONOURED: status.py resolves duplicates latest-wins (latest_by_test),
-        # so a superseded row is not evidence of anything. The dirty-tree assertion below is
-        # therefore scoped to honoured rows. Without that scoping, ONE historical dirty row reds
-        # this gate permanently even after a clean re-run of the same test — and a dirty-then-
-        # clean sequence is the normal shape of "noticed the tree was dirty, fixed it, re-ran".
-        # The row is still kept in the ledger and its run.log stays committed: nothing is erased,
-        # it simply stops being treated as a live claim. Structural assertions (provenance keys,
-        # sentinel, test-still-exists) still apply to EVERY row.
-        honoured_ids = {id(v) for v in rr_latest_by_test(rows).values()}
        for r in rows:
            for key in ("ts_utc", "git_sha", "test", "requirements", "junit_result", "proof",
                        "run_id"):
@@ -407,22 +393,13 @@ try:
                      f"a ledger row for {r.get('test', '?')!r} lacks {key!r} — a row without "
                      "provenance cannot be audited")
            if r.get("proof") == "proven":
-                # L0/L1 rows are recorded proven WITHOUT a sentinel, deliberately, by
-                # record_run.py's own SENTINEL_EXEMPT_LAYERS: run_l1_offline.py renders skips as
-                # JUnit <skipped/>, so those layers have no pass token to attribute. This
-                # assertion had NO layer exemption, so the two halves of the harness contradicted
-                # each other — ANY `--record` run touching L0 or L1 wrote a row this gate then
-                # rejected, which meant a requirement whose verifier set includes an L1 check
-                # (R-NET-06 is one) could never be both PROVEN and L0-green. Honour the same
-                # constant rather than restating the policy here, so the two cannot drift apart.
-                if r.get("layer") not in rr.SENTINEL_EXEMPT_LAYERS:
-                    check(bool(r.get("sentinel")),
-                          f"a ledger row claims `proven` for {r.get('test')!r} with NO sentinel — "
-                          "the sentinel is the only thing separating a real pass from a self-skip")
+                check(bool(r.get("sentinel")),
+                      f"a ledger row claims `proven` for {r.get('test')!r} with NO sentinel — "
+                      "the sentinel is the only thing separating a real pass from a self-skip")
                check(r.get("test") in smap,
                      f"a ledger row claims `proven` for {r.get('test')!r}, which is not a "
                      "test in the suite any more — a proof of something that no longer exists")
-                check(id(r) not in honoured_ids or "-dirty" not in str(r.get("git_sha", "")),
+                check("-dirty" not in str(r.get("git_sha", "")),
                      f"a ledger row claims `proven` for {r.get('test')!r} from a DIRTY tree "
                      f"({r.get('git_sha')}) — what ran is not what is committed")
    else:
@@ -1,99 +0,0 @@
-# =============================================================================
-# L0 STATIC — the bulk tier's capacity strategy is DISTINCT from the replicated tier's
-# Catalog: docs/14 L0 row ("Pre-apply, in CI") — runs WITHOUT a cluster.
-#
-# THE BUG CLASS THIS CLOSES. The `bulk` StorageClass existed from Phase 0 as a stub:
-# `provisioner: nfs.csi.k8s.io`, `server: REPLACE-ME`, `share: REPLACE-ME`, TODO(OQ-2).
-# Four ServiceDefinitions carried a comment promising to attach their media once OQ-2 was
-# resolved, and the ONLY thing keeping a multi-terabyte library off the 3-replica
-# `longhorn` class was those four comments. A bulk volume that lands on the replicated
-# tier does not fail loudly — Longhorn accepts the PVC and starts replicating 3x of data
-# three mini-PCs cannot hold. R-STATE-04 is precisely that the two tiers are distinct, so
-# the distinctness is asserted rather than reviewed.
-#
-# It also pins the four decisions that make the STATIC form of this tier safe, each with a
-# plausible "simplification" that breaks something silently: `Retain` (bulk data pre-exists
-# the cluster and has consumers outside it, so anything else makes `delete pvc` a data-loss
-# operation); NO dynamic provisioning (a dynamic bulk class hands out a FRESH EMPTY
-# directory that mounts and Binds perfectly while the real 8 TiB is never attached — the
-# most deceptive failure available here); `hard` not `soft` (a soft mount hands a server
-# hiccup to the app as a short read, which for a media library is silent corruption); and
-# NFSv4 (mount and lock in-protocol ⇒ ONE reachable port, tcp/2049, which is what the
-# operator must open in pfSense for prod to reach this server at all).
-#
-# AND the EXDEV trap, which is not about storage. Sonarr/Radarr import by hardlinking or
-# atomically renaming, both same-filesystem operations, and to the kernel "same filesystem"
-# means SAME MOUNT. Splitting one export into two subPath mounts looks TIGHTER and breaks
-# link() with EXDEV; the arrs then degrade to copy-then-delete, rewriting every import over
-# NFS and destroying the seeding copy. It surfaces weeks later as "imports are slow and my
-# ratio is gone". A reviewer cannot see that; this check can.
-#
-# GIVEN the repository as authored,
-# WHEN  every `class: bulk` storage entry in catalog/ is compared against the objects the
-#       COMMITTED apps/ goldens carry for it (what Flux reconciles — deliberately not the
-#       renderer, which would only prove it agrees with itself),
-# THEN  (1) no bulk volume is on a Longhorn-provisioned class, and its class is explicitly
-#           EMPTY so nothing can dynamically provision an empty stand-in;
-#       (2) its capacity is declared and is at least an order of magnitude above the same
-#           service's replicated volume — the requirement stated as arithmetic;
-#       (3) PV and PVC are pre-bound from BOTH ends (claimRef + volumeName) and agree;
-#       (4) the PV reclaim policy is Retain;
-#       (5) mountOptions carry `hard` and exactly one `nfsvers=4.x`, and never `soft`;
-#       (6) no workload mounts one export more than once (the EXDEV trap);
-#       (7) a readOnly entry renders nfs.readOnly AND a readOnly volumeMount AND
-#           ReadOnlyMany; a writable one renders neither and is ReadWriteMany; every bulk
-#           access mode is a `...Many`, since one export serves several namespaces and
-#           several nodes;
-#       (8) the PV's nfs.server is a BARE cluster-vars token (never a literal — dev and
-#           prod reach this host by different addresses), that knob has a base default, and
-#           EVERY overlay supplies it;
-#   AND (9) the check is LOAD-BEARING (ci/l0.sh runs it unconditionally, ci/gate.sh blocks
-#           on the l0 stage) and NON-VACUOUS (an empty bulk corpus FAILS rather than
-#           passing trivially, and a floor on the assertion count keeps a pass from
-#           degenerating into a no-op).
-#
-# STATUS: EXECUTABLE. `check.sh` is a pure offline assertion script (standard library +
-# PyYAML); conformance/run.sh --layer l0 runs it directly and this Chainsaw Test wraps the
-# SAME script.
-#
-# NOT proven — the runtime half, which needs a cluster: that the server is reachable, that
-# the export admits these clients, that the PVC binds, that a write succeeds, or that a
-# hardlink between the library and the download path actually works. The uid analysis is
-# reasoning (docs/BULK-NFS-STORAGE.md §6) and the in-cluster probes that would settle it
-# are written down in §9 of that document, unrun. Nor can this prove prod's address is
-# CORRECT — only that it is a supplied knob; while it is a placeholder, prod's pods will
-# not mount, and catching that statically needs the placeholder guard, which is absent from
-# main (§7). Read-only on the media tree is a CLIENT-side guarantee only: the export serves
-# one reader and two writers from the same node addresses, so the server cannot tell them
-# apart (§5).
-# =============================================================================
-apiVersion: chainsaw.kyverno.io/v1alpha1
-kind: Test
-metadata:
-  name: bulk-tier-distinct-from-replicated
-  labels:
-    conformance.home.k3s/R-STATE-04: "true"
-  annotations:
-    conformance.home.k3s/requirements: "R-STATE-04"
-    conformance.home.k3s/layer: "L0"
-    conformance.home.k3s/safety-critical: "false"
-    conformance.home.k3s/executable: "true"
-    conformance.home.k3s/intent: >-
-      Bulk media storage is provably NOT on the 3-replica Longhorn tier, is sized for a
-      different order of magnitude, cannot be silently replaced by a dynamically
-      provisioned empty volume, is never reclaimable by the cluster, mounts hard over
-      NFSv4, and is never split into two mounts of one export — so hardlinked imports keep
-      working. Its server is a per-environment knob, not a literal, because the two
-      clusters reach the same storage host by different addresses.
-spec:
-  steps:
-    - name: assert-bulk-tier-is-distinct-and-statically-bound
-      description: >-
-        Read every `class: bulk` storage entry from catalog/, pair it with the
-        PersistentVolume, PersistentVolumeClaim and volumeMount the committed apps/ goldens
-        carry for it, and assert the tier separation, the static pre-binding, the reclaim
-        and mount-option decisions, the read-only coupling, the one-export-one-mount
-        invariant, and that the server address is supplied by every environment overlay.
-      try:
-        - script:
-            content: bash ./check.sh
@@ -1,480 +0,0 @@
-#!/usr/bin/env bash
-# =============================================================================
-# L0 static check (offline, no cluster) — bulk-tier-distinct-from-replicated
-# Serves: R-STATE-04 ("Bulk media storage has a capacity strategy distinct from the
-#                      replicated tier." — spec docs/04-storage-and-state.md)
-#
-# WHY THIS EXISTS — THE BULK TIER WAS A STUB FOR SIX ROADMAP PHASES, AND STUBS ROT QUIETLY
-# ----------------------------------------------------------------------------------------
-# `platform/longhorn/storageclasses.yaml` declared a `bulk` StorageClass from Phase 0 with
-# `provisioner: nfs.csi.k8s.io`, `server: REPLACE-ME`, `share: REPLACE-ME` and a TODO(OQ-2).
-# Four ServiceDefinitions carried a comment promising to attach their media "once OQ-2 is
-# resolved", and each of them said, correctly, that adding `class: bulk` before then would
-# leave the PVC unbound and the pod Pending. So the tier existed as a NAME with nothing
-# behind it, and the only thing preventing a multi-terabyte media library from being
-# authored onto the 3-replica `longhorn` class was four hand-written comments.
-#
-# That is the failure this check makes impossible to reintroduce. R-STATE-04 is not "bulk
-# storage exists" — it is that bulk storage's capacity strategy is DISTINCT from the
-# replicated tier's. Three mini-PCs cannot hold 3x of 8.33 TiB. A bulk volume that lands on
-# `longhorn` does not fail loudly: Longhorn accepts the PVC and starts replicating, and the
-# tier fills up. Distinctness is therefore a property worth asserting statically, on every
-# gate run, rather than trusting to review.
-#
-# It also pins the four decisions that make the STATIC form of this tier safe, each of
-# which has a plausible-looking "simplification" that silently breaks something:
-#
-#   * RECLAIM POLICY. Bulk data pre-exists the cluster, is larger than the cluster, and has
-#     consumers OUTSIDE it. `Retain` is the whole reason a PV is used here instead of an
-#     inline volume. `Delete` on a dynamically-provisioned bulk class would make `kubectl
-#     delete pvc` a data-loss operation.
-#   * NO DYNAMIC PROVISIONING. A dynamic bulk class hands out FRESH EMPTY directories. The
-#     pod mounts successfully, the PVC is Bound, everything is green — and the 8 TiB of
-#     existing data was never attached. This is the single most deceptive failure available
-#     in this tier, so `storageClassName` must be explicitly empty and binding must be
-#     pinned from BOTH ends (claimRef + volumeName).
-#   * `hard`, NOT `soft`. A soft NFS mount converts a server hiccup into a short read or
-#     EIO handed to the application. For a media library that is silent corruption. `hard`
-#     hangs instead — recoverable, where truncated is not.
-#   * NFSv4. v4 carries mount and lock in-protocol, so the tier needs ONE reachable port
-#     (tcp/2049) instead of v3's rpcbind/111 + mountd + statd on ephemeral numbers. That is
-#     a FIREWALL fact as much as a storage one: prod reaches this server through pfSense,
-#     and the number of ports the operator must open is a consequence of this field.
-#
-# AND ONE INVARIANT THAT IS NOT ABOUT STORAGE AT ALL — the EXDEV trap (assertion 6).
-# Sonarr and Radarr import by hardlinking or atomically renaming a finished download into
-# the library. Both are same-filesystem operations, and to the kernel "same filesystem"
-# means SAME MOUNT. Mounting one export twice (say `subPath: tv` and `subPath: downloads`)
-# looks TIGHTER — less of the tree exposed per mount — and breaks `link()` with EXDEV no
-# matter that the server knows they are one dataset. The arrs do not fail loudly on that:
-# they degrade to copy-then-delete, rewriting every imported file over NFS and destroying
-# the seeding copy. The symptom appears weeks later as "imports are slow and my ratio is
-# gone". A reviewer cannot see it, so this check does: one export, at most one mount, per
-# workload.
-#
-# GIVEN the repository as authored,
-# WHEN  every `class: bulk` storage entry in catalog/ is compared against the manifests
-#       render.py produces for it (read from the committed apps/ goldens, i.e. what Flux
-#       actually reconciles — not from the renderer, which would only agree with itself),
-# THEN  (1) DISTINCTNESS: no bulk volume is on the replicated tier. Its PV/PVC
-#           storageClassName is explicitly empty — never `longhorn`, never any class the
-#           repo declares with the Longhorn provisioner — and its PV's source is an
-#           EXTERNAL one (nfs), not a Longhorn volume;
-#       (2) CAPACITY STRATEGY: every bulk PV declares a capacity, and for each service the
-#           bulk volume is at least an ORDER OF MAGNITUDE larger than that service's own
-#           replicated volume. This is the requirement stated as arithmetic: the two tiers
-#           exist because the data has two magnitudes, so a "bulk" volume sized like a
-#           config PVC means the strategy was named and not actually applied;
-#       (3) NOT DYNAMICALLY PROVISIONED, AND PRE-BOUND BOTH WAYS: every bulk PVC names its
-#           PV by volumeName, every bulk PV names its PVC by claimRef (namespace+name), and
-#           the two agree — so no other claim can win a race, and nothing can provision an
-#           empty replacement;
-#       (4) RECLAIM: every bulk PV is `Retain`;
-#       (5) MOUNT SAFETY: every bulk PV declares mountOptions carrying `hard` and an
-#           `nfsvers=4.x`, and carrying NEITHER `soft` NOR an `nfsvers=3`;
-#       (6) ONE EXPORT, ONE MOUNT PER WORKLOAD (the EXDEV trap above);
-#       (7) READ-ONLY IS REAL AND CONSISTENT: a bulk entry declared readOnly renders BOTH
-#           `spec.nfs.readOnly: true` on the PV AND `readOnly: true` on the volumeMount,
-#           and its access mode is ReadOnlyMany; a writable one renders NEITHER and is
-#           ReadWriteMany. Every bulk access mode is a `...Many` — one export is consumed
-#           by several namespaces and several nodes, which ReadWriteOnce cannot express;
-#       (8) AUTHORED ONCE, PROMOTED (the R-ENV-02 seam this tier depends on): every bulk
-#           PV's `nfs.server` is the BARE cluster-vars token, never a literal address; that
-#           knob has a base default; and EVERY environment overlay supplies it — so the two
-#           clusters, which reach this storage by different addresses, cannot be served by
-#           one hardcoded value;
-#   AND (9) LOAD-BEARING + NON-VACUOUS: ci/l0.sh runs this check unconditionally and
-#           ci/gate.sh blocks the merge when the l0 stage fails; and the corpus is NOT
-#           empty — a repo with zero bulk volumes fails here rather than passing trivially,
-#           because a check that is satisfiable by deleting the feature proves nothing.
-#
-# WHAT THIS DOES *NOT* PROVE (honest gaps)
-#   - Nothing about RUNTIME. Not that the server is reachable, not that the export permits
-#     these clients, not that the PVC binds, not that a write succeeds. The uid analysis in
-#     docs/BULK-NFS-STORAGE.md §6 is reasoning, and the write/hardlink probes in §9 are the
-#     in-cluster proof — they need a cluster and are not run here.
-#   - It cannot prove the prod address is CORRECT, only that it is a supplied knob. While
-#     prod's value is a placeholder, prod's pods will not mount (docs/BULK-NFS-STORAGE.md
-#     §7); catching that needs the placeholder guard, which is absent from main.
-#   - Read-only for the media tree is a CLIENT-side guarantee, and this check can only
-#     assert the client side. The export serves a reader and two writers from the same node
-#     addresses, so the server cannot distinguish them (docs/BULK-NFS-STORAGE.md §5).
-#   - Assertion (2)'s magnitude ratio is a heuristic with a deliberately generous floor. It
-#     catches "bulk volume sized like a config volume", not a subtly wrong capacity — and a
-#     static NFS PV's capacity is advisory anyway, since the ZFS pool is the real limit.
-#   - Assertion (6) reasons over ONE workload's mounts. It cannot see two different
-#     services hardlinking between each other's mounts, which is not a thing that happens.
-# =============================================================================
-set -euo pipefail
-DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-ROOT="$(cd "${DIR}/../../../.." && pwd)"        # repo root
-PYTHON="${PYTHON:-python3}"
-
-cd "${ROOT}"
-
-if [ ! -d "${ROOT}/catalog" ] || [ ! -d "${ROOT}/apps" ]; then
-  echo "check SKIPPED (bulk-tier-distinct-from-replicated): catalog/ or apps/ absent — nothing to audit" >&2
-  exit 3
-fi
-
-"${PYTHON}" - "${ROOT}" <<'PY'
-import re
-import sys
-from pathlib import Path
-
-try:
-    import yaml
-except ImportError:                                  # pragma: no cover
-    print("check FAIL: PyYAML is required (same dependency as catalog/render.py)",
-          file=sys.stderr)
-    raise SystemExit(1)
-
-ROOT = Path(sys.argv[1])
-FAILURES: list[str] = []
-CHECKS = 0
-
-
-def check(cond, msg):
-    global CHECKS
-    CHECKS += 1
-    if not cond:
-        FAILURES.append(msg)
-
-
-def bail(msg):
-    """A failure that makes the rest of the check meaningless: report and stop."""
-    print(f"CHECK FAIL (bulk-tier-distinct-from-replicated): {msg}", file=sys.stderr)
-    raise SystemExit(1)
-
-
-def qty_to_bytes(q):
-    """Kubernetes quantity -> bytes. Only the binary suffixes the schema permits."""
-    m = re.fullmatch(r"([0-9]+)(Mi|Gi|Ti)", str(q).strip())
-    if not m:
-        return None
-    return int(m.group(1)) * {"Mi": 2 ** 20, "Gi": 2 ** 30, "Ti": 2 ** 40}[m.group(2)]
-
-
-# =====================================================================================
-# The replicated tier, named by the repo rather than by this file. Any StorageClass the
-# repo declares with the Longhorn provisioner is "the replicated tier" for assertion (1);
-# deriving it means adding a new Longhorn class cannot create a blind spot here.
-# =====================================================================================
-REPLICATED_CLASSES: set[str] = set()
-LONGHORN_DIR = ROOT / "platform" / "longhorn"
-if LONGHORN_DIR.is_dir():
-    for f in sorted(LONGHORN_DIR.rglob("*.yaml")):
-        for doc in yaml.safe_load_all(f.read_text()):
-            if isinstance(doc, dict) and doc.get("kind") == "StorageClass":
-                if "longhorn" in str(doc.get("provisioner", "")):
-                    REPLICATED_CLASSES.add(doc.get("metadata", {}).get("name", ""))
-# `longhorn` is the name every ServiceDefinition uses for the replicated tier, so it is in
-# the set even if platform/longhorn/ is not present in this checkout.
-REPLICATED_CLASSES.add("longhorn")
-REPLICATED_CLASSES.discard("")
-
-# =====================================================================================
-# The bulk corpus: every `class: bulk` storage entry in every ServiceDefinition, paired
-# with the objects the COMMITTED goldens carry for it. Reading the goldens (not calling
-# the renderer) is deliberate: the goldens are what Flux reconciles, and render==golden is
-# a separate gate stage. Asking the renderer would only prove it agrees with itself.
-# =====================================================================================
-sds: dict[str, dict] = {}
-for f in sorted((ROOT / "catalog").glob("*.yaml")):
-    doc = yaml.safe_load(f.read_text())
-    if isinstance(doc, dict) and doc.get("name"):
-        sds[doc["name"]] = doc
-
-if not sds:
-    bail("no ServiceDefinitions under catalog/ — cannot audit the bulk tier")
-
-# Every rendered app-tier document, indexed by (kind, name).
-rendered: dict[tuple, dict] = {}
-for f in sorted((ROOT / "apps").rglob("*.yaml")):
-    if f.name == "kustomization.yaml" or ".sops." in f.name:
-        continue
-    try:
-        docs = list(yaml.safe_load_all(f.read_text()))
-    except yaml.YAMLError:
-        continue
-    for doc in docs:
-        if isinstance(doc, dict) and doc.get("kind") and doc.get("metadata", {}).get("name"):
-            rendered[(doc["kind"], doc["metadata"]["name"])] = doc
-
-# (svc, vol_entry) for every bulk entry, and the replicated sizes alongside for (2).
-corpus: list[tuple[str, dict]] = []
-replicated_bytes: dict[str, list[int]] = {}
-for svc, sd in sds.items():
-    for vol in sd.get("storage") or []:
-        cls = vol.get("class", "longhorn")
-        if cls == "bulk":
-            corpus.append((svc, vol))
-        else:
-            b = qty_to_bytes(vol.get("size"))
-            if b:
-                replicated_bytes.setdefault(svc, []).append(b)
-
-# ------------------------------------------------------------------- (9) NON-VACUOUS
-# A corpus of zero would make every loop below trivially satisfied. The bulk tier is
-# DELIVERED; a change that removes the last bulk volume must fail here and be argued for,
-# not sail through on an empty iteration.
-check(len(corpus) > 0,
-      "NO `class: bulk` storage entry exists in catalog/ — this check would then pass "
-      "vacuously, which is exactly the posture that let the bulk tier sit as a REPLACE-ME "
-      "stub for six roadmap phases. If the tier is genuinely being retired, retire this "
-      "check in the same change and say so; do not let it pass on an empty corpus")
-if not corpus:
-    bail("empty bulk corpus — refusing to report a vacuous pass (see assertion 9)")
-
-inventory: list[str] = []
-
-for svc, vol in corpus:
-    vname = vol.get("name", "<unnamed>")
-    label = f"{svc}/{vname}"
-    obj = f"{svc}-{vname}"
-    ro = bool(vol.get("readOnly"))
-
-    # `export` is what the schema requires of a bulk entry; without it there is nothing to
-    # attach and the rest of the assertions have no subject.
-    export = vol.get("export")
-    check(bool(export) and str(export).startswith("/"),
-          f"{label}: bulk entry has no absolute `export` path — a bulk volume attaches an "
-          f"EXISTING dataset, so it must say which one (schema requires it when class is bulk)")
-
-    pv = rendered.get(("PersistentVolume", obj))
-    pvc = rendered.get(("PersistentVolumeClaim", obj))
-    check(pv is not None,
-          f"{label}: no PersistentVolume '{obj}' in the committed apps/ goldens. A bulk "
-          f"entry that renders no PV has NO external storage behind it — the pod would get "
-          f"an unbound PVC, or (worse, with a dynamic class) a fresh EMPTY directory")
-    check(pvc is not None,
-          f"{label}: no PersistentVolumeClaim '{obj}' in the committed apps/ goldens")
-    if pv is None or pvc is None:
-        continue
-
-    pvs, pvcs = pv.get("spec", {}) or {}, pvc.get("spec", {}) or {}
-
-    # ------------------------------------------------------ (1) DISTINCT FROM REPLICATED
-    for what, spec in (("PV", pvs), ("PVC", pvcs)):
-        sc = spec.get("storageClassName")
-        check(sc not in REPLICATED_CLASSES,
-              f"{label}: bulk {what} storageClassName is '{sc}', which the repo declares "
-              f"with the Longhorn provisioner. That puts BULK data on the REPLICATED tier "
-              f"— 3x of a multi-terabyte library across three mini-PCs. Longhorn will not "
-              f"refuse it; it will accept the PVC and fill the tier (R-STATE-04)")
-        check(sc == "",
-              f"{label}: bulk {what} storageClassName is {sc!r}, not the explicitly EMPTY "
-              f"string. Empty is what disables dynamic provisioning AND the default-class "
-              f"annotation. Any named class risks a provisioner handing out a fresh EMPTY "
-              f"directory that mounts and binds cleanly while the real data is never attached")
-    check("nfs" in pvs,
-          f"{label}: bulk PV declares no external `nfs` source — the bulk tier's whole "
-          f"premise is that durability lives OUTSIDE the cluster (docs/04)")
-
-    # ------------------------------------------------------------ (4) RECLAIM POLICY
-    check(pvs.get("persistentVolumeReclaimPolicy") == "Retain",
-          f"{label}: bulk PV reclaim policy is "
-          f"{pvs.get('persistentVolumeReclaimPolicy')!r}, not Retain. This data pre-exists "
-          f"the cluster and has consumers outside it; anything but Retain makes `kubectl "
-          f"delete pvc` a data-loss operation")
-
-    # ------------------------------------------------- (3) PRE-BOUND FROM BOTH ENDS
-    check(pvcs.get("volumeName") == obj,
-          f"{label}: bulk PVC does not name its PV via volumeName (got "
-          f"{pvcs.get('volumeName')!r}, expected {obj!r}) — without it the claim can be "
-          f"satisfied by some OTHER volume of matching size and mode")
-    cref = pvs.get("claimRef") or {}
-    check(cref.get("name") == obj and cref.get("namespace") == svc,
-          f"{label}: bulk PV claimRef is {cref!r}, expected namespace={svc!r} name={obj!r} "
-          f"— pre-binding must be pinned from BOTH ends or another PVC can win the race")
-
-    # ------------------------------------------------------------- (5) MOUNT SAFETY
-    mopts = [str(o) for o in (pvs.get("mountOptions") or [])]
-    check(bool(mopts),
-          f"{label}: bulk PV declares no mountOptions — then `hard` vs `soft` and the NFS "
-          f"version are whatever the client happens to default to, and both are decisions "
-          f"with consequences (silent short reads; how many firewall ports must open)")
-    check("hard" in mopts,
-          f"{label}: bulk PV mountOptions do not include `hard` ({mopts}). A soft mount "
-          f"hands a server hiccup to the application as a short read or EIO — silent "
-          f"corruption for a media library. `hard` hangs instead, which is recoverable")
-    check("soft" not in mopts,
-          f"{label}: bulk PV mountOptions include `soft` ({mopts}) — see above; this is the "
-          f"one option that turns an outage into damaged data")
-    vers = [o for o in mopts if o.startswith("nfsvers=")]
-    check(len(vers) == 1 and re.fullmatch(r"nfsvers=4(\.[0-9]+)?", vers[0]),
-          f"{label}: bulk PV must pin exactly one NFSv4 `nfsvers=` option (got {vers}). "
-          f"NFSv4 carries mount and lock in-protocol, so the tier needs ONE reachable port "
-          f"(tcp/2049); v3 also needs rpcbind/111, mountd and statd, which changes what the "
-          f"operator has to open in pfSense for prod to reach this server at all")
-
-    # ---------------------------------------------- (7) READ-ONLY IS REAL + CONSISTENT
-    want_mode = "ReadOnlyMany" if ro else "ReadWriteMany"
-    for what, spec in (("PV", pvs), ("PVC", pvcs)):
-        modes = spec.get("accessModes") or []
-        check(modes == [want_mode],
-              f"{label}: bulk {what} accessModes {modes} != ['{want_mode}']. Every bulk "
-              f"volume is a `...Many` (one export, several namespaces, several nodes — "
-              f"ReadWriteOnce cannot express that), and WHICH `Many` is how the read/write "
-              f"posture stays legible in `kubectl get pv`")
-    nfs_ro = bool((pvs.get("nfs") or {}).get("readOnly"))
-    check(nfs_ro == ro,
-          f"{label}: ServiceDefinition readOnly={ro} but the PV's nfs.readOnly={nfs_ro}. "
-          f"The PV field is what makes the kernel mount `ro`; a mismatch means the declared "
-          f"posture is not the one the kernel enforces")
-
-    # The volumeMount half, read out of the rendered Deployment.
-    dep = rendered.get(("Deployment", svc))
-    check(dep is not None, f"{label}: no rendered Deployment '{svc}' to check the mount against")
-    if dep is not None:
-        containers = (((dep.get("spec") or {}).get("template") or {}).get("spec") or {}).get("containers") or []
-        # The app container is containers[0] by the renderer's contract (a forward-auth
-        # sidecar is appended AFTER it).
-        mounts = [m for m in (containers[0].get("volumeMounts") or []) if m.get("name") == vname] if containers else []
-        check(len(mounts) == 1,
-              f"{label}: the app container has {len(mounts)} volumeMount(s) named {vname!r}, "
-              f"expected exactly 1 — a bulk volume that is declared and never mounted is a "
-              f"PV/PVC pair doing nothing, and two mounts of one volume is assertion (6)")
-        if len(mounts) == 1:
-            m = mounts[0]
-            check(bool(m.get("readOnly")) == ro,
-                  f"{label}: volumeMount readOnly={bool(m.get('readOnly'))} but the "
-                  f"ServiceDefinition says readOnly={ro} — the PV and the mount must agree, "
-                  f"so that dropping one still leaves the other")
-            check(m.get("subPath", vol.get("subPath")) == vol.get("subPath"),
-                  f"{label}: volumeMount subPath {m.get('subPath')!r} does not match the "
-                  f"declared subPath {vol.get('subPath')!r}")
-
-    # ------------------------------------------ (8) AUTHORED ONCE, SUPPLIED PER ENV
-    server = (pvs.get("nfs") or {}).get("server")
-    m = re.fullmatch(r"\$\{([A-Z0-9_]+)\}", str(server or ""))
-    check(m is not None,
-          f"{label}: bulk PV nfs.server is {server!r} — it must be a BARE cluster-vars "
-          f"token, not a literal address. dev and prod reach this storage host by DIFFERENT "
-          f"addresses (dev is on its LAN; prod is across pfSense), so a literal is correct "
-          f"in at most one environment and silently wrong in the other (R-ENV-02)")
-
-    cap = (pvs.get("capacity") or {}).get("storage")
-    req = ((pvcs.get("resources") or {}).get("requests") or {}).get("storage")
-    check(cap is not None and qty_to_bytes(cap) is not None,
-          f"{label}: bulk PV declares no parseable capacity ({cap!r}) — the API requires it, "
-          f"and it is what makes `kubectl get pv` report something true about this tier")
-    check(cap == req,
-          f"{label}: PV capacity {cap!r} != PVC request {req!r}; a request above the "
-          f"capacity never binds")
-
-    inventory.append(
-        f"  {label:<28} export={export}  {'ro' if ro else 'rw'}  {want_mode:<14} "
-        f"cap={cap}  server={server}  opts={','.join(mopts)}"
-    )
-
-    # ---------------------------------- (2) CAPACITY STRATEGY: two tiers, two magnitudes
-    cap_b = qty_to_bytes(cap)
-    reps = replicated_bytes.get(svc) or []
-    if cap_b and reps:
-        biggest = max(reps)
-        # ONE ORDER OF MAGNITUDE. Calibrated against the real corpus rather than guessed:
-        # the media library is ~460x its service's config volume, and the Nextcloud archive
-        # ~13x its 20Gi Longhorn volume — so a floor above ~13x would fire on a legitimate
-        # bulk volume, and this check would be teaching people to weaken it. 10x cannot be
-        # met by accident and still catches the regression that matters: a "bulk" volume
-        # sized like a config PVC, i.e. the strategy named but never applied.
-        check(cap_b >= 10 * biggest,
-              f"{label}: bulk capacity {cap} is less than 10x this service's largest "
-              f"replicated volume ({biggest // 2 ** 30}Gi). R-STATE-04 exists because the "
-              f"two tiers hold data of two different MAGNITUDES — the replicated tier keeps "
-              f"the small config/DB and the bulk tier keeps what cannot be replicated 3x. A "
-              f"bulk volume sized like a config volume means the tier was named, not applied")
-
-# ------------------------------------------------- (6) ONE EXPORT, AT MOST ONE MOUNT
-# The EXDEV trap. Grouped per SERVICE, because the hazard is two mounts inside ONE pod.
-by_service: dict[str, list[dict]] = {}
-for svc, vol in corpus:
-    by_service.setdefault(svc, []).append(vol)
-for svc, vols in sorted(by_service.items()):
-    seen: dict[str, str] = {}
-    for vol in vols:
-        exp = vol.get("export")
-        prev = seen.get(exp)
-        check(prev is None,
-              f"{svc}: mounts the export {exp} TWICE (volumes {prev!r} and "
-              f"{vol.get('name')!r}). Two mounts of one export are two FILESYSTEMS to the "
-              f"kernel, so link() between them fails EXDEV even though the server knows "
-              f"they are one dataset. Sonarr/Radarr do not fail loudly on that — they "
-              f"degrade to copy-then-delete, rewriting every imported file over NFS and "
-              f"destroying the seeding copy. Mount the common ancestor ONCE instead "
-              f"(docs/BULK-NFS-STORAGE.md §4)")
-        seen[exp] = vol.get("name")
-
-# ------------------------------------- (8b) the knob has a base default AND every overlay
-tokens = set()
-for _svc, vol in corpus:
-    pv = rendered.get(("PersistentVolume", f"{_svc}-{vol.get('name')}"))
-    if pv:
-        mm = re.fullmatch(r"\$\{([A-Z0-9_]+)\}", str(((pv.get("spec") or {}).get("nfs") or {}).get("server") or ""))
-        if mm:
-            tokens.add(mm.group(1))
-
-cv = ROOT / "clusters" / "base" / "cluster-vars.yaml"
-base_keys: set[str] = set()
-if cv.is_file():
-    doc = yaml.safe_load(cv.read_text()) or {}
-    base_keys = set((doc.get("data") or {}).keys())
-
-overlays = sorted(
-    d for d in (ROOT / "clusters").iterdir()
-    if cv.is_file() and d.is_dir() and d.name != "base" and (d / "kustomization.yaml").is_file()
-) if (ROOT / "clusters").is_dir() else []
-
-for tok in sorted(tokens):
-    check(tok in base_keys,
-          f"bulk PVs reference ${{{tok}}} but clusters/base/cluster-vars.yaml declares no "
-          f"such knob. Under StrictPostBuildSubstitutions that fails the whole tier; without "
-          f"it the token ships into the cluster as a literal server name")
-    for ov in overlays:
-        text = (ov / "kustomization.yaml").read_text()
-        check(f"/data/{tok}" in text,
-              f"clusters/{ov.name} does not supply ${{{tok}}}. The base default is the DEV "
-              f"value by this repo's convention, so an overlay that omits it silently "
-              f"inherits an address on the WRONG SIDE of pfSense — a mount that either "
-              f"never connects or connects to something unintended (R-ENV-02)")
-
-# --------------------------------------------------------------- (9) LOAD-BEARING
-L0 = (ROOT / "ci" / "l0.sh").read_text() if (ROOT / "ci" / "l0.sh").is_file() else ""
-GATE = (ROOT / "ci" / "gate.sh").read_text() if (ROOT / "ci" / "gate.sh").is_file() else ""
-check("bulk-tier-distinct-from-replicated/check.sh" in L0,
-      "ci/l0.sh does not run bulk-tier-distinct-from-replicated/check.sh. Stage 3b scopes "
-      "the conformance run to TOUCHED requirements, and ci/touched-requirements.sh maps "
-      "neither apps/* nor platform/longhorn/* to the bulk requirement — so a change that "
-      "moves a bulk volume onto the replicated tier could attribute no requirement at all "
-      "and be gated by nothing. It must be invoked directly and unconditionally")
-check(re.search(r"^for s in .*\bl0\b.*; do", GATE, re.M) and re.search(r"fail\).*BLOCK=1", GATE),
-      "ci/gate.sh does not set the merge block when the l0 stage fails")
-
-# ------------------------------------------------------------------------- report
-print("Bulk tier inventory (from catalog/ + the committed apps/ goldens):")
-for line in inventory:
-    print(line)
-print(f"\nreplicated tier StorageClasses (derived from platform/longhorn/): "
-      f"{', '.join(sorted(REPLICATED_CLASSES))}")
-print(f"bulk volumes: {len(corpus)}   distinct exports: "
-      f"{len({v.get('export') for _s, v in corpus})}   "
-      f"server knob(s): {', '.join(sorted(tokens)) or 'NONE'}")
-
-# A floor on the assertion count, for the same reason the corpus may not be empty: a pass
-# must not be able to degenerate into "nothing was actually asked".
-MIN_CHECKS = 40
-check(CHECKS >= MIN_CHECKS,
-      f"only {CHECKS} assertions ran (floor {MIN_CHECKS}) — a pass this thin is not "
-      f"evidence; the corpus or the extraction has degraded")
-
-print()
-if FAILURES:
-    print(f"CHECK FAIL (bulk-tier-distinct-from-replicated): {len(FAILURES)} finding(s)",
-          file=sys.stderr)
-    for f in FAILURES:
-        print(f"  - {f}", file=sys.stderr)
-    raise SystemExit(1)
-
-print(f"check OK: bulk-tier-distinct-from-replicated — {CHECKS} assertions (R-STATE-04)")
-PY
@@ -140,16 +140,8 @@ def docs(path: Path):
        return []


-# Skip dot-DIRECTORIES (.git, .claude, .cache) while still matching dot-FILES such as
-# .sops.yaml. A bare rglob from ROOT also descends into NESTED WORKTREES —
-# .claude/worktrees/agent-*/ created for isolated agent runs — which made this check see
-# 5 CiliumEgressGatewayPolicy objects instead of the 1 authored, failing R-NET-04 for a
-# reason that has nothing to do with egress. .claude/ is gitignored (.gitignore:43), so
-# nothing under it is part of the authored surface by definition.
 TRACKED_YAML = sorted(p for p in ROOT.rglob("*.yaml")
-                      if "_artifacts" not in p.parts
-                      and not any(part.startswith(".")
-                                  for part in p.relative_to(ROOT).parent.parts))
+                      if ".git" not in p.parts and "_artifacts" not in p.parts)
 SCHEMA = ROOT / "schemas" / "servicedefinition.schema.json"
 RENDER = ROOT / "catalog" / "render.py"
 VPN_ROLE = ROOT / "infra" / "ansible" / "roles" / "vpn_egress_gateway"
@@ -1,63 +0,0 @@
-# =============================================================================
-# L0 STATIC — Keycloak client secrets are declared consistently, ordered by a REAL
-# Flux edge, and healed by a reconciler whose behaviour is proven offline
-# Catalog: docs/14 L0 row ("Pre-apply, in CI") — runs WITHOUT a cluster.
-#
-# R-IDP-03 (identity config is declarative) and R-IDP-04 (IdP secrets SOPS-encrypted)
-# are both implemented through ONE mechanism that FAILS OPEN: the KeycloakRealmImport
-# resolves each client's secret from a `(env:NAME)` placeholder mapped to a key of the
-# SOPS-encrypted keycloak-realm-clients Secret. When the operator cannot resolve a
-# placeholder it stores the LITERAL placeholder text as the client secret AND THE IMPORT
-# JOB STILL REPORTS SUCCESS — then rests at Done=True / Started=False and never re-runs.
-#
-# Measured 2026-09-06: prod had all 12 clients holding the literal placeholder (every
-# OIDC login failed with "Invalid client or Invalid client credentials"); dev had exactly
-# one (oauth2-proxy), because that Secret key was added AFTER dev's import completed.
-# Declarative config that silently does not apply is not declarative config.
-#
-# GIVEN the repository as authored,
-# THEN  every confidential client declares a placeholder, every placeholder is defined,
-#       resolves to the keycloak-realm-clients Secret, and names a key that EXISTS in the
-#       SOPS file under the `<clientId>-client-secret` contract the reconciler derives
-#       from; public clients declare no secret; the Flux Kustomization applying the Secret
-#       is a transitive `dependsOn` ancestor (wait: true) of the ones applying the import
-#       and the reconciler — a real edge, not a comment; the reconciler is reconciled from
-#       BOTH cluster entrypoints, mounts only what it needs, carries no ServiceAccount
-#       token and contains no dollar sign; and its own `--selftest` (extracted from the
-#       shipped ConfigMap and executed) proves compare-then-write, idempotence,
-#       no-secret-logging, the endpoint allow-list and fail-closed behaviour.
-#
-# STATUS: EXECUTABLE. `check.sh` is a pure offline assertion script (git + PyYAML +
-# stdlib); conformance/run.sh --layer l0 and ci/l0.sh run it directly, and this Chainsaw
-# Test wraps the SAME script.
-#
-# NOT proven here: that the LIVE realm's stored secrets match git (that is
-# tests/l2-behavioral/keycloak-client-secrets-match-git), that the SOPS ciphertext
-# decrypts to a usable value (key PRESENCE is offline-checkable; plaintext is not), or
-# that the CronJob's schedule fires.
-# =============================================================================
-apiVersion: chainsaw.kyverno.io/v1alpha1
-kind: Test
-metadata:
-  name: keycloak-client-secret-reconciler
-  labels:
-    conformance.home.k3s/R-IDP-03: "true"
-    conformance.home.k3s/R-IDP-04: "true"
-  annotations:
-    conformance.home.k3s/requirements: "R-IDP-03, R-IDP-04"
-    conformance.home.k3s/layer: "L0"
-    conformance.home.k3s/safety-critical: "false"
-    conformance.home.k3s/executable: "true"
-    conformance.home.k3s/intent: >-
-      Every OIDC client's secret is declared once, resolvable, ordered behind a real Flux
-      dependency, and continuously reconciled — so an unresolved import placeholder can
-      neither be merged nor silently persist in the realm.
-spec:
-  steps:
-    - name: audit-declaration-ordering-and-reconciler
-      description: >-
-        Assert the realm import, the SOPS Secret and the reconciler agree; that the Flux
-        ordering is a real dependsOn edge; and run the reconciler's own selftest.
-      try:
-        - script:
-            content: bash ./check.sh
@@ -1,742 +0,0 @@
-#!/usr/bin/env bash
-# =============================================================================
-# L0 static check (offline, no cluster) — keycloak-client-secret-reconciler
-# Serves: R-IDP-03 ("Identity config (realms, clients, groups, roles) is declarative."
-#                   — spec docs/15-identity-and-access.md)
-#         R-IDP-04 ("IdP secrets SOPS-encrypted; MFA available."
-#                   — spec docs/15-identity-and-access.md)
-#
-# WHY THIS EXISTS — THE REALM IMPORT IS A ONE-SHOT THAT FAILS OPEN
-# ---------------------------------------------------------------
-# platform/keycloak/keycloak-realm-import.yaml declares 12 confidential OIDC clients
-# whose secrets are `(env:NAME)` import placeholders, mapped by spec.placeholders to
-# keys of the SOPS-encrypted keycloak-realm-clients Secret. Measured on both clusters
-# on 2026-09-06:
-#
-#   * when the Keycloak Operator cannot resolve a placeholder it stores the LITERAL
-#     placeholder text as the client secret AND THE IMPORT JOB STILL REPORTS SUCCESS;
-#   * the KeycloakRealmImport then rests at Done=True / Started=False and NEVER
-#     re-runs, so it never self-heals.
-#
-# Consequence: on prod all 12 clients held the literal placeholder and every OIDC
-# login on the cluster failed ("Invalid client or Invalid client credentials"); on dev
-# exactly one (oauth2-proxy) was broken, because that Secret key was added AFTER dev's
-# import had already completed and a completed import is never re-run.
-#
-# Nothing in the repo asserted the declaration was self-consistent, and nothing
-# corrected the realm afterwards. This check is the static half of the fix (the
-# reconciler CronJob is the runtime half, and this check proves that too).
-#
-# GIVEN the repository as authored,
-# THEN  (1) DECLARATION IS COMPLETE: every confidential client in the realm import
-#           declares a secret placeholder; every placeholder it uses is defined in
-#           spec.placeholders; every placeholder resolves to the keycloak-realm-clients
-#           Secret; and every referenced key EXISTS in the SOPS file. A client whose
-#           key is missing is exactly the dev failure, and it is now unmergeable.
-#       (2) THE NAMING CONTRACT HOLDS: each key is `<clientId>-client-secret`. That is
-#           what lets the reconciler derive the client set from the mounted Secret with
-#           no second declaration — so it must be enforced, not assumed. No orphan
-#           placeholder, and no orphan Secret key (an orphan key would make every
-#           reconciler run fail with NOT FOUND).
-#       (3) PUBLIC CLIENTS DECLARE NO SECRET (the `kubernetes` client is PKCE-public;
-#           giving it a secret would be a silent downgrade).
-#       (4) ORDERING IS A REAL FLUX EDGE, NOT A COMMENT: the Flux Kustomization that
-#           renders the Secret is a transitive `dependsOn` ancestor — with `wait: true`
-#           — of the one that renders the realm import and the reconciler. The repo's
-#           hard-won rule: a dependency that is only DOCUMENTED is not a dependency.
-#       (5) THE RECONCILER IS WIRED AND SAFE: it is reconciled from BOTH cluster
-#           entrypoints, mounts exactly the two Secrets it needs at the paths its code
-#           defaults to, carries no ServiceAccount token, and contains no dollar sign
-#           (it ships inside a substituted Flux path, where an unescaped dollar-brace
-#           expansion is rewritten before any container runs).
-#       (6) THE RECONCILER'S BEHAVIOUR IS PROVEN, NOT ASSERTED: this check EXTRACTS the
-#           script from the ConfigMap and executes its `--selftest`, which drives the
-#           real code paths against an in-process stub Keycloak and asserts
-#           compare-then-write, idempotence (zero writes on a converged realm),
-#           no-secret-logging, the endpoint allow-list (users/groups/roles are
-#           unreachable) and fail-closed behaviour. What CI proves is byte-for-byte
-#           what ships.
-#      (6b) AND THAT PROOF IS FALSIFIABLE: mutations.py injects plausible regressions
-#           into the extracted script one at a time and requires --selftest to CATCH
-#           every one. A passing suite proves nothing if it cannot fail, and "reported
-#           success while the realm was broken" is the exact failure this component
-#           exists to end — so the same standard is applied to its own test.
-#       (8) THE URL DESIRED STATE IS DERIVED, NOT DECLARED TWICE: the reconciler's
-#           second pass converges the realm's environment-derived client URLs
-#           (redirectUris, webOrigins, rootUrl, baseUrl, adminUrl) on
-#           platform/keycloak/client-urls.yaml. That file is GENERATED from the realm
-#           import by ci/render-keycloak-client-urls.py, and this check RUNS the
-#           renderer with --check, so a hand edit that makes the two disagree is
-#           unmergeable. It further asserts the two files are rendered by the SAME Flux
-#           Kustomization (postBuild.substituteFrom is per-Kustomization, so a split
-#           would let one ${BASE_DOMAIN} resolve to two different values), that the
-#           desired state uses no substitution token the realm definition does not,
-#           that every client it names exists in the import, and that it declares no
-#           field outside the reconciler's five-field write surface.
-#   AND (7) THIS CHECK IS LOAD-BEARING: ci/l0.sh runs it unconditionally, ci/gate.sh
-#           blocks the merge when the l0 stage fails, ci/touched-requirements.sh maps
-#           platform/keycloak/ to these requirement IDs, and an L2 test exists that
-#           audits the LIVE realm. An unreferenced test is precisely the "looks applied
-#           and isn't" failure this repo keeps hitting.
-#
-# NOT proven here (named honestly, in the style of the sibling L0 checks):
-#   - that the LIVE realm's stored secrets OR redirect URIs match git. That is cluster
-#     state; it is conformance/tests/l2-behavioral/keycloak-client-secrets-match-git,
-#     which runs the reconciler's own --audit against the live realm.
-#   - that the hostnames the realm import declares are the ones the Gateway/HTTPRoutes
-#     actually serve. This check proves the realm converges on what GIT says; it cannot
-#     prove git is right about the domain. A BASE_DOMAIN flip that misses an HTTPRoute is
-#     a different invariant.
-#   - that the SOPS ciphertext decrypts to a usable secret. Key PRESENCE is checkable
-#     offline; the plaintext is not, and must not be.
-#   - that the CronJob's image can be pulled or its schedule fires.
-# =============================================================================
-set -euo pipefail
-DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-ROOT="$(cd "${DIR}/../../../.." && pwd)"        # repo root
-PYTHON="${PYTHON:-python3}"
-
-IMPORT_FILE="platform/keycloak/keycloak-realm-import.yaml"
-SOPS_FILE="platform/keycloak/keycloak-realm-clients.sops.yaml"
-RECON_FILE="platform/keycloak/client-secret-reconciler.yaml"
-URLS_FILE="platform/keycloak/client-urls.yaml"
-
-# Honest SKIP (never green) if the thing under test is absent entirely.
-for f in "$IMPORT_FILE" "$SOPS_FILE" "$RECON_FILE" "$URLS_FILE"; do
-  if [ ! -f "${ROOT}/${f}" ]; then
-    echo "check SKIPPED: ${f} absent — nothing to audit" >&2
-    exit 3
-  fi
-done
-
-cd "${ROOT}"
-"${PYTHON}" - "${ROOT}" <<'PY'
-"""Static audit of the Keycloak client-secret declaration, its Flux ordering, and the
-reconciler that heals it. PyYAML (the same dependency tests/l0-static/no-plaintext-secrets
-and least-privilege-rbac already take) parses the manifests; lib/status.py is IMPORTED for
-the Flux graph walk rather than forked, so there is one definition of "what a Flux path
-renders" in the harness."""
-from __future__ import annotations
-
-import json
-import re
-import subprocess
-import sys
-import tempfile
-from pathlib import Path
-
-try:
-    import yaml
-except ImportError:                                  # pragma: no cover
-    print("check FAIL: PyYAML is required (same dependency as catalog/render.py)",
-          file=sys.stderr)
-    raise SystemExit(1)
-
-ROOT = Path(sys.argv[1])
-sys.dont_write_bytecode = True          # keep __pycache__ out of the tracked tree
-sys.path.insert(0, str(ROOT / "conformance" / "lib"))
-import status as st                     # noqa: E402 — reuse the walk, don't fork it
-
-IMPORT_FILE = "platform/keycloak/keycloak-realm-import.yaml"
-SOPS_FILE = "platform/keycloak/keycloak-realm-clients.sops.yaml"
-RECON_FILE = "platform/keycloak/client-secret-reconciler.yaml"
-URLS_FILE = "platform/keycloak/client-urls.yaml"
-SECRET_NAME = "keycloak-realm-clients"
-ADMIN_SECRET = "keycloak-admin-credentials"
-URLS_CM_NAME = "keycloak-client-urls"
-KEY_SUFFIX = "-client-secret"
-SECRETS_MOUNT = "/etc/keycloak-realm-clients"
-URLS_MOUNT = "/etc/keycloak-client-urls"
-URLS_DATA_KEY = "client-urls.json"
-RENDERER = "ci/render-keycloak-client-urls.py"
-DIR_PATH = ROOT / "conformance" / "tests" / "l0-static" / "keycloak-client-secret-reconciler"
-# The ONLY client fields the reconciler may write besides the secret. Kept in lockstep
-# with the renderer and the script; asserted below to be exactly what they agree on.
-URL_LIST_FIELDS = ("redirectUris", "webOrigins")
-URL_STR_FIELDS = ("rootUrl", "baseUrl", "adminUrl")
-URL_FIELDS = URL_LIST_FIELDS + URL_STR_FIELDS
-DOLLAR = chr(36)
-
-FAILURES: list[str] = []
-CHECKS = 0
-
-
-def check(cond, msg):
-    global CHECKS
-    CHECKS += 1
-    if not cond:
-        FAILURES.append(msg)
-
-
-def load(rel):
-    return list(yaml.safe_load_all((ROOT / rel).read_text()))
-
-
-# ================================================================= (1)-(3) declaration
-realm_import = next((d for d in load(IMPORT_FILE)
-                     if d and d.get("kind") == "KeycloakRealmImport"), None)
-check(realm_import is not None, f"{IMPORT_FILE}: no KeycloakRealmImport document found")
-if realm_import is None:
-    print("CHECK FAIL: cannot parse the realm import", file=sys.stderr)
-    raise SystemExit(1)
-
-placeholders = realm_import["spec"].get("placeholders") or {}
-clients = realm_import["spec"]["realm"].get("clients") or []
-check(len(clients) > 0, f"{IMPORT_FILE}: parsed zero clients — the file shape changed and "
-                        f"this check would silently audit nothing")
-
-# The operator's placeholder syntax, spelled without a literal dollar so this file is
-# itself safe to grep for one. Matches `<dollar>(env:NAME)`.
-PLACEHOLDER_RE = re.compile(re.escape(DOLLAR) + r"\(env:([A-Za-z_][A-Za-z0-9_]*)\)")
-
-used_placeholders: set[str] = set()
-declared_pairs: dict[str, str] = {}          # clientId -> secret key in the Secret
-
-for c in clients:
-    cid = c.get("clientId", "<unnamed>")
-    secret = c.get("secret")
-    public = bool(c.get("publicClient"))
-    if public:
-        # (3) a PKCE public client must not carry a secret at all.
-        check(secret is None,
-              f"{IMPORT_FILE}: client '{cid}' is publicClient but declares a secret — a "
-              f"public client holds no secret; declaring one is a silent downgrade")
-        continue
-    # (1) every confidential client must declare one.
-    check(isinstance(secret, str) and secret,
-          f"{IMPORT_FILE}: confidential client '{cid}' declares NO secret. The operator "
-          f"would generate a random one that exists nowhere in git, so nothing could "
-          f"ever reconcile it and the consuming app's mirrored credential would be wrong")
-    if not isinstance(secret, str):
-        continue
-    m = PLACEHOLDER_RE.fullmatch(secret.strip())
-    check(m is not None,
-          f"{IMPORT_FILE}: client '{cid}' has a secret that is not an (env:NAME) "
-          f"placeholder. A literal secret in git violates R-IDP-04/R-SEC-01; the only "
-          f"permitted form is a placeholder resolved from the {SECRET_NAME} Secret")
-    if m is None:
-        continue
-    name = m.group(1)
-    used_placeholders.add(name)
-    # (1) the placeholder must be DEFINED. An undefined one is stored literally by the
-    #     operator, with a successful-looking import — the whole incident.
-    ph = placeholders.get(name)
-    check(ph is not None,
-          f"{IMPORT_FILE}: client '{cid}' uses placeholder '{name}' but spec.placeholders "
-          f"does not define it. The operator stores the LITERAL placeholder as the client "
-          f"secret and STILL reports the import successful (measured 2026-09-06)")
-    if ph is None:
-        continue
-    ref = (ph or {}).get("secret") or {}
-    check(ref.get("name") == SECRET_NAME,
-          f"{IMPORT_FILE}: placeholder '{name}' reads Secret '{ref.get('name')}', not "
-          f"'{SECRET_NAME}'. The reconciler mounts only {SECRET_NAME}, so a second source "
-          f"would be declared but never healed")
-    key = ref.get("key")
-    check(isinstance(key, str) and key,
-          f"{IMPORT_FILE}: placeholder '{name}' names no Secret key")
-    if not isinstance(key, str):
-        continue
-    # (2) the naming contract the reconciler derives its client set from.
-    check(key == cid + KEY_SUFFIX,
-          f"{IMPORT_FILE}: client '{cid}' reads Secret key '{key}', but the reconciler "
-          f"derives the clientId from the key name, so it MUST be "
-          f"'{cid}{KEY_SUFFIX}'. Rename the key (and the SOPS entry) or the reconciler "
-          f"would look up a client that does not exist")
-    declared_pairs[cid] = key
-
-# (2) no orphan placeholder: a defined-but-unused placeholder is dead config that will
-#     quietly rot out of sync with the Secret.
-for name in sorted(set(placeholders) - used_placeholders):
-    check(False,
-          f"{IMPORT_FILE}: spec.placeholders defines '{name}' but no client uses it — "
-          f"dead declaration; remove it or wire it to the client that needs it")
-
-# ---- the SOPS Secret: keys are cleartext in a SOPS-encrypted file; values are not.
-sops_doc = next((d for d in load(SOPS_FILE) if d and d.get("kind") == "Secret"), None)
-check(sops_doc is not None, f"{SOPS_FILE}: no Secret document found")
-sops_keys = set((sops_doc or {}).get("stringData") or {})
-check(len(sops_keys) > 0,
-      f"{SOPS_FILE}: parsed zero stringData keys — the file shape changed and this check "
-      f"would silently audit nothing")
-check((sops_doc or {}).get("metadata", {}).get("name") == SECRET_NAME,
-      f"{SOPS_FILE}: the Secret is not named {SECRET_NAME}")
-
-for cid, key in sorted(declared_pairs.items()):
-    # (1) THE DEV FAILURE, made unmergeable: a client declared in the import whose key
-    #     is absent from the Secret imports "successfully" with a literal placeholder.
-    check(key in sops_keys,
-          f"{SOPS_FILE}: client '{cid}' needs key '{key}' and it is NOT in the Secret. "
-          f"The realm import would still report SUCCESS and store the literal placeholder "
-          f"as that client's secret — this is exactly how dev's oauth2-proxy broke")
-
-# (2) no orphan Secret key: the reconciler treats every `<x>-client-secret` key as a
-#     client that MUST exist, and fails loudly if it does not. Catch it here instead.
-for key in sorted(k for k in sops_keys if k.endswith(KEY_SUFFIX)):
-    cid = key[: -len(KEY_SUFFIX)]
-    check(cid in declared_pairs,
-          f"{SOPS_FILE}: key '{key}' has no matching client in {IMPORT_FILE}. The "
-          f"reconciler would look up client '{cid}', not find it, and fail every run")
-
-# ===================================================== (4) ordering is a real Flux edge
-# Parse contract mirrors lib/status.py's flux_paths(): docs split on a lone `---`; a doc
-# is an entrypoint only if it carries the kustomize.toolkit apiVersion, `kind:
-# Kustomization` and a spec-level `path:`. Overlay `patch: |-` bodies carry Kustomizations
-# WITHOUT a path and are correctly skipped (they are patches, not entrypoints).
-_DEPENDSON_RE = re.compile(r"^(\s*)dependsOn:\s*$", re.M)
-_DEP_NAME_RE = re.compile(r"^\s*(?:-\s*)?name:\s*([^\s#]+)")
-_WAIT_RE = re.compile(r"^\s{2,6}wait:\s*true\s*(?:#.*)?$", re.M)
-
-
-def depends_on(doc: str) -> list[str]:
-    m = _DEPENDSON_RE.search(doc)
-    if not m:
-        return []
-    indent = len(m.group(1))
-    out: list[str] = []
-    for raw in doc[m.end():].splitlines():
-        if raw.strip() == "" or raw.lstrip().startswith("#"):
-            continue
-        if len(raw) - len(raw.lstrip()) <= indent and not raw.lstrip().startswith("-"):
-            break                                    # dedented out of the list
-        nm = _DEP_NAME_RE.match(raw)
-        if nm:
-            out.append(nm.group(1))
-    return out
-
-
-kustomizations: dict[str, dict] = {}
-for yfile in sorted((ROOT / "clusters").rglob("*.yaml")):
-    text = yfile.read_text()
-    if st.FLUX_API not in text:
-        continue
-    for doc in st._DOC_SPLIT_RE.split(text):
-        if st.FLUX_API not in doc or not re.search(r"^kind:\s*Kustomization\s*$", doc, re.M):
-            continue
-        pm = st._PATH_RE.search(doc)
-        if not pm:
-            continue
-        nm = st._NAME_RE.search(doc)
-        name = nm.group(1) if nm else "?"
-        kustomizations[name] = {
-            "file": str(yfile.relative_to(ROOT)),
-            "path": pm.group(1),
-            "dependsOn": depends_on(doc),
-            "wait": bool(_WAIT_RE.search(doc)),
-        }
-
-check(len(kustomizations) > 0,
-      "no Flux Kustomization with a spec.path found under clusters/ — the parser or the "
-      "repo layout changed; this check would silently audit nothing")
-
-
-def direct_render(path_spec: str) -> set[str]:
-    """Files a Flux `spec.path` renders ITSELF — a kustomize walk that deliberately does
-    NOT expand nested Flux Kustomizations.
-
-    lib/status.py's reconciled_files() follows every `spec.path` it discovers, which is
-    right for "is this file applied to the cluster at all" but wrong for OWNERSHIP: the
-    root Kustomization's path is clusters/<env>, so a transitive walk makes the root the
-    owner of every file in the repo and the ordering assertion below becomes vacuous
-    (measured: it reported the root as an owner of all three keycloak files). `resources:`
-    parsing itself is st.kustomize_resources, not forked.
-    """
-    rel = path_spec.lstrip("./") or "."
-    out: set[str] = set()
-    seen: set[str] = set()
-
-    def walk(d: str) -> None:
-        if d in seen:
-            return
-        seen.add(d)
-        kfile = ROOT / d / "kustomization.yaml"
-        if not kfile.is_file():
-            return
-        out.add(str(kfile.relative_to(ROOT)))
-        for item in st.kustomize_resources(kfile):
-            if "://" in item:
-                continue                       # remote base: not repo state
-            target = (ROOT / d / item).resolve()
-            try:
-                trel = target.relative_to(ROOT).as_posix()
-            except ValueError:
-                continue                       # outside the repo
-            if target.is_dir():
-                walk(trel)
-            elif target.is_file():
-                out.add(trel)
-
-    if (ROOT / rel).is_dir():
-        walk(rel)
-    return out
-
-
-rendered_by = {n: direct_render(k["path"]) for n, k in kustomizations.items()}
-
-
-def owners(rel_file: str) -> list[str]:
-    return sorted(n for n, files in rendered_by.items() if rel_file in files)
-
-
-def ancestors(name: str, seen=None) -> set[str]:
-    """Transitive dependsOn closure of `name` (its dependencies, direct and indirect)."""
-    seen = seen if seen is not None else set()
-    for dep in kustomizations.get(name, {}).get("dependsOn", []):
-        if dep in seen:
-            continue
-        seen.add(dep)
-        ancestors(dep, seen)
-    return seen
-
-
-secret_owners = owners(SOPS_FILE)
-import_owners = owners(IMPORT_FILE)
-recon_owners = owners(RECON_FILE)
-
-check(bool(secret_owners),
-      f"{SOPS_FILE} is rendered by NO Flux Kustomization — the client secrets are in git "
-      f"but never applied")
-check(bool(import_owners),
-      f"{IMPORT_FILE} is rendered by NO Flux Kustomization")
-# (5) the reconciler must actually be reconciled, or it is decoration.
-check(bool(recon_owners),
-      f"{RECON_FILE} is rendered by NO Flux Kustomization — the healer would sit in git "
-      f"and never run, which is the 'looks applied and isn't' failure it exists to end")
-
-for consumer_file, consumer_owners in ((IMPORT_FILE, import_owners),
-                                       (RECON_FILE, recon_owners)):
-    for co in consumer_owners:
-        for so in secret_owners:
-            if so == co:
-                continue          # same Kustomization: one atomic apply, ordering moot
-            anc = ancestors(co)
-            check(so in anc,
-                  f"ORDERING IS ONLY DOCUMENTED: '{co}' renders {consumer_file} but does "
-                  f"not (transitively) dependsOn '{so}', which renders the "
-                  f"{SECRET_NAME} Secret. Without that edge the consumer can reconcile "
-                  f"before the secret exists, and the Keycloak Operator's response to a "
-                  f"missing placeholder is to store it LITERALLY and report success")
-            if so in anc:
-                check(kustomizations[so]["wait"],
-                      f"'{so}' (which renders the {SECRET_NAME} Secret) does not set "
-                      f"wait: true, so '{co}' can proceed before it is applied")
-
-# ------ both environments must reconcile the reconciler, not just dev.
-for env in ("dev", "prod"):
-    entry = f"clusters/{env}"
-    if not (ROOT / entry).is_dir():
-        continue
-    applied, _ = st.reconciled_files(ROOT, entry)
-    check(RECON_FILE in applied,
-          f"{RECON_FILE} is not reachable from the {entry} entrypoint — the healer would "
-          f"exist on one cluster only, and prod is where all 12 clients were broken")
-    check(IMPORT_FILE in applied, f"{IMPORT_FILE} is not reachable from {entry}")
-
-# ============================================ (5) the reconciler manifest is safe + wired
-recon_docs = [d for d in load(RECON_FILE) if d]
-cm = next((d for d in recon_docs if d.get("kind") == "ConfigMap"), None)
-cj = next((d for d in recon_docs if d.get("kind") == "CronJob"), None)
-check(cm is not None, f"{RECON_FILE}: no ConfigMap carrying the reconciler script")
-check(cj is not None, f"{RECON_FILE}: no CronJob running the reconciler")
-
-script = ""
-if cm is not None:
-    data = cm.get("data") or {}
-    check(len(data) == 1,
-          f"{RECON_FILE}: the reconciler ConfigMap must carry exactly one script key "
-          f"(found {sorted(data)}) — a second copy would be untested by this gate")
-    script = next(iter(data.values()), "")
-    check(len(script.splitlines()) > 50,
-          f"{RECON_FILE}: the extracted script is implausibly short "
-          f"({len(script.splitlines())} lines) — the ConfigMap shape changed and the "
-          f"selftest below would prove nothing")
-    # THE nextcloud TRAP. This file is rendered by a Flux Kustomization with
-    # postBuild.substituteFrom, so envsubst rewrites it before any container runs. The
-    # script is written with NO dollar sign at all, so the hazard is structural rather
-    # than escaped and no future edit has to reason about escaping.
-    check(DOLLAR not in script,
-          f"{RECON_FILE}: the reconciler script contains a dollar sign. It ships inside a "
-          f"SUBSTITUTED Flux path, where a dollar-brace expansion is rewritten at "
-          f"reconcile time (see apps/wave-1/nextcloud.yaml). Rewrite without it — Python "
-          f"never needs one; chr(36) spells it where a literal is genuinely required")
-
-if cj is not None:
-    pod = cj["spec"]["jobTemplate"]["spec"]["template"]["spec"]
-    ctr = pod["containers"][0]
-    vols = {v["name"]: v for v in pod.get("volumes", [])}
-    mounts = {m["name"]: m for m in ctr.get("volumeMounts", [])}
-    secret_vols = [v for v in vols.values()
-                   if (v.get("secret") or {}).get("secretName") == SECRET_NAME]
-    check(len(secret_vols) == 1,
-          f"{RECON_FILE}: the CronJob must mount the {SECRET_NAME} Secret — that mount IS "
-          f"the desired state it reconciles towards")
-    if secret_vols:
-        mp = mounts.get(secret_vols[0]["name"], {}).get("mountPath")
-        check(mp == SECRETS_MOUNT,
-              f"{RECON_FILE}: {SECRET_NAME} is mounted at {mp}, but the script defaults "
-              f"to {SECRETS_MOUNT} and no env var overrides it")
-    cm_vols = [v for v in vols.values()
-               if (v.get("configMap") or {}).get("name") == (cm or {}).get(
-                   "metadata", {}).get("name")]
-    check(len(cm_vols) == 1,
-          f"{RECON_FILE}: the CronJob does not mount the reconciler ConfigMap, so it "
-          f"would run nothing")
-    # The URL desired state must be MOUNTED at the path the script defaults to, for the
-    # same reason as the Secret: the pod holds no ServiceAccount token and cannot read
-    # the KeycloakRealmImport CR, so an unmounted ConfigMap means pass 2 fails closed on
-    # every run forever -- correct behaviour, but a permanently red reconciler.
-    url_vols = [v for v in vols.values()
-                if (v.get("configMap") or {}).get("name") == URLS_CM_NAME]
-    check(len(url_vols) == 1,
-          f"{RECON_FILE}: the CronJob must mount the {URLS_CM_NAME} ConfigMap — that "
-          f"mount IS the desired state the URL pass reconciles towards, and the pod has "
-          f"no Kubernetes API access to read the realm import instead")
-    if url_vols:
-        mp = mounts.get(url_vols[0]["name"], {}).get("mountPath")
-        check(mp == URLS_MOUNT,
-              f"{RECON_FILE}: {URLS_CM_NAME} is mounted at {mp}, but the script defaults "
-              f"to {URLS_MOUNT}/{URLS_DATA_KEY} and no env var overrides it")
-        check(mounts.get(url_vols[0]["name"], {}).get("readOnly") is True,
-              f"{RECON_FILE}: the {URLS_CM_NAME} mount must be readOnly — the reconciler "
-              f"reads its desired state, it never writes it")
-    env_srcs = {e["name"]: e for e in ctr.get("env", [])}
-    for var in ("KEYCLOAK_ADMIN_USER", "KEYCLOAK_ADMIN_PASSWORD"):
-        ref = (env_srcs.get(var, {}).get("valueFrom") or {}).get("secretKeyRef") or {}
-        check(ref.get("name") == ADMIN_SECRET,
-              f"{RECON_FILE}: {var} must come from the {ADMIN_SECRET} Secret "
-              f"(got {ref.get('name')!r}) — the admin credential is never inlined")
-    for var in ("KEYCLOAK_BASE_URL", "KEYCLOAK_REALM"):
-        check(env_srcs.get(var, {}).get("value"),
-              f"{RECON_FILE}: {var} is not set; the script fails closed without it, but "
-              f"the CronJob should not ship broken")
-    # R-SEC-02: it talks only to Keycloak's REST API and needs no Kubernetes API rights.
-    check(pod.get("automountServiceAccountToken") is False,
-          f"{RECON_FILE}: the reconciler needs NO Kubernetes API access (its inputs are "
-          f"mounted Secrets) — set automountServiceAccountToken: false")
-    check(cj["spec"].get("concurrencyPolicy") == "Forbid",
-          f"{RECON_FILE}: set concurrencyPolicy: Forbid so a run stalled on an "
-          f"unavailable Keycloak is not lapped by the next tick")
-    img = ctr.get("image", "")
-    check(re.search(r":[0-9]", img),
-          f"{RECON_FILE}: container image {img!r} is not pinned to an explicit version")
-
-# ================================================ (8) the URL desired state is DERIVED
-# The reconciler's second pass makes the realm's environment-derived client URLs match
-# platform/keycloak/client-urls.yaml. That file must not be an independent declaration of
-# every public hostname: two declarations that can disagree is the same shape as the bug
-# this whole change closes. It is GENERATED from the realm import, and the generation is
-# gated HERE -- a hand edit to either file that makes them disagree is unmergeable.
-urls_docs = [d for d in load(URLS_FILE) if d]
-urls_cm = next((d for d in urls_docs if d.get("kind") == "ConfigMap"), None)
-check(urls_cm is not None, f"{URLS_FILE}: no ConfigMap document found")
-check((urls_cm or {}).get("metadata", {}).get("name") == URLS_CM_NAME,
-      f"{URLS_FILE}: the ConfigMap must be named {URLS_CM_NAME} (the CronJob mounts it "
-      f"by that name)")
-
-urls_want: dict[str, dict] = {}
-if urls_cm is not None:
-    data = urls_cm.get("data") or {}
-    check(set(data) == {URLS_DATA_KEY},
-          f"{URLS_FILE}: the ConfigMap must carry exactly the key {URLS_DATA_KEY} "
-          f"(found {sorted(data)}) — the script reads that filename from its mount")
-    raw = data.get(URLS_DATA_KEY, "")
-    try:
-        urls_want = json.loads(raw)
-    except ValueError as e:
-        check(False, f"{URLS_FILE}: {URLS_DATA_KEY} is not valid JSON ({e}) — the "
-                     f"reconciler would fail closed on every run")
-    check(isinstance(urls_want, dict) and bool(urls_want),
-          f"{URLS_FILE}: {URLS_DATA_KEY} must be a non-empty JSON object of clients")
-
-# THE GATE: the committed file must be exactly what the renderer produces from the realm
-# import. This is what makes the realm import the single source of truth for hostnames
-# rather than merely the place they were first typed.
-if (ROOT / RENDERER).is_file():
-    proc = subprocess.run([sys.executable, str(ROOT / RENDERER), "--check"],
-                          capture_output=True, text=True, timeout=120, cwd=str(ROOT))
-    check(proc.returncode == 0,
-          f"{URLS_FILE} is NOT what {RENDERER} renders from {IMPORT_FILE}. The URL "
-          f"reconciler would converge the live realm on a desired state the realm import "
-          f"does not declare — two sources of truth for every public hostname, which is "
-          f"the failure class this change exists to close. Regenerate with: "
-          f"python3 {RENDERER} --write\n        "
-          + "\n        ".join((proc.stdout + proc.stderr).strip().splitlines()[:12]))
-else:
-    check(False, f"{RENDERER} is missing — {URLS_FILE} would become a hand-maintained "
-                 f"second declaration of every hostname with nothing keeping it honest")
-
-# Every client named in the desired state must exist in the realm import (otherwise every
-# reconciler run reports NOT FOUND forever), and every field must be one the reconciler is
-# actually allowed to write.
-import_client_ids = {c.get("clientId") for c in clients}
-for cid in sorted(urls_want):
-    check(cid in import_client_ids,
-          f"{URLS_FILE}: declares URLs for client '{cid}', which {IMPORT_FILE} does not "
-          f"declare. The reconciler would look it up, not find it, and fail every run")
-    entry = urls_want[cid] if isinstance(urls_want.get(cid), dict) else {}
-    check(bool(entry), f"{URLS_FILE}: client '{cid}' declares no URL fields")
-    for field in sorted(entry):
-        check(field in URL_FIELDS,
-              f"{URLS_FILE}: client '{cid}' declares field '{field}', which is not one of "
-              f"the writable URL fields ({', '.join(URL_FIELDS)}). Anything else would be "
-              f"a client-representation field written into the realm by a file edit")
-
-# The three places that name the writable field set must agree, or the gate is checking a
-# different contract than the one that ships.
-for field in URL_FIELDS:
-    check(field in script,
-          f"{RECON_FILE}: the shipped script never mentions the writable URL field "
-          f"'{field}', but this check and {RENDERER} both treat it as writable — the "
-          f"three definitions of the write surface have diverged")
-
-# A redirect URI is only as good as the hostname it points at. If the realm import uses a
-# Flux substitution token, the desired-state ConfigMap must carry the SAME token (not a
-# resolved value) and must be rendered by the SAME Kustomization, or the two are
-# substituted differently and disagree about the domain at reconcile time.
-# Scan the DATA PAYLOAD, not the whole file: both files carry header prose that discusses
-# substitution tokens by name, and matching those would be a false positive.
-TOKEN_RE = re.compile(re.escape(DOLLAR) + r"\{([A-Z_][A-Z0-9_]*)\}")
-import_tokens = set(TOKEN_RE.findall(
-    yaml.safe_dump((realm_import or {}).get("spec", {}).get("realm", {}))))
-urls_tokens = set(TOKEN_RE.findall(
-    ((urls_cm or {}).get("data") or {}).get(URLS_DATA_KEY, "")))
-check(urls_tokens <= import_tokens,
-      f"{URLS_FILE} uses substitution token(s) {sorted(urls_tokens - import_tokens)} that "
-      f"the realm definition in {IMPORT_FILE} does not — the desired state would resolve "
-      f"to a hostname the realm import never declared")
-
-urls_owners = owners(URLS_FILE)
-check(bool(urls_owners),
-      f"{URLS_FILE} is rendered by NO Flux Kustomization — the reconciler would mount a "
-      f"ConfigMap that does not exist and fail closed on every run")
-check(set(urls_owners) == set(import_owners),
-      f"{URLS_FILE} is rendered by {urls_owners} but {IMPORT_FILE} by {import_owners}. "
-      f"They MUST be rendered by the same Flux Kustomization: postBuild.substituteFrom is "
-      f"per-Kustomization, so a split would let the same "
-      f"{DOLLAR}{{BASE_DOMAIN}} resolve to two different values — the realm converging on "
-      f"a hostname the HTTPRoutes do not serve")
-for env in ("dev", "prod"):
-    entry = f"clusters/{env}"
-    if not (ROOT / entry).is_dir():
-        continue
-    applied, _ = st.reconciled_files(ROOT, entry)
-    check(URLS_FILE in applied,
-          f"{URLS_FILE} is not reachable from the {entry} entrypoint — the URL pass would "
-          f"fail closed on that cluster forever")
-
-# ============================== (6) the reconciler's BEHAVIOUR, proven by its own selftest
-if script:
-    with tempfile.TemporaryDirectory() as td:
-        p = Path(td) / "reconcile.py"
-        p.write_text(script)
-        proc = subprocess.run([sys.executable, str(p), "--selftest"],
-                              capture_output=True, text=True, timeout=180)
-        tail = (proc.stdout + proc.stderr).strip().splitlines()
-        check(proc.returncode == 0,
-              "the reconciler's --selftest FAILED (compare-then-write / idempotence / "
-              "no-secret-logging / endpoint allow-list / fail-closed):\n        "
-              + "\n        ".join(tail[-12:]))
-        if proc.returncode == 0:
-            print("  reconciler --selftest: " + (tail[-1] if tail else "(no output)"))
-
-        # ---- (6b) THE SELFTEST IS FALSIFIABLE, not decorative -----------------------
-        # A passing suite proves nothing unless it can be shown to FAIL. mutations.py
-        # injects plausible regressions one at a time -- order-sensitive comparison, a
-        # fat round-tripped PUT, additive-only URI sets, a widened allow-list, coupled
-        # passes, fail-open on an unreadable input -- and requires --selftest to catch
-        # every one. A mutation that SURVIVES means the suite does not check that
-        # property, and this check fails naming it.
-        #
-        # This matters more than usual here: the whole incident was a component that
-        # REPORTED SUCCESS while the realm was broken. A green test that cannot fail is
-        # the same lie one level up. ~20s, no network, no cluster.
-        mut = DIR_PATH / "mutations.py"
-        if mut.is_file():
-            mproc = subprocess.run([sys.executable, str(mut), str(p)],
-                                   capture_output=True, text=True, timeout=600)
-            mtail = (mproc.stdout + mproc.stderr).strip().splitlines()
-            check(mproc.returncode == 0,
-                  "the reconciler's --selftest is NOT FALSIFIABLE — at least one injected "
-                  "regression survived it, so the suite does not prove what it claims:\n"
-                  "        " + "\n        ".join(mtail[-14:]))
-            if mproc.returncode == 0:
-                print("  reconciler mutations: "
-                      + (mtail[-1] if mtail else "(no output)"))
-        else:
-            check(False,
-                  "conformance/tests/l0-static/keycloak-client-secret-reconciler/"
-                  "mutations.py is missing — nothing proves the --selftest can fail, and "
-                  "an unfalsifiable proof is the failure mode this component exists to end")
-
-# ======================================================= (7) this check is load-bearing
-L0 = (ROOT / "ci" / "l0.sh").read_text() if (ROOT / "ci" / "l0.sh").is_file() else ""
-GATE = (ROOT / "ci" / "gate.sh").read_text() if (ROOT / "ci" / "gate.sh").is_file() else ""
-TOUCHED = ((ROOT / "ci" / "touched-requirements.sh").read_text()
-           if (ROOT / "ci" / "touched-requirements.sh").is_file() else "")
-check("keycloak-client-secret-reconciler/check.sh" in L0,
-      "ci/l0.sh does not run keycloak-client-secret-reconciler/check.sh — the "
-      "requirement-scoped conformance selection can filter this check out of a gate run, "
-      "so l0.sh must invoke it directly and unconditionally or the gate is not enforcing "
-      "it. An unreferenced test is the failure this whole change exists to end")
-check(re.search(r"^for s in .*\bl0\b.*; do", GATE, re.M) and re.search(r"fail\).*BLOCK=1", GATE),
-      "ci/gate.sh does not set the merge block when the l0 stage fails")
-check(re.search(r"^\s*platform/keycloak/\*\)", TOUCHED, re.M),
-      "ci/touched-requirements.sh has no platform/keycloak/* entry in its path->requirement "
-      "map, so editing the realm import or this reconciler attributes NO requirement and "
-      "the L2 keycloak tests are filtered out of the merge gate's touched subset")
-l2 = ROOT / "conformance/tests/l2-behavioral/keycloak-client-secrets-match-git/chainsaw-test.yaml"
-check(l2.is_file(),
-      "the LIVE-realm counterpart (conformance/tests/l2-behavioral/"
-      "keycloak-client-secrets-match-git) is missing — this static check cannot see a "
-      "realm that drifted after apply, which is the whole failure mode")
-if l2.is_file():
-    l2_text = l2.read_text()
-    check("KEYCLOAK_CLIENT_SECRETS_AUDIT_OK" in l2_text,
-          "the L2 test does not assert the reconciler's secret audit sentinel, so it could "
-          "pass on a Job that never ran the audit")
-    check("KEYCLOAK_CLIENT_URLS_AUDIT_OK" in l2_text,
-          "the L2 test does not assert the reconciler's URL audit sentinel. The live realm "
-          "could hold pre-migration redirect URIs and the test would still pass — which is "
-          "exactly the state prod was measured in")
-    check(URLS_CM_NAME in l2_text,
-          f"the L2 audit Job does not mount the {URLS_CM_NAME} ConfigMap, so its URL pass "
-          f"would fail closed and the test would report a false failure")
-
-# The alert pair must cover the reconciler's silence. Extended, not duplicated: one CronJob
-# gets one Failing/Stale pair, because two pairs keyed to the same job_name regex would
-# both fire for a single cause.
-ALERTS = "platform/monitoring/alerts.yaml"
-alerts_text = (ROOT / ALERTS).read_text() if (ROOT / ALERTS).is_file() else ""
-for rule in ("KeycloakClientSecretReconcilerFailing", "KeycloakClientSecretReconcilerStale"):
-    check(rule in alerts_text,
-          f"{ALERTS} has no {rule} rule — the reconciler's silence would be unobservable, "
-          f"and a healer nobody notices is absent is the 'looks applied and isn't' failure")
-check("redirect" in alerts_text.lower(),
-      f"{ALERTS}: the reconciler alert descriptions never mention redirect URIs, so an "
-      f"operator paged at 3am would look only for a client-secret problem while the actual "
-      f"drift was a hostname")
-
-# The renderer must be a real, runnable, tracked file — not a documented intention.
-check((ROOT / RENDERER).is_file(),
-      f"{RENDERER} is missing; without it {URLS_FILE} cannot be regenerated and the "
-      f"single-source-of-truth gate above has nothing to compare against")
-
-# ------------------------------------------------------------------------------ report
-print(f"realm import: {len(clients)} client(s), {len(declared_pairs)} confidential with a "
-      f"declared secret, {len(placeholders)} placeholder(s)")
-print(f"{SECRET_NAME} Secret: {len(sops_keys)} key(s) in git (names only; values stay "
-      f"SOPS-encrypted)")
-print("flux ordering: " + ", ".join(
-    f"{f.split('/')[-1]} <- {owners(f)}" for f in (SOPS_FILE, IMPORT_FILE, RECON_FILE)))
-
-print()
-if not FAILURES:
-    print(f"check OK: keycloak-client-secret-reconciler — {CHECKS} assertions "
-          f"(R-IDP-03, R-IDP-04)")
-    raise SystemExit(0)
-print(f"CHECK FAIL (keycloak-client-secret-reconciler): {len(FAILURES)} failed, "
-      f"{CHECKS - len(FAILURES)} passed", file=sys.stderr)
-for f in FAILURES:
-    print(f"  FAIL {f}", file=sys.stderr)
-raise SystemExit(1)
-PY
@@ -1,190 +0,0 @@
-#!/usr/bin/env python3
-# =============================================================================
-# mutations.py -- prove the reconciler's --selftest IS FALSIFIABLE.
-#
-# WHY THIS EXISTS
-# ---------------
-# A selftest that passes proves nothing on its own: a selftest asserting `True ==
-# True` also passes. The reconciler exists because a one-shot import REPORTED
-# SUCCESS while the realm was broken, and the same trap applies one level up -- a
-# green test suite that cannot fail is the same lie in a different place.
-#
-# So this injects a series of PLAUSIBLE regressions into the shipped script, one at
-# a time, and requires that --selftest FAILS for every one. A mutation that
-# SURVIVES means the suite does not actually check that property, and the check
-# fails, naming it.
-#
-# Each mutation is a mistake a reasonable person could make while editing this
-# code -- not a nonsense edit. Several were live design questions while it was
-# written (order-sensitivity, additive-vs-exact URI sets, whether the two passes
-# should share a failure path); the mutation is the record that the answer is
-# enforced and not merely commented.
-#
-# Invoked by check.sh with the script EXTRACTED from the shipped ConfigMap, so what
-# is proven falsifiable is byte-for-byte what runs on the cluster.
-#
-#   python3 mutations.py <path-to-extracted-reconciler.py>
-#
-# Runtime is ~20s (one subprocess per mutation, each running the full selftest
-# against an in-process stub Keycloak; no network, no cluster).
-# =============================================================================
-from __future__ import annotations
-
-import re
-import subprocess
-import sys
-import tempfile
-from pathlib import Path
-
-
-def dedent4(text: str) -> str:
-    """Anchors are written with the YAML file's indentation. The script extracted from
-    the ConfigMap block scalar has 4 fewer leading spaces on every line, so anchors are
-    authored to match the YAML (easy to diff against the manifest) and dedented here."""
-    return "\n".join(ln[4:] if ln.startswith("    ") else ln for ln in text.split("\n"))
-
-
-# (name, the mistake being modelled, anchor, replacement)
-MUTATIONS: list[tuple[str, str, str, str]] = [
-    ("order-sensitive URL comparison",
-     "compare redirectUris in the order Keycloak returns them -- Keycloak does not "
-     "preserve the authored order, so a converged realm is rewritten on every run "
-     "forever while every run still reports success",
-     "if sorted(stored) != sorted(declared_value):",
-     "if stored != declared_value:"),
-
-    ("fat round-tripped URL PUT",
-     "PUT the whole client representation back instead of only the drifted fields -- "
-     "the live server rejects it with 400 'Cannot parse the JSON', and even if it did "
-     "not, the blast radius would cover every field of the client",
-     "            rep = {\"clientId\": client.get(\"clientId\")}\n"
-     "            for field in sorted(drift):\n"
-     "                rep[field] = want[cid][field]",
-     "            rep = dict(client)\n"
-     "            for field in sorted(drift):\n"
-     "                rep[field] = want[cid][field]"),
-
-    ("write unconditionally (compare-then-write removed)",
-     "drop the converged short-circuit -- every client is rewritten on every run, so "
-     "the CronJob becomes a permanent write load against the IdP database",
-     "            drift = url_drift(client, want[cid])\n            if not drift:",
-     "            drift = url_drift(client, want[cid]) or dict(\n"
-     "                (f, (\"\", want[cid][f])) for f in want[cid])\n            if False:"),
-
-    ("additive-only URI reconciliation",
-     "union the declared URIs with the stored ones -- a redirect URI REMOVED from git "
-     "is never removed from the realm, so a stale wildcard keeps working and quietly "
-     "widens what the realm will redirect to",
-     "                rep[field] = want[cid][field]",
-     "                rep[field] = (sorted(set(want[cid][field]) | set(client.get(field) or []))\n"
-     "                              if field in URL_LIST_FIELDS else want[cid][field])"),
-
-    ("URL pass fails OPEN on an unreadable input",
-     "treat a missing ConfigMap as 'nothing declared' -- the run reports success while "
-     "reconciling nothing, which is precisely the fail-open of the realm import",
-     "        except OSError as e:\n"
-     "            url_input_fail = (\"cannot read %s (%s) -- the keycloak-client-urls ConfigMap \"\n"
-     "                              \"is not mounted\" % (URLS_FILE, e.__class__.__name__))",
-     "        except OSError:\n            want_urls = {}"),
-
-    ("URL pass skips public clients",
-     "copy the secret pass's publicClient skip into the URL pass -- the `kubernetes` "
-     "client is PKCE-public, so kubectl's redirect URIs would never converge",
-     "            drift = url_drift(client, want[cid])",
-     "            if client.get(\"publicClient\"):\n                continue\n"
-     "            drift = url_drift(client, want[cid])"),
-
-    ("the two passes are coupled",
-     "abort the whole run when either input is bad -- a typo in a hostname ConfigMap "
-     "would stop client-SECRET repair, making credential healing depend on a file that "
-     "has nothing to do with credentials",
-     "        if want_secrets is None and want_urls is None:",
-     "        if want_secrets is None or want_urls is None:"),
-
-    ("unmanaged fields forced to empty",
-     "iterate every writable field instead of only the declared ones -- a rootUrl or "
-     "adminUrl the realm holds and git does not declare would be WIPED, turning "
-     "'not declared' into 'must be empty'",
-     "        for field, declared_value in sorted(wanted.items()):",
-     "        for field in URL_FIELDS:\n"
-     "            declared_value = wanted.get(\n"
-     "                field, [] if field in URL_LIST_FIELDS else \"\")"),
-
-    ("no readback verification",
-     "trust the 204 -- a server that accepts the PUT and does not apply it is reported "
-     "as REPAIRED, which is 'reports success while the realm never converged' again",
-     "            still = url_drift(after[0], want[cid])",
-     "            still = {}"),
-
-    ("allow-list widened to the whole realm",
-     "add a realm-level endpoint 'just in case' -- /users, /groups, /roles and "
-     "/authentication stop being structurally unreachable",
-     "        r\"/admin/realms/[^/]+/clients\",",
-     "        r\"/admin/realms/[^/]+\",\n        r\"/admin/realms/[^/]+/clients\","),
-
-    ("desired-state field validation removed",
-     "accept any field name from the ConfigMap -- an arbitrary client field becomes "
-     "writable through a file edit, bypassing the five-field write surface entirely",
-     "                if field not in URL_FIELDS:",
-     "                if False:"),
-
-    ("URL drift detected but never repaired",
-     "leave the audit-mode early-continue in place on the write path -- drift is "
-     "reported in the log and silently not fixed",
-     "            if audit:\n                continue\n            # MINIMAL representation",
-     "            if True:\n                continue\n            # MINIMAL representation"),
-]
-
-
-def main() -> int:
-    if len(sys.argv) != 2:
-        print("usage: mutations.py <extracted-reconciler.py>", file=sys.stderr)
-        return 2
-    src = Path(sys.argv[1]).read_text()
-
-    problems: list[str] = []
-    killed = 0
-    for name, why, old, new in MUTATIONS:
-        old, new = dedent4(old), dedent4(new)
-        # An anchor that no longer matches is a FAILURE, not a skip: it means the code
-        # moved and this mutation silently stopped proving anything.
-        n = src.count(old)
-        if n != 1:
-            problems.append("%s: anchor matches %d times (expected exactly 1) -- the "
-                            "code moved and this mutation no longer proves anything. "
-                            "Re-anchor it against the current source." % (name, n))
-            continue
-        with tempfile.NamedTemporaryFile("w", suffix=".py", delete=False) as fh:
-            fh.write(src.replace(old, new, 1))
-            path = fh.name
-        try:
-            proc = subprocess.run([sys.executable, path, "--selftest"],
-                                  capture_output=True, text=True, timeout=240)
-        finally:
-            Path(path).unlink(missing_ok=True)
-        if proc.returncode == 0:
-            problems.append("%s SURVIVED -- --selftest still passed with this regression "
-                            "applied, so nothing in the suite actually checks it. The "
-                            "mistake it models: %s" % (name, why))
-            continue
-        killed += 1
-        out = proc.stdout + proc.stderr
-        m = re.search(r"SELFTEST FAIL: (\d+) of \d+", out)
-        how = ("%s assertion(s) caught it" % m.group(1) if m
-               else "the selftest aborted (caught, less cleanly)")
-        print("  killed  %-42s %s" % (name, how))
-
-    print()
-    if problems:
-        print("MUTATION CHECK FAIL: %d of %d mutation(s) not proven"
-              % (len(problems), len(MUTATIONS)), file=sys.stderr)
-        for p in problems:
-            print("  FAIL %s" % p, file=sys.stderr)
-        return 1
-    print("MUTATION CHECK OK: all %d injected regressions were caught by --selftest "
-          "(the suite is falsifiable, not decorative)" % killed)
-    return 0
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())
@@ -48,10 +48,7 @@
 #           sync.yaml, every patch it applies targets an ENVIRONMENT-SCOPED object
 #           (cluster-vars, a Flux Kustomization's sops decryption, or the Git source),
 #           prod's Git ref is the SAME ref dev tracks (no prod-only branch to develop on),
-#           and no prod-only manifest exists anywhere in the rendered configuration — a file
-#           whose NAME carries the environment token is excused ONLY when the tree proves it
-#           is an OVERLAY-SELECTED half that every environment applies (see the four
-#           conditions at "THE ONE STRUCTURAL EXCEPTION" below); anything else still fails;
+#           and no prod-only manifest exists anywhere in the rendered configuration;
 #       (6) prod artifacts exist only WITH a declaration of readiness: the spec roadmap
 #           carries the phase that opens prod build (checked when the spec repo is mounted),
 #           and the repository records that the operator opened it.
@@ -440,172 +437,12 @@ tracked = [f for f in subprocess.run(["git", "-C", str(ROOT), "ls-files", "-z"],
                                     capture_output=True, text=True, check=True
                                     ).stdout.split("\0") if f]
 RENDERED_DIRS = ("platform/", "platform-config", "apps/", "catalog/", "clusters/base/")
-
-# THE ONE STRUCTURAL EXCEPTION: AN OVERLAY-SELECTED ENVIRONMENT HALF.
-# ------------------------------------------------------------------
-# A filename carrying an environment token is a HEURISTIC for the harm R-ENV-03 names
-# (content only prod runs, in a path every environment renders). It is not the harm
-# itself. There is exactly one shape where a `-prod` name is the OPPOSITE of a prod-only
-# fork: a credential that is TWO Secret objects, BOTH applied to BOTH clusters as base
-# resources, where a cluster-vars knob decides only which one the shared consumer
-# REFERENCES (clusters/base/cluster-vars.yaml, BACKUP_CREDENTIALS_SUFFIX: dev backs up to
-# MinIO, prod to Cloudflare R2, and one shared Secret meant either environment's rotation
-# broke the other). Nothing is forked: every environment renders the identical manifests
-# and the identical object set, which is precisely the R-ENV-02 contract.
-#
-# So the exception is not a path allowlist — it is the PROPERTY that makes that shape
-# safe, asserted from the tree, and a file is excused ONLY when all of it holds:
-#
-#   (A) it declares nothing but Secrets whose names end in the environment suffix — the
-#       only kind of object this repo has a selection mechanism for;
-#   (1) it is listed in its own directory's kustomization.yaml, so EVERY environment
-#       applies it, and every environment's value of the selector resolves to an object
-#       that some resource of that same kustomization declares (same namespace) — no
-#       environment is left selecting a Secret that does not exist;
-#   (2) nothing references the suffixed name literally: every reference is a substituted
-#       expression `<base-name>${VAR}`, and the suffixed FILE is named only by the
-#       kustomization that applies it — so no shared manifest hardcodes prod's half;
-#   (3) that VAR is defined for the base AND for every overlay (StrictPostBuildSubstitutions
-#       fails on an UNDEFINED variable, so an overlay that forgot it would break that
-#       cluster's reconciliation), and prod's value is the one that selects THIS file.
-#
-# References are read from PARSED YAML VALUES, never from raw text: the mechanism is
-# documented by name in comments beside the resources (cluster-vars says so explicitly),
-# and a name in a comment references nothing. Anything that fails any condition still
-# fails, naming the condition it broke.
-ENV_SUFFIX = f"-{prod_token}"
-# Every environment that renders the shared configuration.
-ENVS = ["base"] + sorted(p.parent.name
-                         for p in (ROOT / "clusters").glob("*/kustomization.yaml")
-                         if p.parent.name != "base")
-
-
-def _docs(rel: str):
-    p = ROOT / rel
-    if not p.is_file():
-        return None
-    try:
-        return [d for d in yaml.safe_load_all(p.read_text(errors="replace"))
-                if isinstance(d, dict)]
-    except Exception:
-        return None
-
-
-def _strings(node):
-    if isinstance(node, str):
-        yield node
-    elif isinstance(node, dict):
-        for k, v in node.items():
-            yield from _strings(k)
-            yield from _strings(v)
-    elif isinstance(node, (list, tuple)):
-        for v in node:
-            yield from _strings(v)
-
-
-RENDERED_YAML = [f for f in tracked
-                 if f.startswith(RENDERED_DIRS) and f.endswith((".yaml", ".yml"))]
-VALUE_STRINGS = {f: list(_strings(_docs(f) or [])) for f in RENDERED_YAML}
-
-
-def overlay_selected(f: str) -> list[str]:
-    """[] if f is an overlay-SELECTED environment half; else the conditions it breaks."""
-    docs = _docs(f)
-    objs = [((d.get("kind")), (d.get("metadata") or {}).get("name"),
-             (d.get("metadata") or {}).get("namespace")) for d in (docs or [])]
-    if not objs or any(k != "Secret" or not isinstance(n, str) or not n.endswith(ENV_SUFFIX)
-                       for k, n, _ in objs):
-        return [f"(A) it does not consist solely of Secret objects whose names end in "
-                f"{ENV_SUFFIX!r} — an overlay can select which SECRET a shared manifest "
-                f"references, and nothing else, so any other prod-suffixed object in a "
-                f"shared path is prod-only content"]
-
-    reasons: list[str] = []
-    d = str(Path(f).parent)
-    kfile = f"{d}/kustomization.yaml"
-    kdocs = _docs(kfile) or []
-    res = [str(r) for r in ((kdocs[0].get("resources") if kdocs else None) or [])]
-    if Path(f).name not in res:
-        reasons.append(f"(1) {kfile} does not list it in `resources`, so it is not applied "
-                       f"in every environment — one environment would carry an object the "
-                       f"other does not")
-
-    # Every Secret the applying kustomization declares: name -> namespace.
-    declared: dict[str, object] = {}
-    for r in res:
-        for dd in (_docs(f"{d}/{r}") or []):
-            if dd.get("kind") == "Secret":
-                md = dd.get("metadata") or {}
-                declared[str(md.get("name"))] = md.get("namespace")
-
-    for _, name, ns in objs:
-        base_name = name[:-len(ENV_SUFFIX)]
-        pat = re.escape(base_name) + r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}"
-        sel = sorted({m for ss in VALUE_STRINGS.values() for s in ss
-                      for m in re.findall(pat, s)})
-        # (2) literal references to the suffixed OBJECT — anywhere but this file, except
-        # the applying kustomization naming this FILE (which is how it gets applied).
-        lit = sorted({f2 for f2, ss in VALUE_STRINGS.items() for s in ss
-                      if name in s and f2 != f
-                      and not (f2 == kfile and s == Path(f).name)})
-        if lit:
-            reasons.append(f"(2) {', '.join(lit)} names the Secret {name!r} literally — a "
-                           f"shared manifest that hardcodes prod's half makes every "
-                           f"environment use prod's credential; reference "
-                           f"{base_name}${{...}} instead")
-        # (2) literal references to the suffixed FILE from anywhere but its kustomization.
-        stray = sorted({f2 for f2, ss in VALUE_STRINGS.items() for s in ss
-                        if Path(f).name in s and f2 not in (f, kfile)})
-        if stray:
-            reasons.append(f"(2) {', '.join(stray)} names the file {Path(f).name!r} "
-                           f"literally — only {kfile} may apply it")
-        if not sel:
-            reasons.append(f"(2) no manifest references {base_name}${{...}}: nothing selects "
-                           f"between the halves, so this file is prod-only content that "
-                           f"only prod could ever consume")
-            continue
-        if len(sel) > 1:
-            reasons.append(f"(2) {base_name} is selected by more than one variable {sel} — "
-                           f"they can drift apart and leave one consumer on the other "
-                           f"environment's credential")
-        var = sel[0]
-        vals = env_values(var)
-        missing = [e for e in ENVS if e not in vals]
-        if missing:
-            reasons.append(f"(3) ${{{var}}} is not defined for {missing} (defined for "
-                           f"{sorted(vals)}) — StrictPostBuildSubstitutions fails on an "
-                           f"undefined variable, so that environment would stop "
-                           f"reconciling, or silently inherit the other's credential")
-        for e, v in sorted(vals.items()):
-            want = base_name + v
-            if want not in declared:
-                reasons.append(f"(1) environment {e!r} selects Secret {want!r} via "
-                               f"${{{var}}}={v!r}, which no resource of {kfile} declares — "
-                               f"that environment renders a dangling reference")
-            elif declared[want] != ns:
-                reasons.append(f"(1) environment {e!r} selects Secret {want!r}, declared in "
-                               f"namespace {declared[want]!r}, but this half is in {ns!r} — "
-                               f"the halves are not interchangeable")
-        if prod_overlay and prod_overlay in vals and base_name + vals[prod_overlay] != name:
-            reasons.append(f"(3) the prod overlay's ${{{var}}}={vals[prod_overlay]!r} selects "
-                           f"{base_name + vals[prod_overlay]!r}, not this file's {name!r} — "
-                           f"prod's half is applied but no environment selects it")
-    return reasons
-
-
-flagged = [f for f in tracked
-           if f.startswith(RENDERED_DIRS)
-           and re.search(rf"(^|[/_.-]){re.escape(prod_token)}([/_.-]|$)", f, re.I)]
-for f in flagged:
-    why = overlay_selected(f)
-    check(not why,
-          f"{f} is a prod-specific file inside the shared configuration — every environment "
-          f"renders these paths, so prod-only content here is development against prod. It is "
-          f"not an overlay-selected environment half either: " + " AND ".join(why))
-    if not why:
-        note(f"{f} carries the {prod_token!r} token but is an OVERLAY-SELECTED half: both "
-             f"halves are applied in every environment and only the cluster-vars selector "
-             f"differs (R-ENV-02), so it is not prod-only content")
+for f in tracked:
+    if f.startswith(RENDERED_DIRS) and re.search(rf"(^|[/_.-]){re.escape(prod_token)}([/_.-]|$)",
+                                                 f, re.I):
+        check(False,
+              f"{f} is a prod-specific file inside the shared configuration — every environment "
+              "renders these paths, so prod-only content here is development against prod")
 check(len(tracked) > 50, f"git ls-files returned only {len(tracked)} paths — not a real checkout")

 # --------------------------- (6) prod work exists only with a declaration of readiness
@@ -1,242 +0,0 @@
-# =============================================================================
-# L2 BEHAVIORAL — every OIDC client's STORED secret AND redirect URIs equal what git
-#                 declares                          (R-IDP-02, R-IDP-04)   [RUNNABLE]
-# Spec: docs/15 identity-and-access; ADR-0014.
-# Impl: platform/keycloak/client-secret-reconciler.yaml (the CronJob + its script).
-#
-# WHY THIS TEST EXISTS
-# --------------------
-# The KeycloakRealmImport resolves each confidential client's secret from a `(env:NAME)`
-# placeholder mapped to a key of the SOPS-encrypted keycloak-realm-clients Secret. When
-# the Keycloak Operator cannot resolve a placeholder it stores the LITERAL placeholder
-# text as the client secret AND THE IMPORT JOB STILL REPORTS SUCCESS; the CR then rests
-# at Done=True / Started=False and never re-runs. Measured 2026-09-06: prod had all 12
-# clients holding the literal placeholder — every OIDC login on the cluster failed with
-# "Invalid client or Invalid client credentials" — and dev had one (oauth2-proxy), whose
-# Secret key was added after dev's import had already completed.
-#
-# Every SIGNAL the cluster offered was green: the import Job succeeded, the CR read
-# Done=True, the Keycloak pods were healthy, and the existing keycloak-config-as-code
-# test (which asserts Done=True) PASSED throughout. The only observable was a human
-# failing to log in. This test is that observable, automated: it compares STORED against
-# DECLARED for every client and fails on any difference, so the class cannot sit
-# undetected for months again.
-#
-# GIVEN the `homelab` realm and the SOPS-encrypted keycloak-realm-clients Secret,
-# WHEN  the reconciler's script is run in --audit mode (READ-ONLY: it cannot write, and
-#       its endpoint allow-list makes users/groups/roles unreachable),
-#   ->  THEN every declared client's stored secret matches git, and none holds an
-#       unresolved import placeholder. Emits KEYCLOAK_CLIENT_SECRETS_AUDIT_OK.
-#   ->  AND every declared client URL — redirectUris, webOrigins, rootUrl, baseUrl,
-#       adminUrl — matches git. Emits KEYCLOAK_CLIENT_URLS_AUDIT_OK.
-#
-# THE URL HALF EXISTS FOR THE SAME REASON, ONE FIELD OVER. The realm import is templated
-# per environment, so moving prod's BASE_DOMAIN re-renders every declared redirect URI in
-# git — and reaches the realm never, because a completed import does not re-run. Measured
-# on prod: stored redirect URIs still on the PRE-migration hostname while git declared the
-# new one. Every OIDC login would fail with a redirect_uri mismatch, and once again every
-# signal would be green: the CR reads Done=True, the pods are healthy, and the
-# secret-only version of this very test would have PASSED. Asserting only the secret
-# sentinel is what would have let that ship.
-#
-# WHY IT RUNS THE SHIPPED SCRIPT rather than re-implementing the comparison: a detector
-# that duplicates the healer's logic drifts from it. This audits with the same code, in
-# the same mode the L0 gate selftests, so a passing audit is evidence about the healer
-# too. It runs as a Job IN the keycloak namespace (not in the ephemeral chainsaw
-# namespace) because the desired state is the mounted keycloak-realm-clients Secret plus
-# the keycloak-client-urls ConfigMap, and the admin credential is
-# keycloak-admin-credentials — all namespace-local, which is also why neither this Job nor
-# the CronJob needs any Kubernetes API access.
-#
-# chainsaw v0.2.12: Job + status assert; the leftover-Job delete keeps a re-run from
-# colliding with an immutable Job left by a previous failure.
-# =============================================================================
-apiVersion: chainsaw.kyverno.io/v1alpha1
-kind: Test
-metadata:
-  name: keycloak-client-secrets-match-git
-  labels:
-    conformance.home.k3s/R-IDP-02: "true"
-    conformance.home.k3s/R-IDP-04: "true"
-  annotations:
-    conformance.home.k3s/requirements: "R-IDP-02, R-IDP-04"
-    conformance.home.k3s/layer: "L2"
-    conformance.home.k3s/safety-critical: "false"
-    conformance.home.k3s/executable: "true"
-    conformance.home.k3s/intent: >-
-      Every OIDC client's stored secret equals the SOPS-declared one and none holds an
-      unresolved import placeholder — the observable that was missing when all 12 prod
-      clients were broken behind a green realm import.
-spec:
-  timeouts:
-    apply: 60s
-    assert: 5m
-    exec: 5m
-  steps:
-    - name: given-the-reconciler-is-reconciled-onto-the-cluster
-      description: >-
-        The audit Job runs the CronJob's own script from its ConfigMap; if Flux has not
-        applied the reconciler, there is nothing to audit WITH and that is the failure.
-      try:
-        - assert:
-            resource:
-              apiVersion: v1
-              kind: ConfigMap
-              metadata:
-                name: keycloak-client-secret-reconciler
-                namespace: keycloak
-        - assert:
-            resource:
-              apiVersion: batch/v1
-              kind: CronJob
-              metadata:
-                name: keycloak-client-secret-reconciler
-                namespace: keycloak
-              spec:
-                concurrencyPolicy: Forbid
-    - name: then-every-stored-client-secret-matches-git
-      try:
-        - script:
-            content: |
-              kubectl -n keycloak delete job keycloak-client-secret-audit \
-                --ignore-not-found --wait=true
-        - apply:
-            resource:
-              apiVersion: batch/v1
-              kind: Job
-              metadata:
-                name: keycloak-client-secret-audit
-                namespace: keycloak
-                labels:
-                  app.kubernetes.io/name: keycloak-client-secret-audit
-                  app.kubernetes.io/part-of: conformance
-              spec:
-                backoffLimit: 0
-                activeDeadlineSeconds: 240
-                template:
-                  metadata:
-                    labels:
-                      app.kubernetes.io/name: keycloak-client-secret-audit
-                  spec:
-                    restartPolicy: Never
-                    automountServiceAccountToken: false
-                    securityContext:
-                      runAsNonRoot: true
-                      runAsUser: 65532
-                      runAsGroup: 65532
-                      seccompProfile:
-                        type: RuntimeDefault
-                    containers:
-                      - name: audit
-                        image: python:3.13.15-alpine3.23
-                        command:
-                          - python3
-                          - /opt/reconciler/reconcile-keycloak-clients.py
-                          # READ-ONLY. --audit never issues a write on any path; the
-                          # script's --selftest proves that (L0 gate).
-                          - --audit
-                        env:
-                          - name: KEYCLOAK_BASE_URL
-                            value: http://keycloak-service.keycloak.svc.cluster.local:8080
-                          - name: KEYCLOAK_REALM
-                            value: homelab
-                          - name: PYTHONDONTWRITEBYTECODE
-                            value: "1"
-                          - name: PYTHONUNBUFFERED
-                            value: "1"
-                          - name: KEYCLOAK_ADMIN_USER
-                            valueFrom:
-                              secretKeyRef:
-                                name: keycloak-admin-credentials
-                                key: admin-user
-                          - name: KEYCLOAK_ADMIN_PASSWORD
-                            valueFrom:
-                              secretKeyRef:
-                                name: keycloak-admin-credentials
-                                key: admin-password
-                        volumeMounts:
-                          - name: script
-                            mountPath: /opt/reconciler
-                            readOnly: true
-                          - name: client-secrets
-                            mountPath: /etc/keycloak-realm-clients
-                            readOnly: true
-                          # The URL pass's desired state. Without this mount the audit
-                          # fails closed on pass 2 and this test would report a false
-                          # failure rather than auditing the realm's redirect URIs.
-                          - name: client-urls
-                            mountPath: /etc/keycloak-client-urls
-                            readOnly: true
-                        securityContext:
-                          allowPrivilegeEscalation: false
-                          readOnlyRootFilesystem: true
-                          capabilities:
-                            drop:
-                              - ALL
-                        resources:
-                          requests:
-                            cpu: 10m
-                            memory: 48Mi
-                          limits:
-                            memory: 128Mi
-                    volumes:
-                      - name: script
-                        configMap:
-                          name: keycloak-client-secret-reconciler
-                          defaultMode: 0444
-                      - name: client-secrets
-                        secret:
-                          secretName: keycloak-realm-clients
-                          defaultMode: 0400
-                      - name: client-urls
-                        configMap:
-                          name: keycloak-client-urls
-                          defaultMode: 0444
-        - assert:
-            resource:
-              apiVersion: batch/v1
-              kind: Job
-              metadata:
-                name: keycloak-client-secret-audit
-                namespace: keycloak
-              status:
-                succeeded: 1
-        # The Job's exit status already encodes the verdict, but the sentinel is what the
-        # run LEDGER records (conformance/runs/*/run.log): a JUnit alone cannot tell a real
-        # audit from a Job that succeeded without auditing anything.
-        - script:
-            content: |
-              set -u
-              out="$(kubectl -n keycloak logs job/keycloak-client-secret-audit \
-                       --tail=-1 2>&1 || true)"
-              echo "$out"
-              echo "$out" | grep -q 'KEYCLOAK_CLIENT_SECRETS_AUDIT_OK' \
-                || { echo "FAIL: the audit did not report every client secret matching git"; exit 1; }
-              echo "$out" | grep -q 'UNRESOLVED IMPORT PLACEHOLDER' \
-                && { echo "FAIL: a client still holds the literal import placeholder"; exit 1; }
-              # The URL half. A realm still holding PRE-migration redirect URIs is the
-              # state prod was measured in: every OIDC login fails with a redirect_uri
-              # mismatch while the KeycloakRealmImport CR still reads Done=True. Asserting
-              # only the secret sentinel would pass on exactly that realm.
-              echo "$out" | grep -q 'KEYCLOAK_CLIENT_URLS_AUDIT_OK' \
-                || { echo "FAIL: the audit did not report every client URL matching git"; exit 1; }
-              echo "$out" | grep -q 'DRIFT -- realm has' \
-                && { echo "FAIL: a client URL still differs from the declared one"; exit 1; }
-              echo KEYCLOAK_CLIENT_SECRET_DRIFT_NONE_OK
-            check:
-              (contains($stdout, 'KEYCLOAK_CLIENT_SECRET_DRIFT_NONE_OK')): true
-      catch:
-        - describe:
-            apiVersion: batch/v1
-            kind: Job
-            namespace: keycloak
-            name: keycloak-client-secret-audit
-        - podLogs:
-            namespace: keycloak
-            selector: app.kubernetes.io/name=keycloak-client-secret-audit
-            tail: 60
-      finally:
-        - script:
-            content: |
-              kubectl -n keycloak delete job keycloak-client-secret-audit \
-                --ignore-not-found --wait=false
@@ -11,18 +11,6 @@
 #
 # Goes GREEN once Keycloak is up + the realm client-secret hand-off (keycloak-realm-clients) is filled
 # so the placeholder resolves and the import completes. chainsaw v0.2.12: script + block contains check.
-#
-# NOT PROVEN HERE — READ THIS BEFORE TRUSTING A GREEN RUN (added 2026-09-06).
-# Done=True means the import JOB EXITED 0. It does NOT mean the placeholders resolved: when the
-# operator cannot resolve one it stores the LITERAL `(env:...)` text as the client secret and the
-# import still reports success, then rests at Done=True / Started=False and never re-runs. THIS
-# TEST PASSED THROUGHOUT the 2026-09-06 outage in which all 12 prod clients held the literal
-# placeholder and every OIDC login on the cluster failed. Done=True is a necessary, badly
-# insufficient condition.
-# The sufficient check is tests/l2-behavioral/keycloak-client-secrets-match-git, which compares
-# each client's STORED secret against the SOPS-declared one; the static half is
-# tests/l0-static/keycloak-client-secret-reconciler. Do not "simplify" by folding those into this
-# test: the whole lesson is that the import's own success signal is not evidence.
 # =============================================================================
 apiVersion: chainsaw.kyverno.io/v1alpha1
 kind: Test
@@ -10,31 +10,8 @@
 #       egress-gateway node is STOPPED (fault injection) —
 # THEN  the vpn-class workload reaches NOTHING on the plain internet: its curl FAILS
 #       and, critically, does NOT return the WAN IP (a WAN IP = LEAK = test FAILS).
-#       vpn-class egress is blackholed while tun0 is absent (ADR-0015 invariant 3/4:
-#       vpn fails CLOSED).
-#
-#       MECHANISM CORRECTED 2026-09-12: this comment used to credit "the host nftables
-#       kill-switch". That table never sees this traffic — Cilium's eBPF egress-gateway
-#       datapath (bpf.masquerade) bpf_redirect()s the steered skbs to the egress device
-#       without invoking netfilter POSTROUTING, measured with counters on the gateway
-#       node (0 packets for the pod CIDR and the PIA range in forward and postrouting,
-#       against 45166 on the same hook for WAN traffic). What fails closed is CILIUM's
-#       own drop when it cannot use the cached egress interface; tcpdump on both the WAN
-#       and tun0 during the stop caught 0 packets. The ASSERTION below is unchanged and
-#       still the right one — only the explanation was wrong.
-#
-#       PREREQUISITE, AND IT IS NOT OPTIONAL: this test cannot run at all unless a node
-#       carries homelab.keithjackson.dev/egress-gateway=true (the fault Job is pinned to
-#       it by nodeSelector, and with no gateway there is no tunnel to stop).
-#
-#       RECOVERY DEPENDS ON THE RE-STEER PATH. Stopping and starting the tunnel gives
-#       tun0 a NEW ifindex, and Cilium does not re-resolve `interface: tun0` on its own,
-#       so before 2026-09-12 the recovery step could not pass: measured, the map stayed
-#       on the old ifindex and vpn-class egress never came back. roles/vpn_egress_gateway
-#       now publishes the live uplink as a node label on every connect, which makes
-#       Cilium re-resolve; with it, stop->start recovers unaided (map 1989 -> 1990, PIA IP
-#       returns). If this test's recovery step ever fails again, check
-#       `journalctl -u vpn-egress-resteer` on the gateway node first.
+#       The host nftables kill-switch blackholes egress-class=vpn traffic while tun0
+#       is absent (ADR-0015 invariant 3/4: vpn fails CLOSED).
 # AND   the `internet`-class workload is UNAFFECTED (invariant 3: internet fails OPEN).
 # WHEN  the tunnel RECOVERS (service restarted), THEN vpn-class egress returns (PIA IP).
 #
@@ -1,337 +0,0 @@
-# Bulk NFS storage — the media library and the legacy Nextcloud archive
-
-The operator-facing companion to the `class: bulk` storage entries in `catalog/{plex,sonarr,radarr,nextcloud}.yaml`.
-Everything here was **measured** on the storage host on 2026-09-07, over read-only SSH; nothing is
-assumed. It resolves the long-standing **OQ-2** stub (`platform/longhorn/storageclasses.yaml`) for the
-case that actually exists here — attaching datasets that already hold data — and leaves dynamic bulk
-provisioning unimplemented, because nothing needs it.
-
---
-
-## 1. The storage host, and why prod cannot reach it
-
-The bulk data lives on the **Proxmox host's ZFS pool**. That host has exactly one interface:
-
-```
-vmbr0  192.168.1.9/24        default via 192.168.1.1   (pfSense LAN)
-```
-
-| | nodes | reaches 192.168.1.9? |
-|---|---|---|
-| **dev** | 192.168.1.40 / .41 / .42 | **Yes** — same L2 segment, and it already holds an NFS export from this host (`/mnt/Storage/k8s-volumes`, in `/etc/exports`) |
-| **prod** | 192.168.2.10 / .11 / .12 | **No** — 192.168.2.0/24 is pfSense's **WAN**. Ports 22 and 8006 answer only because pfSense port-forwards them via 192.168.2.102. 2049 and 111 do not. |
-
-Opening the storage path for prod is a **pending operator decision**. Its two candidate outcomes give
-different values for the `BULK_NFS_SERVER` cluster-var:
-
-| outcome | `BULK_NFS_SERVER` for prod |
-|---|---|
-| port-forward `192.168.2.102:2049` → `192.168.1.9:2049` | `192.168.2.102` |
-| a WAN→LAN route to 192.168.1.0/24 | `192.168.1.9` |
-
-Until it lands, `clusters/prod` holds `TBD-PROD-BULK-NFS-SERVER`. **See §7 for why that placeholder
-behaves differently from the repo's other ones.**
-
-### It only needs ONE port
-
-The PVs mount `nfsvers=4.1`. **NFSv4 carries mount and lock in-protocol**, so the only thing that
-must be reachable is **tcp/2049** — no rpcbind/111, no mountd, no statd. NFSv3 would have needed
-four ports, three of them on ephemeral numbers, which is a materially worse firewall story. The
-server advertises `+3 +4 +4.1 +4.2` (`/proc/fs/nfsd/versions`), so v4.1 is available and is the
-conservative floor.
-
-### The export's client list is the SAME under both outcomes — measured, not assumed
-
-**pfSense DNATs the existing port-forward WITHOUT source NAT.** Connecting from this workstation
-(192.168.2.132) to `192.168.2.102:22`, the storage host logged the peer as `192.168.2.132`:
-
-```
-SSH_CONNECTION=192.168.2.132 39768 192.168.1.9 22
-ss -tn  ->  192.168.1.9:22   192.168.2.132:39768
-```
-
-The original WAN-side source address survives the translation (return traffic finds its way back via
-the host's default route, 192.168.1.1). So under **either** outcome the NFS server will see prod's
-**real** node addresses — 192.168.2.10/.11/.12 — and the `sharenfs` change in §5 is the same either
-way. **It does not have to wait for the routing decision.**
-
---
-
-## 2. The layout, as found
-
-### `Storage/plex` → `/mnt/Storage/plex` — 8.33 T used, 6.91 T avail
-
-```
-/mnt/Storage/plex/                 drwxr-xr-x  1000:3000
-├── downloads/                     drwxr-xr-x  1000:1000
-│   ├── complete/                  drwxr-xr-x  1000:3000
-│   │   ├── radarr/                drwxr-xr-x  1000:3000
-│   │   └── tv-sonarr/             drwxr-xr-x  1000:3000
-│   ├── incomplete/                drwxrwxr-x  1000:3000
-│   ├── downloads -> /shared/downloads     (symlink OUTSIDE the export — see §8)
-│   └── <in-flight release dirs>   drwxr-xr-x  1000:1000
-├── movies/    1165 entries        drwxr-xr-x  1000:3000
-├── tv/         107 entries        drwxr-xr-x  1000:3000
-└── photos/                        drwxr-xr-x  1000:3000
-```
-
-**The single most important fact about this layout: `downloads/` is a SIBLING of `tv/` and `movies/`
-inside the same dataset.** Sonarr's import source and its library are on one filesystem on the
-server, and so are Radarr's. That is what makes hardlinked / atomically-renamed imports possible at
-all — and §4 is about not throwing it away on the client side.
-
-Note the download client's completed folders are already named `radarr` and `tv-sonarr`, i.e. this
-tree has been driven by an arr stack for a long time. The k3s arrs are joining an existing layout,
-not defining one.
-
-### `Storage/nextcloud` → `/mnt/Storage/nextcloud` — 270 G used
-
-```
-/mnt/Storage/nextcloud/            drwxr-xr-x  root:root
-└── nextcloud/                     drwxr-xr-x  1000:3000
-    ├── config/          103 M     drwxr-xr-x  1000:3000
-    ├── config_old/      3.4 G     drwxr-xr-x  1000:3000
-    └── config_backup/   267 G     drwxr-xr-x  1000:3000
-```
-
-**There is no `data/` directory.** 99% of the dataset is `config_backup`. This is the old, non-k3s
-Nextcloud's configuration plus a backup blob — an **archive**, not a live user-data tree. That is
-why it is mounted read-only; see §6.
-
---
-
-## 3. What each app mounts, and why
-
-| app | export | mountPath | subPath | mode | access mode |
-|---|---|---|---|---|---|
-| plex | `/mnt/Storage/plex` | `/media` | — | **ro** | ReadOnlyMany |
-| sonarr | `/mnt/Storage/plex` | `/media` | — | rw | ReadWriteMany |
-| radarr | `/mnt/Storage/plex` | `/media` | — | rw | ReadWriteMany |
-| nextcloud | `/mnt/Storage/nextcloud` | `/mnt/legacy-nextcloud` | `nextcloud` | **ro** | ReadOnlyMany |
-
-In-app paths that follow: Plex libraries at `/media/movies`, `/media/tv`, `/media/photos`; Sonarr
-Root Folder `/media/tv`; Radarr Root Folder `/media/movies`; both importing from
-`/media/downloads/complete/{tv-sonarr,radarr}`.
-
-**All three media consumers use the identical mountPath on purpose.** When qbittorrent is eventually
-given the same export (it is not in this change), the arrs and the download client will already agree
-on absolute paths, which is the second precondition for hardlinked imports — otherwise every import
-needs a Remote Path Mapping and quietly falls back to copying when one is missing.
-
-**`/media` and not `/data`, and that is not cosmetic.** The arrs' Longhorn config PVC renders at the
-renderer default `/data/config`. A media mount at `/data` would be mounted **over** it and mask the
-config volume entirely. See §8 for the separate latent bug that discovery exposed.
-
---
-
-## 4. Why the arrs mount the export ROOT, not two subPaths
-
-This is the design decision most likely to be "simplified" later by someone who has not read this.
-
-Sonarr and Radarr import a finished download into the library by **hardlinking** it (preserving the
-seeding copy) or by an **atomic rename**. Both are same-filesystem operations, and **to the kernel,
-"same filesystem" means "same mount."**
-
-The tempting narrower shape is two subPath volumes — `subPath: tv` and `subPath: downloads`. Those
-are **two separate mounts** of the same export. `link()` across them returns **EXDEV** no matter that
-the server knows perfectly well they are one dataset. Sonarr does not fail loudly at that: it
-degrades to copy-then-delete. Every import then rewrites the entire file over NFS and destroys the
-seeding copy. The symptom is "imports are slow and my ratio is gone", weeks later.
-
-The smallest subtree containing both `tv/` and `downloads/` **is the export root**. So the root mount
-is the *minimal* mount that preserves hardlinks: least privilege bounded below by a hard requirement,
-not a shortcut.
-
-The cost is stated rather than hidden: Sonarr can also write `/media/movies` and Radarr `/media/tv`.
-Narrowing that would cost hardlinks, which is the worse trade.
-
---
-
-## 5. The export change the operator must apply — STRICTLY ADDITIVE
-
-**Do not apply this from the repo.** It is an operator step on the storage host, recorded here so the
-exact value is reviewable. The existing consumers **must keep working**: 192.168.1.6, 10.123.1.10 and
-192.168.1.18 on the media tree, and 192.168.1.6 and 192.168.1.5 on the Nextcloud dataset, are the
-live non-k3s prod stack.
-
-Current values (`zfs get sharenfs Storage`):
-
-```
-Storage/plex       rw=@192.168.1.6:@10.123.1.10:@192.168.1.18,no_subtree_check,no_root_squash
-Storage/nextcloud  rw=@192.168.1.6:@192.168.1.5,no_subtree_check,no_root_squash
-```
-
-### `Storage/plex` — append the six k3s node addresses to the existing `rw=` list
-
-```sh
-zfs set sharenfs='rw=@192.168.1.6:@10.123.1.10:@192.168.1.18:@192.168.1.40:@192.168.1.41:@192.168.1.42:@192.168.2.10:@192.168.2.11:@192.168.2.12,no_subtree_check,no_root_squash' Storage/plex
-```
-
-Every pre-existing host is preserved verbatim and no option changes; only clients are added.
-
-### `Storage/nextcloud` — add the k3s nodes to a **`ro=`** list, not to `rw=`
-
-```sh
-zfs set sharenfs='rw=@192.168.1.6:@192.168.1.5,ro=@192.168.1.40:@192.168.1.41:@192.168.1.42:@192.168.2.10:@192.168.2.11:@192.168.2.12,no_subtree_check,no_root_squash' Storage/nextcloud
-```
-
-This is the one place read-only can be enforced **server-side**, and it is worth taking: only
-Nextcloud consumes this dataset from k3s, so the export itself can refuse writes. Even a
-misconfigured client cannot then damage the archive.
-
-**The media tree cannot get the same treatment.** Plex is read-only and the arrs are read-write, but
-all three may be scheduled on any node, so the server sees the same source addresses for all of
-them and cannot distinguish the reader from the writers. **Plex's read-only posture is therefore a
-client-side guarantee** (`spec.nfs.readOnly: true` on `plex-media`, plus the volumeMount flag) — an
-honest limitation, not an oversight. If it ever needs to be server-enforced, the lever is a
-dedicated node pool for Plex, which is a much larger change.
-
-Per-node addresses rather than `@192.168.2.0/24` follows the precedent already on this host
-(`/etc/exports` lists the dev nodes individually, as `/32`). A `/24` would work and would survive
-adding a node; it also grants the whole subnet. Per-node is the tighter default.
-
-### Verifying, without changing anything
-
-```sh
-exportfs -v                  # the new client lines appear, existing ones unchanged
-zfs get sharenfs Storage/plex Storage/nextcloud
-```
-
---
-
-## 6. uid / gid — the failure this design exists to avoid
-
-The export uses `sec=sys` and `no_root_squash`, so **the client's numeric uid is passed through
-verbatim** and the server's on-disk ownership is the entire access story. The media tree is
-`0755 1000:3000` throughout (some directories `1000:1000`), so **only uid 1000 can write it**.
-
-**`fsGroup` cannot fix this, and reaching for it is how this gets misdiagnosed.** The in-tree NFS
-volume plugin performs **no ownership management**, so `podSecurityContext.fsGroup` is a **no-op on
-an NFS volume**. There is no recursive chown, no `g+rwX`, nothing. The container's own uid has to be
-right.
-
-| app | container uid | writes? | outcome |
-|---|---|---|---|
-| plex | 1000 (`PLEX_UID`/`PLEX_GID`, already set) | no | reads fine — the tree is 0755, so world-readable anyway |
-| sonarr | **1000 via `PUID`** (added) | yes | matches the owner ⇒ writes fine |
-| radarr | **1000 via `PUID`** (added) | yes | matches the owner ⇒ writes fine |
-| nextcloud | 33 (`www-data`) | no | reads and traverses 0755 fine |
-
-**Without the added `PUID`/`PGID`, sonarr and radarr would have failed exactly the way this tier
-always fails: the mount succeeds, the pod is Ready, and every import fails EACCES.** LinuxServer
-images default to uid/gid **911**, which owns nothing on that tree.
-
-`PGID: 3000` is chosen over 1000 so files these write are group-owned by the dominant group already
-on the tree, keeping them readable to the live non-k3s consumers.
-
-`podSecurityContext.fsGroup: 3000` is also added to sonarr and radarr — **for the Longhorn config
-PVC, not for the NFS mount.** fsGroup does work on a Longhorn filesystem, and it makes `/data/config`
-writable by the now-uid-1000 process without depending on the image re-chowning it after `PUID`
-changed.
-
-Nextcloud needs no uid work at all, which is one more reason its mount is read-only.
-
---
-
-## 7. The prod placeholder does NOT fail at apply time — read this before merging
-
-Every other `TBD-PROD-*` in `clusters/prod/kustomization.yaml` is rejected loudly by CRD validation
-(Cilium, LB-IPAM, CNPG, Velero) because it is not a valid IP or URL. **This one is not.**
-
-A PersistentVolume's `spec.nfs.server` is an **unvalidated string**. The API server accepts
-`TBD-PROD-BULK-NFS-SERVER`, the PV is created, and the PVC **binds** to it — binding is pure
-control-plane and never touches the server. The failure appears only when a pod mounts:
-
-```
-MountVolume.SetUp failed ... Failed to resolve server TBD-PROD-BULK-NFS-SERVER
-```
-
-so **plex, sonarr, radarr and nextcloud would sit in ContainerCreating on prod.**
-
-**Consequently: set `BULK_NFS_SERVER` in `clusters/prod/kustomization.yaml` BEFORE prod reconciles a
-revision that carries these bulk volumes.** Dev renders a working value and is where these manifests
-are meant to be proven first.
-
-### The static guard this convention leans on is missing from `main`
-
-`bootstrap/standup.sh` — which carries the `*_is_placeholder` guard family — **does not exist on
-`main`** (verified: `git ls-tree main bootstrap/` lists only `README.md`, `flux/`, `gitea/`, `sops/`;
-the file lives on unmerged prod branches). So
-`conformance/tests/l0-static/placeholder-guard-coupling/check.sh` exits **3 SKIPPED**, and **nothing
-in the gate currently refuses a placeholder-shaped overlay value.**
-
-When `standup.sh` lands, that check's assertions (2) and (3) cover this knob automatically — its
-independent detector's SEED vocabulary already matches `TBD` — **provided** the guard gains an arm
-for `BULK_NFS_SERVER` and a consumer that calls it. Without the arm, assertion (2) is what fails, by
-design.
-
---
-
-## 8. Loose ends and things deliberately not done
-
-1. **`/data/config` is very likely the wrong home for the arrs' config PVC — a pre-existing latent
-   bug, not fixed here.** LinuxServer's arr images run with `-data=/config`, so the app writes
-   `/config`, which is the container's **ephemeral layer**, while the Longhorn PVC mounted at
-   `/data/config` sits unused. This is the exact failure `catalog/plex.yaml`'s `mountPath` comment
-   records having already been bitten by ("the PVC sat unused and Plex wrote to the container's
-   EPHEMERAL layer"). Plex was fixed; sonarr, radarr, qbittorrent and jackett were not. Fixing it
-   means those apps start from an empty config volume, so it belongs in its own change with the
-   operator present. **This change does not make it worse** — that is why the media mount is `/media`.
-2. **qbittorrent has no bulk mount.** Today the arrs' import source is written by the **legacy
-   non-k3s** download client, which is coherent while the legacy stack stays live. Giving the k3s
-   qbittorrent the same export at the same `/media` path is the natural follow-up, and the paths were
-   chosen so it needs no Remote Path Mapping.
-3. **`/mnt/Storage/plex/downloads/downloads` is a symlink to `/shared/downloads`**, which is outside
-   the export. Over NFS it is a dangling symlink on every client. Harmless, but expect Sonarr to log
-   it while scanning `downloads/`.
-4. **`nfsvers=4.2` was not chosen**, though the server advertises it. Nothing in these workloads uses
-   what 4.2 adds (server-side copy, sparse-file `SEEK`/`ALLOCATE`), and 4.1 is the version the
-   existing `bulk` StorageClass stub already named.
-5. **`nconnect` was not set.** It would likely help throughput on the media tree, but it is a tuning
-   change that should be measured, not guessed, and it changes client behaviour under server
-   failure.
-6. **Nothing was verified against a live cluster.** No `kubectl` context is usable from this
-   workstation (browser-OIDC only), and this change is manifest-authoring under an explicit
-   do-not-apply constraint. Every claim about the *host* is measured; every claim about *binding and
-   mounting* is reasoned from the manifests.
-7. **The bulk-tier requirement ID is deliberately not typed on any non-`.md` line of this change.**
-   `ci/touched-requirements.sh` attributes requirements from R-IDs appearing in added lines, and the
-   in-cluster conformance test for the bulk tier is still a gap, so typing it would select a
-   requirement the gate cannot satisfy — the hazard `catalog/qbittorrent.yaml` already warns about.
-   Scaffolding that test needs a cluster and is the correct follow-up.
-
---
-
-## 9. Validating on dev, once the export is extended
-
-Dev can prove all of this before prod's network path exists. Read-only checks first:
-
-```sh
-infra/devcontainer/dev.sh run kubectl --context k3s-dev get pv
-infra/devcontainer/dev.sh run kubectl --context k3s-dev get pvc -A | grep -E 'media|legacy-archive'
-infra/devcontainer/dev.sh run kubectl --context k3s-dev -n sonarr get pod -o wide
-```
-
-Expect **four** PVs — `plex-media`, `sonarr-media`, `radarr-media`, `nextcloud-legacy-archive` (three
-of them naming the *same* export, one per namespace, which is the cross-namespace model) — each
-`Bound` to the same-named PVC, with `RECLAIM POLICY Retain` and an empty `STORAGECLASS`.
-
-Then the two things that actually matter, from inside a pod:
-
-```sh
-# (a) the writer can write — this is the uid check, and it is the one that fails in practice
-infra/devcontainer/dev.sh run kubectl --context k3s-dev -n sonarr exec deploy/sonarr -- \
-  sh -c 'id; touch /media/tv/.k3s-write-probe && rm /media/tv/.k3s-write-probe && echo WRITE-OK'
-
-# (b) hardlinks work BETWEEN the library and the download path — the root-mount check
-infra/devcontainer/dev.sh run kubectl --context k3s-dev -n sonarr exec deploy/sonarr -- \
-  sh -c 'set -e; d=/media/downloads/complete/tv-sonarr/.probe; t=/media/tv/.probe-link;
-         echo x > "$d"; ln "$d" "$t"; ls -i "$d" "$t"; rm -f "$d" "$t"; echo HARDLINK-OK'
-
-# (c) the reader really is read-only
-infra/devcontainer/dev.sh run kubectl --context k3s-dev -n plex exec deploy/plex -- \
-  sh -c 'touch /media/.probe 2>&1 | head -1'      # expect "Read-only file system"
-```
-
-`(b)` printing two identical inode numbers is the proof that §4's argument holds in practice. If it
-prints `EXDEV`, something has split the single mount back into two.
@@ -1,319 +0,0 @@
-# Decommission Proxmox `192.168.2.102` → TrueNAS SCALE on the N5 Air
-
-**Status:** plan. **Drafted** 2026-09-06, **revised** 2026-09-06 after the operator settled the
-target architecture.
-
-| | |
-|---|---|
-| **N5 Air** | TrueNAS SCALE. **Storage only** — no VMs, no hypervisor. |
-| **Compute** | k3s **prod** cluster (`192.168.2.10/.11/.12`). Plex, sonarr, radarr, jackett, qbittorrent, seerr and nextcloud are already modelled in `apps/wave-0/`. |
-| **Dev k3s** | Does not move to the NAS. |
-| **Proxmox host** | **Decommissioned.** |
-| **Drives** | Physically relocated. None purchased. |
-
-All figures measured on the live host, not inferred.
-
---
-
-## Already done — 2026-09-06
-
-**Reclaimed 3.56 TB.** `Storage/plex@pre-cleanup-20260828` destroyed after confirming no holds and
-no clones. Async free drained in ~90 seconds.
-
-| | before | after |
-|---|---|---|
-| `Storage` allocated | 12.3 T (75%) | **8.76 T (53%)** |
-| `Storage` free | 4.04 T | **7.60 T** |
-| `Storage/plex` used | 11.9 T | 8.33 T (0 B in snapshots) |
-
-That free space is not a nicety — **it is what lets this whole migration happen with no network
-transfer at all.** See §3.
-
---
-
-## 1. The plan changed shape
-
-The first draft carried the VMs across on their own zvols and put Proxmox on the NAS. With no
-hypervisor on the far side, that inverts:
-
-> **Every zvol is now dead weight, and every service's data has to be drained onto a plain dataset
-> before the Proxmox box dies.**
-
-Dead-weight zvols, once their contents are extracted:
-
-| zvol | size | after extraction |
-|---|---|---|
-| `Storage:vm-101/102/105-disk-0,-disk-1` (dev k3s) | 585 G | delete |
-| `Storage:vm-112-disk-0` (docker-services) | 65 G | delete |
-| `Storage:vm-103-disk-0` (nextcloud boot) | 81 G | delete |
-| `Storage2:vm-103-disk-0` (**nextcloud data**) | 4.06 T | extract, then delete |
-
-The good news is the shape of the drain. Everything that must be rescued is either already on
-`Storage`, or can be copied onto `Storage` **locally, inside the same chassis, onto the very drives
-that are moving.** With 7.60 T free there is room. **Nothing crosses the network.**
-
---
-
-## 2. Findings
-
-### 2a. "The path lies" — two confirmed instances, assume a third
-
-**Games — 422 GB, not on the Storage pool.** Challenged and re-verified; the test is decisive:
-
-```
-findmnt -T /mnt/Storage/Games   →  TARGET /                  SOURCE rpool/ROOT/pve-1
-findmnt -T /mnt/Storage/plex    →  TARGET /mnt/Storage/plex  SOURCE Storage/plex
-```
-
-`Games` is a plain directory on the Proxmox **root** filesystem that merely sits inside the Storage
-mount tree. The pool accounting closes on it exactly:
-
-```
-rpool ALLOC 540 G  =  ROOT 431 G  +  data 105 G  +  var-lib-vz 3.75 G
-      ROOT 431 G  =  Games 422 G  +  /usr 3.5 G  +  /var 5 G  +  /boot 0.4 G
-```
-
-The two Microns are a **mirror**, so the pool is ~888 G usable, not ~1.9 T — 422 G is 48% of it, and
-`zpool list` agrees at 60% capacity overall. Surprising, but the numbers do match.
-
-**Gitea — 13 MB in an anonymous Docker volume, not where its compose file points.**
-`/root/docker-compose.yml` in LXC 106 bind-mounts `/opt/gitea/data:/var/lib/data`. That path holds
-**512 bytes**. The real Gitea state is in an unnamed Docker volume:
-
-```
-.../volumes/8a1d2616…/_data/git/repositories/   12 M   home-network.git, home-network-uds.git, jakman85/…
-.../volumes/8a1d2616…/_data/data/gitea.db       11 M   SQLite, mtime today 19:00 — live
-```
-
-Copying `/opt/gitea` would migrate an empty directory and silently lose all three repositories.
-
-**Rule for the rest of the drain:** for every service, verify where the bytes actually are before
-trusting the path in a compose file, an export list or a mount table. Two for two so far.
-
-### 2b. Retiring pfSense is easier than rehoming it
-
-`pfsense` is VM 100 and **is** `192.168.1.1`. Decommissioning the host takes the entire
-`192.168.1.0/24` with it:
-
-| | |
-|---|---|
-| `.1` pfSense · `.4` iDRAC · `.9` the Proxmox host itself | die with the box |
-| `.5` nextcloud · `.6` plex · `.7` gitea · `.18` docker-services/Pi-hole | data drains, service moves to k3s |
-| `.8` unifi · `.20` monitoring · `.21` auth · `.10` semaphore | rehome or retire (§2e) |
-| `.49` kube-vip VIP · `.50/.51/.52` dev k3s | dev cluster is not moving; retire with the box |
-
-**But pfSense exists only to contain that lab LAN.** Prod k3s is already on `192.168.2.0/24` —
-pfSense's *WAN* side — and the NAS will be too. Once the lab is gone, nothing lives behind pfSense.
-**Retire it; do not rehome it.** That is only clean because everything else is leaving `1.0/24`
-anyway, which is exactly what this migration does.
-
-**A documented blocker dissolves with it.** `infra/PROD-STANDUP.md` records that prod k3s cannot
-reach the S3 backup endpoint at `192.168.1.9:9000` because prod sits on pfSense's WAN side and the
-crossing rule does not exist. With the NAS on `192.168.2.0/24`, prod reaches its storage directly
-and no pfSense rule is ever needed.
-
-**Three things to close before pfSense goes:**
-
-1. The SOHO router NATs inbound **443** to `192.168.2.102` for the legacy stack. It must be
-   repointed at prod's Gateway (`192.168.2.20`). `infra/PROD-PROVISIONING.md` §10a already flags
-   that the apex flip and the legacy cutover are the same event — **this migration is that event.**
-2. **VLANs 5 and 123** (minecraft; the arr VM, which reaches Plex media as `10.123.1.10`) are
-   pfSense-side constructs. They disappear; the k3s apps need no VLAN.
-3. The **OpenVPN split-tunnel egress gateway** currently runs on `k3s-dev-3` (ADR-0015). It is
-   already modelled for prod in `platform/vpn-egress/`, but confirm prod's gateway is live before
-   the dev node goes away.
-
-### 2c. Nextcloud is the critical path
-
-Its user data is **inside a filesystem, inside LVM, inside a 4 TB zvol, inside a VM**:
-
-```
-Storage2/vm-103-disk-0  →  zd224  →  PV /dev/zd224  →  VG vg_storage  →  LV lv_storage (<4.00 T)
-```
-
-With no hypervisor on the far side this must be copied out **at the file level, while the VM is
-still running** — it gates the decommission and it is the longest single task in the plan.
-
-**One number is missing and I could not get it.** VM 103's QEMU guest agent is not running
-(`qm agent 103 ping` → *"QEMU guest agent is not running"*) and its SSH does not accept the
-`home_network` key. The zvol shows `referenced 3.94 T`, but it is attached **without `discard=on`**
-and is thick-provisioned (`refreservation 4.06 T`), so deletions inside the guest were never
-returned to ZFS — 3.94 T is an **upper bound**, possibly far above the truth.
-
-**Run this inside the Nextcloud VM and the last open question closes:**
-
-```bash
-df -h /            # or wherever lv_storage is mounted
-lsblk -f /dev/mapper/vg_storage-lv_storage
-```
-
-Separately: `Storage/nextcloud` is 270 G, and **267 G of it is a directory called `config_backup`**
-(`config` itself is 103 M). Almost certainly a stale dump — likely another easy reclaim, but confirm
-before deleting.
-
-### 2d. Storage2 — and the two oldest drives — can probably be retired
-
-After the drain and the zvol deletions, `Storage` would hold roughly:
-
-| | |
-|---|---|
-| plex | 8.33 T |
-| nextcloud data | ≤ 3.94 T (real number pending §2c) |
-| Games | 0.42 T |
-| docker-volumes, other_containers, k8s-volumes, gitea | ~0.02 T |
-| **total** | **≤ 12.7 T of 16.4 T (≤ 77%)** |
-
-If the real Nextcloud figure comes in under ~2 T — which the missing-TRIM situation makes plausible
-— everything fits on the 18 TB mirror at a comfortable ~65%, and **`Storage2` can be retired
-entirely.** That would remove the two **6.0-year-old** Seagates, by a wide margin the most
-failure-prone hardware in the estate, and free two bays. This is the single best safety outcome
-available, and it costs nothing.
-
-### 2e. Services that simply end
-
-Decommissioning kills these; none has a home in the target architecture unless you give it one:
-
-| guest | note |
-|---|---|
-| `docker-services` Pi-hole (`192.168.1.18`) | **LAN DNS.** Confirm what `192.168.2.0/24` uses for DNS before the box dies. |
-| `unifi` (VM 121) | Network controller — needs a home or a cloud key. |
-| `auth` (VM 122) | Superseded by Keycloak on prod k3s. |
-| `monitoring` (VM 120) | Superseded by the k3s platform monitoring stack. |
-| `semaphore` (LXC 110) | 432 MB resident of an 8 GB reservation; low value. |
-| `minecraft` (LXC 108) | Operator call. Its 3.5 G subvol is on `Storage` and rides along. |
-| **versitygw S3** on `192.168.1.9:9000` | **Velero + CNPG backup target.** Dies with the box. TrueNAS must take this role (MinIO app) or prod loses its backup destination — which `infra/PROD-STANDUP.md` already records as uncovered. |
-
-### 2f. Still true from the first draft
-
- **No VM backups exist.** `/etc/pve/jobs.cfg` empty, no PBS, newest dump log April 2025.
- **`ata-WDC_…3FGHAB2T` has 36 ZFS read errors** on pristine SMART (0 reallocated / pending /
-  uncorrectable, `PASSED`) — points at cable or controller, not platter. Re-verify after the move.
- Drive ages: WD 18 TB pair **4.75 y**, Seagate 8 TB pair **6.0 y**.
-
---
-
-## 3. TrueNAS SCALE specifics
-
-**ZFS compatibility: verified, not assumed.** The only non-default features *active* on either pool
-are `block_cloning`, `head_errlog`, `vdev_zaps_v2`, `zilsaxattr`, `zstd_compress`, `log_spacemap`,
-`project_quota`, `spacemap_v2` — all OpenZFS **2.2** era or older. Nothing 2.3- or 2.4-only is
-active, even though the pools were last written by `zfs-2.4.3-pve1`. Any TrueNAS SCALE from 24.10
-onward imports them cleanly.
-
-**Mountpoints mostly already match TrueNAS convention** — a lucky accident worth knowing:
-
-| dataset | current mountpoint | TrueNAS wants | |
-|---|---|---|---|
-| `Storage/plex`, `/nextcloud`, `/docker-volumes`, `/other_containers`, `/k8s-volumes`, `/k3s-backups` | `/mnt/Storage/…` | `/mnt/Storage/…` | already correct |
-| `Storage` (pool root) | `/Storage` | `/mnt/Storage` | fix on import |
-| `Storage2` (pool root) | `/Storage2` | `/mnt/Storage2` | fix on import |
-| `Storage/subvol-108-disk-0` | `/Storage/subvol-108-disk-0` | — | rename or delete |
-
-**Clear the ZFS `sharenfs` properties.** `Storage/plex`, `Storage/nextcloud` and
-`Storage/docker-volumes` carry exports as ZFS properties; `Storage/k8s-volumes` is in
-`/etc/exports`. TrueNAS generates exports from its own middleware database and will fight
-properties it did not set. Wipe them and recreate the shares in the UI.
-
-**Fix the export targets while you are there.** Today's lists are stale: `/mnt/k3s` →
-`192.168.1.25/.26/.27` and `k8s-volumes` → `192.168.1.40/.41/.42` are hosts that do not answer ARP.
-The new exports target **`192.168.2.10/.11/.12`**.
-
---
-
-## 4. Sequence
-
-### Phase A — drain, locally, no downtime, no network
-
-Everything below is a copy from somewhere else in the chassis onto `Storage`, which is moving.
-
-1. **Games** — `rpool` → new dataset.
-   ```bash
-   zfs create Storage/games
-   rsync -aHAX --info=progress2 /mnt/Storage/Games/ /mnt/Storage/games/
-   ```
-2. **Gitea** — copy the *Docker volume*, not `/opt/gitea` (§2a). Stop the container first so
-   `gitea.db` is quiesced.
-   ```bash
-   pct exec 106 -- docker stop gitea
-   zfs create Storage/gitea
-   rsync -aHAX /rpool/data/subvol-106-disk-0/var/lib/docker/volumes/8a1d2616*/_data/ /mnt/Storage/gitea/
-   ```
-3. **Nextcloud** — the long one. Copy `lv_storage`'s contents to a new dataset at file level from
-   inside the running VM, over its existing NFS mount of the host. Verify §2c's size first so you
-   know whether it fits (it will, in 7.60 T free).
-4. **Confirm `config_backup`** (267 G) is disposable; drop it if so.
-5. **Verify each service's real data location before copying it** — the §2a rule.
-
-### Phase B — prove the compute side before the storage moves
-
-Bring the `apps/wave-0` manifests up on prod k3s while the old box is still alive, so the cutover is
-a storage swap against known-good workloads rather than two migrations at once.
-
-Wrinkle: prod cannot reach `192.168.1.9` today (§2b). Either add a temporary pfSense rule for the
-test, or accept that app bring-up happens after the NAS is live on `192.168.2.0/24` — in which case
-stage the app configs now and keep the window short.
-
-### Phase C — cutover
-
-1. Stop all guests; reverse `startup:` order, pfSense last.
-2. Export cleanly — **never just cut power:**
-   ```bash
-   zpool export Storage && zpool export Storage2
-   zpool status                  # both must be gone before power-off
-   ```
-3. **Label every drive** with pool and serial before it leaves the chassis
-   (`3FGJ6M9T`/`3FGHAB2T` = Storage; `WKD1S0TR`/`WKD1S10M` = Storage2).
-4. Move the four HDDs — cold, upright. The 6-year-old Seagates are the fragile ones.
-5. Install TrueNAS SCALE on the shipped 64 GB M.2 (Slot A). The five bays take the four HDDs with
-   one spare; the Micron SSDs are **not needed** in the target — leave them in the old chassis.
-6. **Import read-only first**, verify, then import for real:
-   ```bash
-   zpool import                          # both pools listed
-   zpool import -o readonly=on Storage
-   zfs list                              # verify datasets, sizes
-   zpool export Storage && zpool import Storage
-   ```
-
-### Phase D — repoint, reclaim, retire
-
-1. Recreate NFS shares in the TrueNAS UI targeting `192.168.2.10/.11/.12`; clear the old `sharenfs`
-   properties.
-2. Point the `apps/wave-0` PVs at the NAS; bring Plex, the arr stack and Nextcloud up on prod k3s.
-3. Stand up MinIO on TrueNAS to replace versitygw, and re-enable Velero + CNPG backups (§2e).
-4. **Only once everything is verified from k3s:** delete the dead zvols (§1) — ~731 G on `Storage`,
-   4.06 T on `Storage2`.
-5. Re-evaluate §2d. If it fits, evacuate `Storage2`, retire the 6-year-old Seagates, keep two bays
-   free.
-6. Repoint the SOHO 443 NAT, then power off the Proxmox host.
-7. `zpool status` — confirm the 36 read errors did **not** follow `3FGHAB2T` into the new chassis.
-   Full scrub on the new hardware.
-
---
-
-## 5. Rollback, and the one-way door
-
-Until Phase D step 4, the old chassis is intact: `rpool`, the Proxmox install, both Microns and
-every `/etc/pve` config untouched. Put the four HDDs back and it boots into the previous world.
-
-> ### Do not run `zpool upgrade`
->
-> `zpool status Storage2` already nags — *"Some supported and requested features are not enabled…
-> action: Enable all features using 'zpool upgrade'"* — and TrueNAS will offer it in the UI after
-> import. It is the only irreversible step here: run it and the pools can never go back to the
-> Proxmox host. Leave it until the NAS has been serving k3s for weeks.
-
-Note the asymmetry that makes this safe: TrueNAS's OpenZFS **reads** these pools happily, because
-newer ZFS always reads older pools. The reverse is not true.
-
---
-
-## 6. Open items
-
-1. **The Nextcloud number** (§2c) — one `df -h` inside VM 103. It decides whether `Storage2` and the
-   two oldest drives can be retired.
-2. **Is `config_backup` (267 G) disposable?**
-3. **What serves DNS on `192.168.2.0/24`** once Pi-hole dies?
-4. **UniFi controller** — rehome, cloud key, or drop?
-5. **Minecraft** — keep or retire?
-6. **Gitea** — its whole state is 3 repos and an 11 MB SQLite DB, and prod k3s already runs its own
-   Gitea. Is the legacy one worth migrating at all, or does it just end?
@@ -0,0 +1,315 @@
+# Off-cluster backup on Cloudflare R2 — operator procedure
+
+**Owner:** WS-BACKUP · **Serves:** R-STATE-03 (DR restore), R-IDP-05 (Keycloak DB), R-ENV-02 (one
+environment-agnostic path) · **Status:** authored and staged, **NOT switched on**.
+
+> **Read this first.** Velero-against-R2 and CNPG/barman-against-R2 are **UNPROVEN in this repo**.
+> Nothing here claims they work. What *is* proven is in [§7](#7-what-is-proven-and-what-is-not).
+> R-STATE-03 is currently PROVEN **against MinIO only** and must be re-proven against R2 before this
+> migration can be called done.
+
+## Why R2 at all
+
+The off-cluster target used to be a MinIO on the Proxmox host (`http://192.168.1.9:9000`). Two things
+killed it, both recorded on 2026-09-05:
+
+1. **Prod physically cannot reach it.** Prod is on the SOHO network `192.168.2.0/24`, which is
+   pfSense's **WAN**; the MinIO sits inside pfSense's LAN `192.168.1.0/24`. Measured *from a pod inside
+   prod* — the workstation's own reachability proves nothing about pod egress and has misled this
+   project once already — `curl http://192.168.1.9:9000/minio/health/live` timed out after 15 s.
+   Reaching it would need a WAN→LAN rule that does not exist.
+2. **Dev's "off-cluster" backup was not off-cluster.** That MinIO runs on the *same Proxmox host* as
+   dev's VMs, so it shared a failure domain with the thing it was backing up.
+
+R2 needs no firewall change, is genuinely off-site, and is already trusted with the OpenTofu remote
+state (`infra/tofu/REMOTE-STATE.md`) — the same account, the same kind of token. It is explicitly an
+interim decision: *"that's where backups will go for the time being until a better on-prem backup
+solution is created."*
+
+---
+
+## 1. Cloudflare: buckets, token, account id
+
+All of this is in the Cloudflare dashboard; none of it touches the cluster.
+
+1. **R2 → Create bucket** → `velero`. Location hint: whatever is nearest; jurisdiction default.
+2. **R2 → Create bucket** → `cnpg`.
+   Both names are fixed by the manifests: `bucket: velero` in `platform/velero/helmrelease.yaml`, and
+   `cnpg` via `CNPG_BACKUP_PATH` (`s3://cnpg/dev`, `s3://cnpg/prod`). Use exactly those names or change
+   the manifests — do not improvise.
+3. **R2 → Manage API Tokens → Create API Token**
+   - Permission: **Object Read & Write** (not "Admin Read & Write" — the cluster never needs to create
+     or delete buckets, and a token that can is a token that can delete your backups).
+   - **Specify bucket(s):** `velero` **and** `cnpg`. One token scoped to both — the manifests already
+     assume one shared key, and `s3check` proves both buckets with it.
+   - Copy the **Access Key ID** and **Secret Access Key**. They are shown **once**. Put them in the
+     password manager before you do anything else.
+4. **Account id** — 32 hex characters, visible in the dashboard URL and in the R2 S3 endpoint the
+   token page shows you.
+
+   **This repo already knows it.** `infra/tofu/backend.r2.hcl` commits
+   `https://7ce452a1b3d6c86daf2c020edcb82d37.r2.cloudflarestorage.com` for the OpenTofu state bucket,
+   with the (correct) note that an account id is not a credential. **If the backup buckets live in the
+   same Cloudflare account as the tofu state — they should; there is no reason for a second account —
+   then the endpoint is already known and you do not need to look anything up:**
+
+   ```
+   https://7ce452a1b3d6c86daf2c020edcb82d37.r2.cloudflarestorage.com
+   ```
+
+   Confirm it matches the endpoint on the token page. If it does not, you have two accounts and the
+   `velero`/`cnpg` buckets are in the other one — use that one's id, and say so in the DECISIONS-LOG.
+
+The endpoint form is always `https://<account-id>.r2.cloudflarestorage.com` — **no bucket in the
+hostname** (path-style; see [§6](#6-r2-specific-gotchas)).
+
+---
+
+## 2. Repo values to set
+
+Only **one** non-secret value is missing, and it is not in a secret file:
+
+| File | Key | From | To |
+|---|---|---|---|
+| `clusters/base/cluster-vars.yaml` | `S3_BACKUP_ENDPOINT` | `https://TBD-R2-ACCOUNT-ID.r2.cloudflarestorage.com` | `https://<account-id>.r2.cloudflarestorage.com` |
+
+Already authored, **do not change**: `S3_BACKUP_REGION: "auto"`, `VELERO_BACKUP_PREFIX` (dev `dev` /
+prod `prod`), `CNPG_BACKUP_PATH` (`s3://cnpg/dev` / `s3://cnpg/prod`), and `checksumAlgorithm: ""` on
+the Velero BackupStorageLocation.
+
+The placeholder is deliberately loud, and the guards match it as a **substring** — `bootstrap/standup.sh`
+refuses to probe it and refuses to resume `velero` while it is there.
+
+---
+
+## 3. Credentials: four Secrets, one command
+
+The R2 key is read by **four** committed SOPS Secrets, in **two** different shapes. No manifest changes
+are needed — only the encrypted values change:
+
+| File | Consumer | Shape |
+|---|---|---|
+| `platform/velero/velero-cloud.sops.yaml` | Velero (`velero-cloud-credentials`, ns `velero`) | `stringData.cloud` — an **AWS INI file**: `[default]` + `aws_access_key_id=` / `aws_secret_access_key=` |
+| `platform/keycloak/keycloak-db-backup-credentials.sops.yaml` | CNPG `keycloak-db` (ns `keycloak`) | `ACCESS_KEY_ID` / `ACCESS_SECRET_KEY` |
+| `apps/wave-0/nextcloud-db-backup-credentials.sops.yaml` | CNPG `nextcloud-db` | `ACCESS_KEY_ID` / `ACCESS_SECRET_KEY` |
+| `apps/wave-0/semaphore-db-backup-credentials.sops.yaml` | CNPG `semaphore-db` | `ACCESS_KEY_ID` / `ACCESS_SECRET_KEY` |
+
+A Secret is namespace-scoped and a Pod reads only its own namespace's, which is why there are four
+copies of one key rather than one.
+
+### The one command
+
+```bash
+./infra/devcontainer/dev.sh run platform/backup/set-r2-credentials.sh
+```
+
+It prompts for the two values (never echoed, never in shell history, never written to a file), writes
+them into all four Secrets with `sops set` — which encrypts in place, so **no decrypted copy of any
+Secret is ever created** — then decrypts each file back and proves all four now hold the same key by
+comparing 32-bit fingerprints. It never prints a credential.
+
+There is deliberately **no plaintext template to fill in and encrypt**. The
+`cp x.example x.yaml && $EDITOR && sops -e -i` pattern leaves a plaintext Kubernetes Secret sitting in
+the working tree, one `git add -A` away from being in git forever; `sops set` removes that window
+entirely. If you would rather edit by hand, `sops platform/velero/velero-cloud.sops.yaml` decrypts to
+`$EDITOR` and re-encrypts on save — also with no plaintext on disk — but then you must repeat it four
+times and keep them identical yourself.
+
+To check the four files agree without changing anything:
+
+```bash
+./infra/devcontainer/dev.sh run platform/backup/set-r2-credentials.sh --check
+```
+
+### What is NOT changed
+
+`platform/backup/minio-credentials.sops.yaml` keeps the **MinIO** key. That is not an oversight — it
+is what you need to read the pre-migration backups ([§8](#8-going-back-to-minio)). Its
+`.yaml.example` sibling documents the MinIO shape and stays accurate for the MinIO target.
+
+Review the diff before committing: you should see **only** `ENC[...]` blobs and `lastmodified`/`mac`
+change in four files. If a key name or a `metadata.name` moved, stop.
+
+---
+
+## 4. Prove the target BEFORE merging
+
+```bash
+K=infra/ansible/.cache/dev.kubeconfig        # or prod.kubeconfig
+./infra/devcontainer/dev.sh run bootstrap/standup.sh \
+  --kubeconfig $K --expect-server https://192.168.2.50:6443 \
+  --s3-endpoint https://<account-id>.r2.cloudflarestorage.com s3check
+```
+
+`--s3-endpoint` probes a **candidate** endpoint instead of the live `cluster-vars` value, so the target
+is proven before it is authored. Run it against **both** clusters — prod's egress is the whole reason
+this migration exists, and it is a different network position from dev's.
+
+`s3check` runs a throwaway pod per credential and, for each, proves:
+
+1. the committed credential **loads and authenticates**, in the exact shape its consumer reads it
+   (Velero's INI file mounted from the `cloud` key; CNPG's discrete `ACCESS_KEY_ID`/`ACCESS_SECRET_KEY`);
+2. the bucket **exists and is listable** under this environment's own prefix (`ListObjectsV2`, the
+   operation Velero's BSL sync and Barman's WAL scan both perform);
+3. **PUT → GET → byte-compare (sha256 + length) → DELETE → confirm-gone** of one throwaway object at
+   `<prefix>/.s3check/<pod>.probe` — a genuine write, not a HEAD. A HEAD passes with a read-only token
+   and with R2 rejecting every write;
+4. whether the **R2 checksum trap** ([§6](#6-r2-specific-gotchas)) is present, probed on purpose by
+   repeating the PUT with checksums *enabled*, and named explicitly either way.
+
+It **fails closed**: a pod that cannot be created, an empty log, a missing verdict line, a short
+assertion count, an OK-line count that disagrees with the claimed count, or any single failed
+assertion all exit non-zero. There is no "reachability unknown, carrying on". Each failure names its
+own cause and fix — wrong key, unscoped token, missing bucket, no pod egress, checksum rejection.
+
+It writes and deletes exactly one object per bucket and leaves no pod behind. It applies nothing and
+does not enable `velero`.
+
+---
+
+## 5. The verification sequence — and it ends in a RESTORE
+
+**A backup that has never been restored is not a backup.** R-STATE-03 is PROVEN against MinIO; that
+proof does not transfer to a different object store with a different S3 implementation.
+
+1. **`s3check` on dev, then on prod.** Both must pass. (§4)
+2. Merge the endpoint + the four re-encrypted Secrets. Let Flux reconcile.
+3. **Resume `velero`** — `bootstrap/standup.sh … resume` does it once the endpoint is no longer a
+   placeholder, or `flux resume kustomization velero`.
+4. `velero backup-location get` → `default` **Available**. *Available only means the BSL could be
+   read;* it is not a backup.
+5. `velero backup create --from-schedule daily` → wait for **Completed**, then
+   `velero backup logs <name>`. Watch specifically for `PodVolumeBackup` failures: the PVC-data path
+   is kopia, a **different S3 client** from the one the BSL uses (see §7), so metadata can succeed
+   while volume data fails.
+6. **Restore.** `velero restore create --from-backup <name> --include-namespaces <scratch-ns>` into a
+   *scratch* namespace, and verify the PVC contents actually arrived — not just that the Restore says
+   Completed. This is the R-STATE-03 re-proof.
+7. **CNPG PITR.** Confirm WAL is arriving:
+   `kubectl -n keycloak get cluster keycloak-db -o jsonpath='{.status.conditions}'` (look for
+   `ContinuousArchiving`), and that objects exist under `s3://cnpg/<env>/keycloak-db/`. Then do the
+   restore that matters: a `Cluster` with `bootstrap.recovery` from the barman object store into a
+   scratch namespace, and check the recovered database actually contains data. The existing
+   `conformance/tests/l2-behavioral/keycloak-db-dr` and `l3-fault-injection/dr-restore` tests are the
+   harness for this; they must be re-run **against R2**, and their previous PROVEN state records MinIO,
+   not R2.
+8. Only after 6 and 7 pass may this migration be described as done, and the ledger updated. Until
+   then the honest status is "R2 accepts objects; recovery from R2 is untested".
+
+---
+
+## 6. R2-specific gotchas
+
+**`region: auto`.** R2 accepts nothing else, and S3 clients require *some* region. Authored as
+`S3_BACKUP_REGION: "auto"`. This repo already learned it — `infra/tofu/backend.tf` carries
+`region = "auto"  # R2 ignores region, but the S3 backend requires a value.`
+
+**`checksumAlgorithm: ""` — do not delete this line.** `velero-plugin-for-aws`'s own
+`backupstoragelocation.md` for **v1.14.2** (the pinned version) documents:
+
+> `checksumAlgorithm` — The checksum algorithm to use for uploading objects to S3. Supported values
+> "CRC32", "CRC32C", "SHA1", "SHA256". If the value is set as empty string "", no checksum will be
+> calculated and attached to the request headers. **Optional (defaults to "CRC32")**
+
+So *omitting* the key means CRC32 **is** sent: `x-amz-sdk-checksum-algorithm` +
+`x-amz-checksum-crc32`. This repo has already measured that R2 rejects AWS checksums — that is exactly
+why `infra/tofu/backend.tf` needs `skip_s3_checksum = true  # R2 rejects AWS trailing (streaming)
+checksums.` for the state bucket. The plugin's own compatibility table lists `checksumAlgorithm=""` as
+the remedy for the same failure on Ceph S3, Backblaze B2 and IBM COS. If backups start failing with
+**HTTP 400 `InvalidArgument`** after a plugin bump, look here first. `s3check` probes for this on
+purpose and names it.
+
+**Path-style.** `s3ForcePathStyle: "true"` is authored and correct for R2 as well as MinIO. Never put
+the bucket in the hostname.
+
+**No bucket auto-creation.** Velero does not create buckets and the token deliberately cannot. A
+missing bucket is a `NoSuchBucket` at runtime; `s3check` catches it first.
+
+**Token scope is per-bucket.** A token scoped only to `velero` authenticates fine and then 403s on
+`cnpg` — the two consumers fail independently and quietly. `s3check` proves both.
+
+**Both clusters share one endpoint.** Isolation is by path only (`VELERO_BACKUP_PREFIX`,
+`CNPG_BACKUP_PATH`), because R2 is one environment-independent URL. Those two knobs are load-bearing:
+without them prod would adopt and TTL-delete dev's Velero backups, and both clusters' identically-named
+CNPG Clusters would interleave their WAL into one Barman stream, destroying PITR for both.
+
+---
+
+## 7. What is proven, and what is not
+
+**Proven (2026-09-05, WS-BACKUP, against the live dev cluster and its MinIO at
+`http://192.168.1.9:9000`):**
+
+- the rewritten `s3check` itself: 7 assertions per credential path, both credential shapes, from
+  in-cluster pods — credential load, bucket list, PUT/GET/byte-compare/DELETE/confirm-gone, no object
+  and no pod left behind;
+- its failure paths: the `TBD` placeholder is refused; an unreachable endpoint (`:9099`) produces a
+  named no-egress failure and exit 1, not a pass;
+- MinIO accepts a checksum-bearing PUT, i.e. the trap is a *non-*MinIO problem, and
+  `checksumAlgorithm: ""` is harmless there.
+
+**Established from the pinned versions (documentation and image inspection, not measured against R2):**
+
+- `velero-plugin-for-aws` **v1.14.2** pairs with Velero **v1.18.x** (upstream compatibility table) and
+  defaults `checksumAlgorithm` to `CRC32` — so the authored `""` is load-bearing, not decoration.
+- That plugin's "Non-AWS S3 compatible provider known issues with plugin v1.14.x" table **does not
+  mention Cloudflare R2 at all** — neither as working nor as broken. There is no upstream statement to
+  lean on in either direction.
+- **The PVC-data path is a different S3 client.** `checksumAlgorithm` is a BackupStorageLocation
+  *object-store plugin* setting. Velero's File System Backup uploads through the unified repository's
+  kopia S3 backend (`pkg/repository/udmrepo/kopialib/backend/s3.go` in Velero v1.18.0), which
+  references only endpoint/region-style options and has **no checksum knob at all**. So R2
+  compatibility for backup *metadata* and for *volume data* are two separate questions, and only step
+  5 of §5 answers the second.
+- **CNPG/barman is likely clear of the checksum trap, by version.** Inspected inside the live
+  `ghcr.io/cloudnative-pg/postgresql:16.4` image on dev: `barman-cloud-wal-archive 3.11.1`,
+  `botocore 1.35.60`, `boto3 1.35.60`. That botocore predates the 1.36 line in which botocore began
+  attaching CRC32 request checksums by default (the change that introduced
+  `AWS_REQUEST_CHECKSUM_CALCULATION`), so this image should not send them.
+  **Forward risk:** barman-cloud exposes **no** `checksumAlgorithm` equivalent. If the CNPG image is
+  ever bumped to one carrying botocore ≥ 1.36, WAL archiving to R2 could start failing with 400
+  `InvalidArgument` and there is no config knob to turn it off — only `AWS_REQUEST_CHECKSUM_CALCULATION=when_required`
+  in the Pod environment, which CNPG does not currently set. Treat a CNPG image bump as a change that
+  must be re-verified against R2.
+
+**Not proven, at all:**
+
+- that Velero can write a usable backup to R2 (metadata or kopia volume data);
+- that CNPG/barman can archive WAL to R2 and that a `bootstrap.recovery` from R2 succeeds;
+- that a restore from R2 yields correct data. R-STATE-03's PROVEN state is **MinIO-only**.
+
+---
+
+## 8. Going back to MinIO
+
+The MinIO copy is **not deleted** and stays restorable. To read pre-migration backups:
+
+1. `clusters/base/cluster-vars.yaml` → `S3_BACKUP_ENDPOINT: "http://192.168.1.9:9000"`
+2. `VELERO_BACKUP_PREFIX` → `""` (the MinIO backups are at the **bucket root**, not under `dev/`) and
+   `CNPG_BACKUP_PATH` → `s3://cnpg` (Barman objects are at `s3://cnpg/<clusterName>/`).
+3. Restore the MinIO credential into the four Secrets. `platform/backup/minio-credentials.sops.yaml`
+   still holds it: `sops -d --extract '["stringData"]["ACCESS_KEY_ID"]'` it and feed the two values to
+   `platform/backup/set-r2-credentials.sh` (the script cares about the *shape*, not the provider).
+4. `S3_BACKUP_REGION` may stay `auto` — MinIO ignores the region.
+5. **Dev only.** Prod cannot reach that MinIO at all (§Why R2); pointing prod back is not an option.
+
+Reverting for a restore and then returning to R2 costs another PITR reset. Prefer restoring from MinIO
+into a *scratch* namespace on dev with a second, temporary BackupStorageLocation over flipping the
+whole cluster's endpoint back and forth.
+
+---
+
+## 9. Merging this resets dev's PITR window
+
+Say it out loud before merging, because it is not recoverable by retrying:
+
+- Dev's R2 prefix starts **empty**. Velero writes a **fresh kopia repository** — the first backup is a
+  full upload of every included PVC, and no earlier restore point exists in R2.
+- Every CNPG Cluster takes **new base backups** (`ScheduledBackup … immediate: true`), and the 30-day
+  Barman retention window (`retentionPolicy: "30d"`) rebuilds **from zero**. For roughly a month, dev's
+  point-in-time recovery reaches back only as far as the merge.
+- The MinIO copy still holds the old window (§8), but reaching it means reverting the endpoint — so it
+  is a deliberate excursion, not a fallback that happens automatically.
+
+**Do not merge this while dev's proven MinIO restore is the only recovery path you are relying on.**
+The same applies to prod the day it first backs up anywhere.
@@ -102,12 +102,10 @@ check runs instead of silently skipping.
 > `KUBECONFIG` to `dev.kubeconfig`, leaving an undocumented — and misleadingly named —
 > `DEV_KUBECONFIG=<prod path>` as the only route to prod. Select with **`CLUSTER=dev|prod`**;
 > **dev remains the default**, so every existing invocation means what it always meant. Inside the
-> container `KUBECONFIG` is a chain of a tiny per-invocation selector file over a merged config
-> carrying both clusters as contexts **`k3s-dev`** and **`k3s-prod`** (never `default`), regenerated
-> each run by [`infra/devcontainer/kubeconfig.sh`](devcontainer/kubeconfig.sh) into the gitignored
-> `infra/ansible/.cache/clusters.kubeconfig`. The merged file deliberately carries **no**
-> `current-context`: it is shared by every container, and stamping the current cluster into it made
-> concurrent `CLUSTER=dev` and `CLUSTER=prod` runs silently swap clusters mid-command. Naming matters: k3s writes *both* source kubeconfigs
+> container `KUBECONFIG` is a merged config carrying both clusters as contexts **`k3s-dev`** and
+> **`k3s-prod`** (never `default`), regenerated each run by
+> [`infra/devcontainer/kubeconfig.sh`](devcontainer/kubeconfig.sh) into the gitignored
+> `infra/ansible/.cache/clusters.kubeconfig`. Naming matters: k3s writes *both* source kubeconfigs
 > with context, cluster and user all called `default`, and kubectl's merge keeps the first
 > duplicate — so a plain `KUBECONFIG=dev:prod` would silently resolve `default` to **dev** while you
 > believed you were on prod. Requesting a cluster whose kubeconfig is absent is a hard error, never
@@ -10,6 +10,11 @@ hardware until the blockers in §7 are closed and `make -C ci gate` is green.
 > **R-ENV-01/02/03** — the very requirements this document serves. We proceed with prep work
 > knowingly; see `DECISIONS-LOG.md` (2026-08-31 Phase-7 entry).

+> **Looking for the commands?** This document is the *decision record* — why prod is built the way it
+> is, what was measured, and which hypotheses are dead. The **executable sequence** lives in
+> [`PROD-STANDUP.md`](PROD-STANDUP.md): every command, in order, driven by `bootstrap/standup.sh`.
+> Start there; come back here for the reasoning behind any step.
+
 Prod is **3× mini-PC bare metal** ([spec docs/02](../../home-network-k3s/docs/02-environments-and-provisioning.md)).
 The dev path (OpenTofu → Proxmox VM → cloud-init) **does not apply**: there is no hypervisor API to
 call. Stage 1 "Create" is a human imaging a box. Stages 2–4 (baseline, cluster-join, manage) reuse
@@ -28,9 +33,9 @@ Fill a row per node as it lands. Node 1 is as-found on 2026-08-31.
 | Model | HP EliteDesk 800 G5 Desktop Mini | HP EliteDesk 800 G5 Desktop Mini | HP EliteDesk 800 G5 Desktop Mini |
 | CPU | i7-9700T, 8c/8t, VT-x | i7-9700T, 8c/8t, VT-x | i7-9700T, 8c/8t, VT-x |
 | RAM | 31 GiB (2×16 GB Micron) | **2.6 GiB usable of 16 GB — see §0b** | 31.1 GiB usable of 32 GB (2×16 GB Crucial `CT16G4SFD8266.C16FE`, **a matched pair**) |
-| Disks | 1× 466 GB SATA SSD (`Samsung SSD 850 EVO 500GB`) — **replaced 2026-09-07, was `C400-MTFDDAC512M` (~2011); see §0c** | 1× 477 GB **NVMe** (`PM9A1 Samsung`) | 1× 466 GB SATA SSD (`WDC WDS500G2B0A-00SM50`) |
-| Drive health | PASSED, **0 reallocated / 0 grown bad blocks**, `Wear_Leveling_Count` raw 106, but **77,047 power-on hours** (~8.8 y) over only 219 power cycles | **96% life**, 2142 h, PASSED | **100% life**, 3213 h, PASSED, 0 grown bad blocks / 0 erase / 0 program fails, 0 CRC |
-| OS | Debian 13 (trixie), kernel **7.1.8+deb13-amd64** | Debian 13 (trixie), kernel **7.1.8+deb13-amd64** | Debian 13 (trixie), kernel **7.1.8+deb13-amd64** |
+| Disks | 1× 477 GB SATA SSD (`C400-MTFDDAC512M`, ~2011) | 1× 477 GB **NVMe** (`PM9A1 Samsung`) | 1× 466 GB SATA SSD (`WDC WDS500G2B0A-00SM50`) |
+| Drive health | 69% life, 15 grown bad blocks, 187 erase fails | **96% life**, 2142 h, PASSED | **100% life**, 3213 h, PASSED, 0 grown bad blocks / 0 erase / 0 program fails, 0 CRC |
+| OS | Debian 13 (trixie), kernel 6.12 | Debian 13 (trixie), kernel 6.12 | Debian 13 (trixie), kernel 6.12.107+deb13-amd64 |
 | Firmware | HP R21 Ver. 02.26.00 | HP R21 Ver. 02.26.00 | HP R21 Ver. 02.26.00 |
 | `/dev/kvm` | present | present | present |
 | Serial | MXL1052NGN | MXL0073949 | 8CC015307N |
@@ -48,47 +53,6 @@ modern NVMe; node 3 a third distinct model, a WD Blue 3D SATA SSD in excellent h
 an identical disk layout across all three because Longhorn replica placement assumes symmetry — so
 this is a live input to the disk decision, not a footnote. Node 2's NVMe is the drive node 1 should
 have had; node 3 does not have it either. Node 3 is also the only node with **two matched DIMMs**
-
-### §0c — node 1's drive was replaced (2026-09-07)
-
-**The paragraph above is history, not current state.** On 2026-09-07 node 1's kubelet stopped
-posting status while the host stayed up and SSH-able. Root cause was **etcd starved of disk I/O**,
-not memory: `slow fdatasync` at 3.5–6.7 s against an expected 1 s, `apply request took too long` at
-8.3 s against 100 ms, then `etcdserver: too many requests`. No OOM, MCE or hung task in the kernel
-log. All three nodes place etcd's WAL (`/data/k3s`) and every Longhorn replica (`/data/longhorn`) on
-the SAME partition; node 1's was the 2011 C400 with 61,440 reallocated sectors and ~31% of its
-endurance consumed, and its `/proc/pressure/io` `full avg300` ran **11.36** against ~1% steady state
-on the other two. The failure is self-reinforcing: the node dropping out degrades Longhorn volumes,
-whose rebuilds add write load to the very disk etcd needs.
-
-The operator replaced the drive with a **Samsung SSD 850 EVO 500GB** and reinstalled Debian.
-Measured under the same load class — an active 18-replica Longhorn rebuild, with node 1 as the
-*write target*, i.e. the heavier side:
-
-| node | drive | `full avg300` during rebuild |
-|---|---|---|
-| `k3s-prod-1` | Samsung 850 EVO (new) | **3.79** |
-| `k3s-prod-2` | Samsung PM9A1 NVMe | 0.33 |
-| `k3s-prod-3` | WDC WDS500G2B0A | 2.47 |
-| *`k3s-prod-1` before* | *C400-MTFDDAC512MAM* | ***11.36*** |
-
-Roughly a **3× improvement**, moving node 1 from ~4× worse than its peers into the same band as
-node 3, and it stayed `Ready` throughout. **The 850 EVO is not new**: 77,047 power-on hours over 219
-power cycles — it evidently idled in an always-on system — but its wear tells the opposite story to
-the C400's (0 reallocated vs 61,440; wear-leveling 106 vs 949).
-
-**§2's exit condition is NOT met by this replacement.** etcd and Longhorn still share one SATA
-spindle. The contention is reduced, not eliminated; giving etcd its own device remains the fix, and
-node 1 still has exactly one disk. The three nodes also remain asymmetric — now 850 EVO / PM9A1
-NVMe / WD Blue.
-
-The rebuild itself surfaced two things worth carrying forward. Longhorn refused the new disk with
-`DiskFilesystemChanged` — "record diskUUID doesn't match the one on the disk" — because its node CR
-still held the old drive's `diskUUID`; the remedy is disable → remove the disk entry → re-add, in
-that order (the validating webhook enforces it). And the Ansible inventory still listed node 1 in
-`[k3s_prod_primary]`, which decides both which play gets `--cluster-init` *and* what `k3s_api_host`
-resolves to — so re-provisioning it unchanged would have pointed a rebuilt, empty node at itself and
-founded a second cluster beside the live one. The founder now sits on node 2.
 (2× Crucial 2Rx8 DDR4-2666) and the only one with its **full** RAM complement of 32 GB actually
 usable. **That is luck, not immunity:** node 3's firmware publishes an EFI Memory Attributes Table
 too (`MEMATTR=0xbeece018`), the kernel simply parsed it cleanly on this boot, so node 3 is exposed to
@@ -820,74 +784,6 @@ disk each, already fully partitioned, no free slot verified populated.
  physically) and flip `node_baseline_etcd_source` back to `device` — the role still
  supports that path unchanged, and it is the dev shape.

-#### THE MEASUREMENT CAME BACK — 2026-09-07. It does not hold on node 1.
-
-Not under a deliberate test: in production, 2 days 7 hours after the cluster came up.
-**06:05:21Z `k3s-prod-1` went NotReady** (`Kubelet stopped posting node status`) while the
-host stayed up, pingable and SSH-able. From the k3s journal:
-
-```
-slow fdatasync                took=3.56s / 6.71s / 4.95s / 3.40s   (etcd expects 1s)
-apply request took too long   took=8.30s                           (expects 100ms)
-etcdserver: too many requests
-Failed to check local etcd status for learner management: context deadline exceeded
-```
-
-No OOM, no MCE, no hung task in the kernel log — **this is not §0b and not memory**. Recovery
-required a manual `systemctl restart k3s`; 18 Longhorn volumes then rebuilt, and I/O pressure
-only fell once the rebuilds finished. The failure is **self-reinforcing**: the node dropping
-out degrades volumes → Longhorn rebuilds → more write load on the disk etcd needs → etcd
-stalls further.
-
-`/proc/pressure/io` **full avg300**, all three nodes, during the incident:
-
-| node | device | drive | IO PSI full avg300 |
-|---|---|---|---|
-| k3s-prod-1 | `/dev/sda5` | C400-MTFDDAC512MAM (2011 consumer SATA SSD, no PLP) | **11.36** |
-| k3s-prod-2 | `/dev/nvme0n1p5` | Samsung PM9A1 NVMe | 0.75 |
-| k3s-prod-3 | `/dev/sda5` | WDC WDS500G2B0A (WD Blue SATA) | 1.29 |
-
-The prediction in the paragraph above — "every prod node is SSD or NVMe end-to-end, where
-concurrent rebuild I/O costs latency rather than collapsing it" — was **right about node 2 and
-wrong about node 1**. A 2011 consumer SATA SSD with 61,440 reallocated sectors, 31% of its
-endurance consumed and no power-loss protection is not the "SSD" that reasoning assumed. §0's
-"three nodes, three different drive models" is the fault line, exactly where §0 said it would
-be.
-
-**The exit condition is therefore OPEN and the second bullet applies.** Replacing node 1's
-drive, or giving etcd its own device, is the fix, and it is the operator's.
-
-Pending that, three SOFTWARE mitigations shipped on 2026-09-07 (branch
-`fix/etcd-io-priority`). **None of them is the fix**, and each says so where it lives:
-
-1. **Observability first** — `NodeIOPressureHigh` / `NodeIOPressureCritical` in
-   `platform/monitoring/alerts.yaml`, plus an I/O row on the node-pressure dashboard. The
-   stack alerted on CPU and memory PSI and had **no I/O alert at all**, so the first signal
-   of this outage was a node already out of the cluster. Thresholds (5% / 10% full IO PSI)
-   come from the numbers in the table above. Note the etcd histograms this section's first
-   bullet asks for are **not available**: kube-prometheus-stack's `kubeEtcd` scrape job is
-   deliberately disabled for k3s (`platform/monitoring/kube-prometheus-stack.yaml`), so PSI
-   is the signal that actually exists. Enabling the etcd scrape would make that bullet
-   directly measurable and is worth doing.
-2. **Longhorn concurrency ceilings** — `platform/longhorn/helmrelease.yaml`. Honestly:
-   `concurrentReplicaRebuildPerNodeLimit: 1` was **already in force during this outage** and
-   held; the 18 rebuilds ran one at a time and the node stalled anyway. Longhorn offers no
-   further rebuild lever for the v1 data engine (`replicaRebuildingBandwidthLimit` is
-   v2/SPDK only; `rebuildConcurrentSyncLimit` is already at its floor of 1). What was
-   tightened is every *other* Longhorn I/O source — restore, snapshot purge, backup, engine
-   upgrade — each of which defaulted to 5 concurrent operations per node and each of which
-   could reproduce this on its own.
-3. **Block-layer arbitration** — `roles/etcd_io_priority`, enabled for `[k3s_prod]`, applied
-   only when the operator runs `playbooks/etcd-io-priority.yml`. BFQ + `IOWeight` so etcd's
-   cgroup outweighs `kubepods.slice`. Its README is explicit that most of the obvious knobs
-   here (`IOSchedulingClass`, `io.latency`, bare `IOWeight`) are **inert on these nodes**,
-   and that even this one cannot arbitrate the ext4 journal both tenants share.
-
-The last point is the one that matters for this section: **no cgroup or scheduler control
-separates two tenants on one filesystem.** etcd's `fdatasync` waits on a jbd2 journal commit,
-jbd2 is a kernel thread in the root cgroup, and Longhorn's writes dirty the same journal. The
-software mitigations raise the load level at which this happens. They do not remove it.
-
 **~~Known, separate consequence — `/var` is only 12 GiB.~~ RESOLVED 2026-09-04 by moving
 k3s's data dir.** This was recorded here on 2026-09-03 as a capacity item "to watch or
 repartition". Re-measuring it against dev turned it from a watch item into a blocker: the
@@ -1369,6 +1265,45 @@ no extra vars is byte-for-byte the dev behaviour it always had (verified by a be
            `inventory/group_vars/k3s_prod/vars.yml` overrides it to `clusters/prod` — the same
            mechanism as `kube_vip_address` / `k3s_data_dir` / `cluster_kubeconfig_name`. Dev has no
            override anywhere, so dev resolves `clusters/dev` exactly as before.
+- [x] **The Kyverno CRDs had to be applied BY HAND, and nothing in the repo did it** — AUTHORED
+      2026-09-04 (`bootstrap/flux/preapply-crds.sh`), **not yet run against prod**. This is a hard
+      **prerequisite of a reconcilable cluster**, not a nice-to-have: `platform/kyverno/admission/
+      helmrelease.yaml` sets `crds: {install: false}` — deliberately, because the CRD bundle is
+      **5,636,155 bytes across 22 CRDs** at chart 3.9.0 (measured 2026-09-04 with `helm template
+      kyverno kyverno --repo https://kyverno.github.io/kyverno/ --version 3.9.x --include-crds`), far
+      past kubectl's 256 KiB `last-applied-configuration` annotation limit, so Helm's client-side apply
+      fails `metadata.annotations: Too long` and **stalls** the HelmRelease. The consequence nobody
+      had automated: with CRD install off, **nothing installs those CRDs at all**. On a cold cluster
+      Kyverno's Deployments come up with no API types, `platform` never goes Ready, and all 9
+      downstream Kustomizations stall behind it. Dev only ever had them because the command was once
+      run in somebody's shell — the exact undocumented manual step this project exists to delete.
+      - **The fix, in git:** `bootstrap/flux/bootstrap.sh` now runs `./preapply-crds.sh` as step
+        **[2/4]** — after `flux install`, before `kubectl apply -k ${FLUX_ENV_PATH}`. It renders the
+        chart, keeps only `kind: CustomResourceDefinition` documents, applies them
+        `--server-side --force-conflicts` (no annotation, so no size limit), then waits for
+        `condition=Established`. The chart name, version constraint, repo URL and namespace are
+        **derived from that same HelmRelease/HelmRepository** — there is no second copy of the version
+        to drift, and a Renovate bump of the one pinned line carries this step with it.
+      - **No environment fork (R-ENV-02):** the script names no environment and takes no overlay
+        input; it reads repo files and acts on `$KUBECONFIG`. Dev and prod run the identical step, so
+        `roles/flux_bootstrap` needed **no change** — it already calls `bootstrap.sh`, the single
+        entrypoint.
+      - **It cannot silently skip.** Three guards, and deliberately no opt-out flag: a derived
+        coordinate that is empty/`null` aborts; a render that yields **0** CRDs aborts (an empty
+        stream would otherwise `kubectl apply` cleanly and exit 0 — the silent-success failure mode
+        that has bitten this repo repeatedly); and every applied CRD must reach `Established`.
+      - **Toolchain:** needs `helm` and `yq` on PATH in addition to `flux`/`kubectl` — both are baked
+        into `infra/devcontainer` (`HELM_VERSION=3.16.0`, `YQ_VERSION=4.44.6`), and `bootstrap.sh`
+        now fails at step 0 if either is missing. It also needs **outbound network to
+        `kyverno.github.io`** from wherever the play runs (the Ansible role delegates to localhost).
+      - **PM to verify in the container** (the authoring agent ran it only with a stubbed `kubectl`,
+        never against a cluster): pinned `helm` 3.16.0 renders the same 22 CRDs, and the run is a
+        clean no-op the second time.
+      - **Unblocking the already-bootstrapped prod cluster** does not need a full re-bootstrap:
+        ```bash
+        # inside infra/devcontainer, KUBECONFIG=.cache/prod.kubeconfig
+        bash bootstrap/flux/preapply-crds.sh
+        ```
 - [ ] Confirm prod's backup target: dev's S3 (`versitygw` on the Proxmox host, `192.168.1.9`) is on
      the dev side of the network. Off-cluster backup is DR-critical (R-STATE-03).

@@ -1381,7 +1316,11 @@ single-server-etcd window, and avoids a later member-add dance.
 1. Per-node §1–§5 on each of the three boxes.
 2. §7 repo work lands on `main` with the gate green.
 3. `node-baseline` → `k3s` (all three) → `kube-vip` → `cilium`.
-4. `flux bootstrap ... --path=clusters/prod`.
+4. `flux bootstrap ... --path=clusters/prod` — i.e. `playbooks/flux-bootstrap.yml -e k3s_hosts=k3s_prod`,
+   which runs `bootstrap/flux/bootstrap.sh` with `FLUX_ENV_PATH=clusters/prod`. That script's step
+   **[2/4]** pre-applies the 22 Kyverno CRDs server-side; **without it `platform` cannot go Ready**
+   and the 9 tiers below it never reconcile (§7). It needs `helm` + `yq` on PATH and outbound network
+   to the chart repo.
 5. Conformance against prod (§9).

 ## 9. Exit criteria (Phase 7)
@@ -1397,7 +1336,122 @@ single-server-etcd window, and avoids a later member-add dance.
      distinct `EXTERNALDNS_TXT_OWNER_ID`, prod wildcard cert `*.prod.home.keithjackson.dev`, and
      prod's own Keycloak realm + per-service client secrets.

-## 10. DNS/TLS environment split — **the 2026-09-04 incident, and phase 1 of the fix**
+## 9a. PROD BRING-UP STATE (2026-09-04) — what came up, and the TWO blockers that need the operator
+
+Prod's `platform` tier was unsuspended after the Kyverno CRDs were pre-applied (see §7). Result:
+**8 of 9 HelmReleases Ready, 76/76 pods Running** — cert-manager, CNPG, Cilium, Longhorn,
+kube-prometheus-stack, Loki, Alloy, external-dns. `platform` itself is **not** Ready, and the 10
+downstream tiers remain blocked behind it. Two distinct problems, both needing an operator decision.
+
+### The CRD pre-apply itself: WORKED, and is now proven against a live API server
+
+22 CRDs applied server-side and all reached `Established`; a second run was a clean no-op (`rc=0`),
+so the idempotency claim is proven, not asserted. Reproduced under the **pinned** toolchain
+(helm 3.16.0, yq 4.44.6) matching an independent helm 3.20.0 run byte-for-byte: `3.9.x` → 3.9.0,
+22 CRDs, 5,636,155 bytes.
+
+**The CRDs were the real cause of a 4½-hour crash loop.** Kyverno's four Deployments had **55
+restarts each** and stabilised within minutes of the CRD apply — its controllers were coming up with
+no API types to watch. That is the failure mode §7 predicted, observed.
+
+### BLOCKER 1 — Kyverno's Helm release is stuck in `uninstalling`
+
+Diagnosis chain, from the HelmRelease conditions:
+
+```
+Released:   InstallFailed   — Helm install failed ...: context deadline exceeded
+Remediated: UninstallFailed — Helm uninstall remediation ... timed out waiting for the condition
+Ready:      StateError      — unable to determine state for release with status 'uninstalling'
+```
+
+The install timed out **because** the pods were crash-looping without CRDs. `install.remediation.
+retries: 3` then triggered an uninstall, and that uninstall itself timed out — leaving
+`sh.helm.release.v1.kyverno.v1` with `status=uninstalling`. helm-controller cannot proceed from that
+state, so `platform` (which has `wait: true`) never goes Ready and all 10 tiers below it stay blocked.
+
+**The Deployments are healthy and at the right chart version** (kyverno-3.9.0), so this is purely a
+stale Helm bookkeeping record, not a broken workload.
+
+**Remediation requires a destructive command and was deliberately NOT run unattended:**
+
+```
+helm uninstall kyverno -n kyverno --no-hooks --wait=false   # clear the stuck release
+flux reconcile helmrelease kyverno -n kyverno --force        # let Flux install cold, CRDs now present
+```
+
+This is safe on prod specifically because prod carries no real workloads yet, and a clean cold
+install is the stronger outcome: it *proves* the bootstrap path this repo now automates. Note the
+`failurePolicy: Fail` webhooks (`kyverno-policy-*`, `kyverno-exception-*`, `kyverno-cleanup-*`,
+`kyverno-global-context-*`) gate only **Kyverno's own** resource types, so a brief absence does not
+block ordinary workload admission. The pre-applied CRDs live **outside** the Helm release and
+survive the uninstall, which is exactly why the pre-apply had to be a separate step.
+
+### BLOCKER 2 — prod's external-dns is an ARMED footgun: the installed release predates the fix
+
+**This is the more dangerous of the two.** Prod's live external-dns Deployment renders:
+
+```
+--policy=sync  --registry=txt  --domain-filter=keithjackson.dev  --provider=cloudflare
+```
+
+**and NO `--txt-owner-id`.** Absent that flag external-dns falls back to the chart default `default`
+— which is precisely the owner under which **all 36 of dev's TXT registry entries are recorded**.
+Scaling this Deployment to 1 would make prod claim dev's records and, under `policy: sync`,
+**delete them: the 2026-09-04 incident, exactly.**
+
+The repo is CORRECT and is not the problem. `platform/externaldns/helmrelease.yaml` sets
+`txtOwnerId: "${EXTERNALDNS_TXT_OWNER_ID}"`, prod's applied `cluster-vars` resolves it to `prod`, and
+the **live HelmRelease `spec.values` carries `txtOwnerId: "prod"`**. Verified against the real chart:
+`helm template external-dns --version 1.21.1` with `txtOwnerId: prod` renders `--txt-owner-id=prod`,
+so the values key is right and an upgrade *would* fix it.
+
+**The fault is Helm/Flux state, not authoring.** `helm get values external-dns` shows the INSTALLED
+release carries `policy` and `domainFilters` but **no `txtOwnerId`** — it predates `c8c1d29`. Yet
+helm-controller reports `Ready=True`, `observedGeneration=3`, and a history entry whose
+`configDigest` **equals** `lastAttemptedConfigDigest`, i.e. **it believes it has already deployed the
+current config.** So it will never upgrade on its own, and the stale release persists indefinitely.
+
+**The only thing preventing the incident was `replicas=0` — manual drift from the incident recovery,
+which any Helm upgrade would reset to the chart default of 1.** That is not a guard; it is luck.
+
+Containment applied (reversible, and it costs nothing because prod publishes no hostnames yet — the
+app tiers are suspended): **`flux suspend helmrelease external-dns -n external-dns`**, with
+`replicas` left at 0. Flux can now neither upgrade the release nor reset the replica count.
+
+**Verified remediation, in this order — the ordering is the whole point:**
+
+1. Force the upgrade so the release picks up `txtOwnerId`, but **do not let the pod run before the
+   flag is confirmed present.** The naive `resume` + `reconcile --force` resets `replicas` to 1 and
+   starts the pod in the same operation, so there is no window in which to check — a race against a
+   process whose failure mode is deleting dev's DNS.
+2. Confirm the live Deployment args contain **`--txt-owner-id=prod`**.
+3. Only then allow it to scale to 1, and immediately diff the Cloudflare zone against a
+   pre-recorded snapshot.
+
+**Dev was verified unharmed throughout this work:** the Cloudflare zone was snapshotted before
+unsuspending `platform` and re-queried after — **57 records, byte-for-byte identical, all 36 TXT
+entries still `owner=default`.** Snapshots are taken against the **Cloudflare API**, never a local
+resolver (a pfSense override produced a false healthy reading during the incident).
+
+> ### THE APEX INVERSION IS NOT ON THIS BRANCH
+>
+> Sections 10 and 10a below describe the **target** DNS layout — prod on the bare apex, dev on
+> `dev.home.` — and the ordered runbook for getting there. **That change is deliberately not authored
+> on this branch.** It is one commit, `9137bbd`, kept on branch **`migration/dns-apex-inversion`**,
+> because merging it *is* dev's migration: `clusters/dev/sync.yaml` reconciles `./clusters/dev` from
+> `main` every 10 minutes, so a merge executes the cutover on the live dev cluster unattended.
+>
+> On this branch **prod is authored on `prod.home.keithjackson.dev`**, which is what prod actually
+> runs. Read what follows as the plan, not as a description of the current authored state. The
+> inversion is applied as step **D3** of the §10a runbook, operator present.
+
+## 10. DNS/TLS environment split — **the 2026-09-04 incident, and the target architecture**
+
+> **Current state (2026-09-04):** the target below is **AUTHORED, NOT APPLIED.** Prod's
+> `platform` tier and its 10 downstream tiers are SUSPENDED. Dev is **live on the apex**. The
+> ordered cutover is **§10a**, and merging the authored change to `main` is step **D3** of it,
+> not step one — dev self-reconciles `main`, so a merge outside the runbook window executes
+> the migration unattended.

 ### What happened

@@ -1518,26 +1572,205 @@ Deliberately not attempted here: dev is **live** (18 Longhorn volumes, working S
 Gitea that is currently this repo's GitOps source. Moving it is a migration, not a variable change.
 Required, roughly in order:

-1. **`BASE_DOMAIN` → `dev.home.keithjackson.dev`** in `clusters/dev`, plus `k3s_base_domain` for the
-   dev inventory group.
-2. **A new `*.dev.home.keithjackson.dev` wildcard cert** issued and Ready *before* cutover.
-3. **Keycloak issuer change.** The realm's issuer URL is baked into every token; every OIDC client
-   (`grafana`, `gitea`, `nextcloud`, `semaphore`, `headlamp`, `crafty`, `pihole`, `jackett`,
-   `qbittorrent`, `radarr`, `sonarr`, and the `kubernetes` public client) needs its redirect URIs
-   updated in the same change, and the k3s apiserver `oidc-issuer-url` with it — which means a
-   **rolling k3s restart** on the dev servers.
-4. **kubectl SSO re-issue** for every user (issuer changed).
-5. **`gitea.dev.home.keithjackson.dev`** — this is the GitOps source URL in
-   `clusters/dev/kustomization.yaml`. Flux must be able to resolve and trust the new host *before*
-   the URL is switched, or dev stops reconciling and the change cannot be pushed to fix itself.
-   Sequence this last, and keep the bootstrap Gitea available as the recovery path (docs/05).
-6. **`EXTERNALDNS_TXT_OWNER_ID` for dev.** Once dev's names move, its existing `default`-owned
-   records are for names dev no longer serves; retire them deliberately rather than letting a
-   renamed owner orphan them.
-7. **Apex CNAMEs** `<svc>.home.keithjackson.dev` → `<svc>.prod.home.keithjackson.dev`. This is what
-   actually **moves user traffic to prod**, and it must not land before prod serves those services.
-8. **The L2 behavioural conformance tests hardcode `home.keithjackson.dev`** (`dns-exposure`,
-   `dns-internal-resolution`, `gateway-ingress-reachable`, `oidc-sso-grafana`, `sso-forward-auth`,
-   `trusted-tls`). They are test inputs, not reconciled manifests, so they were left alone in phase 1
-   — but they will fail against prod and must be parameterized before §9's "suite passes green
-   against the prod cluster" can be claimed.
+**(2b) AND THE MERGE ITSELF IS THE TRIGGER.** `clusters/dev/sync.yaml` is a self-managing root
+Kustomization reconciling `./clusters/dev` from **`main`** every 10 minutes, and prod tracks `main`
+too. **A merge to `main` therefore *executes* this migration on the live dev cluster, unattended.**
+The authored change is a *target*, not a deployment. **Do not merge these files to `main` outside the
+runbook window.** Park them on a branch until the operator is at the keyboard and step **D1** has
+been done.
+
+### D — dev vacates the apex (operator present, in this order)
+
+- [ ] **D0. Snapshot the escape routes before touching anything.** Confirm a working break-glass
+      **cert-based** admin kubeconfig for dev (not the OIDC one — OIDC breaks in D6). Confirm the
+      retained bootstrap Gitea can be started (`bootstrap/gitea/bootstrap-gitea.sh`; dev's historical
+      recovery host was `192.168.2.103:3000`, prod's current bootstrap is `192.168.2.132:3000`).
+      Record dev's current Cloudflare record set (names + TXT registry owners) so D7 has a checklist
+      and so an accidental deletion is detectable.
+- [ ] **D1. Break the automatic trigger.** Either `flux suspend kustomization flux-system -n
+      flux-system` on dev (stops the root from self-applying the merge), **or** re-point dev's
+      `GitRepository` url at the bootstrap Gitea for the window. Do this **before** the merge, not
+      after. This is what converts "merging strands dev" into "merging is inert until you say go".
+- [ ] **D2. Publish and prove the new host set *while dev still serves the apex*.** The new names
+      must exist before anything depends on them:
+      - a `*.dev.home.keithjackson.dev` + `dev.home.keithjackson.dev` **wildcard Certificate issued
+        and `Ready`** (LE prod, DNS-01) — see the ACME note below before ordering it;
+      - public DNS for `gitea.dev.home.keithjackson.dev` resolving to dev's Gateway VIP
+        (192.168.1.53), **checked against Cloudflare / a public resolver, never through the pfSense
+        override** — that mistake was made on 2026-09-04 and produced a false healthy reading;
+      - `curl -sS https://gitea.dev.home.keithjackson.dev/api/v1/version` succeeding with a
+        publicly-trusted cert from a host on the dev LAN.
+      Until D2 is green, D3 is not safe.
+- [ ] **D3. Merge the authored change to `main`** (`clusters/base/cluster-vars.yaml`,
+      `clusters/dev/`, `clusters/prod/`, the two Ansible files, `platform/externaldns`,
+      `catalog/render.py`). With D1 in place nothing moves yet.
+- [ ] **D4. Release dev's reconciliation deliberately.** Resume the root Kustomization (or restore
+      the in-cluster url, now pointing at `gitea.dev.home.keithjackson.dev`) and watch
+      `flux get kustomizations -A` converge. Dev's Gateway listeners, HTTPRoutes, coredns-custom zone
+      and Homepage links all move here. Dev's external-dns publishes the new record set under owner
+      `dev`.
+- [ ] **D5. Keycloak: issuer + every client.** Dev's realm issuer becomes
+      `https://keycloak.dev.home.keithjackson.dev/realms/homelab`. The issuer is baked into every
+      token, and **~12 OIDC clients** need their redirect URIs moved in the same change: `grafana`,
+      `gitea`, `nextcloud`, `semaphore`, `headlamp`, `crafty`, `pihole`, `jackett`, `qbittorrent`,
+      `radarr`, `sonarr`, plus the `kubernetes` public client. Most are reconciled from
+      `platform/keycloak/keycloak-realm-import.yaml` and move with D4; **headlamp's is the exception**
+      — its issuer/callback live in a SOPS-encrypted Secret and must be re-encrypted by hand.
+- [ ] **D6. Rolling k3s restart + kubectl SSO re-issue.** `k3s_base_domain` now renders a different
+      `oidc-issuer-url` into `/etc/rancher/k3s/config.yaml`, so the `k3s_server` role is **no longer
+      a no-op on dev**: it rewrites the file and fires the Restart handler on all three servers. Run
+      it **one server at a time**, confirming `kubectl get nodes` between each. Every user's OIDC
+      kubeconfig is invalidated and must be re-issued; the cert-based admin kubeconfig keeps working
+      throughout, which is what makes this step recoverable. **Do not reboot nodes** — restart the
+      `k3s` service only.
+- [ ] **D7. Retire dev's old apex records.** They are owned by `default`, which no environment now
+      claims, so nothing removes them automatically. Delete the `<svc>.home.keithjackson.dev` A
+      records, the apex record, **and their TXT registry entries** (`external-dns-*` TXT), plus any
+      stale `_acme-challenge.home.keithjackson.dev`. *Alternative, if you prefer external-dns to do
+      its own cleanup:* set dev's `EXTERNALDNS_TXT_OWNER_ID` back to `default` for exactly one
+      reconcile after D4 — `policy: sync` will then garbage-collect the apex set it no longer has
+      sources for — and only then switch it to `dev`. That is two commits and needs watching; the
+      direct deletion is preferred because it is verifiable in one place.
+- [ ] **D8. Verify the apex is EMPTY, against Cloudflare.** `dig +short <svc>.home.keithjackson.dev
+      @1.1.1.1` returns nothing for every service name, and no TXT registry entry for those names
+      remains. **This is the gate on the whole P sequence.** A pfSense/PiHole override is not
+      evidence about public DNS.
+
+### D0/D1 AS EXECUTED (2026-09-04) — and the self-reference that would have undone it
+
+D1 was taken via the **repoint** option, not the suspend option, because it is strictly stronger:
+suspending only defers the strand, while repointing removes the dependency that causes it. Dev's
+`GitRepository` now reads **`http://192.168.2.132:3000/Home-Network/home-network-k3s-impl.git`,
+branch `dev-live`** — IP-addressed, plain HTTP, so it depends on **neither DNS nor TLS**, which is
+exactly the property needed during a DNS/TLS migration. **A merge to `main` is now inert for dev.**
+
+**THE TRAP, and it is not obvious: a bare copy of `main@4809199` on the bootstrap Gitea would NOT
+have held.** The root Kustomization renders `./clusters/dev`, and `clusters/dev/kustomization.yaml`
+*contains the patch that sets `GitRepository.spec.url`*. So Flux would have fetched from the
+bootstrap Gitea, applied the manifests it found there, and those manifests would have set the url
+straight back to `gitea.home.keithjackson.dev` — reverting the repoint on the next reconcile and
+re-arming the strand. **The branch must declare its own source.** `dev-live` (`a4a3c2e`) therefore
+carries exactly one content change on top of `4809199`: `clusters/dev/kustomization.yaml`'s url +
+`ref.branch`. Proven by `kubectl kustomize clusters/dev` at both revisions differing in **exactly
+two lines** out of 357. The same property makes the revert self-sustaining in reverse, since
+`main@4809199` carries the `gitea.home…` url. Revert command: see `REVERT.txt` in the run scratchpad.
+
+Verified after the cutover: **13/13 Kustomizations Ready** on `dev-live@a4a3c2e`, `cluster-vars`
+byte-identical to the D0 snapshot (27/27 keys, `BASE_DOMAIN` still `home.keithjackson.dev`),
+**18/18 Longhorn volumes attached+healthy**, all 16 HTTPRoute hostnames still on the apex,
+`wildcard-home` still Ready, and the Cloudflare record set re-queried and diffed **byte-for-byte
+against the D0 snapshot — identical** (57 records, 36 TXT all `owner=default`).
+
+> **NEW DEPENDENCY, AND IT IS THE WEAK LINK.** `192.168.2.132` is a **DHCP lease on the
+> workstation's wifi** (`wlp0s20f3`, NetworkManager `ipv4.method=auto` — confirmed, not assumed). A
+> lease change, a wifi drop, or the workstation going offline **stalls dev's fetching**. The cluster
+> keeps running and serving; only reconciliation pauses, and the fix is a `kubectl patch` of the url.
+> Two consequences: (a) a **static reservation** for this host is worth having before any repeat of
+> this procedure, and (b) **keep the window short** — the exit is D4, which returns dev to its own
+> in-cluster Gitea at `gitea.dev.home.keithjackson.dev`.
+>
+> Pointing Flux at dev's Gitea **Service** instead (`gitea.gitea.svc.cluster.local:3000`) would drop
+> the DNS, TLS *and* workstation dependencies all at once, and is the better long-term shape. It was
+> **not** done here: namespace `gitea` carries a `default-deny` NetworkPolicy plus a
+> `gitea-allow-gateway-ingress` CiliumNetworkPolicy admitting **only** the Gateway, so
+> source-controller cannot reach the Service without a new authored policy. Worth doing deliberately,
+> not mid-migration.
+
+### P — prod takes the apex
+
+- [ ] **P1. Unsuspend prod**, in dependency order: `flux resume kustomization platform`, then the
+      10 downstream tiers.
+      **CORRECTION (2026-09-04) — the D8 gate is on prod taking the APEX, not on prod coming up.**
+      This step originally read "only now unsuspend prod", which silently assumed prod would be
+      serving `*.home.keithjackson.dev`. It is not: prod runs revision `c8c1d29`, where
+      `BASE_DOMAIN=prod.home.keithjackson.dev` and `EXTERNALDNS_TXT_OWNER_ID=prod`. Prod's names are
+      therefore a **disjoint subtree** under a **distinct registry owner**, so neither incident mode
+      of §10a applies: it cannot claim dev's `default`-owned records, and its wildcard
+      `*.prod.home.keithjackson.dev` is a **different name set** from dev's, so it has its own
+      Let's Encrypt duplicate-certificate bucket and races no `_acme-challenge` with dev.
+      **Prod may therefore be brought up in full while dev still holds the apex** — and should be,
+      because it proves the entire platform stack end-to-end *before* the apex flip, turning the
+      cutover into a DNS/TLS change on a known-good cluster instead of a bring-up and a cutover at
+      once. What D8 gates is **only** the later move of prod's `BASE_DOMAIN` to the bare apex.
+      Verify the two guard values live in prod's applied `cluster-vars` before unsuspending; if
+      `EXTERNALDNS_TXT_OWNER_ID` is not `prod`, STOP — that was the root cause of the incident.
+- [ ] **P2. Prod's external-dns publishes under owner `prod`.** Check the logs for a matching hosted
+      zone (F-DNS1 fail-closed mode: no zone selected ⇒ nothing created — which is safe, but means
+      the filter was narrowed against instruction). Confirm the new TXT registry entries read owner
+      `prod` and that no record owned by `dev` was touched.
+- [ ] **P3. Prod's wildcard cert `*.home.keithjackson.dev` + `home.keithjackson.dev` is `Ready`** and
+      solved DNS-01 without colliding with any `_acme-challenge` dev still holds.
+- [ ] **P4. Prod's kubectl SSO works without an `/etc/hosts` pin.** Prod nodes resolve
+      `keycloak.home.keithjackson.dev` through **public** upstreams (§4), so this only becomes true
+      once P2 has published that name at prod's Gateway VIP (192.168.2.20). Before then a prod node
+      resolving its own issuer reaches **dev**. Use the break-glass cert kubeconfig until P2 is done.
+- [ ] **P5. Cut prod's GitOps source over** from the workstation bootstrap Gitea
+      (`http://192.168.2.132:3000/...`) to `https://gitea.home.keithjackson.dev/...` once prod runs
+      its own Gitea — the block in `clusters/prod/kustomization.yaml` owns that procedure. Note the
+      inversion **simplified** this: prod's apex Gitea name and dev's `gitea.dev.home...` are now
+      different hosts, so prod pointing at the apex means "prod's own Gitea", never dev's.
+
+### ACME: the duplicate-certificate limit
+
+Let's Encrypt rate-limits **duplicate certificates — 5 per week for an identical set of names.**
+Prod's wildcard is `*.home.keithjackson.dev` + `home.keithjackson.dev`, which is **exactly the name
+set dev holds today**. Every prod order therefore counts against the same bucket as dev's historical
+renewals of that pair, and a retry loop during P3 can exhaust it and leave prod with **no trusted
+cert for up to a week**. Mitigations:
+
+- Order prod's cert **after** D8, so dev is no longer renewing the same pair.
+- Let dev's old `*.home.keithjackson.dev` `Certificate`/Secret be **deleted** in D4/D7 rather than
+  left to renew in the background.
+- Rehearse against **letsencrypt-staging** first (`ACME_CLUSTER_ISSUER`) — staging has its own,
+  far looser limits — and only switch prod to `letsencrypt-prod` when the DNS-01 solve is proven.
+- Do **not** delete-and-recreate prod's `Certificate` to "retry"; each recreation is a fresh order.
+
+### The L2 tests hardcode the apex
+
+Six behavioural (L2) chainsaw tests embed `home.keithjackson.dev` as a literal:
+
+| test | file |
+|---|---|
+| `dns-exposure` | `conformance/tests/l2-behavioral/dns-exposure/chainsaw-test.yaml` |
+| `dns-internal-resolution` | `conformance/tests/l2-behavioral/dns-internal-resolution/chainsaw-test.yaml` |
+| `gateway-ingress-reachable` | `conformance/tests/l2-behavioral/gateway-ingress-reachable/chainsaw-test.yaml` (+ `fixtures/app-and-probe.yaml`) |
+| `oidc-sso-grafana` | `conformance/tests/l2-behavioral/oidc-sso-grafana/chainsaw-test.yaml` |
+| `sso-forward-auth` | `conformance/tests/l2-behavioral/sso-forward-auth/chainsaw-test.yaml` |
+| `trusted-tls` | `conformance/tests/l2-behavioral/trusted-tls/chainsaw-test.yaml` |
+
+They are **test inputs, not reconciled manifests**, so they are deliberately left alone by the
+authoring change: they are the only thing still asserting the apex, and while dev is live on the apex
+they are *correct*. Under the new direction the literal happens to become right for **prod** and
+wrong for **dev**, so they must be **parameterized per environment** (not merely edited) before
+either §9's *"suite passes green against the prod cluster"* or a green L2 run against migrated dev
+can be claimed. Until then, expect the dev L2 suite to fail between D4 and that parameterization —
+a known, scheduled failure, not a regression.
+
+The L1 *rendering* tests need no change: their goldens carry `${BASE_DOMAIN}` as a token, and the
+literal hostnames in them come from each test's own fixture `ingressRoutes[].host`, which is
+environment-independent by construction.
+
+### Offline proof carried out for the authoring change (no cluster touched)
+
+All 12 `substituteFrom` paths were rendered for **both** overlays through the real toolchain
+(`kubectl kustomize --load-restrictor LoadRestrictionsNone <spec.path> | flux envsubst --strict`,
+with each overlay's effective `cluster-vars` taken from `kubectl kustomize clusters/<env>`), before
+and after the change.
+
+- **Dev's diff is *only* the hostname move.** Proven by inverse substitution rather than by reading
+  the diff: applying `s/dev.home.keithjackson.dev/home.keithjackson.dev/` to the new render restores
+  the baseline **byte-for-byte** in all 12 paths, plus 2 declared `txtOwnerId: default → dev` lines.
+  145 changed line pairs, all classified: 143 hostname-move, 2 declared owner-id, **0 unexplained**,
+  and **0 bare-apex remnants**. The 2 paths with no change (`cilium`, `velero`) provably contain no
+  hostname at all (`grep -c keithjackson.dev` = 0).
+- **Prod lands on the bare apex:** **0** occurrences of `prod.home.keithjackson.dev` and **0** of
+  `dev.home.keithjackson.dev` anywhere in prod's rendered output; 143 apex names.
+- **Certs:** dev `*.dev.home.keithjackson.dev` + `dev.home.keithjackson.dev`; prod
+  `*.home.keithjackson.dev` + `home.keithjackson.dev`.
+- **Positive controls** (the harness is provably not blind): perturbing an unrelated knob
+  (`GATEWAY_VIP`) surfaces 1 unexplained line; perturbing `BASE_DOMAIN` to a wrong value fails 10 of
+  12 paths; injecting a single un-moved hostname is caught by the independent bare-apex scan (which
+  the byte-identity check alone cannot see — which is why both exist).
+- Gates: `ci/l0.sh`, `ci/l1.sh`, `ci/render-golden.sh`, `ci/lint.sh`,
+  `flux-postbuild-substitution/check.sh` (254 assertions), `catalog/render.py --selftest`,
+  `ci/spec-immutability.sh` — all **PASS**.
@@ -0,0 +1,519 @@
+# Prod cluster standup — the complete command sequence
+
+**Every command below is copy-pasteable and runs from the repo root**, through the pinned dev
+container. Nothing here needs a command invented on the spot; that is the point. If a step is not in
+this file it is not part of standup.
+
+State as of **2026-09-06**: phases **A**, **B** and **C** are DONE. The cluster is live — k3s
+v1.36.4 HA (3 etcd members), kube-vip, Cilium 1.20.1, Longhorn — with **12/13 Flux tiers Ready**
+(`velero` deliberately suspended), **112 pods Running**, an issued Let's Encrypt wildcard, and
+**all three nodes addressing their full 32 GB** (node 2's DIMM upgrade completed 2026-09-06 and is
+confirmed at ~`32638256Ki`, matching the other two to within 4 KiB — the §0b memattr bug is not
+biting on the backports kernel).
+
+Storage survived two unplanned node events (an ungraceful power-down of prod-3, then prod-2's memory
+upgrade) and converged to **17/17 volumes healthy, replicas symmetric across all three nodes** —
+though only after a leftover cordon was cleared; read
+[§Node maintenance](#node-maintenance--draining-rebooting-and-the-cordon-that-stops-self-healing)
+before doing node work, that cordon silently prevents Longhorn from rebuilding.
+
+What remains is in [§Blockers](#blockers), and every item there needs an operator decision:
+no backup coverage (R2 credentials), prod still on `prod.home.` rather than the apex, and — found
+2026-09-05 — **prod's UniFi VM has no disk** because Kyverno blocks CDI's generated importer pod.
+
+> **The one guard that matters.** Dev is LIVE and shares the `keithjackson.dev` Cloudflare zone with
+> prod. On 2026-09-04 prod deleted every one of dev's public DNS records. `bootstrap/standup.sh`
+> therefore **requires** `--expect-server` and compares it against the live API server before running
+> anything. There is no override flag. Do not add one.
+
+## Constants
+
+| | |
+|---|---|
+| kube-API (kube-vip VIP) | `https://192.168.2.9:6443` |
+| Gateway / ingress VIP | `192.168.2.20` |
+| Nodes | `k3s-prod-1/2/3` — `192.168.2.10 / .11 / .12` |
+| Break-glass kubeconfig | `infra/ansible/.cache/prod.kubeconfig` (cert-based; survives OIDC/DNS failure) |
+| GitOps source (interim) | `http://192.168.2.132:3000/Home-Network/home-network-k3s-impl.git` branch `main` |
+| `BASE_DOMAIN` | `prod.home.keithjackson.dev` — **not** the apex; see §Blockers |
+
+Container paths: the repo is mounted at `/work`, so kubeconfig paths passed to the script are
+**repo-relative** (`infra/ansible/.cache/prod.kubeconfig`).
+
+---
+
+## Phase A — bare metal to a joined cluster (Ansible)
+
+Prerequisites that are **not** commands: firmware/BIOS (§1), disk layout (§2), OS install (§3),
+network (§4), and SSH + the vaulted `ansible_become_password` in
+`inventory/group_vars/k3s_prod/vault.yml` (§5). Ansible cannot escalate on prod without that vault
+file. See `infra/PROD-PROVISIONING.md`.
+
+**Install the backports kernel first.** All three prod nodes need ≥6.13 because HP's R21 firmware
+publishes a corrupt EFI Memory Attributes Table and Linux 6.12's `efi_memattr_init()` does not
+validate it, reserving everything above the table (fixed upstream in 6.13 by `8fbe4c49c0cc`, absent
+from `linux-6.12.y`). Prod runs `7.1.8+deb13-amd64` from trixie-backports.
+
+```bash
+timeout 3600 infra/devcontainer/dev.sh run env ANSIBLE_CONFIG=/work/infra/ansible/ansible.cfg \
+  ansible-playbook /work/infra/ansible/playbooks/node-baseline.yml \
+  -i /work/infra/ansible/inventory/hosts.ini -e k3s_hosts=k3s_prod --tags kernel
+```
+
+Then the full chain — baseline, k3s HA, Cilium, verify, Flux hand-off. **All three `-e` flags are
+required**: a play's `hosts:` is templated before any host exists, so `group_vars` cannot retarget it.
+
+```bash
+timeout 3600 infra/devcontainer/dev.sh run env ANSIBLE_CONFIG=/work/infra/ansible/ansible.cfg \
+  ansible-playbook /work/infra/ansible/playbooks/site.yml \
+  -i /work/infra/ansible/inventory/hosts.ini \
+  -e k3s_hosts=k3s_prod -e k3s_primary_hosts=k3s_prod_primary -e k3s_secondary_hosts=k3s_prod_secondary
+```
+
+kube-vip is a separate play (not imported by `site.yml`):
+
+```bash
+timeout 1800 infra/devcontainer/dev.sh run env ANSIBLE_CONFIG=/work/infra/ansible/ansible.cfg \
+  ansible-playbook /work/infra/ansible/playbooks/kube-vip.yml \
+  -i /work/infra/ansible/inventory/hosts.ini -e k3s_hosts=k3s_prod
+```
+
+Hardware health (read-only; `node_health` asserts the memattr fix took and that all RAM is addressed).
+
+> **The flag is `node_health_hosts`, NOT `k3s_hosts`.** Diagnostic plays deliberately keep their own
+> target var so they stay independently pointable while a bring-up run is scoped elsewhere
+> (`node-health.yml` → `node_health_hosts`, `memattr-survey.yml` → `memattr_survey_hosts`,
+> `vpn.egress.gateway.yml` → `vpn_egress_gateway_hosts`). The consequence is a **silent** one: passing
+> `k3s_hosts=k3s_prod` here does not error, it falls through to the `k3s_servers` default and runs
+> against **DEV's three nodes**. Verified — this exact mistake was made while writing this file and
+> caught only by `--list-hosts`. **Always dry-run the target first:**
+>
+> ```bash
+> timeout 200 infra/devcontainer/dev.sh run env ANSIBLE_CONFIG=/work/infra/ansible/ansible.cfg \
+>   ansible-playbook /work/infra/ansible/playbooks/node-health.yml \
+>   -i /work/infra/ansible/inventory/hosts.ini -e node_health_hosts=k3s_prod --list-hosts
+> ```
+
+```bash
+timeout 900 infra/devcontainer/dev.sh run env ANSIBLE_CONFIG=/work/infra/ansible/ansible.cfg \
+  ansible-playbook /work/infra/ansible/playbooks/node-health.yml \
+  -i /work/infra/ansible/inventory/hosts.ini -e node_health_hosts=k3s_prod
+```
+
+## Phase B — Flux bootstrap
+
+`flux-bootstrap.yml` (last play of `site.yml`) shells to `bootstrap/flux/bootstrap.sh`, the single
+documented entrypoint. That script now runs the CRD pre-apply as step **[2/4]**, so a cold bootstrap
+no longer strands `platform`. The **age private key Secret must exist first** — bootstrap verifies it
+and refuses to continue rather than silently skipping decryption
+(`bootstrap/sops/age-key-secret.md`).
+
+## Phase C — bring the reconciliation graph up
+
+One script, six steps, each independently runnable and idempotent. **Run them in this order.**
+
+```bash
+K=infra/ansible/.cache/prod.kubeconfig
+S=https://192.168.2.9:6443
+```
+
+### C1 — preflight (read-only, changes nothing)
+
+Asserts the guards that prevent a repeat of the DNS incident and reports every tier's state. **Fails
+closed** if `EXTERNALDNS_TXT_OWNER_ID` is empty, still the base sentinel, or `default` (dev's owner).
+
+```bash
+timeout 300 infra/devcontainer/dev.sh run bootstrap/standup.sh \
+  --kubeconfig infra/ansible/.cache/prod.kubeconfig --expect-server https://192.168.2.9:6443 preflight
+```
+
+### C2 — pre-apply chart CRDs, and break the CDI cold-start deadlock (idempotent)
+
+```bash
+timeout 600 infra/devcontainer/dev.sh run bootstrap/standup.sh \
+  --kubeconfig infra/ansible/.cache/prod.kubeconfig --expect-server https://192.168.2.9:6443 crds
+```
+
+Two cold-start problems, same root cause: **Flux server-side dry-runs every object in a Kustomization
+before applying any of them, so one unknown `kind` aborts the entire tier.**
+
+1. **Kyverno** — its ~5.6 MiB CRD bundle exceeds kubectl's 256 KiB annotation limit, so the chart sets
+   `crds.install: false` and nothing else installs them. `bootstrap/flux/preapply-crds.sh` applies them
+   server-side, deriving the version from the HelmRelease Flux reconciles (no second pin).
+
+2. **CDI — a true deadlock.** `platform-config` applies the CDI CR *and*
+   `platform/kubevirt/cdi-storageprofiles.yaml` in the **same pass**. But `StorageProfile`'s CRD does
+   not exist until `cdi-operator` has reconciled that CR. So the dry-run fails on the unknown kind →
+   the CR is never applied → the CRD is never created → `platform-config` can never converge, and
+   every tier below it stays blocked. **Invisible on an established cluster** (dev's CRDs have been
+   there for months) **and fatal on a cold one.** Observed on prod 2026-09-05: `cdi-operator`
+   Available for 12h with **no CDI CR** and no `storageprofiles` CRD.
+
+   The step breaks it by applying **only the CR** — the minimum — and leaving everything else to
+   Flux. It carries no `${}` tokens, so applying it here cannot diverge from what Flux would render.
+   The CRD appeared in 10s.
+
+   **REPAIRED IN THE GRAPH (2026-09-05).** `cdi-storageprofiles.yaml` now lives in its own tier,
+   `./platform-config-cdi`, with `dependsOn: platform-config` and `wait: true`. Because `dependsOn` is
+   **revision**-gated and `platform-config` runs `wait: true`, the CDI CR must be applied *and healthy*
+   — which is precisely the event that creates the `StorageProfile` CRD — before the profiles are
+   applied. A cold cluster now orders itself. The `crds` step is retained as a fallback for a cluster
+   already stuck in the broken state, and because it is idempotent.
+
+   **The generalisable rule, now recorded in `clusters/README.md`:** a CR whose CRD is created by a
+   *controller at runtime* is **second-order** and needs its own pass. `cdi-operator.yaml` registers
+   exactly one CRD (`cdis.cdi.kubevirt.io`); `storageprofiles`, `datavolumes`, `datasources` and
+   `dataimportcrons` appear only after cdi-operator reconciles the CDI CR. An audit of
+   `platform-config` found `StorageProfile` is the only second-order kind there.
+
+   `apps-vms` gained `dependsOn: platform-config-cdi` — load-bearing, not cosmetic:
+   `apps/vms/unifi/20-datavolume.yaml` provisions on `storageClassName: longhorn-vm`, and its comment
+   previously claimed `platform-config` guaranteed the profiles, which the move would have silently
+   falsified.
+
+#### Merge precondition and the one-time transient
+
+`platform-config` has `prune: true`, so on the first reconcile after merge it **deletes** both
+StorageProfiles, CDI recreates them within seconds with an empty spec, and `platform-config-cdi`
+re-applies the overrides ~30–60s later (kustomize-controller's dependency requeue).
+
+**Measured on dev, so the exposure is known rather than assumed** — comparing our override against
+what CDI infers unaided for the same Longhorn provisioner:
+
+| profile | `spec` (ours) | CDI-derived `status` |
+|---|---|---|
+| `longhorn-vm` (we override) | RWX / Block, `snapshot` | **ReadWriteMany** / Block, `snapshot` |
+| `longhorn-static` (no override) | *unset* | **ReadWriteOnce** / Block, `snapshot` |
+
+So **`cloneStrategy` is not at risk** — CDI infers `snapshot` for Longhorn anyway. **`accessModes`
+is:** unaided, CDI produces **RWO**. A DataVolume provisioned on `longhorn-vm` *inside the window*
+would be stamped RWO/Block and be **permanently non-live-migratable**. Flux can no longer cause that
+(`apps-vms` now gates on the new tier); **do not start a VM disk import, clone or DataVolume by hand
+during the window.**
+
+- [ ] **Before merging, confirm `flux get kustomization platform-config` is Ready on the target.** If
+      it is not, the profiles are pruned and sit at CDI defaults until it recovers — and prod was in
+      exactly that state hours earlier. The fallback is the two-step the Cilium move used: merge with
+      `platform-config.prune: false`, confirm the ownership label transferred, restore `prune: true` in
+      a second commit.
+- [ ] After: both profiles carry `spec.cloneStrategy: snapshot`, `longhorn-vm` carries
+      `claimPropertySets` RWX/Block, and both are labelled
+      `kustomize.toolkit.fluxcd.io/name=platform-config-cdi`.
+
+### C3 — unwedge stuck Helm releases
+
+A release left in `uninstalling` / `pending-*` cannot be advanced by helm-controller: it reports
+`StateError` forever, so `platform` (`wait: true`) never goes Ready and all 10 tiers below it stall.
+**Dry run by default** — it prints what it would do and changes nothing:
+
+```bash
+timeout 600 infra/devcontainer/dev.sh run bootstrap/standup.sh \
+  --kubeconfig infra/ansible/.cache/prod.kubeconfig --expect-server https://192.168.2.9:6443 unwedge
+```
+
+To act (this **deletes** the release's Kubernetes objects and lets Flux install cold). Pre-applied
+CRDs live **outside** the Helm release and survive, which is exactly why C2 comes first:
+
+```bash
+timeout 900 infra/devcontainer/dev.sh run bootstrap/standup.sh \
+  --kubeconfig infra/ansible/.cache/prod.kubeconfig --expect-server https://192.168.2.9:6443 \
+  --confirm-destructive unwedge
+```
+
+### C4 — external-dns safe upgrade
+
+**The most dangerous step in the repo, and the ordering is the entire mitigation.** A Helm upgrade
+resets `replicas` to the chart default (1) and applies the new args in ONE operation — there is no
+window in which to check the result first. If `--txt-owner-id` were missing, the pod starts as owner
+`default`, claims dev's TXT registry entries, and under `policy: sync` **deletes the records behind
+them**.
+
+So this step **proves the outcome before causing it**: it renders the chart with the *live*
+HelmRelease values and asserts `--txt-owner-id=<owner>` appears, aborting if not. Rendering is
+deterministic and touches nothing, which converts a race into a precondition. It then snapshots the
+Cloudflare zone, upgrades, re-verifies the live args (scaling to 0 and failing if they are wrong),
+and diffs the zone.
+
+```bash
+timeout 900 infra/devcontainer/dev.sh run bootstrap/standup.sh \
+  --kubeconfig infra/ansible/.cache/prod.kubeconfig --expect-server https://192.168.2.9:6443 externaldns
+```
+
+Expected diff: **additions only, all under `prod.home.keithjackson.dev`, all TXT entries owner
+`prod`.** Any change to a name outside that subtree, or to a TXT entry owned by another cluster, is a
+regression — stop and investigate.
+
+### C5 — resume the tiers, in dependency order
+
+`platform → platform-config → {oauth2-proxy, headlamp, homepage, crafty, apps-wave-0, apps-vms} →
+apps-wave-1 → apps-wave-2`. Waits for each to go Ready and **stops at the first that does not** — a
+partial, understood bring-up beats a forced green. `velero` is skipped while `S3_BACKUP_ENDPOINT` is
+a placeholder (see §Blockers). Long-running; expect it to background.
+
+```bash
+timeout 3600 infra/devcontainer/dev.sh run bootstrap/standup.sh \
+  --kubeconfig infra/ansible/.cache/prod.kubeconfig --expect-server https://192.168.2.9:6443 resume
+```
+
+### C6 — verify
+
+```bash
+timeout 900 infra/devcontainer/dev.sh run bootstrap/standup.sh \
+  --kubeconfig infra/ansible/.cache/prod.kubeconfig --expect-server https://192.168.2.9:6443 verify
+```
+
+### Everything at once
+
+Runs C1→C6 with `unwedge` in **dry-run** mode (no `--confirm-destructive`), so it stops short of the
+destructive step by design:
+
+```bash
+timeout 3600 infra/devcontainer/dev.sh run bootstrap/standup.sh \
+  --kubeconfig infra/ansible/.cache/prod.kubeconfig --expect-server https://192.168.2.9:6443 all
+```
+
+---
+
+## VERIFIED WORKING — 2026-09-05, standup complete
+
+| | |
+|---|---|
+| Kustomizations | **12/13 Ready** (`velero` deliberately suspended — see §Blockers) |
+| Pods | **113 total, 0 not Running/Completed** |
+| Wildcard cert | `wildcard-home` **Ready**, Let's Encrypt **prod**, `*.prod.home.keithjackson.dev` |
+| Gateway VIP | `192.168.2.20` · crafty L4 `192.168.2.22` · unifi L4 `192.168.2.21` |
+| Published DNS | **36 records** under `prod.home.keithjackson.dev`, every TXT registry entry `owner=prod` |
+| Dev's records | **57, byte-identical to the pre-standup baseline — nothing added, changed or removed** |
+
+End-to-end HTTPS **through the workstation's own resolver — no `--resolve` override, no `-k`** (after
+installing the dnsmasq profile below). Every request landed on `192.168.2.20`:
+
+```
+prod.home.keithjackson.dev  200   headlamp.  200   keycloak.  302 (login)   nextcloud. 302
+gitea.       200            semaphore. 200        grafana.   302 (SSO)      pihole.    302
+```
+
+The served certificate is prod's own, confirmed by SNI — **not** dev's wildcard, which is what the
+resolver bug previously made it look like:
+
+```
+subject = CN=*.prod.home.keithjackson.dev
+issuer  = C=US, O=Let's Encrypt, CN=YR1          (LE production)
+SAN     = *.prod.home.keithjackson.dev, prod.home.keithjackson.dev
+valid   = 2026-09-05 .. 2026-12-04
+```
+
+Dev, checked in the same pass, still serves `CN=*.home.keithjackson.dev` — **unchanged**.
+
+> Ignore `curl: (91) No OCSP response received` if you pass `--cert-status`: Let's Encrypt retired
+> OCSP responders in 2025, so there is nothing to staple. It is not a certificate fault, and the plain
+> requests above all validated normally.
+
+### Workstation DNS — install the prod dnsmasq profile
+
+**CORRECTION (operator, 2026-09-05): this is the workstation's own dnsmasq, NOT pfSense.** Earlier
+notes in this repo called it a "pfSense override"; that was wrong and the topology below is why.
+
+`/etc/NetworkManager/dnsmasq.d/10-home.keithjackson.dev.conf` (the dev profile) contains:
+
+```
+address=/.home.keithjackson.dev/192.168.2.50
+```
+
+dnsmasq's `address=` matches the named domain **and every subdomain**, so that one line also captures
+the whole `prod.home.keithjackson.dev` tree and answers it with **dev's** address, while Cloudflare
+correctly returns prod's `192.168.2.20`.
+
+**Fix — one command, already authored in this repo:**
+
+```bash
+sudo infra/workstation/dnsmasq/install.sh
+```
+
+It installs `20-prod.home.keithjackson.dev.conf` and reloads the live resolver. dnsmasq resolves
+`address=` by **longest match**, so the prod rules win without editing or reordering the dev profile.
+Verified against a throwaway dnsmasq loaded with both profiles:
+
+| query | answer |
+|---|---|
+| `prod.home.keithjackson.dev`, `gitea.prod…`, `keycloak.prod…` | `192.168.2.20` (Gateway) |
+| `unifi.prod.home.keithjackson.dev` | `192.168.2.21` (L4) |
+| `crafty-mc.prod.home.keithjackson.dev` | `192.168.2.22` (L4) |
+| `gitea.home.keithjackson.dev`, `home.keithjackson.dev` | `192.168.2.50` — **dev, unchanged** |
+
+> **Install to `/etc/NetworkManager/dnsmasq.d/`, never `/etc/dnsmasq.d/`.** The standalone
+> `dnsmasq.service` is **inactive**; the live resolver is the one NetworkManager spawns
+> (`dns=dnsmasq`) with `--conf-dir=/etc/NetworkManager/dnsmasq.d`. A file in the other directory
+> changes nothing and looks like it should. `install.sh` pins the destination for exactly this reason.
+
+**Until it is installed, verify prod only with an explicit resolve** — and note *why* this trap is
+convincing: dev serves a `*.home.keithjackson.dev` wildcard cert, which matches
+`prod.home.keithjackson.dev` (one label), so the prod **apex returns 200 over valid TLS** while
+`keycloak.prod.home.keithjackson.dev` (two labels) fails with a certificate-name error. It reads like
+a broken prod certificate. It is the wrong server answering.
+
+```bash
+curl -sS --resolve gitea.prod.home.keithjackson.dev:443:192.168.2.20 \
+  https://gitea.prod.home.keithjackson.dev/
+```
+
+## NODE MAINTENANCE — draining, rebooting, and the cordon that stops self-healing
+
+Two unplanned node events on 2026-09-05/06 (an ungraceful power-down of prod-3, then a memory upgrade
+on prod-2) produced one finding worth more than either incident.
+
+### A failed `kubectl drain` still cordons — and on this cluster a cordon stops storage self-healing
+
+`kubectl drain k3s-prod-2` fails on this cluster with the usual two objections (emptyDir pods,
+DaemonSet pods) — **but it cordons the node first, and the cordon persists after the failure.**
+Longhorn's `disable-scheduling-on-cordoned-node` is **`true`**, so a cordoned node receives no replica
+scheduling. Observed consequence: after prod-2's reboot, **all 17 volumes sat `attached/degraded` with
+prod-2 holding 2 of 18 replicas and ZERO rebuilds in flight.** Nothing was driving recovery, every pod
+was `Running`, and no alert fired. `kubectl uncordon` started the rebuilds immediately; they converged
+17/17 healthy in ~6 minutes.
+
+**So: after any drain — successful or not — `kubectl uncordon <node>` is not tidying up, it is what
+re-enables storage redundancy.** Check `kubectl get nodes -o custom-columns=N:.metadata.name,\
+U:.spec.unschedulable` before believing a green pod list.
+
+### `drain` cannot complete on this cluster, and the two usual flags do not fix it
+
+`--ignore-daemonsets --delete-emptydir-data` clears the two reported objections and then **hangs
+indefinitely**: all three `instance-manager-*` PDBs report **`ALLOWED DISRUPTIONS = 0`** and
+`node-drain-policy` is `block-if-contains-last-replica`. Longhorn is refusing to let an
+instance-manager be evicted while it runs volume engines. That is data protection, not a bug.
+
+**Requesting replica eviction does not rescue it either:** with **3 nodes and 3 replicas** there is no
+fourth node to move a replica to, so eviction can never complete. This is structural, not a setting.
+
+Options, in order of preference:
+
+1. **Just reboot it (cordon, reboot, uncordon).** Proven twice on this cluster: Longhorn resyncs on
+   return, and `replica-replenishment-wait-interval` is `600`s, so a reboot completed inside ~10
+   minutes does not even trigger full rebuilds. This is the normal path for firmware, memory or kernel
+   work.
+2. **If the node must be genuinely empty**, first scale to 0 the workloads holding engines on it —
+   that detaches them, the PDBs go to 1 allowed disruption, and the drain completes. Find them with:
+   `kubectl -n longhorn-system get engines.longhorn.io -o json` filtered on `.spec.nodeID`.
+3. Changing `node-drain-policy` to `always-allow` is **not** advised at 3 nodes/3 replicas — it trades
+   the protection for exactly the data risk the default exists to prevent.
+
+### Check the kube-vip leader before rebooting a server
+
+`kube-vip` holds the API VIP (`192.168.2.9`) via a leader lease, and it is a **DaemonSet** — so
+`--ignore-daemonsets` leaves it running and a *drain* does not move the VIP. A **reboot** does: the
+lease fails over and `kubectl` blips for a few seconds. Harmless, but do not mistake it for a fault.
+Identify the holder: `kubectl -n kube-system logs -l app.kubernetes.io/name=kube-vip-ds | grep -i leader`.
+
+### Post-maintenance checklist
+
+- [ ] `kubectl uncordon <node>` — **and confirm** `.spec.unschedulable` is clear on all three.
+- [ ] Longhorn converged: `kubectl -n longhorn-system get volumes.longhorn.io` shows **0 degraded**.
+      Rebuilds are serial, so allow minutes, and watch rather than assume.
+- [ ] Replica symmetry: each node runs the same count
+      (`get replicas.longhorn.io` grouped by `.spec.nodeID` / `.status.currentState`).
+- [ ] All three nodes report full memory — prod's HP firmware publishes a corrupt EFI memattr table and
+      a wrong kernel silently strands RAM (§0b). `k3s-prod-*` should each show ~`32638256Ki`.
+- [ ] Pods: none `Running`-but-not-`Ready`.
+
+> **KNOWN, NOT FIXED — pod distribution does not rebalance.** After prod-2's cordon+reboot the spread
+> was **52 / 10 / 50**. Kubernetes does not move running pods back, and for the many SINGLE-REPLICA
+> apps here a spread constraint would change nothing (one replica lives on one node either way). It
+> self-corrects as pods restart. It is recorded because it concentrates failure: at the time of writing
+> **all three CNPG databases run on prod-1**, so losing that node takes Keycloak, Nextcloud and
+> Semaphore together. The real fix is database HA (`instances: 2`), which is a resource decision the
+> operator owns — see §Blockers.
+
+## Blockers
+
+### 1. Backups: endpoint DECIDED, but prod cannot reach it — **prod has no backup coverage**
+
+**Operator ruling 2026-09-05: "the S3 backup location is the same as Dev."** Prod's
+`S3_BACKUP_ENDPOINT` is now `http://192.168.1.9:9000`, and the `TBD-PROD-*` placeholder is retired.
+
+**It is not reachable from prod.** Measured from a throwaway pod **inside the prod cluster** — not
+from the workstation, which proves nothing about pod egress:
+
+```
+curl http://192.168.1.9:9000/minio/health/live   →  connection timed out after 15s
+```
+
+**This is expected topology, not a misconfiguration** (operator, 2026-09-05): the k3s prod cluster is
+**not behind pfSense**. It sits on the SOHO router network `192.168.2.0/24`, which is pfSense's
+**WAN** side; pfSense *contains* `192.168.1.0/24`, the dev LAN where MinIO lives. Prod reaching
+`192.168.1.9` therefore means crossing **into** pfSense from its WAN — which requires a deliberate
+pfSense rule or port-forward that does not exist. **Operator decision required**, and worth weighing:
+a prod backup that lands inside the dev LAN is arguably not off-cluster DR at all.
+
+Re-test with `standup.sh --s3-endpoint http://192.168.1.9:9000 s3check` — it fails closed and says so.
+
+> **Do not confuse this with the SOHO router's inbound NAT.** That router forwards inbound **443** to
+> the pfSense WAN address `192.168.2.102` to serve the **existing, non-k3s production environment**,
+> which must stay live on 443 until the k3s cutover is deliberately made. Nothing in this runbook
+> touches that NAT, and prod's Gateway on `192.168.2.20:443` does not contend with it.
+
+Until that routing exists, `velero` stays suspended and **`apps-wave-0` still applies GREEN** — the
+token *is* defined, so strict substitution succeeds — while every write to that endpoint fails at
+**runtime**. A green `apps-wave-0` does not mean backups work. **Treat prod as having no backup
+coverage.**
+
+#### Sharing one MinIO is only safe because of two new knobs
+
+Nothing about the backup *paths* was per-environment before 2026-09-05. Enabling prod against dev's
+MinIO without these would have been the external-dns incident again, in a worse place:
+
+| | before | after |
+|---|---|---|
+| Velero | `bucket: velero`, **no prefix**, both clusters run schedules named `daily` | `prefix: ${VELERO_BACKUP_PREFIX}` — dev `""`, prod `prod` |
+| CNPG | `destinationPath: s3://cnpg`, serverName defaults to the **Cluster name** | `${CNPG_BACKUP_PATH}` — dev `s3://cnpg`, prod `s3://cnpg/prod` |
+
+- **Velero** syncs backups it finds in its BSL into itself as `Backup` CRs and **garbage-collects them
+  when their TTL expires**. Prod would have adopted dev's backups and deleted them on schedule.
+- **CNPG is the worse one.** Both environments have `keycloak-db`, `nextcloud-db` and `semaphore-db`,
+  so prod's Postgres would archive WAL into the *exact prefix* dev's does. Two servers interleaving
+  WAL into one Barman stream **corrupts the chain and destroys PITR** — and dev's restore is
+  **proven** (R-STATE-03).
+
+**The base values are dev's current layout on purpose and must stay that way.** Dev's existing
+backups live at the bucket root and at `s3://cnpg`; giving dev a prefix would **orphan the very
+backups its proven restore depends on** — the same reasoning that keeps dev's
+`EXTERNALDNS_TXT_OWNER_ID` at `default`. Prod has no history to preserve, so it takes the nested paths.
+
+> **Delivery note.** Prod reconciles the bootstrap Gitea's `main` at `c8c1d29`. This change sits on a
+> later branch that also carries the DNS inversion, which must **not** reach prod (it would move prod
+> to the apex and collide with dev). Delivering it needs a `c8c1d29`-based branch carrying only these
+> commits. No urgency: velero cannot be enabled until the routing above is fixed.
+
+### 2. Prod is on `prod.home.keithjackson.dev`, not the apex
+
+The operator's target layout is dev under `*.dev.home.keithjackson.dev` and prod on the **apex**.
+Prod cannot take the apex until dev vacates it, because both clusters would request a
+`letsencrypt-prod` wildcard for the *same name set* and race each other's `_acme-challenge` TXT
+records, so neither reliably solves DNS-01.
+
+**This does not block phase C.** Prod on its own subtree with `txtOwnerId: prod` collides with
+nothing, and standing it up first is strictly better: it proves the whole stack before the flip, so
+the cutover becomes a DNS/TLS change on a known-good cluster instead of a bring-up and a cutover at
+once. The apex move is `infra/PROD-PROVISIONING.md` §10a, sequence **D** (dev vacates) then **P**.
+
+> **THE APEX FLIP IS ALSO THE LEGACY-PROD CUTOVER — they are the same event.** The SOHO router NATs
+> inbound **443** to the pfSense WAN address `192.168.2.102`, serving the **existing, non-k3s
+> production environment**, which the operator requires to stay live until the switch is deliberately
+> made. So §10a sequence **P** is not merely a DNS/TLS change on the k3s side: taking
+> `home.keithjackson.dev` means public traffic must also stop resolving to / being forwarded to the
+> legacy stack. Sequence **P** as written does **not** cover repointing that NAT, and the two must be
+> planned together or the apex will resolve to k3s while inbound 443 still lands on the legacy box.
+>
+> Nothing in phase C touches it: prod's Gateway is `192.168.2.20:443`, a different address from the
+> NAT target, and prod's wildcard cert is issued by **DNS-01**, which needs no inbound 443 at all.
+
+---
+
+## Not covered here
+
+- **Dev's migration off the apex** — §10a sequence D. Touches Keycloak's issuer, ~12 OIDC client
+  redirect URIs, and a rolling k3s restart that invalidates every OIDC kubeconfig. Operator-present.
+- **Prod's GitOps cutover** (P5) from the workstation bootstrap Gitea to prod's own in-cluster Gitea.
+  `clusters/prod/kustomization.yaml` owns that procedure.
+- **Six L2 conformance tests hardcode the apex** and must be parameterized per environment before
+  §9's "suite passes green against the prod cluster" can be claimed.
@@ -27,7 +27,6 @@ ansible-playbook playbooks/site.yml        # core bring-up, in order (baseline
 # — or step by step —
 ansible-playbook playbooks/node-baseline.yml    # 0. packages, kernel modules, Longhorn disk, swap-off
 ansible-playbook playbooks/k3s.yml         # 1. HA k3s (primary --cluster-init, then joins)
-ansible-playbook playbooks/etcd-io-priority.yml # 1b. shared etcd/Longhorn device I/O weights (PROD ONLY; a no-op on dev)
 ansible-playbook playbooks/cilium.yml      # 2. Cilium CNI (cluster is NotReady until this)
 ansible-playbook playbooks/verify.yml      # 3. gate #1: 3 nodes Ready, Hubble up, no kube-proxy
 ansible-playbook playbooks/flux-bootstrap.yml   # 4. hand off to bootstrap/ (SOPS then Flux)
@@ -72,7 +71,6 @@ should return a PIA exit IP (not the home ISP).
 |---|---|
 | `k3s.yml` / `k3s_server` | k3s server install with `--flannel-backend=none --disable-kube-proxy --disable-network-policy --disable=traefik,servicelb`; primary `--cluster-init`, secondaries join via the primary's node-token (fetched, never pre-stored) |
 | `cilium.yml` / `cilium` | Gateway API CRDs, then Helm `cilium` with `kubeProxyReplacement`, `k8sServiceHost/Port`, `gatewayAPI`, `egressGateway`, `hubble`, `l2announcements` |
-| `etcd-io-priority.yml` / `etcd_io_priority` | **prod only** (master switch defaults false; `group_vars/k3s_prod` sets it true). Switches the disk carrying `k3s_data_dir` to **BFQ**, drops `nr_requests` to 32, and gives `system.slice` + `k3s.service` `IOWeight=10000` so the embedded etcd outweighs `kubepods.slice` — then reads every value back out of the kernel and **fails** if it did not land. Mitigation for the 2026-09-07 k3s-prod-1 etcd WAL-fsync starvation. Restarts nothing. NVMe-backed nodes are skipped, not decorated. See `roles/etcd_io_priority/README.md` for the mechanisms in this space that are **inert** here and must not be "fixed" back in |
 | `verify.yml` / `cluster_verify` | waits nodes Ready + Cilium/Hubble rollout; asserts kube-proxy absent |
 | `flux-bootstrap.yml` / `flux_bootstrap` | calls `bootstrap/` (SOPS age key, then `flux bootstrap --path=clusters/dev`) |

@@ -122,20 +120,10 @@ for PM reconciliation.
 ## Targeting prod (R-ENV-02: same playbooks, same roles, scoped overrides)

 `inventory/hosts.ini` carries a `[k3s_prod_primary]` / `[k3s_prod_secondary]` /
-`[k3s_prod:children]` section for the three bare-metal nodes — `k3s-prod-1` `192.168.2.10`,
-`k3s-prod-2` `.11`, `k3s-prod-3` `.12` — with its own `group_vars/k3s_prod/` (vaulted
-`ansible_become_password` + non-secret overrides).
-
-**All three are live in the inventory.** This paragraph used to say `k3s-prod-2`/`-3` "are
-addressed but not yet racked, so their inventory lines stay commented out"; that stopped being
-true on 2026-09-03 when node 3 came online, and the group has been complete since. Corrected
-2026-09-07 because the stale sentence is the kind that gets used to reason about blast radius:
-`-e k3s_hosts=k3s_prod` targets **three** hosts, not one. Confirm it, every time, before running
-anything:
-
-```bash
-ansible-playbook playbooks/<play>.yml -e k3s_hosts=k3s_prod --limit k3s-prod-3 --list-hosts
-```
+`[k3s_prod:children]` section for the three planned bare-metal nodes (`k3s-prod-1` at
+`192.168.2.10` exists and is reachable today; `k3s-prod-2`/`-3` at `.11`/`.12` are addressed but
+not yet racked, so their inventory lines stay commented out), with its own `group_vars/k3s_prod/`
+(vaulted `ansible_become_password` + non-secret overrides).

 **Nothing here is forked for prod.** Every node-host play takes its target group from a variable
 that defaults to the dev group, so the dev commands above are unchanged, and prod is the same
@@ -165,11 +153,6 @@ ansible-playbook playbooks/kube-vip.yml        -e k3s_hosts=k3s_prod
 ansible-playbook playbooks/verify.yml          -e k3s_hosts=k3s_prod
 ansible-playbook playbooks/flux-bootstrap.yml  -e k3s_hosts=k3s_prod   # see caveat below

-# prod — I/O arbitration for the shared etcd/Longhorn disk. ONE NODE AT A TIME, node 3 first
-# (WD Blue, IO PSI 1.29 during the incident) before node 1 (the 2011 C400, 11.36).
-ansible-playbook playbooks/etcd-io-priority.yml -e k3s_hosts=k3s_prod --limit k3s-prod-3 --list-hosts
-ansible-playbook playbooks/etcd-io-priority.yml -e k3s_hosts=k3s_prod --limit k3s-prod-3
-
 # prod — k3s.yml needs both halves of the primary/secondary split
 ansible-playbook playbooks/k3s.yml \
  -e k3s_primary_hosts=k3s_prod_primary -e k3s_secondary_hosts=k3s_prod_secondary
@@ -2,7 +2,8 @@
 # Ansible Galaxy collections required by infra/ansible. Install with:
 #   ansible-galaxy collection install -r collections/requirements.yml
 collections:
-  # node_baseline role: community.general.modprobe, community.general.filesystem
+  # node_baseline role: community.general.modprobe, community.general.filesystem,
+  #                      community.general.timezone
  - name: community.general
  # node_baseline role: ansible.posix.mount
  - name: ansible.posix
@@ -334,13 +334,7 @@ node_baseline_dns_search: []
 # prod takes prod.home.keithjackson.dev. Consequence here: prod's Keycloak issuer is
 # https://keycloak.prod.home.keithjackson.dev/realms/homelab, so the `kubernetes` OIDC client
 # in prod's OWN Keycloak realm is what prod's kubectl SSO authenticates against.
-# [parity] MUST equal BASE_DOMAIN in clusters/prod/kustomization.yaml. This value renders the
-# kube-apiserver's `oidc-issuer-url`, which is INSTALL-TIME config in /etc/rancher/k3s/config.yaml
-# — changing it here does NOT take effect until the k3s_server role re-renders that file and k3s
-# is restarted (serial: 1). Until that rolling restart completes the apiserver still trusts the
-# OLD issuer while Keycloak serves the NEW one, so kubectl OIDC logins fail in between. Cert-based
-# kubeconfigs are unaffected.
-k3s_base_domain: "home.keithjackson.dev"
+k3s_base_domain: "prod.home.keithjackson.dev"

 # --- Flux bootstrap: WHICH overlay this environment reconciles from (R-ENV-02) -----
 # roles/flux_bootstrap defaults flux_bootstrap_env_path to `clusters/dev` (dev is LIVE
@@ -363,29 +357,3 @@ k3s_base_domain: "home.keithjackson.dev"
 #     from the prod nodes — it is outside every cluster and can simply be powered off;
 #   - the `sops-age` Secret is installed out of band on prod (bootstrap/sops).
 flux_bootstrap_env_path: "clusters/prod"
-
-# --- shared etcd/Longhorn device I/O arbitration (2026-09-07 incident) --------------
-# TURNS ON roles/etcd_io_priority FOR PROD ONLY. Role default is false, and dev must stay
-# that way: dev has a DEDICATED etcd device (/dev/vdc mounted at server/db,
-# docs/plans/ssd-etcd-isolation.md) so it has nothing to arbitrate, and pulling a LIVE
-# cluster onto a different I/O scheduler as a side effect of sharing a role would be a real
-# change for no benefit. Same seam as node_baseline_backports_kernel above.
-#
-# WHY PROD NEEDS IT. The §2 ruling put /data/k3s (etcd WAL) and /data/longhorn (every
-# replica) on ONE partition per node and recorded the contention as "accepted to be
-# MEASURED, not assumed". It has now been measured, the hard way: 2026-09-07 06:05:21Z
-# k3s-prod-1 went NotReady with the host still up, because etcd could not finish a WAL
-# fdatasync (3.4-6.7 s against a 1 s expectation) while Longhorn wrote to the same disk.
-# /proc/pressure/io FULL avg300 was 11.36 there against 0.75 (node 2, NVMe) and 1.29
-# (node 3). See PROD-PROVISIONING.md §2 "THE MEASUREMENT CAME BACK".
-#
-# WHAT ENABLING THIS DOES *NOT* DO: nothing, until the operator runs
-# playbooks/etcd-io-priority.yml. It is not applied by any other play except through
-# site.yml, and site.yml is not run against prod casually. Apply to k3s-prod-3 (the WD Blue,
-# IO PSI 1.29) FIRST and watch NodeIOPressureHigh for a day — the role's own README explains
-# why, what it cannot fix, and how to roll it back in three commands.
-#
-# k3s-prod-2 is NVMe-backed; the role detects that and deliberately installs nothing there
-# rather than leaving an inert drop-in behind, so this group-wide `true` is still correct
-# for it.
-etcd_io_priority_enabled: true
@@ -1,49 +0,0 @@
-# group_vars for the `vpn_egress_gateway_prod` node — ADR-0015 host tunnel secrets (PROD)
-
-Prod's egress gateway is `k3s-prod-3`, in its **own** inventory group
-(`[vpn_egress_gateway_prod]`) so that the flagless
-`ansible-playbook playbooks/vpn.egress.gateway.yml` — which defaults to the **dev** group —
-can never touch a prod node. Prod is reached only by asking for it:
-
-```bash
-cd infra/ansible
-ansible-playbook playbooks/vpn.egress.gateway.yml \
-  -e vpn_egress_gateway_hosts=vpn_egress_gateway_prod --limit k3s-prod-3 --list-hosts
-# then drop --list-hosts
-```
-
-A separate group needs its own `group_vars`, because the dev group's vault is attached to the
-dev group and is not visible to a prod host. The required keys are **identical** to dev's —
-see `../vpn_egress_gateway/README.md` for what each one is:
-
-| Vault key |
-|---|
-| `vault_vpn_egress_gateway_pia_username` |
-| `vault_vpn_egress_gateway_pia_password` |
-| `vault_vpn_egress_gateway_pia_ovpn` |
-
-## How this file was created (2026-09-12)
-
-`vault.yml` here is a **verbatim copy of the dev group's encrypted blob**. That is deliberate:
-
- the operator confirmed the **same PIA account** applies to prod and is well inside its
-  connection limit, so the values are the same values;
- copying the *ciphertext* means the plaintext was never decrypted to disk, never passed
-  through a shell, and never appeared in an agent transcript. The repo-wide vault passphrase
-  (`ansible.cfg` → `vault_password_file`) decrypts both files identically.
-
-The two files are now **independent**: rotating the PIA credentials means re-encrypting
-*both*, and that is the intended behaviour — prod and dev are separate environments and a
-credential change should be a deliberate act per environment, not an accident of sharing a
-file. If prod is ever moved to its own PIA account (recommended if the connection limit ever
-bites, or to keep the public egress IPs of the two environments independent), re-create this
-file alone:
-
-```bash
-cd infra/ansible
-ansible-vault create inventory/group_vars/vpn_egress_gateway_prod/vault.yml
-```
-
-## Notes
- `vault.yml` is skipped by the yamllint gate (`ci/yamllint.yaml` ignores `**/vault.yml`).
- **Never** run `ansible-inventory --list`: it prints DECRYPTED vault values.
@@ -1,22 +0,0 @@
---
-# Non-secret group_vars for the PROD egress gateway (k3s-prod-3). Credentials live in the
-# vaulted vault.yml beside this file.
-
-# PROD DIALS ITS OWN PIA SERVER — and it has to.
-#
-# The vaulted .ovpn pins ONE server, 158.173.241.112. Measured 2026-09-12: with dev
-# (k3s-dev-3) and prod (k3s-prod-3) both dialling it, PIA handed BOTH sessions the SAME
-# tunnel address — `Server-10779-2a ... ifconfig 10.72.192.7` in both journals. Each
-# reconnect steals that address from the other; the loser then sees no data for 60s, hits
-# `--ping-restart`, reconnects and steals it back. Both tunnels flapped on a ~2-minute cycle
-# (dev 22:57 / 22:59 / 23:01 UTC, prod interleaved a minute off) and neither environment had
-# reliable vpn-class egress. This is NOT the account's connection limit — that has room.
-#
-# 158.173.241.135 is another member of the same PIA Stockholm LB pool: `sweden.privacy.network`
-# resolved from k3s-prod-3 to 158.173.241.{34,55,112,135,239}. Any server in the pool accepts
-# the same credentials and the same inline CA, so only the address differs. Dev keeps the
-# .ovpn's pinned .112 untouched, which is why the override lives HERE and not in the role.
-#
-# A public server address is not a credential, so this is plain group_vars, not vault.
-# If prod is ever moved to its own PIA account, this can go away again.
-vpn_egress_gateway_pia_remote: "158.173.241.135 1198"
@@ -1,131 +0,0 @@
-$ANSIBLE_VAULT;1.1;AES256
-38306636313935333030303664653739316233663139616563663233643262373637306331653264
-6538363539383464353964663632633561323562383038340a396233633232356137313435336231
-36333663363130643532613566643339373365626462313537653033306363336564343139373537
-3164323463616632630a373732333536613936366463313439343664363530383664636535616261
-31306332666666643637346364386537343062383033663530346163373536323639373366333661
-36303831323662333232633034393362333561643637623033656565303461336465333664366332
-36336466656437343431363735646661643837666335633464666130643866313830366465323166
-65616537343630303431623537303766383762646533616539613235343938656237613934623565
-35373835633539653333343232626336326638303961373230663561313330313632393066323130
-31383838386262393730666231376362383132646230383964313439663935613861323137646331
-36626365323566316436386662383434366664356166636431663332393665323734623735303239
-62383031613237333136613837356166343436306137353935613661613264326466346337346530
-65356630373235313131656131383965313366613965386662343961653864373964346337363562
-66626635363438656430306231323764346564666132633966353861643262353131623134656137
-39306134626163346336613732326264396438623630656534396565343763303761333538316661
-30623466366232323531633262623434643366613833353564386163303966316537623934336163
-66323933653036366535663266666531666236646431383066616262353733316165666434616534
-35386633313464363233326663363634626239616364383866653135386435636130316335666430
-38323363386462373862366230393764386338643466363161333733653065383966663935303964
-64333461333437653433373131343736333831313236306262373631353461356237386163623166
-30663761306534316439356365396663336135333537663366353866336536663934646133333332
-65353236366435326630653830346631643236326337626435333236386634333137623138316131
-61323365303461353932626336323534633533646233653135396665373765666337333934356662
-36646339376564366439646137306364623165626263383634326337333566663236346366366334
-37316166393763663139336538663266333736623661383062383732303133326233313232323836
-34356130396131346533633336373736663366366231343162633064343565613937366434613664
-31656365303439643837323135613936646133306163663265373933633231616135353335376435
-61633965356538613636663564346566313261306235313438316639626131303566363365663764
-64316562383361373663653034393333653831666239363037343636666364666536343534613066
-61373335353464633963623161353866333038643062303931653134626334393864663461653531
-65373766336437376435616138666563623337373038623566666136626637633463323965666139
-38336139613635383066336164373061663266326235363731663565613336623936373531383966
-62323330636263306665343462666639643530336633346531636263616436356631343763343432
-61636264316566663933346163373163653734623031393962343635653765653964633137346433
-62616138333536336438363564356636316531346566663066376234356230626165386165336165
-38656166633839656438323966376561306338623237303530376231393663663661343362653664
-37613433653634363534306432383061373165613266386562323632636537373736656137633639
-33666634316465383066333737616636363866626330313232326434666164316439333433633461
-35336161313039646264613835613336326332326464313339636532653837653430393939633462
-66656563626163376134623437633332633535336233656237303962623063363134616431623235
-62323265666136323366343565656236646430393237356333613836343438353164313535616138
-34613134663465333734663337303663316161313439626332326334363835376361366463363631
-30396336303761383465373439313666656237356433303532373162306166356165316164656334
-37326163393230306161653332323462333238313166373134343839323461363936353737613662
-61643938333464366433306363633531383264613030316631303034376131323135313136316461
-31666632393631376133373533616639396361633762636663323463363433626537656333653837
-38633632623139356630616637663331616461383361383733613563383165363139343035383333
-32653563303038336137336333653561363037633634323662616666356534323138353438616463
-64383639353733333537616632346230383330356166663433373734386435376563333038333934
-37616536333132303166366235613265336432616239393730353036616337386463363330306532
-65613066363434346562633630336135313736393137656230306163383730356133326561333431
-35393466323936636562616336353934626262336363306335636331666633623432343134396334
-37343765646134306638313636326531616665323136643264353666323236343239633639356537
-61343037666232386135303363363938353864396461363830373633363730323165303165633430
-32656532633761316132613463616365353066393165353331646665613239333738653861363134
-30343939333633366532656430303133643337356336336638653266653766353637356561663636
-61626632333438313436323465643434666334393034366166343930323661386664623663396435
-61633334656464623330623333346165363563356363656663373335646433343762663964323735
-34316239663164623431353164306235386661666632383730376139626666313935663462326433
-36313039326635396435646466306366343438656334346231393066326638313337373933393231
-35373731336164396165303935316366343565363631353137393333613363656564343063373933
-61643033373839656532373330666436396632626663353436356231323166326365323832346461
-31366630626635323465333238613734313265356231313338356431393338353166633939383334
-64333634373638383937663933336563663132366534643434373032333934383533383063643036
-39323534326139633931623831656565333930323362366539346239646464353536346639343436
-37663337396565363065343532616437333666316266633632653661396535636461653335363538
-37303263383030646233313239643536326434323933616135373835393965346462373739383230
-33383863353761346232643835633139363231663162353937343363653339393838616535393436
-38303737633231626236316336353832306232633032653032633365386563626434646139633063
-31313436343063643766663465626330643366313663633733663762653631633966383736336138
-36353738383865613537366364666665326464626365643032666630643039326538663531353964
-33646561303435666237303763663634613136633662633536303066363764383039373236666135
-30626639313531353134353730323334623631396463323035313733333965623665333466366164
-62646539393966626337333637646438313635373666306538356631396237323739613032353832
-31383439343531373561353038363535353266333265336434303934343239633334383865333436
-31616537666332373332623932623162333138373563363336303735396431396464613364643162
-61323737643134653763373232383134643931343366353437633632326138353237336333616238
-32613239316666343761363863376563613638366432313461643535353731383631633233366234
-39663632613062656337616561636533633264373930376437373132303830313863303734346632
-33313431336661663634663861643364363834626638666565383635343433323961623765633265
-34386631653833653865383865636364653265363162323462376364313964626633313635313765
-38633131653836396463393136333932346532333631626461373739346139306465653532366632
-65363333633964616632613830386566623865356131646338653236373861346430363164393634
-64373434663531643436616664393936396564666561313862636133306537373137636234626232
-61313861626235636235353431353738303161373939373966383139323833346662373731646434
-31356133623331323661666234613466366130336662653764363738626531346531333339396266
-32636564653732396339383533343266633730666434303161626362366134353361643362626533
-32333132656636333736616661323664316234393937313666653762316463306437633163383030
-65383962346233663061356338306164343363323736646165613730373166333533373635373039
-39343262393031626637623030373535663539613361613366333863336531663637323035376564
-30626232623661623065643136353237316333323764663633643132653264303162366134616564
-31366665313830323263373835613964386132333538396165663538653865356236646234373465
-30383533316532363133323465363063373962353763346366383563613862336465366565653637
-62346365373030383135323933366138656463323864663363626330653036623739343035336135
-62643262336436336462386430363430363139396566343963373664373334366430643830313439
-66613362396137313661666635643962346662376137393066653765636330366462633565346663
-38633864366666316534653665303664393863633464373638393463343537386437383039373530
-66623063653536666364303531373931613433383139376664633934376662336231643762353830
-31653065303430393864383532333766613965383831616563653233313338343861663334373465
-39363361313532333034613034666265386430333466396163303263636133316263313538656634
-65313464316463623134356562366261663635323530393137313233316638643364646639343935
-38313131633132636463323231393332363766653935323965373134653336376131636163376566
-33623934346431313433386430656264316566383436663535343532303763316437303436336465
-39626231303931303130643237383663666338313662663231313935643334393362323130356530
-39656330616138373962633732313436666662616165316236323930613134643161613464636536
-36613735636233616164396630333634333935393938623061313963636461316564313662353034
-33666434326165643964303561303062323061633766353761313464376262333361323532616361
-39313832396134643966353239333066346239626534303830396165396530623135333834313439
-37373661643731623935666437313465633565303232366638393837303462613035393031333061
-39666433373662633730363665313666346165366138306636623263633537346631666132306238
-34393363373335663631393135646231313064383837396164613536643533383464313536663561
-38633131303966356534636539333733336464393236623238373766616537613437343232303437
-64323636613862663135323034636239333433333165313239336239666665616631623934316637
-61303237336334316265643964663830336539353164343063636664633430326139356534313036
-65393464653330333365336335363531303536313439363066376438666133376664346336633937
-66346165666337666134333963613662303534656339373731663534656533316338353763643930
-38313239313637326435323463333863663433623366316532333065396635663966343835636630
-34643631613431333431636330623564663662383063353138656436336333333262306633666163
-38303032316133666533393663343366333162626436333762306263616362643936383637653330
-31643334613766636231333461643863333838343961383762316162636634323134313461313962
-35363062663934336632383931333262643534336563376537343265633434336439623131613337
-32626461643263623664353639613361393735663266373930366134633534343466346461313032
-34363162343464353037616637316636393139343563303331376362653238346631333137393164
-30306636366636366562656361346637356364666136623131316138393262633963356138383232
-64383932623831376632336331343333393739613663613763303035643136653965386435366663
-35313161383236356563653931313933633737646363363339336639613961343934373637643034
-63333036343532323166366132656538643331303131393737656338323939396461393764393134
-33663539633930636265393363326638636130313036333536306362663633616563316465616330
-35353539303335336565373962393433333539613439353765643061383132386133396339616137
-3265326634643062346666356263326639373130333362333064
@@ -31,48 +31,13 @@ k3s_secondary
 [hypervisor]
 localhost ansible_connection=local

-# VPN egress-gateway node — runs the node-level split-tunnel OpenVPN + kill-switch
-# (ADR-0015, playbooks/vpn.egress.gateway.yml). EXACTLY ONE host belongs here.
-#
-# THIS GROUP IS SINGLETON BY DESIGN — a second host does NOT buy failover. Cilium OSS has no
-# egress-gateway failover: the CRD's own `egressGateway` description says "In case multiple nodes
-# are a match for the given set of labels, the first node in lexical ordering based on their name
-# will be selected". Two labelled nodes therefore means one silently wins by NAME ORDER (a labelled
-# `k3s-prod-1` beats a labelled `k3s-prod-3`) and the other carries no traffic. See
-# platform/vpn-egress/README.md "Resilience — there is no gateway failover" for the full statement
-# and the ADR-0016 escalation. Members are already defined above with their ansible_host under
-# k3s_secondary; this group only tags the egress role target.
+# VPN egress-gateway node(s) — run the node-level split-tunnel OpenVPN + kill-switch
+# (ADR-0015, playbooks/vpn.egress.gateway.yml). Initially just k3s-dev-3 (single gateway;
+# add a second host here for Cilium Egress Gateway HA). Members are already defined above
+# with their ansible_host under k3s_secondary; this group only tags the egress role target.
 [vpn_egress_gateway]
 k3s-dev-3

-# PROD's egress-gateway node — a SEPARATE group on purpose (2026-09-12).
-#
-# It is NOT a member of [vpn_egress_gateway] above, and k3s-prod-3 must never be added
-# there: that group is the DEFAULT target of playbooks/vpn.egress.gateway.yml, so a
-# flagless `ansible-playbook playbooks/vpn.egress.gateway.yml` would then silently
-# reconfigure a prod node's kill-switch datapath. Prod is reached only by asking for it:
-#
-#   ansible-playbook playbooks/vpn.egress.gateway.yml \
-#     -e vpn_egress_gateway_hosts=vpn_egress_gateway_prod --limit k3s-prod-3
-#
-# (`--list-hosts` first, every time. The play's target var is
-# `vpn_egress_gateway_hosts`, NOT the cluster-wide `k3s_hosts`.)
-#
-# WHY NODE 3, AND WHY EXACTLY ONE:
-#   * node 1's SSD was replaced 2026-09-11 and it has the least operational history;
-#   * node 2 carries the §0b memattr history;
-#   * node 3 has a single uplink (eno1 192.168.2.12 — its wireless is gone: no wl* netdev
-#     exists, the /etc/network/interfaces wlp1s0 stanza is commented out, and the
-#     192.168.2.195 address the earlier note below claimed is NOT present. Verified
-#     2026-09-12) which is what a kill-switch datapath wants.
-#   * LEXICAL ORDERING HAZARD: Cilium OSS picks "the first node in lexical ordering" when
-#     several match the policy's nodeSelector, so labelling k3s-prod-1 would SILENTLY take
-#     the gateway role from k3s-prod-3. Exactly one host here, forever.
-# Its PIA credentials are its own vaulted group_vars/vpn_egress_gateway_prod/vault.yml —
-# the dev group's vault does not apply to a prod host.
-[vpn_egress_gateway_prod]
-k3s-prod-3
-
 # The Proxmox HYPERVISOR host itself (NOT a k3s node) — target of playbooks/backup-s3.yml,
 # which stands up the off-cluster versitygw S3 backup target for Velero + CNPG. Reached DIRECTLY
 # over SSH as root (its own 192.168.2.102:22, the pfSense NAT to the host's 192.168.1.9), so
@@ -120,17 +85,9 @@ proxmox ansible_host=192.168.2.102 ansible_port=22 ansible_user=root ansible_ssh
 # =============================================================================

 [k3s_prod_primary]
-# FOUNDER MOVED TO NODE 2 (2026-09-07). Node 1 founded this cluster with --cluster-init,
-# then its SSD was replaced and Debian reinstalled, so it is now a JOINING node like any
-# other. This group decides two things at once: which play sets k3s_server_mode: primary
-# (-> --cluster-init, which FOUNDS A NEW CLUSTER) and what k3s_api_host resolves to (the
-# --server join target). Leaving node 1 here would have pointed a rebuilt, empty node at
-# ITSELF and founded a second cluster beside the live one. Node 2 is the senior surviving
-# server; its own --cluster-init is inert because /etc/rancher/k3s/k3s.yaml already exists.
-k3s-prod-2 ansible_host=192.168.2.11 ansible_port=22 k3s_node_ip=192.168.2.11
+k3s-prod-1 ansible_host=192.168.2.10 ansible_port=22 k3s_node_ip=192.168.2.10

 [k3s_prod_secondary]
-k3s-prod-1 ansible_host=192.168.2.10 ansible_port=22 k3s_node_ip=192.168.2.10
 # Uncomment a node only once it is actually on the bench and answering SSH, so no play can
 # target a host that isn't there. k3s-prod-2 came online 2026-09-01 (imaged identically to
 # node 1: same HP EliteDesk 800 G5 Mini, same BIOS R21 02.26.00, same i7-9700T, same
@@ -144,13 +101,9 @@ k3s-prod-1 ansible_host=192.168.2.10 ansible_port=22 k3s_node_ip=192.168.2.10
 # needs the same >= 6.13 kernel. Its disk is a THIRD distinct model (WDC WDS500G2B0A SATA
 # SSD) — all three nodes differ on disk, so the §2 Longhorn/etcd symmetry gap is now
 # three-way, not a node-1 outlier.
-# NOTE (CORRECTED 2026-09-12): this used to read "node 3 also has wlp1s0 up on
-# 192.168.2.195 (wifi)". It does not — re-verified before making node 3 the VPN egress
-# gateway, because a second live uplink in a kill-switch datapath is not a detail: there is
-# NO wl* netdev on the node at all, no phy80211 device, the wlp1s0 stanza in
-# /etc/network/interfaces is commented out, 192.168.2.195 is assigned nowhere, and the only
-# IPv4 addresses are lo and eno1 192.168.2.12. `ansible_host` is that WIRED eno1 address —
-# the same directly-routable 192.168.2.x LAN as nodes 1 and 2.
+# NOTE: node 3 also has wlp1s0 up on 192.168.2.195 (wifi). `ansible_host` is deliberately
+# the WIRED eno1 address — the same directly-routable 192.168.2.x LAN as nodes 1 and 2.
+k3s-prod-2 ansible_host=192.168.2.11 ansible_port=22 k3s_node_ip=192.168.2.11
 k3s-prod-3 ansible_host=192.168.2.12 ansible_port=22 k3s_node_ip=192.168.2.12

 # Umbrella group group_vars/k3s_prod/ attaches to (mirrors dev's [k3s_servers:children]
@@ -1,38 +0,0 @@
---
-# Give the k3s server process — and therefore the EMBEDDED ETCD inside it — a larger share of
-# the block device it shares with Longhorn. Mitigates the 2026-09-07 k3s-prod-1 outage, where
-# etcd's WAL fdatasync took 3.4-6.7 s (expects 1 s) and the node went NotReady while the host
-# stayed up. See roles/etcd_io_priority/README.md for the mechanism, the measurements, and the
-# list of things in this space that do NOT work on these nodes.
-#
-# NO-OP BY DEFAULT. The role's master switch (etcd_io_priority_enabled) is false in
-# defaults/main.yml and true only in inventory/group_vars/k3s_prod/vars.yml, so running this
-# with no extra vars against dev changes nothing on any node — dev already has a dedicated
-# etcd device and has nothing to arbitrate.
-#
-# TARGET (R-ENV-02): same `k3s_hosts` seam every other whole-cluster play uses.
-#   ansible-playbook playbooks/etcd-io-priority.yml                       # dev, a no-op
-#   ansible-playbook playbooks/etcd-io-priority.yml -e k3s_hosts=k3s_prod # prod
-#
-# RUN IT ON ONE NODE FIRST, AND CONFIRM THE TARGET BEFORE YOU RUN IT AT ALL. A prior mistake in
-# this repo silently targeted dev's three nodes; --list-hosts is how that is caught:
-#   ansible-playbook playbooks/etcd-io-priority.yml -e k3s_hosts=k3s_prod \
-#     --limit k3s-prod-3 --list-hosts
-# k3s-prod-3 first, deliberately: it is the WD Blue SATA node (IO PSI 1.29 during the incident),
-# so it exercises the identical code path as the failing node 1 (11.36) at a fraction of the
-# stakes. Watch NodeIOPressureHigh / the "Nodes — CPU / Memory / I-O Pressure" dashboard for a
-# day before touching node 1.
-#
-# ROLLBACK is immediate and needs no playbook — see the header of
-# /usr/local/sbin/k3s-etcd-io-priority on the node.
-
- name: I/O priority for the shared etcd + Longhorn device
-  hosts: "{{ k3s_hosts | default('k3s_servers') }}"
-  become: true
-  # serial: 1 — one node at a time. Nothing here restarts k3s, but changing a live node's I/O
-  # scheduler is a real change to a cluster member, and the whole point is that a bad outcome
-  # on one node must never be simultaneous across all three.
-  serial: 1
-  gather_facts: true
-  roles:
-    - role: etcd_io_priority
@@ -23,17 +23,6 @@
 - name: Import - install k3s HA control plane
  ansible.builtin.import_playbook: k3s.yml

-# ADDED 2026-09-07, after the k3s-prod-1 etcd I/O-starvation outage. It must come AFTER
-# k3s.yml because it writes an IOWeight drop-in for k3s.service and verifies the value in
-# that unit's LIVE cgroup — on a cold build the cgroup does not exist until k3s runs, and
-# the role reports that rather than asserting on it. It restarts nothing.
-#
-# A NO-OP FOR DEV, by design: roles/etcd_io_priority defaults its master switch to false and
-# only inventory/group_vars/k3s_prod/vars.yml sets it true. Dev has a dedicated etcd device
-# and must not be dragged onto a different I/O scheduler by a shared role.
- name: Import - I/O arbitration for the shared etcd + Longhorn device (prod only)
-  ansible.builtin.import_playbook: etcd-io-priority.yml
-
 # ADDED 2026-09-06, AND ITS POSITION IS LOAD-BEARING: kube-vip MUST come up between k3s and
 # Cilium. Cilium's k8sServiceHost is now the control-plane VIP (k3s_apiserver_endpoint =
 # kube_vip_address) rather than the founder's own node IP, so installing Cilium before the
@@ -11,25 +11,13 @@
 #   vault_vpn_egress_gateway_pia_username, vault_vpn_egress_gateway_pia_password, vault_vpn_egress_gateway_pia_ovpn
 # Run:  ansible-playbook playbooks/vpn.egress.gateway.yml
 #
-# TARGET (R-ENV-02): the group is a var, defaulting to dev's `vpn_egress_gateway`. PROD has
-# its own group as of 2026-09-12 — `vpn_egress_gateway_prod` (k3s-prod-3), with its own
-# vaulted credentials in group_vars/vpn_egress_gateway_prod/:
-#
-#   ansible-playbook playbooks/vpn.egress.gateway.yml \
-#     -e vpn_egress_gateway_hosts=vpn_egress_gateway_prod --limit k3s-prod-3 --list-hosts
-#
-# The var is `vpn_egress_gateway_hosts`, NOT the cluster-wide `k3s_hosts`. Prod is
-# deliberately NOT a member of the default group, so a flagless run cannot reach it — and
-# `--list-hosts` before every run is the habit that catches a mistyped flag, which in this
-# repo has silently targeted all three dev nodes before.
-#
-# THE LABEL IS WHAT ARMS THE DATAPATH. The CiliumEgressGatewayPolicy is already reconciled
-# on both clusters, so the instant a node carries
-# `homelab.keithjackson.dev/egress-gateway=true` steering goes live. The role sequences
-# kill-switch -> tunnel -> re-steer -> label, with the label LAST, on purpose. Never apply
-# that label by hand, and never run this play with tags that skip the tunnel or kill-switch
-# tasks. Exactly ONE node per cluster may carry it (Cilium OSS picks the lexically first
-# match, so a labelled k3s-prod-1 would silently steal the role from k3s-prod-3).
+# TARGET (R-ENV-02): the group is a var, defaulting to dev's `vpn_egress_gateway`. Prod
+# has NOT decided whether it runs an egress gateway at all, nor on which node
+# (infra/PROD-PROVISIONING.md §6, last box) — so there is deliberately no prod egress
+# group in the inventory yet. When there is, point this at it with
+#   -e vpn_egress_gateway_hosts=<that group>
+# and it needs its OWN vaulted PIA credentials group_vars; this play's vault is scoped to
+# the dev group.
 - name: Configure the node-level VPN egress gateway
  hosts: "{{ vpn_egress_gateway_hosts | default('vpn_egress_gateway') }}"
  become: true
@@ -1,142 +0,0 @@
-# roles/etcd_io_priority
-
-Give the k3s server process — and therefore the **embedded etcd** it hosts — a larger share of
-the block device it shares with Longhorn.
-
-## The incident
-
-2026-09-07 06:05:21Z, `k3s-prod-1` went NotReady (`Kubelet stopped posting node status`) while
-the host stayed up, pingable and SSH-able. From the k3s journal:
-
-```
-slow fdatasync                took=3.56s / 6.71s / 4.95s / 3.40s   (etcd expects 1s)
-apply request took too long   took=8.30s                           (expects 100ms)
-etcdserver: too many requests
-Failed to check local etcd status for learner management: context deadline exceeded
-```
-
-No OOM, no MCE, no hung task. All three prod nodes put `/data/k3s` (etcd's WAL **and** the
-containerd image store) and `/data/longhorn` (every replica) in two directories on **one
-partition**:
-
-| node | device | drive | `/proc/pressure/io` full avg300 |
-|---|---|---|---|
-| k3s-prod-1 | `/dev/sda5` | C400-MTFDDAC512MAM — 2011 consumer SATA SSD, no PLP, 61,440 reallocated sectors | **11.36** |
-| k3s-prod-2 | `/dev/nvme0n1p5` | Samsung PM9A1 NVMe | 0.75 |
-| k3s-prod-3 | `/dev/sda5` | WDC WDS500G2B0A (WD Blue SATA) | 1.29 |
-
-Recovery took a manual `systemctl restart k3s`; 18 Longhorn volumes then rebuilt onto the same
-disk etcd was trying to fsync on. The failure is self-reinforcing.
-
-## What this role does
-
-Two things, both instantly reversible, neither requiring a k3s restart:
-
-1. **`k3s-etcd-io-priority.service`** (oneshot, ordered `Before=k3s.service`) switches the
-   shared device to the **BFQ** I/O scheduler and lowers `nr_requests` from 64 to 32.
-2. **`IOWeight=10000` drop-ins** on `system.slice` and `k3s.service`, which BFQ turns into
-   `io.bfq.weight=1000` — a 10:1 share against the default-weighted `kubepods.slice` where
-   Longhorn's instance-managers live.
-
-Then it **reads every one of those values back out of the kernel and asserts on them**. If the
-scheduler did not switch, or the weight did not land, the play fails. A mitigation that looks
-applied and is not is worse than none.
-
-## Why not the obvious things — all checked on the live nodes, all inert
-
-**`IOSchedulingClass=realtime` on `k3s.service`.** The scheduler here (`mq-deadline`, and BFQ
-too) *does* honour the I/O priority class, so this looks like the answer. It is not: I/O
-priority is inherited across `fork` and preserved across `exec` — `block/blk-ioc.c`'s
-`__copy_io()` copies `ioc->ioprio` into the child — and **containerd is a child of the k3s
-server process**. Verified on k3s-prod-1: `ps -eo pid,ppid,comm` shows containerd's PPID is the
-`k3s-server` PID (84553 → 84333), and
-
-```
-$ ionice -c2 -n0 -- sh -c 'ionice -p $$; sh -c "ionice -p \$\$"'
-best-effort: prio 0
-best-effort: prio 0
-```
-
-confirms inheritance through two levels of fork+exec on that node. Putting k3s in the realtime
-class therefore puts **every container on the node** in the realtime class, Longhorn included —
-no relative gain, and it demotes the rest of `system.slice` (sshd, journald) for nothing.
-Getting this to work would need containerd out of k3s's process tree, i.e. an external
-containerd via `--container-runtime-endpoint`. That is an architecture change, not a tweak.
-
-**`IOSchedulingPriority` (the level within a class).** `mq-deadline` buckets requests by
-priority **class** only — `IOPRIO_CLASS_RT/BE/IDLE` map to three `dd_per_prio` queues and the
-level inside a class is not read. So best-effort 0 versus best-effort 4 is a no-op here.
-
-**`io.latency` (blk-iolatency).** The ideal mechanism for this problem — scheduler-independent,
-protects one cgroup's latency target by throttling its siblings. **Not available:** Debian
-trixie's kernel does not build it. `grep CONFIG_BLK_CGROUP /boot/config-7.1.8+deb13-amd64`
-returns `CONFIG_BLK_CGROUP_IOCOST=y` and no `IOLATENCY`, and there is no `io.latency` file
-anywhere under `/sys/fs/cgroup`.
-
-**`IOWeight=` on the stock configuration.** systemd writes `IOWeight=` to both `io.weight`
-(blk-iocost) and `io.bfq.weight` (BFQ). On these nodes as shipped, **neither reader exists**:
-the scheduler is `mq-deadline`, which has no cgroup awareness, and blk-iocost is compiled in but
-disabled (`/sys/fs/cgroup/io.cost.qos` is empty). Adding a drop-in without switching the
-scheduler would be pure decoration. That is why the scheduler change and the weights are one
-role and one verification, not two independent knobs.
-
-**`IOWeight=1000` instead of `10000`.** systemd rescales `IOWeight`'s 1..10000 range onto
-`io.bfq.weight`'s 1..1000, so 1000 maps to roughly 101 — BFQ's default for an unweighted cgroup
-is 100. Only the top of the range separates anything. The role asserts the resulting
-`io.bfq.weight` is ≥ `etcd_io_priority_min_bfq_weight` (500) rather than trusting that mapping.
-
-**A udev rule instead of a unit.** `ATTR{queue/scheduler}="bfq"` on the *boot* disk fires its
-`add` event in the initramfs, where `/etc/udev/rules.d` is absent and `bfq.ko` is not loaded —
-it silently does nothing and the disk comes up on `mq-deadline` with no error anywhere.
-
-**A drop-in on `kubepods.slice`** (to lower Longhorn instead of raising etcd). `kubepods.slice`
-is a **transient** unit the kubelet creates through the systemd API (`systemctl cat
-kubepods.slice` shows `/run/systemd/transient/kubepods.slice`), and whether systemd merges an
-`/etc` drop-in into a transient unit was not verified on these nodes. Raising `system.slice`
-gets the same ratio through a unit systemd definitely owns.
-
-## What this role does NOT fix
-
- **It does not separate etcd from Longhorn.** Both still write to one ext4 filesystem on one
-  partition. Weights arbitrate the *device queue*; they cannot arbitrate the **shared ext4
-  journal**. `fdatasync` on the WAL waits on a jbd2 commit, jbd2 is a kernel thread in the root
-  cgroup, and Longhorn's replica writes dirty the same journal. That serialization point is
-  invisible to every cgroup control in this file and is a hard limit of co-tenancy on one
-  filesystem.
- **It is unproven on this hardware.** BFQ is the right mechanism on paper and it is what
-  systemd's `IOWeight` was built for, but nothing here has been measured on a C400 under a real
-  Longhorn rebuild. The role fails loudly if the settings do not *land*; it cannot tell you they
-  *helped*. Apply to `k3s-prod-3` first and watch the `NodeIOPressureHigh` alert and the
-  "Nodes — CPU / Memory / I-O Pressure" dashboard before touching node 1.
- **BFQ has costs.** It disables writeback throttling (`wbt_lat_usec` is 2000 on sda today),
-  spends more CPU per request, and gives up some peak throughput. On a device that is already
-  the bottleneck this is the right trade; it is still a trade.
- **k3s-prod-2 gets nothing.** Its data dir is on NVMe, BFQ is a poor fit for NVMe, and its IO
-  PSI was 0.75 during the incident. The role detects this and installs nothing rather than
-  leaving inert files behind.
- **The real fix is hardware and is the operator's**: replace the C400, or give etcd its own
-  device and flip `node_baseline_etcd_source` back to `device` — the path
-  `roles/node_baseline` still supports unchanged, and the exit condition already written into
-  `infra/PROD-PROVISIONING.md` §2.
-
-## Running it
-
-```bash
-cd infra/ansible
-
-# ALWAYS confirm the target first — a prior mistake in this repo silently hit dev.
-ansible-playbook playbooks/etcd-io-priority.yml -e k3s_hosts=k3s_prod \
-  --limit k3s-prod-3 --list-hosts
-
-ansible-playbook playbooks/etcd-io-priority.yml -e k3s_hosts=k3s_prod --limit k3s-prod-3
-```
-
-Rollback, per node, immediate:
-
-```bash
-systemctl disable --now k3s-etcd-io-priority.service
-printf 'mq-deadline\n' > /sys/block/sda/queue/scheduler
-printf '64\n'          > /sys/block/sda/queue/nr_requests
-rm -f /etc/systemd/system/{system.slice,k3s.service}.d/10-etcd-io-weight.conf
-systemctl daemon-reload
-```
@@ -1,84 +0,0 @@
---
-# etcd_io_priority — give the k3s process (and therefore the embedded etcd it hosts) a
-# larger share of the block device it SHARES with Longhorn, on nodes where those two
-# tenants sit on one partition.
-#
-# READ roles/etcd_io_priority/README.md BEFORE CHANGING ANY OF THESE. Two of the obvious
-# knobs in this space are INERT on these nodes and are deliberately absent:
-#   * IOSchedulingClass= / IOSchedulingPriority= on k3s.service. I/O priority is inherited
-#     across fork and preserved across exec (block/blk-ioc.c __copy_io copies
-#     ioc->ioprio), and containerd is a CHILD of the k3s server process — verified live on
-#     k3s-prod-1: `ps -eo pid,ppid,comm` shows containerd's PPID is the k3s-server PID, and
-#     `ionice -c2 -n0 -- sh -c 'ionice -p $$; sh -c "ionice -p \$\$"'` reports best-effort
-#     prio 0 at BOTH levels. So raising k3s to the realtime I/O class raises every container
-#     on the node with it, Longhorn's instance-managers included, and buys nothing against
-#     the thing we are trying to beat.
-#   * io.latency (blk-iolatency), which WOULD be the ideal scheduler-independent protection.
-#     Not available: CONFIG_BLK_CGROUP_IOLATENCY is absent from Debian trixie's kernel
-#     (`grep CONFIG_BLK_CGROUP /boot/config-7.1.8+deb13-amd64` returns IOCOST but not
-#     IOLATENCY), and there is no /sys/fs/cgroup/**/io.latency on any prod node.
-# What is left, and what this role does, is cgroup-proportional weighting — which requires
-# a scheduler that implements it.
-
-# MASTER SWITCH. Default FALSE so the LIVE dev cluster is untouched by a shared role:
-# dev already has a DEDICATED etcd device (/dev/vdc mounted at server/db,
-# docs/plans/ssd-etcd-isolation.md), so it has nothing to arbitrate and no reason to take a
-# scheduler change. Set true for the [k3s_prod] group in
-# inventory/group_vars/k3s_prod/vars.yml — the same seam node_baseline_backports_kernel,
-# kube_vip_address and k3s_data_dir use.
-etcd_io_priority_enabled: false
-
-# THE I/O SCHEDULER, AND WHY IT HAS TO CHANGE FOR ANY OF THIS TO WORK.
-# Measured on k3s-prod-1: `cat /sys/block/sda/queue/scheduler` -> `none [mq-deadline]`.
-# mq-deadline honours the I/O priority CLASS (RT/BE/IDLE) but knows nothing about cgroups,
-# and cgroup v2's io.weight is implemented by blk-iocost — which is compiled in
-# (CONFIG_BLK_CGROUP_IOCOST=y) but DISABLED: /sys/fs/cgroup/io.cost.qos is empty. So on the
-# nodes as they stand today, systemd's IOWeight= writes a number nothing reads.
-# BFQ is the one scheduler here that implements cgroup weights (CONFIG_IOSCHED_BFQ=m,
-# CONFIG_BFQ_GROUP_IOSCHED=y — module present at
-# /lib/modules/7.1.8+deb13-amd64/kernel/block/bfq.ko.xz). Under BFQ, systemd's IOWeight=
-# lands in io.bfq.weight and becomes real.
-#
-# THE COSTS, STATED: BFQ turns OFF writeback throttling (wbt_lat_usec currently 2000 on
-# sda), costs some peak throughput and some CPU, and is a poor fit for NVMe — which is why
-# the role SKIPS the whole scheduler+weight path on an NVMe-backed node (k3s-prod-2), rather
-# than pretending to help there. Set to "" to install nothing but still run the report.
-etcd_io_priority_scheduler: bfq
-
-# Block-layer queue depth for the shared device. sda's current value is 64 (blk-mq derives
-# it from the SATA NCQ depth). A shallower queue bounds how many Longhorn writes can already
-# be sitting in front of an etcd WAL flush at the DEVICE, below any scheduler's reach — the
-# one lever in this file that does not depend on cgroups or on the scheduler honouring
-# anything. Costs sequential throughput. Set to "" to leave nr_requests alone.
-etcd_io_priority_nr_requests: 32
-
-# systemd IOWeight= for system.slice (which contains k3s.service, hence etcd) and for
-# k3s.service itself. THE VALUE IS 10000 — systemd's maximum — AND THAT IS NOT COSMETIC.
-# systemd rescales IOWeight (1..10000) onto io.bfq.weight (1..1000) linearly, so IOWeight=1000
-# maps to roughly 101 — indistinguishable from BFQ's own default of 100 for an unweighted
-# cgroup. Only near the top of the range does the weight actually separate: 10000 -> 1000,
-# a 10:1 share against the default-weighted kubepods.slice. A "reasonable-looking" 500 or
-# 1000 here would be a change that reads as applied and does nothing.
-etcd_io_priority_weight: 10000
-
-# FAIL-LOUD FLOOR for the verification step. The role reads io.bfq.weight back out of the
-# live cgroups and asserts it is at least this. It exists because the rescaling above is
-# systemd's business, not ours: if a systemd version maps IOWeight differently, this role
-# must FAIL rather than leave a drop-in that looks applied. 500 = 5x an unweighted sibling.
-etcd_io_priority_min_bfq_weight: 500
-
-# The k3s data dir whose backing device is the one to protect. Defers to the CLUSTER-WIDE
-# k3s_data_dir (group_vars/k3s_prod sets /data/k3s; the roles/k3s_server default restates
-# k3s's own /var/lib/rancher/k3s for dev) — NOT a second independent variable, for the same
-# reason node_baseline_k3s_data_dir defers to it.
-etcd_io_priority_k3s_data_dir: "{{ k3s_data_dir | default('/var/lib/rancher/k3s') }}"
-
-# The systemd units that get an IOWeight drop-in, and the section each one takes.
-# BOTH ENTRIES ARE LOAD-BEARING — see tasks/main.yml "THE CGROUP WEIGHTS" for why weighting
-# k3s.service alone would arbitrate k3s against sshd and not against Longhorn at all.
-# system.slice must come first so the parent is weighted before its child.
-etcd_io_priority_weighted_units:
-  - unit: system.slice
-    section: Slice
-  - unit: k3s.service
-    section: Service
@@ -1,18 +0,0 @@
---
-# Re-run the oneshot so a changed script/unit takes effect NOW rather than at the next boot.
-# It only writes two sysfs files; there is nothing to drain and nothing to restart.
- name: Apply k3s etcd I/O settings
-  ansible.builtin.systemd_service:
-    name: k3s-etcd-io-priority.service
-    state: restarted
-    daemon_reload: true
-
-# daemon-reload re-realizes cgroup attributes for units that are already running, which is how
-# a changed IOWeight= reaches a LIVE k3s.service without restarting it. It is deliberately NOT
-# a `systemctl restart k3s`: restarting the k3s server to apply a performance tweak is exactly
-# the disruption this role exists to prevent, and on the primary it bounces etcd. If the reload
-# turns out not to be enough, the role's verification step FAILS and tells the operator to do a
-# rolling restart deliberately — it never quietly leaves the weight unapplied.
- name: Reload systemd for I/O weights
-  ansible.builtin.systemd_service:
-    daemon_reload: true
@@ -1,238 +0,0 @@
---
-# etcd_io_priority — see README.md in this directory for the incident, the mechanism, and the
-# honest list of what this does NOT achieve. Everything runs under one `when` on the master
-# switch, so a run against dev (default false) touches nothing at all.
-#
-# TAG: `io-priority`. Every task carries it, so this role can be run alone against a live
-# cluster without dragging in any other role's work:
-#   ansible-playbook playbooks/etcd-io-priority.yml -e k3s_hosts=k3s_prod --limit k3s-prod-3
-#
-# ALWAYS --limit AND ALWAYS CHECK --list-hosts FIRST. This role changes the I/O scheduler of a
-# live node's boot disk; it is safe and instantly reversible, but it is a real change and the
-# blast radius must be one node at a time (playbooks/etcd-io-priority.yml sets serial: 1 too).
-
- name: I/O priority for the shared etcd device
-  when: etcd_io_priority_enabled | bool
-  tags: [io-priority]
-  block:
-
-    # -------------------------------------------------------------------------------------
-    # DISCOVER THE DEVICE. Never hardcoded, and never taken from inventory: k3s-prod-1 and -3
-    # boot from SATA (/dev/sda5) while k3s-prod-2 boots from NVMe (/dev/nvme0n1p5), and a
-    # per-host var freezes today's hardware into inventory where it goes stale silently the
-    # first time a disk is swapped. Inventory declares the INTENT ("the disk under k3s's data
-    # dir"); the node answers what that is. Same reasoning as roles/node_baseline's disk
-    # discovery.
-    # -------------------------------------------------------------------------------------
-    - name: Find the filesystem source backing the k3s data dir
-      ansible.builtin.command:
-        cmd: "findmnt --noheadings --output SOURCE --target {{ etcd_io_priority_k3s_data_dir }}"
-      register: _eio_findmnt
-      changed_when: false
-      check_mode: false
-
-    - name: Find the whole-disk parent of that filesystem source
-      ansible.builtin.command:
-        cmd: "lsblk --noheadings --output PKNAME {{ _eio_findmnt.stdout | trim }}"
-      register: _eio_pkname
-      changed_when: false
-      check_mode: false
-
-    # PKNAME is empty when the source IS already a whole disk (no partition), so fall back to
-    # its basename rather than producing an empty device name.
-    - name: Resolve the block device to tune
-      ansible.builtin.set_fact:
-        _eio_device: >-
-          {{ (_eio_pkname.stdout_lines | map('trim') | reject('equalto', '') | list
-              + [_eio_findmnt.stdout | trim | basename]) | first }}
-
-    # -------------------------------------------------------------------------------------
-    # NVMe IS DELIBERATELY LEFT ALONE, and this is the whole reason the role discovers the
-    # device instead of assuming it. BFQ is a bad fit for NVMe (single software queue, real
-    # throughput cost) and k3s-prod-2 — the NVMe node — measured /proc/pressure/io full avg300
-    # of 0.75 during the 2026-09-07 incident against k3s-prod-1's 11.36. It does not have this
-    # problem. Installing an inert weight drop-in there would be exactly the false confidence
-    # this role exists to avoid, so nothing is written on such a node.
-    # -------------------------------------------------------------------------------------
-    - name: Decide whether a scheduler change applies to this device
-      ansible.builtin.set_fact:
-        _eio_scheduler: "{{ '' if _eio_device is match('nvme') else etcd_io_priority_scheduler }}"
-
-    - name: Report that this node is intentionally skipped (NVMe-backed)
-      ansible.builtin.debug:
-        msg: >-
-          {{ inventory_hostname }}: k3s data dir {{ etcd_io_priority_k3s_data_dir }} is on
-          /dev/{{ _eio_device }} (NVMe). BFQ and its cgroup weights are not applied to NVMe —
-          nothing installed here. This is intentional, not a failure.
-      when: not _eio_scheduler
-
-    # -------------------------------------------------------------------------------------
-    # THE BLOCK-LAYER SETTINGS (scheduler + queue depth).
-    # -------------------------------------------------------------------------------------
-    - name: Install the block-layer settings script
-      ansible.builtin.template:
-        src: k3s-etcd-io-priority.sh.j2
-        dest: /usr/local/sbin/k3s-etcd-io-priority
-        owner: root
-        group: root
-        mode: "0755"
-      when: _eio_scheduler | length > 0
-      notify: Apply k3s etcd I/O settings
-
-    - name: Install the boot-time unit that applies them
-      ansible.builtin.template:
-        src: k3s-etcd-io-priority.service.j2
-        dest: /etc/systemd/system/k3s-etcd-io-priority.service
-        owner: root
-        group: root
-        mode: "0644"
-      when: _eio_scheduler | length > 0
-      notify: Apply k3s etcd I/O settings
-
-    - name: Enable and start the block-layer settings unit
-      ansible.builtin.systemd_service:
-        name: k3s-etcd-io-priority.service
-        enabled: true
-        state: started
-        daemon_reload: true
-      when: _eio_scheduler | length > 0
-
-    # -------------------------------------------------------------------------------------
-    # THE CGROUP WEIGHTS.
-    #
-    # BOTH LEVELS ARE REQUIRED AND NEITHER IS SUFFICIENT. cgroup v2 weights are hierarchical:
-    # k3s.service's weight only decides its share WITHIN system.slice, and system.slice's
-    # weight is what competes with kubepods.slice (where Longhorn's instance-managers live).
-    # Weighting only k3s.service — the intuitive thing to do — would arbitrate k3s against
-    # sshd and journald and would not touch Longhorn at all.
-    #
-    # kubepods.slice is deliberately NOT given a drop-in of its own. It is a TRANSIENT unit
-    # created by the kubelet through the systemd API (`systemctl cat kubepods.slice` on
-    # k3s-prod-1 shows /run/systemd/transient/kubepods.slice), and whether systemd merges an
-    # /etc drop-in into a transient unit is not something this change verified on these nodes.
-    # Raising system.slice achieves the same ratio through a unit systemd definitely owns.
-    # -------------------------------------------------------------------------------------
-    - name: Ensure the systemd drop-in directories exist
-      ansible.builtin.file:
-        path: "/etc/systemd/system/{{ item.unit }}.d"
-        state: directory
-        owner: root
-        group: root
-        mode: "0755"
-      loop: "{{ etcd_io_priority_weighted_units }}"
-      loop_control:
-        label: "{{ item.unit }}"
-      when: _eio_scheduler | length > 0
-
-    - name: Write the IOWeight drop-ins
-      ansible.builtin.template:
-        src: io-weight.conf.j2
-        dest: "/etc/systemd/system/{{ item.unit }}.d/10-etcd-io-weight.conf"
-        owner: root
-        group: root
-        mode: "0644"
-      vars:
-        _eio_dropin_unit: "{{ item.unit }}"
-        _eio_dropin_section: "{{ item.section }}"
-      loop: "{{ etcd_io_priority_weighted_units }}"
-      loop_control:
-        label: "{{ item.unit }}"
-      when: _eio_scheduler | length > 0
-      notify: Reload systemd for I/O weights
-
-    # Handlers must run BEFORE the verification below, or the verification reads the state from
-    # before this run's changes and reports a stale pass.
-    - name: Apply pending I/O changes before verifying them
-      ansible.builtin.meta: flush_handlers
-
-    # -------------------------------------------------------------------------------------
-    # VERIFY — the point of the role, not a formality.
-    #
-    # This repo has repeatedly shipped configuration that looked applied and was not (the
-    # Keycloak reconciler that could not read its own Secret; Longhorn's orphanAutoDeletion,
-    # silently dropped by the chart for two minor versions). Every mechanism here has a
-    # plausible silent-failure mode: the scheduler module may not load; a systemd version may
-    # rescale IOWeight differently; daemon-reload may not re-realize a running unit's cgroup.
-    # So each is READ BACK OUT OF THE KERNEL and asserted. A failure here means the mitigation
-    # is not in place — which is strictly better than believing that it is.
-    # -------------------------------------------------------------------------------------
-    - name: Read back the live block-layer state
-      ansible.builtin.command:
-        cmd: "cat /sys/block/{{ _eio_device }}/queue/scheduler /sys/block/{{ _eio_device }}/queue/nr_requests"
-      register: _eio_queue_state
-      changed_when: false
-      check_mode: false
-      when: _eio_scheduler | length > 0
-
-    - name: VERIFY the scheduler and queue depth actually took effect
-      ansible.builtin.assert:
-        that:
-          - "'[' ~ _eio_scheduler ~ ']' in _eio_queue_state.stdout_lines[0]"
-          - "etcd_io_priority_nr_requests | string | length == 0
-             or _eio_queue_state.stdout_lines[1] | trim | int == etcd_io_priority_nr_requests | int"
-        fail_msg: >-
-          /dev/{{ _eio_device }} did not take the requested settings. Wanted scheduler
-          '{{ _eio_scheduler }}' and nr_requests '{{ etcd_io_priority_nr_requests }}'; the kernel
-          reports {{ _eio_queue_state.stdout_lines | join(' / ') }}. The cgroup weights below are
-          MEANINGLESS without the scheduler, so this is a hard failure, not a warning. Check
-          `journalctl -u k3s-etcd-io-priority` and that bfq.ko exists for this kernel.
-        success_msg: >-
-          /dev/{{ _eio_device }}: scheduler {{ _eio_scheduler }}, nr_requests
-          {{ _eio_queue_state.stdout_lines[1] | trim }}.
-      when: _eio_scheduler | length > 0
-
-    # k3s.service's cgroup does not exist until k3s is running, so a cold-build run legitimately
-    # has nothing to read there. Reported, not asserted away.
-    - name: Check whether k3s.service has a live cgroup to verify
-      ansible.builtin.stat:
-        path: /sys/fs/cgroup/system.slice/k3s.service
-      register: _eio_k3s_cgroup
-      when: _eio_scheduler | length > 0
-
-    - name: Report that k3s is not running, so its own cgroup weight cannot be verified yet
-      ansible.builtin.debug:
-        msg: >-
-          {{ inventory_hostname }}: /sys/fs/cgroup/system.slice/k3s.service does not exist — k3s
-          is not running on this node, so its drop-in is written but UNVERIFIED. Re-run
-          playbooks/etcd-io-priority.yml once k3s is up.
-      when:
-        - _eio_scheduler | length > 0
-        - not (_eio_k3s_cgroup.stat.exists | default(false))
-
-    - name: Read back the live cgroup I/O weights
-      ansible.builtin.command:
-        cmd: "cat /sys/fs/cgroup/{{ item }}/io.bfq.weight"
-      register: _eio_bfq_weights
-      changed_when: false
-      check_mode: false
-      failed_when: false
-      # `| default(false)`: Ansible templates `loop:` BEFORE evaluating `when:`, so on an NVMe
-      # node — where the stat task above is skipped and _eio_k3s_cgroup has no `.stat` at all —
-      # a bare `_eio_k3s_cgroup.stat.exists` raises even though this task will not run.
-      loop: >-
-        {{ ['system.slice']
-           + (['system.slice/k3s.service']
-              if (_eio_k3s_cgroup.stat.exists | default(false)) else []) }}
-      when: _eio_scheduler | length > 0
-
-    - name: VERIFY every live cgroup weight actually carries the boost
-      ansible.builtin.assert:
-        that:
-          - item.rc == 0
-          - item.stdout | trim | int >= etcd_io_priority_min_bfq_weight | int
-        fail_msg: >-
-          /sys/fs/cgroup/{{ item.item }}/io.bfq.weight reads
-          '{{ item.stdout | default('<unreadable>') | trim }}' (rc={{ item.rc }}), below the
-          required floor of {{ etcd_io_priority_min_bfq_weight }}. IOWeight={{ etcd_io_priority_weight }}
-          did NOT reach the kernel. Most likely causes, in order: (1) systemd rescaled the value
-          differently from what defaults/main.yml assumes — compare
-          `systemctl show {{ item.item | basename }} -p IOWeight` against the file; (2) the cgroup
-          was not re-realized by daemon-reload, in which case a ROLLING `systemctl restart k3s`
-          (one node at a time — etcd quorum is 2/3) applies it; (3) BFQ is not active on the
-          device, which the previous assert should already have caught. The drop-in on disk is
-          NOT the mitigation — this file is.
-        success_msg: "{{ item.item }} io.bfq.weight = {{ item.stdout | default('') | trim }}"
-      loop: "{{ _eio_bfq_weights.results | default([]) }}"
-      loop_control:
-        label: "{{ item.item | default('') }}"
-      when: _eio_scheduler | length > 0
@@ -1,21 +0,0 @@
-# {{ ansible_managed }}
-#
-# Proportional share of {{ _eio_device }} for {{ _eio_dropin_unit }}, which is where the k3s server
-# process — and therefore the EMBEDDED ETCD it hosts — does its WAL writes. Its competitor is
-# kubepods.slice (Longhorn's instance-managers), which stays at the default weight.
-#
-# THIS ONLY WORKS BECAUSE THE SCHEDULER IS {{ _eio_scheduler | upper }}. systemd writes IOWeight= to both
-# io.weight (blk-iocost, which is compiled in but DISABLED here — /sys/fs/cgroup/io.cost.qos is
-# empty) and io.bfq.weight (BFQ). Under the stock mq-deadline this file is a no-op; the sibling
-# k3s-etcd-io-priority.service is what makes it mean something. roles/etcd_io_priority verifies
-# io.bfq.weight after writing this and FAILS the play if the value did not land.
-#
-# NOT IOSchedulingClass=. I/O priority is inherited across fork and kept across exec, and
-# containerd is a child of the k3s server process — so realtime here would be realtime for every
-# container on the node, Longhorn included. See roles/etcd_io_priority/README.md.
-#
-# {{ etcd_io_priority_weight }} is systemd's MAXIMUM and that is deliberate: systemd rescales
-# 1..10000 onto io.bfq.weight's 1..1000, so IOWeight=1000 would map to ~101 — BFQ's unweighted
-# default is 100. Only the top of the range actually separates this cgroup from its siblings.
-[{{ _eio_dropin_section }}]
-IOWeight={{ etcd_io_priority_weight }}
@@ -1,24 +0,0 @@
-# {{ ansible_managed }}
-#
-# WHY A ONESHOT UNIT AND NOT A UDEV RULE. The obvious form of this is
-# `ACTION=="add|change", KERNEL=="{{ _eio_device }}", ATTR{queue/scheduler}="{{ _eio_scheduler }}"`,
-# and it is unreliable for the ONE device that matters: {{ _eio_device }} is the boot disk, so its
-# `add` event fires in the initramfs, where /etc/udev/rules.d is not present and bfq.ko is not
-# loaded — the ATTR assignment silently does nothing and the disk comes up on mq-deadline with
-# no error anywhere. A unit ordered Before=k3s.service runs after modules are loadable, fails
-# LOUDLY if the scheduler is unavailable, and is visible in `systemctl status`.
-[Unit]
-Description=Block-layer I/O settings for the shared etcd + Longhorn device ({{ _eio_device }})
-Documentation=file:///etc/systemd/system/k3s-etcd-io-priority.service
-DefaultDependencies=no
-After=systemd-modules-load.service
-Before=k3s.service shutdown.target
-Conflicts=shutdown.target
-
-[Service]
-Type=oneshot
-RemainAfterExit=yes
-ExecStart=/usr/local/sbin/k3s-etcd-io-priority
-
-[Install]
-WantedBy=multi-user.target
@@ -1,38 +0,0 @@
-#!/bin/sh
-# {{ ansible_managed }}
-#
-# Block-layer settings for the device that carries BOTH etcd's WAL ({{ etcd_io_priority_k3s_data_dir }})
-# and every Longhorn replica on this node. Run at boot by k3s-etcd-io-priority.service, and
-# safe to run by hand at any time — it is idempotent and takes effect immediately.
-#
-# Rollback (immediate, no reboot):
-#   printf 'mq-deadline\n' > /sys/block/{{ _eio_device }}/queue/scheduler
-#   printf '64\n'          > /sys/block/{{ _eio_device }}/queue/nr_requests
-# and `systemctl disable --now k3s-etcd-io-priority.service` so it does not come back.
-set -eu
-
-DEV="{{ _eio_device }}"
-QUEUE="/sys/block/${DEV}/queue"
-
-if [ ! -d "${QUEUE}" ]; then
-    echo "k3s-etcd-io-priority: /sys/block/${DEV} does not exist — device renamed?" >&2
-    exit 1
-fi
-
-{% if _eio_scheduler %}
-# BFQ is a module here (CONFIG_IOSCHED_BFQ=m); loading it is what makes it selectable at all.
-modprobe -q {{ _eio_scheduler }} 2>/dev/null || true
-
-if ! grep -qw '{{ _eio_scheduler }}' "${QUEUE}/scheduler"; then
-    echo "k3s-etcd-io-priority: scheduler '{{ _eio_scheduler }}' not available for ${DEV}: $(cat "${QUEUE}/scheduler")" >&2
-    exit 1
-fi
-printf '%s\n' '{{ _eio_scheduler }}' > "${QUEUE}/scheduler"
-{% endif %}
-
-{% if etcd_io_priority_nr_requests %}
-# ORDER MATTERS: switching the elevator resets nr_requests, so this must come second.
-printf '%s\n' '{{ etcd_io_priority_nr_requests }}' > "${QUEUE}/nr_requests"
-{% endif %}
-
-exit 0
@@ -81,6 +81,45 @@ node_baseline_kernel_modules:
  - dm_crypt
  - nfs

+# --- NODE CLOCK / TIMEZONE (tag `system`) ---------------------------------------------
+# EVERY NODE IS UTC. Measured drift that made this a knob rather than an assumption: the
+# three prod nodes disagreed on their local zone (read off k3s's own etcd snapshot
+# annotations, 2026-09-05) —
+#   k3s-prod-1  2026-09-05T16:31:28-05:00   (Central)
+#   k3s-prod-2  2026-09-05T16:29:07-05:00   (Central)
+#   k3s-prod-3  2026-09-05T17:27:35-04:00   (Eastern)
+# In UTC those three are ~4 minutes apart, which is the CORRECT staggering: nothing was
+# functionally wrong, because Kubernetes, etcd and the container runtime are UTC internally.
+# What IS wrong is that any timestamp a node renders in LOCAL time — journald's default
+# output, `ls -l`, a shell prompt, an etcd snapshot filename — cannot be compared across
+# nodes without knowing each node's zone. Correlating three nodes' logs during an incident
+# is exactly when nobody has that knowledge to hand, and this cluster took two unplanned
+# node reboots in the 24h before this was written.
+#
+# WHY UTC AND NOT "STANDARDISE ON CENTRAL". Picking one local zone leaves the class of
+# problem intact (a fourth node, or a DST boundary, re-creates it) and adds a translation
+# step to every log read, because every timestamp the PLATFORM emits — Kubernetes events,
+# Prometheus, Loki, Velero, CNPG WAL, the etcd snapshot annotations above — is already UTC.
+# UTC removes the class: node-local time and platform time become the same clock, and there
+# is no DST discontinuity in which a repeated or skipped hour breaks log ordering. This also
+# matches the only timezone convention the repo already had — catalog/plex.yaml records
+# "no other service in the repo sets a timezone ... so use the safe default" and both it and
+# apps/wave-0/plex.yaml set TZ=Etc/UTC. (platform/crafty sets America/Chicago, deliberately:
+# that is a USER-FACING game server's schedule, not a node's log clock.)
+#
+# WHY A KNOB AT ALL (R-ENV-02). The value is a node-level Ansible fact, so its override seam
+# is inventory/group_vars/<env>, NOT clusters/base/cluster-vars.yaml (a Flux postBuild
+# ConfigMap no Ansible code reads) — the same seam node_baseline_backports_kernel and
+# node_baseline_dns_servers use. NO ENVIRONMENT OVERRIDES IT TODAY, and one should not have
+# to: dev and prod both want UTC and there is nothing environment-specific about a node
+# clock. It is a variable rather than a literal in the task so a genuinely
+# operator-facing one-off host (never a k3s node) can differ without forking the role.
+#
+# Must be a zoneinfo name present on the node (`timedatectl list-timezones`); the assert in
+# tasks/main.yml rejects an empty or obviously-malformed value before the module runs, and
+# community.general.timezone itself refuses a name /usr/share/zoneinfo does not have.
+node_baseline_timezone: Etc/UTC
+
 # --- DATA DISKS (tasks/disks.yml, tag `disks`) ----------------------------------------
 # The SHAPE of a node's storage is DECLARED per environment, never inferred from whether
 # a variable happens to be set. Two enums, defaulting to dev's two-dedicated-disks shape:
@@ -11,13 +11,3 @@
  ansible.builtin.systemd:
    name: systemd-resolved
    state: restarted
-
-# Masking a unit writes a symlink to /dev/null into /etc/systemd/system; the RUNNING
-# systemd keeps its old view of the unit until it re-reads unit files. The mask is read
-# fresh at the next boot either way, so this only exists so the running manager does not
-# disagree with on-disk state in between. `daemon_reload` re-parses unit files and
-# re-runs generators — it starts and stops nothing, so it cannot disturb k3s or Longhorn.
-# Fires only when a mask actually changed, so a re-run of the role is silent.
- name: Reload systemd unit files
-  ansible.builtin.systemd_service:
-    daemon_reload: true
@@ -331,15 +331,6 @@
 # The devices this role must never, under any circumstances, hand to e2label. Built from
 # gathered facts (`gather_facts: true` in the play) plus the live swap list, so it is the
 # NODE's account of what is load-bearing, not a hardcoded list of partition names.
-#
-# EXPECT THIS LIST TO BE EMPTY. Since the swap tasks in tasks/main.yml mask swap.target,
-# a baselined node has no active swap for `swapon` to report, so the swap half of
-# node_baseline_lh_protected_devices below contributes nothing. That is not a weakened
-# guard: the claim CANDIDATE is derived from whatever is mounted at
-# node_baseline_longhorn_claim_mountpoint, and a swap partition is neither mounted nor an
-# ext4 filesystem, so it can never be the candidate in the first place. The task stays
-# because the list must be the node's own account of what is load-bearing — including on
-# a node reached before baseline has run, or one an operator has deliberately swapped on.
 - name: "Longhorn (claim): list the devices currently backing active swap"
  ansible.builtin.command: swapon --noheadings --show=NAME
  register: node_baseline_lh_swapdevs
@@ -262,6 +262,52 @@
      vars:
        newest: "{{ (node_baseline_installed_kernels.stdout_lines | default([]) | first) | default('', true) }}"

+# =====================================================================================
+# NODE CLOCK — tag `system`: every node renders time in the SAME zone, and that zone is UTC
+# =====================================================================================
+# MEASURED DRIFT (2026-09-05), read off k3s's own etcd snapshot annotations on live prod:
+#   k3s-prod-1  2026-09-05T16:31:28-05:00   Central
+#   k3s-prod-2  2026-09-05T16:29:07-05:00   Central
+#   k3s-prod-3  2026-09-05T17:27:35-04:00   Eastern  <-- a different UTC offset
+# Functionally harmless (in UTC those three are ~4 minutes apart, which is the correct
+# staggering — Kubernetes, etcd and containerd are UTC internally). Operationally it is a
+# trap: any timestamp a node renders in LOCAL time (journald's default output, `ls -l`, an
+# etcd snapshot filename) is not comparable across nodes without knowing each node's zone,
+# and cross-node log correlation during an incident is precisely when nobody knows that.
+#
+# Rationale for UTC rather than standardising on one local zone, and for this being a knob
+# whose override seam is inventory/group_vars/<env>: see node_baseline_timezone in
+# defaults/main.yml. NOT a cluster-vars knob — no Ansible code reads that ConfigMap.
+#
+# IDEMPOTENCE / BLAST RADIUS. community.general.timezone is a converge-to-state module: it
+# reports `changed` only when /etc/localtime + /etc/timezone do not already name this zone,
+# and it does NOT step the clock (no time jump, no NTP interaction) — only the RENDERING of
+# an unchanged monotonic clock moves. No reboot is required. Already-running services that
+# render local time keep the old zone until they are next restarted (the module restarts
+# cron itself, which is the one scheduler on these nodes that cares); k3s, etcd and the
+# kubelet are unaffected either way because they emit UTC regardless.
+- name: Assert this node's timezone has actually been decided
+  ansible.builtin.assert:
+    that:
+      - node_baseline_timezone is string
+      - node_baseline_timezone | length > 0
+      - "'BLOCKED' not in node_baseline_timezone"
+      - node_baseline_timezone is match('^[A-Za-z][A-Za-z0-9+_-]*(/[A-Za-z0-9+_.-]+)*$')
+    fail_msg: >-
+      node_baseline_timezone is not a usable zoneinfo name (got:
+      '{{ node_baseline_timezone | default("<undefined>") }}'). It must be a name
+      /usr/share/zoneinfo actually carries — `timedatectl list-timezones` on the node — and
+      the repo default (Etc/UTC) is almost certainly what you want; see the rationale in
+      roles/node_baseline/defaults/main.yml. Empty is NOT "leave it alone": a node whose zone
+      nobody has decided is the config drift this task exists to remove.
+    quiet: true
+  tags: [system]
+
+- name: Pin the node timezone (cross-node log correlation; UTC by default)
+  community.general.timezone:
+    name: "{{ node_baseline_timezone }}"
+  tags: [system]
+
 - name: Persist kernel modules for k3s/Cilium/Longhorn
  ansible.builtin.copy:
    dest: /etc/modules-load.d/k3s-cilium-longhorn.conf
@@ -646,109 +692,9 @@
      tags: [disks]
  tags: [disks]

-# =====================================================================================
-# SWAP — off NOW, and still off AFTER A REBOOT — tag `system`
-# =====================================================================================
-# `swapoff -a` plus commenting the fstab line is NOT durable on these nodes, and never
-# was. Measured 2026-09-07: all three prod nodes were running with 977 MiB of swap
-# ACTIVE (k3s-prod-1 /dev/sda4, k3s-prod-3 /dev/sda4, k3s-prod-2 /dev/nvme0n1p4) while
-# their fstab swap line was commented out by this very role. The role reported success
-# and the setting silently expired at the next boot, on every node.
-#
-# CAUSE. systemd-gpt-auto-generator (systemd 257, Debian 13) claims any GPT partition
-# whose partition TYPE GUID is 0657fd6d-a4ab-43c4-84e5-0933c84b4f4f ("Linux swap"),
-# synthesises a .swap unit for it into /run/systemd/generator.late/ and drops a symlink
-# to it into swap.target.wants/. It decides that from the PARTITION TABLE and never
-# consults /etc/fstab, so neutralising fstab cannot stop it. Re-running the generator by
-# hand on k3s-prod-1 says so in as many words:
-#     Dissecting swap partition with label n/a and UUID acd3876c-fd8e-4389-b856-...
-#     Probed fstype 'swap' on partition /dev/disk/by-diskseq/1-part4.
-#     Adding swap: /dev/disk/by-diskseq/1-part4
-#
-# MECHANISM — mask swap.target.
-# `systemctl show` on the generated unit says its ONLY inbound dependency is
-# `WantedBy=swap.target`: Requires/RequiredBy/BoundBy are empty, and the matching
-# .device unit has an empty `Wants=`, so a device appearing does NOT pull its swap unit
-# in by itself. swap.target is therefore the single choke point that every swap
-# activation path goes through — the gpt-auto units, the units systemd-fstab-generator
-# builds from /etc/fstab, and the ones systemd-cryptsetup-generator builds for encrypted
-# swap. Masking it is:
-#   * DEVICE-SHAPE INDEPENDENT. No device name appears anywhere in the change, so one
-#     task is correct on prod's sda4 AND nvme0n1p4 and on dev's vda VMs, and there is
-#     nothing to keep in sync when a disk is replaced. (The generated unit is named after
-#     the kernel's DISKSEQ, not the disk — it is `dev-disk-by\x2ddiskseq-1\x2dpart4.swap`
-#     on all three prod nodes, NVMe and SATA alike — but diskseq is assigned per boot in
-#     device-probe order, so it is not a name worth depending on either.)
-#   * BOOT-SAFE. swap.target is reached by `Wants=`/`After=` from sysinit.target and by
-#     `After=` from tmp.mount, and `RequiredBy=` is EMPTY (verified on all three prod
-#     nodes and all three dev nodes). A masked unit reached only through Wants= is
-#     skipped with a log line; it cannot fail the boot transaction. Ordering deps on a
-#     unit that never starts are no-ops.
-#   * INERT WHEN THE DEVICE IS ABSENT OR RENAMED. The mask names no device, so there is
-#     nothing to go stale: a disk swap, a new partition layout or a restore onto
-#     different hardware all stay swap-free without touching this role.
-# The per-unit masks that follow are a SECOND, independent barrier over the units that
-# exist right now, so `systemctl status <unit>` says "masked" to whoever looks, and an
-# operator who unmasks swap.target does not silently get swap back. Their failure mode is
-# benign in the other direction too: a mask whose unit no longer exists is an unreferenced
-# symlink to /dev/null, and the only thing it could ever shadow is a .swap unit.
-#
-# ALTERNATIVES REJECTED.
-#   * `systemd.gpt_auto=0` on the kernel cmdline — disables GPT auto-discovery WHOLESALE,
-#     not just swap, and needs a GRUB edit + update-grub + a reboot before it is even
-#     true. On prod that happens to be swap-only today (root= is already on the cmdline
-#     so root auto-detection is off, and the /boot/efi fstab entry already suppresses the
-#     ESP/XBOOTLDR mounts — the generator says so in its debug output), but on DEV the
-#     root partition carries the discoverable GUID 4f68bce3-e8cd-4db1-96e7-fbcaf984b709
-#     ("Linux root (x86-64)"), so the same flag would switch off root auto-discovery
-#     there. Editing a bootloader to fix a swap unit is the wrong blast radius, and the
-#     change would not be verifiable without the reboot it depends on.
-#   * Clearing or retyping the swap partition's GPT type GUID — a partition-table write
-#     to the live boot disk of a node currently serving etcd and Longhorn replicas, to
-#     fix something a symlink fixes. It would also decide, irreversibly and without
-#     asking, that those 977 MiB are never swap again and are simply unused; that is the
-#     operator's call to make, not this role's, and it needs a maintenance window.
-#   * `systemd.swap=0` — read by systemd-fstab-generator ONLY. The swap here does not
-#     come from fstab, so it would change nothing at all.
-#
-# DEV. Dev's nodes have no swap partition, no swap unit and no fstab swap line at all
-# (checked on k3s-dev-1/2/3), so the mask is inert there: it flips to masked once and
-# every later run is changed=0. It is correct rather than merely harmless — if a future
-# dev image ever ships a swap partition, the same mask already covers it.
-
- name: Mask swap.target so no generator can pull a swap unit in at boot
-  ansible.builtin.systemd_service:
-    name: swap.target
-    masked: true
-  notify: Reload systemd unit files
-  tags: [system]
-
- name: Find the .swap units systemd's generators produced for this boot
-  ansible.builtin.find:
-    # gpt-auto writes to generator.late; systemd-fstab-generator and
-    # systemd-cryptsetup-generator (the other two that can emit a .swap unit) write to
-    # generator. /run/systemd/generator.early is deliberately NOT listed: nothing that
-    # makes swap units runs at early priority, and `find` warns on a path that does not
-    # exist. Anything a future early generator invented is still caught by the
-    # swap.target mask above, which is the mechanism this task only reinforces.
-    paths:
-      - /run/systemd/generator
-      - /run/systemd/generator.late
-    patterns: "*.swap"
-    file_type: file
-  register: node_baseline_generated_swap
+- name: Disable swap now
+  ansible.builtin.command: swapoff -a
  changed_when: false
-  check_mode: false
-  tags: [system]
-
- name: Mask each generated .swap unit (second barrier, independent of swap.target)
-  ansible.builtin.systemd_service:
-    name: "{{ item }}"
-    masked: true
-  loop: "{{ node_baseline_generated_swap.files | map(attribute='path') | map('basename') | unique | sort }}"
-  loop_control:
-    label: "{{ item }}"
-  notify: Reload systemd unit files
  tags: [system]

 - name: Neutralize swap entries in fstab (kubelet requires swap off)
@@ -758,23 +704,6 @@
    replace: '# \1  # disabled for k3s/kubelet'
  tags: [system]

-# The masks above only decide what the NEXT boot does. Anything swapped on already stays
-# on until it is told otherwise, so read the live state and act only when there is
-# something to act on — that is what makes a second run honestly changed=0 instead of
-# `changed_when: false` hiding a command that ran anyway.
- name: Read the swap devices that are active right now
-  ansible.builtin.command: swapon --noheadings --show=NAME
-  register: node_baseline_swap_active
-  changed_when: false
-  check_mode: false
-  tags: [system]
-
- name: Disable swap now
-  ansible.builtin.command: swapoff -a
-  when: node_baseline_swap_active.stdout | trim | length > 0
-  changed_when: true
-  tags: [system]
-
 # =====================================================================================
 # DNS RESILIENCE — tag `dns` (R-DNS-01 / R-DNS-03)
 # =====================================================================================
@@ -47,7 +47,7 @@ Serves **R-ENV-01** (you cannot certify a node you have not measured), **R-ENV-0
 | `storage.io_errors` | kernel SCSI/ATA `ioerr_cnt` / `iotmo_cnt` per disk (no root needed) |
 | `storage.capacity`, `storage.inodes` | worst mounted filesystem, space and inodes |
 | `storage.layout` | physical disks, and whether any device is genuinely free for `node_baseline_longhorn_device` / `node_baseline_etcd_device` |
-| `memory.available`, `memory.swap`, `memory.ecc` | headroom, kubelet's swap-off prerequisite **and whether it survives a reboot** (see [Swap off is not the same as swap disabled](#swap-off-is-not-the-same-as-swap-disabled)), EDAC correctable/uncorrectable counters |
+| `memory.available`, `memory.swap`, `memory.ecc` | headroom, kubelet's swap-off prerequisite, EDAC correctable/uncorrectable counters |
 | `memory.physical_total` | **is this machine using the RAM it has?** `/proc/zoneinfo` present-vs-managed pages (no root, metal or VM) and `MemTotal` vs the DIMM capacity SMBIOS reports (`dmidecode -t 17`, root, bare metal). Cause-independent by design — see [The EFI memattr check](#the-efi-memattr-check) |
 | `cpu.throttle`, `cpu.inventory`, `cpu.virtualisation` | thermal throttle events since boot, topology + frequency scaling, VT-x/AMD-V + `/dev/kvm` (KubeVirt, R-PLAT-03) |
 | `thermal.max` | hottest sensor across thermal zones and hwmon chips |
@@ -59,37 +59,6 @@ Serves **R-ENV-01** (you cannot certify a node you have not measured), **R-ENV-0
 Thresholds live in `defaults/main.yml` (`node_health_thresholds`); no verdict is hardcoded
 in task logic.

-### Swap off is not the same as swap disabled
-
-`SwapTotal == 0` only says nothing has been swapped on *yet this boot*. It is not evidence
-that swap is disabled, and on these nodes it was not: measured 2026-09-07, all three prod
-nodes came back from a reboot with 977 MiB of swap active (`k3s-prod-1` `/dev/sda4`,
-`k3s-prod-3` `/dev/sda4`, `k3s-prod-2` `/dev/nvme0n1p4`) even though `roles/node_baseline`
-had run `swapoff -a` and commented their `/etc/fstab` swap line.
-
-The cause is `systemd-gpt-auto-generator`. It claims any GPT partition whose partition
-**type GUID** is `0657fd6d-a4ab-43c4-84e5-0933c84b4f4f` ("Linux swap") straight from the
-partition table, synthesises a `.swap` unit into `/run/systemd/generator.late/` and links
-it into `swap.target.wants/`. `/etc/fstab` is never consulted, so neutralising fstab
-cannot stop it. `roles/node_baseline` masks `swap.target` — the one unit every generated
-`.swap` unit is `WantedBy` — and masks the generated units as a second barrier.
-
-So `memory.swap` reports on both halves, from `memory.swap_persistence` in the collector
-(all world-readable, no root):
-
-| fact | meaning |
-|---|---|
-| `swap_target_masked` | is `/etc/systemd/system/swap.target` a symlink to `/dev/null`? |
-| `generated_units` | `.swap` units systemd's generators built for this boot |
-| `masked_units` | `.swap` units masked in `/etc/systemd/system` |
-| `unmasked_generated_units` | `generated_units` minus `masked_units` — what a reboot would bring back |
-| `fstab_active_swap_lines` | uncommented `swap` lines left in `/etc/fstab` |
-
-A node with no swap active but an unmasked generated unit and an unmasked `swap.target`
-**WARNs**: it is clean now and will be swapping again after the next reboot. The check
-used to blame that state on "this node has not been baselined yet", which sent the reader
-back to re-run the role that had already run.
-

 ## The EFI memattr check

@@ -301,94 +301,9 @@ def collect_memory():
    # Firmware's own inventory of what is physically installed. Root-only; attempted
    # here anyway so the unprivileged pass records the exact reason it could not read.
    mem["devices"] = collect_memory_devices()
-    # "Is swap off right now?" and "will it still be off after a reboot?" are DIFFERENT
-    # questions, and only the second one is kubelet's actual prerequisite. See below.
-    mem["swap_persistence"] = collect_swap_persistence()
    return mem


-# --- Will swap still be off after the next reboot? -----------------------------------
-# SwapTotal=0 does not mean swap is DISABLED — it means nothing has been swapped on yet
-# this boot. Measured 2026-09-07 on all three prod nodes: swap was active again after a
-# reboot, 977 MiB each, with /etc/fstab's swap line commented out by node_baseline. The
-# cause is systemd-gpt-auto-generator, which claims any GPT partition typed
-# 0657fd6d-a4ab-43c4-84e5-0933c84b4f4f ("Linux swap") straight from the PARTITION TABLE,
-# synthesises a .swap unit into /run/systemd/generator.late/ and links it into
-# swap.target.wants/. It never reads fstab, so neutralising fstab cannot stop it.
-#
-# roles/node_baseline masks swap.target — the single unit every generated .swap unit is
-# `WantedBy` — and masks the generated units themselves as a second barrier. This
-# collects the facts that let the report tell "off, and staying off" apart from "off
-# until the next reboot", which a /proc/meminfo reading alone can never do.
-#
-# All of it is world-readable (the generator dirs are 0755 root:root), so it works in the
-# UNPRIVILEGED pass: no root, no systemctl, no parsing of systemctl output.
-def _is_systemd_mask(path):
-    """True when `path` is a systemd mask, i.e. a symlink to /dev/null."""
-    try:
-        return os.path.islink(path) and os.path.realpath(path) == "/dev/null"
-    except OSError:
-        return False
-
-
-def collect_swap_persistence():
-    out = {
-        "readable": False,
-        "swap_target_masked": None,
-        # .swap units systemd's generators built for THIS boot (gpt-auto, fstab,
-        # cryptsetup). These are the ones that would be activated again next boot.
-        "generated_units": [],
-        # .swap units masked in /etc/systemd/system — what node_baseline wrote.
-        "masked_units": [],
-        # generated MINUS masked: the units that would still come back. Empty is good.
-        "unmasked_generated_units": [],
-        "fstab_active_swap_lines": None,
-    }
-
-    generated = set()
-    for directory in (
-        "/run/systemd/generator.early",
-        "/run/systemd/generator",
-        "/run/systemd/generator.late",
-    ):
-        try:
-            names = os.listdir(directory)
-        except FileNotFoundError:
-            # A generator produced nothing. That is a real answer, not a read failure:
-            # dev's VMs have no swap partition, so gpt-auto emits no unit at all.
-            continue
-        except OSError as exc:
-            UNREADABLE.append({"path": directory, "reason": "read error: %s" % exc})
-            return out
-        for name in names:
-            if name.endswith(".swap") and os.path.isfile(os.path.join(directory, name)):
-                generated.add(name)
-
-    masked = set()
-    try:
-        for name in os.listdir("/etc/systemd/system"):
-            if name.endswith(".swap") and _is_systemd_mask("/etc/systemd/system/" + name):
-                masked.add(name)
-    except OSError as exc:
-        UNREADABLE.append({"path": "/etc/systemd/system", "reason": "read error: %s" % exc})
-        return out
-
-    out["swap_target_masked"] = _is_systemd_mask("/etc/systemd/system/swap.target")
-    out["generated_units"] = sorted(generated)
-    out["masked_units"] = sorted(masked)
-    out["unmasked_generated_units"] = sorted(generated - masked)
-    out["readable"] = True
-
-    fstab = read_text("/etc/fstab")
-    if fstab is not None:
-        out["fstab_active_swap_lines"] = sum(
-            1
-            for line in fstab.splitlines()
-            if not line.lstrip().startswith("#") and re.search(r"\sswap\s", line)
-        )
-    return out
-
-
 # --- Memory the kernel has but never hands out --------------------------------------
 # /proc/zoneinfo is WORLD-READABLE on every Linux, VM or metal — no root, no dmidecode,
 # no DMI. Per zone it reports:
@@ -499,76 +499,25 @@
            if phys_status in ['WARN', 'FAIL'] else '')
         ~ (phys_cause if phys_status in ['WARN', 'FAIL'] else '') }}

-# WHY THIS CHECK IS NOT JUST `SwapTotal == 0`. It used to be, and it read the answer
-# wrong in both directions. It called active swap "this node has not been baselined yet",
-# which was a MIS-DIAGNOSIS — measured 2026-09-07, all three prod nodes had been
-# baselined, had their fstab swap line commented out by node_baseline, and were STILL
-# swapping 977 MiB each after a reboot. Whoever read that WARN would have re-run the very
-# role that had already run, and watched swap come back at the next boot again.
-#
-# The real mechanism is systemd-gpt-auto-generator: it claims any GPT partition typed
-# 0657fd6d-a4ab-43c4-84e5-0933c84b4f4f ("Linux swap") from the PARTITION TABLE, builds a
-# .swap unit into /run/systemd/generator.late/ and links it into swap.target.wants/. It
-# never reads /etc/fstab, so `swapoff -a` + a commented fstab line only holds until the
-# next boot. roles/node_baseline now masks swap.target (the single unit those generated
-# units are `WantedBy`) plus the generated units themselves.
-#
-# The other direction matters just as much: SwapTotal == 0 is NOT proof that swap is
-# disabled — a node freshly `swapoff`-ed reads zero and swaps again on reboot. So this
-# check now asks both questions, and WARNs on a node that is clean right now but has an
-# unmasked generated swap unit waiting for the next boot.
- name: Assess — swap (kubelet requires it off, and off ACROSS REBOOTS)
+- name: Assess — swap (kubelet requires it off)
  ansible.builtin.set_fact:
    node_health_checks: "{{ node_health_checks + [{
      'id': 'memory.swap',
-      'title': 'Swap disabled, and disabled across reboots (kubelet prerequisite)',
+      'title': 'Swap disabled (kubelet prerequisite)',
      'status': swap_status,
      'detail': swap_detail }] }}"
  vars:
    ansible_become: false
    m: "{{ node_health_facts.memory }}"
-    p: "{{ node_health_facts.memory.swap_persistence | default({}, true) }}"
-    # Durability was actually measurable on this node (the collector could read
-    # /etc/systemd/system and the generator dirs). When false, say so rather than
-    # implying a clean bill from a failed read.
-    swap_durable_known: "{{ (p.readable | default(false)) | bool }}"
-    # Units a reboot would bring back: produced by a generator, not masked, and with
-    # swap.target itself unmasked so something can still pull them in.
-    swap_will_return: >-
-      {{ (swap_durable_known | bool)
-         and not (p.swap_target_masked | default(false) | bool)
-         and (p.unmasked_generated_units | default([], true) | length) > 0 }}
-    swap_units_returning: "{{ p.unmasked_generated_units | default([], true) | join(', ') }}"
-    swap_active_gib: "{{ (m.swap_total_kb | int / 1048576) | round(2) | string }}"
    swap_status: >-
      {{ 'SKIP' if m.swap_total_kb is not number
-         else ('WARN' if (m.swap_total_kb | int) > 0
-               else ('WARN' if (swap_will_return | bool) else 'PASS')) }}
-    swap_cause: >-
-      systemd-gpt-auto-generator re-creates a .swap unit for the GPT partition typed
-      0657fd6d-a4ab-43c4-84e5-0933c84b4f4f ("Linux swap") at EVERY boot, straight from
-      the partition table and without reading /etc/fstab. This is NOT "the node has not
-      been baselined" — `swapoff -a` and a commented fstab line simply do not survive a
-      reboot. The fix is roles/node_baseline masking swap.target and the generated units:
-      re-run `node-baseline.yml --tags system` against this node.
+         else ('PASS' if m.swap_total_kb | int == 0 else 'WARN') }}
    swap_detail: >-
      {{ 'swap state unreadable' if m.swap_total_kb is not number
-         else ((swap_active_gib ~ ' GiB swap active'
-                ~ (' via ' ~ swap_units_returning if swap_units_returning else '')
-                ~ ' — ' ~ swap_cause)
-               if (m.swap_total_kb | int) > 0
-               else ('no swap active, but it would come back at the next boot via '
-                     ~ swap_units_returning ~ ' — ' ~ swap_cause
-                     if (swap_will_return | bool)
-                     else ('no swap active'
-                           ~ (' and none can be activated: swap.target is masked'
-                              if (p.swap_target_masked | default(false) | bool)
-                              else (' and no swap unit exists on this node'
-                                    if (p.generated_units | default([], true) | length) == 0
-                                    else ' and every generated swap unit is masked'))
-                           if (swap_durable_known | bool)
-                           else 'no swap active (durability NOT verified — the collector
-                                 could not read /etc/systemd/system or /run/systemd)'))) }}
+         else ('no swap active' if m.swap_total_kb | int == 0
+               else ((m.swap_total_kb | int / 1048576) | round(2) | string
+                     ~ ' GiB swap active — roles/node_baseline runs swapoff and neutralises
+                       the fstab entry; this node has not been baselined yet')) }}

 - name: Assess — ECC / EDAC correctable errors
  ansible.builtin.set_fact:
@@ -12,19 +12,12 @@ separate GitOps authored against the shared **interface contract** below.
 > once the tunnel is live in dev/test.

 ## Interface contract (shared with the Cilium-policy agent)
- **Inventory groups** — `vpn_egress_gateway` (dev: `k3s-dev-3`) and `vpn_egress_gateway_prod`
-  (prod: `k3s-prod-3`). Each holds **exactly one** host, and prod is a separate group so the
-  flagless playbook run cannot reach it. Cilium OSS resolves a multi-match nodeSelector to "the
-  first node in lexical ordering", so a second labelled node does not add failover — it silently
-  steals the role.
+- **Inventory group** `vpn_egress_gateway` — the egress-gateway node(s) (initially `k3s-dev-3`).
 - **k8s node label** `homelab.keithjackson.dev/egress-gateway=true` — set by this role; the
-  `CiliumEgressGatewayPolicy` selects the gateway node by it. **This label is what arms the
-  datapath**, which is why the role applies it LAST.
+  `CiliumEgressGatewayPolicy` selects the gateway node by it.
 - **Host tunnel iface** `tun0` — the Cilium policy SNATs vpn-class pods out of it.
- **k8s node label** `homelab.keithjackson.dev/egress-uplink=<tun0 addr>-<ifindex>` — set by this
-  role's re-steer script. Its VALUE is the uplink Cilium was last told about; its CHANGE is what
-  makes Cilium re-resolve `interface: tun0`. Cilium caches that resolution and never revisits it
-  on its own, so without this the gateway is dead after its first reconnect. See "Re-steering".
+- **Optional fwmark** `0x1c8` — a contract *hook* (see kill-switch layering) the Cilium side
+  MAY stamp on vpn-class packets for a tunnel-down-independent host backstop.

 ## How the four ADR-0015 invariants are met

@@ -52,42 +45,19 @@ Two independent layers, both scoped so they can only ever affect vpn-class traff
 2. **nftables backstop (`templates/vpn-egress-killswitch.nft.j2`).** Own `inet
   vpn_egress_gateway_killswitch` table, base chain **policy `accept`** (so it can only *drop* the
   narrow matches, never break anything). In `postrouting`:
-   - `oifname tun0 ip saddr @vpn_egress_gateway_src counter accept` — tunnel up: let
-     tunnel-sourced traffic out.
-   - `ip saddr @vpn_egress_gateway_src oifname != tun0 counter drop` — **fail closed:** a
-     tunnel-sourced packet trying to leave any other interface is dropped (no WAN leak). Armed
-     even while the tunnel is down, because the source set is retained on tunnel-down (below).
+   - `oifname tun0 ip saddr @vpn_egress_gateway_src accept` — tunnel up: let steered traffic out.
+   - `ip saddr @vpn_egress_gateway_src oifname != tun0 drop` — **fail closed:** a tunnel-sourced packet
+     trying to leave any other interface is dropped (no WAN leak). This rule stays **armed even
+     while the tunnel is down** because the source set is retained on tunnel-down (see below).
+   - `meta mark 0x1c8 oifname != tun0 drop` — the optional fwmark backstop.

-   **Both rules carry a `counter`, and both were exercised on k3s-prod-3 (2026-09-12):** a
-   source-bound `ping -I 10.72.192.7` with the ip-rule present got 3/3 replies at 144 ms and
-   moved `killswitch_accept`; with the ip-rule deleted, `ip route get` confirmed the packet
-   would go `via 192.168.2.1 dev eno1` and `killswitch_drop` moved **0 -> 3 packets / 252
-   bytes** with 100% loss. The kill-switch is real, not aspirational — but read the next
-   paragraph for what it can and cannot reach.
-
-   **SCOPE, HONESTLY — it does NOT cover vpn-class POD traffic.** A counters-only diagnostic
-   table on the dev gateway recorded, while a vpn-class pod egressed: 0 packets in `forward`
-   and 0 in `postrouting` for the pod CIDR *and* for the PIA range, against 45166 packets on
-   that same postrouting hook for WAN traffic and 16 for a node-originated curl out tun0.
-   Cilium's eBPF egress-gateway datapath (`bpf.masquerade: true`) `bpf_redirect()`s those skbs
-   straight to the egress device, so netfilter POSTROUTING is never invoked for them. What
-   makes vpn-class fail closed is **Cilium's own drop** when the cached egress interface is
-   unusable — tcpdump during a tunnel stop and during a tunnel restart showed 0 packets leaving
-   either the WAN or tun0. This table's real, narrow guarantee is NODE-originated traffic
-   sourced from a tunnel address.
-
-`@vpn_egress_gateway_src` holds **every tun0 address this node has had**. The up-script **adds**
-the current tunnel source on every connect and never flushes (it used to flush first, which left
-a brief moment where a packet still sourced from the previous address matched neither the
-ip-rule nor the drop rule); the down-script **intentionally retains it**. Node traffic and
-`internet`-class traffic are never in the set and are never targeted — so they always hit
+`@vpn_egress_gateway_src` holds the **last live tun0 address**. The up-script **flushes then
+re-adds** the current tunnel source on every connect; the down-script **intentionally retains
+it** (see "Explicit fail-closed backstop" below). Node traffic and `internet`-class traffic are
+never in the set, never carry the mark, and are never targeted — so they always hit
 `policy accept` and **fail open toward the WAN** (invariant 4), tunnel up or down. The opposite
 directions are structural, not incidental.

-A kill-switch **reload no longer disarms the guard**: the `.nft` file uses `table` + `flush
-table`, which clears rules but leaves a named set's elements intact, so an Ansible run cannot
-empty the set mid-flight.
-
 ### Explicit, edge-independent fail-closed backstop (tunnel-down)
 Earlier the down-script flushed `@vpn_egress_gateway_src`, which **disarmed** the
 `ip saddr @vpn_egress_gateway_src oifname != tun0 drop` rule at exactly the moment the tunnel was
@@ -97,15 +67,12 @@ guarantee that could leak if the edge router NATed that stale source. The down-s
 vpn-class packet that would leave the WAN while the tunnel is down is **explicitly dropped** in
 the host nft table — defense-in-depth, independent of edge-router behavior.

-The set is now **ADDITIVE** (2026-09-12): the up-script only ever `nft add element`s the live
-source. It used to `nft flush set` first, and that flush was itself a hole — between a reconnect
-and Cilium refreshing its cached egress IP, a packet still sourced from the PREVIOUS address
-matched neither the ip-rule (deleted by the down-script) nor the drop rule (the address had just
-been flushed out of the set). Additive closes it, `add element` is idempotent, and an extra
-element can only ever *tighten* the rule. The set stays scoped to tunnel sources; node and
-internet-class traffic are never members, so **no invariant is weakened** (invariants 3 & 4
-upheld). The routing blackhole in table `51820` remains as the second, independent fail-closed
-layer.
+This is safe precisely because the **up-script flushes the set and re-adds the (possibly new)
+tunnel source on every reconnect** (`nft flush set …` then `nft add element …`), so a retained
+stale address self-corrects the instant the tunnel comes back. The set remains scoped to
+vpn-class only; node and internet-class traffic are never members, so **no invariant is
+weakened — only vpn-class is ever affected** (invariants 3 & 4 upheld). The routing blackhole in
+table `51820` remains as the second, independent fail-closed layer.

 ## Why the node default route is provably unaffected (design-level verification)
 - OpenVPN runs with `route-noexec` and `pull-filter ignore "redirect-gateway"` → it adds **zero**
@@ -115,79 +82,40 @@ layer.
  ip-rules, never `table main` / `table default`.
 - The ip-rule matches **`from <tun0-addr>`** — an address the node itself never sources traffic
  from — so node-originated and internet-class packets never enter the tunnel table.
- The nft table is **policy accept** and matches only `@vpn_egress_gateway_src`.
+- The nft table is **policy accept** and matches only `@vpn_egress_gateway_src` / the fwmark.
 - Live check (post-deploy): `ip route show table main default` still points at the WAN NIC, and
  `curl ipinfo` from the node shows the WAN IP, with the tunnel both up and forced down.
-  Confirmed on `k3s-prod-3` 2026-09-12: `default via 192.168.2.1 dev eno1` with the tunnel up,
-  node curl = the WAN IP, k3s `/readyz` = ok.

-## Re-steering — Cilium does not follow tun0, and that is the whole ball game
-
-Cilium resolves the policy's `interface: tun0` **once**, caching an (egress IP, egress ifindex)
-pair in its BPF egress map, and does not re-resolve when tun0 changes. Upstream says so: *"it
-does not automatically respond to a change in the gateway node's network configuration... You can
-force a fresh selection by re-applying the Egress Gateway policy."* Measured on `k3s-dev-3`:
-
-| what | observed |
-|---|---|
-| after `systemctl restart vpn-egress-tunnel` | node tun0 at ifindex 1987, `cilium-dbg bpf egress list` still 1986 |
-| vpn-class pod egress in that state | times out; tcpdump shows **0** packets out the WAN and **0** out tun0 (no leak, no service) |
-| `egress-gateway-reconciliation-trigger-interval: 1s` | does **not** help — dev has that setting; still stale 20s+ later. It coalesces reconciliations that k8s object events already triggered; netlink link events are not among them |
-| stale egress **IP** with a live ifindex | does NOT leak either — the redirect follows the cached *ifindex*, so packets leave tun0 with the previous source (0 packets on the WAN, verified) |
-| touching any node label on the gateway | cached ifindex 1986 -> 1987 within 10s, egress recovers immediately |
-
-So the failure mode is a **silent outage on every tunnel restart or reboot**, not a leak. The fix
-is `templates/vpn-egress-resteer.sh.j2`, which publishes `<tun0 addr>-<ifindex>` as the
-`egress-uplink` node label:
-
- **a node label, not an annotation on the policy** — the policy is Flux-managed GitOps and
-  writing to it from a node invites drift/ownership churn; node labels are already this role's
-  territory, and the label value doubles as the diagnostic (`kubectl get nodes -L
-  homelab.keithjackson.dev/egress-uplink`);
- **fired by the OpenVPN up-script** on every connect, via `systemctl start --no-block` so it can
-  never block or fail the tunnel and does not need the kube-API to be up at that instant;
- **plus `vpn-egress-resteer.timer`** (2 min) as the convergence backstop for the boot case where
-  the API is not up yet and every in-script retry is exhausted;
- **idempotent** — if the label already records the live uplink it exits without writing, so the
-  timer is free to run often.
-
-Proof it works: `systemctl stop` then `start` of the tunnel now recovers on its own (map
-1989 -> 1990, PIA IP returns). Before the fix the identical sequence stayed broken indefinitely —
-which also means the **recovery half of the `vpn-killswitch` conformance test could not have
-passed** on the pre-fix implementation.
+## Kill-switch layering & ESCALATION (honest limit)
+- The nft `@vpn_egress_gateway_src` rule and the routing blackhole robustly cover the case where the
+  tunnel is up but a vpn-class packet is (mis)routed toward the WAN, and flaps.
+- The **tunnel fully-down** case relies on **Cilium Egress Gateway itself failing closed** when
+  its egress interface `tun0` is absent (Cilium 1.16 drops egress-gateway-marked traffic with no
+  valid egress path rather than falling back to the node's default route) **plus** the routing
+  blackhole. That is the ADR's expected behavior.
+- For a host backstop that is **fully independent of Cilium even in a fail-open regression** when
+  the tunnel is down (no tun0 address to key on), the host needs a stable marker on vpn-class
+  packets. The only contract-available marker is the **`0x1c8` fwmark hook** — which the
+  *Cilium-policy* side must stamp. **ESCALATION for the PM:** either (a) confirm the Cilium
+  Egress Gateway 1.16 fail-closed + routing-blackhole guarantee is accepted as sufficient for
+  R-NET-06, or (b) add the shared fwmark to the interface contract so the CiliumEgressGatewayPolicy
+  marks vpn-class traffic and this role's fwmark rule becomes the tunnel-down-independent backstop.
+  This is flagged rather than worked around — no spec was weakened to reach done.

 ## What requires a LIVE tunnel to validate (cannot be tested without operator creds)
 1. The PIA `.ovpn` + username/password actually authenticate and `tun0` comes up as a host iface.
 2. `egress-class`: a vpn-class pod's public IP is the PIA Stockholm IP; internet-class shows the WAN IP.
 3. `vpn-killswitch`: forcing `tun0`/the tunnel service down blackholes vpn-class egress (no leak)
   while internet-class is unaffected — the R-NET-06 fault-injection gate.
-4. ~~That Cilium Egress Gateway SNAT to `tun0` traverses host `postrouting` so the nft set-scoped
-   rules actually see the packets (eBPF datapath interaction).~~ **ANSWERED 2026-09-12: it does
-   NOT.** Counters proved the steered pod packets never enter netfilter at all. See "SCOPE,
-   HONESTLY" above. The question is closed; the guard's claim is narrowed rather than pretended.
+4. That Cilium Egress Gateway SNAT to `tun0` traverses host `postrouting` so the nft set-scoped
+   rules actually see the packets (eBPF datapath interaction — see escalation).
 5. MTU/PMTU across `pod → tun0 → PIA` (in-pod gluetun needed 1300 for Cilium VXLAN; on the node
   the tunnel rides the physical WAN, so `vpn_egress_gateway_tun_mtu` defaults empty — verify live).
 6. Port forwarding (PIA NAT-PMP) for qBittorrent — ADR-0015 Phase-6 follow-up, not in this role.

 ## Files
 - `defaults/main.yml` — all `vpn_egress_gateway_*` vars; the three vault-backed credential vars.
- `tasks/main.yml` — packages, rt_tables, config/scripts/units, enable+start, re-steer, node label
-  (kill-switch -> tunnel -> re-steer -> **label last**, because the label is what arms steering).
+- `tasks/main.yml` — packages, rt_tables, config/scripts/units, enable+start, node label.
 - `handlers/main.yml` — daemon-reload, kill-switch reload, tunnel restart.
- `templates/` — wrapper `.conf`, credentials, up/down scripts, nft ruleset, the kill-switch and
-  tunnel units, and `vpn-egress-resteer.{sh,service,timer}`.
- Playbook: `playbooks/vpn.egress.gateway.yml`. Vault keys:
-  `inventory/group_vars/vpn_egress_gateway/README.md` (dev) and
-  `inventory/group_vars/vpn_egress_gateway_prod/README.md` (prod).
-
-## Running it against prod
-```bash
-cd infra/ansible
-ansible-playbook playbooks/vpn.egress.gateway.yml \
-  -e vpn_egress_gateway_hosts=vpn_egress_gateway_prod --limit k3s-prod-3 --list-hosts
-# confirm it says exactly `k3s-prod-3`, then drop --list-hosts
-```
-The target var is `vpn_egress_gateway_hosts`, **not** the cluster-wide `k3s_hosts`. Never apply
-the `egress-gateway` label by hand, and never run this role with tags that skip the tunnel or
-kill-switch tasks — the label is the last step precisely so nothing is steered at a node that is
-not ready to carry it.
+- `templates/` — wrapper `.conf`, credentials, up/down scripts, nft ruleset, both systemd units.
+- Playbook: `playbooks/vpn.egress.gateway.yml`. Vault keys: `inventory/group_vars/vpn_egress_gateway/README.md`.
@@ -20,23 +20,6 @@ vpn_egress_gateway_pia_password: "{{ vault_vpn_egress_gateway_pia_password }}"
 # The FULL text of the PIA .ovpn (inline <ca>/<crl-verify>, cipher/auth, remote line).
 vpn_egress_gateway_pia_ovpn: "{{ vault_vpn_egress_gateway_pia_ovpn }}"

-# OVERRIDE THE .ovpn's PINNED `remote` — "<host-or-ip> <port>", empty to honour the .ovpn.
-#
-# WHY THIS EXISTS (measured 2026-09-12): the operator's .ovpn pins ONE PIA server IP. Point
-# two gateways at the same vaulted .ovpn and both dial the SAME server, and PIA hands BOTH
-# THE SAME tunnel address — dev and prod were both `Server-10779-2a`
-# (158.173.241.112:1198) with `ifconfig 10.72.192.7`. Each reconnect then steals that
-# address from the other, the loser sees no data for 60s, hits `--ping-restart`, reconnects
-# and steals it back: both tunnels flapped on a ~2-minute cycle and neither was reliably
-# up. It is NOT a connection-LIMIT problem (the account has room) — it is one pinned server.
-#
-# So each environment's gateway needs its own server. The PIA Stockholm LB pool
-# (`sweden.privacy.network`) resolved to 158.173.241.{34,55,112,135,239}; any PIA server in
-# the pool accepts the same credentials and CA, so this is a per-environment knob rather
-# than a second vaulted .ovpn. NOT a secret — a public server address — so it belongs in
-# group_vars/<group>/vars.yml, not the vault.
-vpn_egress_gateway_pia_remote: ""
-
 # --- on-disk layout -------------------------------------------------------------
 vpn_egress_gateway_config_dir: /etc/openvpn/client
 vpn_egress_gateway_scripts_dir: /etc/openvpn/scripts
@@ -63,39 +46,17 @@ vpn_egress_gateway_blackhole_metric: 4000
 # (Cilium VXLAN 1450 → needed 1300) no shrink is required by default.
 vpn_egress_gateway_tun_mtu: ""

-# --- nftables kill-switch -------------------------------------------------------
-# SCOPE, HONESTLY (measured 2026-09-12, see the .nft template header): this table guards
-# NODE-ORIGINATED traffic sourced from a tunnel address. It does NOT and cannot guard the
-# egress-class=vpn POD traffic, because Cilium's eBPF egress-gateway datapath
-# (bpf.masquerade) redirects those packets to the egress device without ever invoking
-# netfilter POSTROUTING. vpn-class fails closed via Cilium's own drop, not via this table.
+# --- nftables kill-switch (invariants 3 & 4 — scoped to vpn-class ONLY) ----------
 vpn_egress_gateway_nft_table: vpn_egress_gateway_killswitch
-# Named set of tunnel source addresses, ADDITIVE (the up-script adds, never flushes; the
-# down-script retains). Every element only tightens the `oifname != tun0 drop` rule.
+# Named set holding the LIVE tun0 source address (maintained by the up/down scripts).
+# The kill-switch matches ONLY packets SNAT'd to this address by Cilium Egress Gateway
+# — i.e. exactly the egress-class=vpn traffic. Node / internet-class traffic is never
+# in this set and is never matched (fails OPEN toward the WAN).
 vpn_egress_gateway_nft_src_set: vpn_egress_gateway_src
-# REMOVED 2026-09-12: vpn_egress_gateway_fwmark ("0x1c8"). It backed a kill-switch rule
-# for a mark that nothing stamps — Cilium OSS does not mark egress-gateway traffic with
-# it (the only Cilium mark on the gateway node is 0x200/0xf00, the L7-proxy rule), and the
-# traffic never reaches netfilter anyway. A dead rule matching an arbitrary mark is a
-# hazard, not a backstop, so both the rule and the variable are gone.
-
-# --- re-steer: keep Cilium's cached egress interface in sync with tun0 -----------
-# Cilium resolves `interface: tun0` ONCE into a cached (egress IP, egress ifindex) and does
-# not re-resolve when tun0 changes; every tunnel restart gives tun0 a NEW ifindex, which
-# silently blackholes vpn-class egress. The re-steer script publishes the live uplink as a
-# node label; that node update makes the gateway node re-process the policy. Verified on
-# k3s-dev-3: cached ifindex moved 1986 -> 1987 within 10s and egress recovered.
-# The label VALUE is "<tun0 address>-<ifindex>", so `kubectl get nodes -L <this key>` shows
-# which uplink Cilium was last told about.
-vpn_egress_gateway_uplink_label_key: homelab.keithjackson.dev/egress-uplink
-# The kube-API is routinely not reachable yet when OpenVPN connects during boot, so the
-# script retries. Bounded so a genuinely broken API does not hang a systemd job forever.
-vpn_egress_gateway_resteer_retries: 30
-vpn_egress_gateway_resteer_retry_seconds: 10
-# Convergence backstop if the event-driven trigger is ever lost (no-op when already in
-# sync, so it is cheap to run often).
-vpn_egress_gateway_resteer_timer_boot: 2min
-vpn_egress_gateway_resteer_timer_interval: 2min
+# Contract-hook fwmark: if the CiliumEgressGatewayPolicy side stamps vpn-class packets
+# with this mark, the kill-switch also drops marked packets not leaving via tun0 — an
+# extra, tunnel-down-independent backstop. See README "Kill-switch layering / escalation".
+vpn_egress_gateway_fwmark: "0x1c8"

 # --- kubernetes node label (steering contract) ----------------------------------
 # The label the parallel CiliumEgressGatewayPolicy selects the gateway node by.
@@ -28,18 +28,9 @@
    - "{{ vpn_egress_gateway_config_dir }}"
    - "{{ vpn_egress_gateway_scripts_dir }}"

-# `vpn_egress_gateway_pia_remote`, when set, rewrites ONLY the `remote <host> <port>` line so
-# this environment's gateway dials its own PIA server. Two gateways sharing the .ovpn's single
-# pinned server are handed the SAME tunnel address and evict each other every ~2 minutes — see
-# the variable's comment in defaults/main.yml. The regex is anchored to a line that STARTS with
-# `remote` followed by whitespace, so `remote-cert-tls` / `remote-random` are untouched.
 - name: Deploy the operator's proven PIA .ovpn (from Ansible vault)
  ansible.builtin.copy:
-    content: >-
-      {{ vpn_egress_gateway_pia_ovpn if (vpn_egress_gateway_pia_remote | length == 0)
-         else (vpn_egress_gateway_pia_ovpn
-               | regex_replace('(?m)^[ \t]*remote[ \t]+\S+.*$',
-                               'remote ' ~ vpn_egress_gateway_pia_remote)) }}
+    content: "{{ vpn_egress_gateway_pia_ovpn }}"
    dest: "{{ vpn_egress_gateway_ovpn_path }}"
    owner: root
    group: root
@@ -78,17 +69,6 @@
    - vpn-egress-down.sh
  notify: Restart vpn-egress tunnel

-# The re-steer script is what keeps Cilium's cached egress interface in sync with tun0.
-# It is NOT notified to the tunnel: it is triggered BY the tunnel's up-script and by its
-# own timer, and re-deploying it must never bounce the tunnel.
- name: Deploy the Cilium egress-gateway re-steer script
-  ansible.builtin.template:
-    src: vpn-egress-resteer.sh.j2
-    dest: "{{ vpn_egress_gateway_scripts_dir }}/vpn-egress-resteer.sh"
-    owner: root
-    group: root
-    mode: "0750"
-
 - name: Deploy the scoped nftables kill-switch ruleset
  ansible.builtin.template:
    src: vpn-egress-killswitch.nft.j2
@@ -120,18 +100,6 @@
    - Reload systemd
    - Restart vpn-egress tunnel

- name: Deploy the egress-gateway re-steer unit and its convergence timer
-  ansible.builtin.template:
-    src: "{{ item }}.j2"
-    dest: "/etc/systemd/system/{{ item }}"
-    owner: root
-    group: root
-    mode: "0644"
-  loop:
-    - vpn-egress-resteer.service
-    - vpn-egress-resteer.timer
-  notify: Reload systemd
-
 - name: Flush handlers so units/routing exist before enable+start
  ansible.builtin.meta: flush_handlers

@@ -147,27 +115,6 @@
    enabled: true
    state: started

-# Enabled BEFORE the node label below: the label is what arms steering, and by then the
-# re-steer path must already exist so Cilium is told about the live tun0 rather than
-# caching whatever it resolves once and never revisiting it.
- name: Enable and start the re-steer convergence timer
-  ansible.builtin.systemd_service:
-    name: vpn-egress-resteer.timer
-    enabled: true
-    state: started
-
-# Publish the live uplink immediately rather than waiting for the timer's first tick, so a
-# first-time stand-up (or a re-run after a tunnel restart) converges within seconds.
-# `started`, NOT `restarted`: the tunnel's up-script fires this same unit on connect, and
-# `restarted` SIGTERMs that in-flight run (logging a spurious unit failure) instead of
-# letting it finish. The unit is a oneshot without RemainAfterExit, so it is `inactive`
-# once it has run and `started` genuinely re-runs it.
- name: Re-steer Cilium onto the live tun0 now
-  ansible.builtin.systemd_service:
-    name: vpn-egress-resteer.service
-    state: started
-  changed_when: false
-
 # --- steering contract: label the k8s node the Cilium Egress Gateway policy selects ----
 - name: Read the current egress-gateway node label
  ansible.builtin.command:
@@ -8,20 +8,15 @@
 # then (2) remove the tunnel default from the dedicated table. The permanent
 # `blackhole default` in that table is LEFT in place as one fail-closed fallback.
 #
-# FAIL-CLOSED BACKSTOP: the kill-switch source set is INTENTIONALLY RETAINED here (we do
-# NOT flush it). Keeping every tunnel address this node has had in
+# EXPLICIT FAIL-CLOSED BACKSTOP: the kill-switch source set is INTENTIONALLY RETAINED
+# here (we do NOT flush it). Keeping the last live tun0 address in
 # `{{ vpn_egress_gateway_nft_src_set }}` keeps the nft rule
 # `ip saddr @{{ vpn_egress_gateway_nft_src_set }} oifname != tun0 drop` ARMED while the
-# tunnel is down. The up-script is ADDITIVE (2026-09-12 — it used to flush first, which
-# briefly left a previous tunnel address matched by nothing), so the set only ever grows
-# and only ever tightens the rule.
-#
-# SCOPE, HONESTLY: that rule covers NODE-ORIGINATED traffic sourced from a tunnel address.
-# It does NOT cover the egress-class=vpn POD traffic — Cilium's eBPF egress-gateway
-# datapath never presents those packets to netfilter (measured; see the kill-switch .nft
-# header). vpn-class pods fail closed because Cilium drops them when it cannot reach the
-# cached egress interface — verified with tcpdump on both the WAN and tun0 during a tunnel
-# stop and a tunnel restart: zero packets left either device.
+# tunnel is down, so any vpn-class packet that would leave the WAN is EXPLICITLY dropped
+# — defense-in-depth that does not depend on the stale SNAT source being non-routable or
+# on the edge router refusing to NAT it. This is safe because the up-script FLUSHES then
+# re-adds the (possibly new) tunnel source on every reconnect, so the set self-corrects.
+# The set is scoped to vpn-class only; node and internet-class traffic are never in it.
 # The node's main-table WAN default is never touched here.
 set -uo pipefail

@@ -35,8 +30,8 @@ SRC="${ifconfig_local:-}"

 # 1) Stop steering to the (now dead) tunnel source by dropping the ip-rule.
 #    NOTE: the nft source set is deliberately NOT flushed here — see header. Retaining the
-#    tun0 addresses keeps the `oifname != tun0 drop` rule armed for node-originated
-#    tunnel-sourced traffic until the up-script re-registers on reconnect.
+#    last tun0 address keeps the `oifname != tun0 drop` rule armed as the explicit,
+#    edge-independent fail-closed backstop until the up-script re-registers on reconnect.
 if [ -n "${SRC}" ]; then
  while ip rule del from "${SRC}" lookup "${TABLE_ID}" priority "${RULE_PRIO}" 2>/dev/null; do :; done
 fi
@@ -1,93 +1,40 @@
 #!/usr/sbin/nft -f
 # {{ ansible_managed }}
-# ADR-0015 kill-switch — SCOPED TO tunnel-sourced traffic ONLY, fails CLOSED.
-#
-# =============================================================================
-# READ THIS BEFORE TRUSTING THIS FILE (measured 2026-09-12 on k3s-dev-3)
-# =============================================================================
-# This table does NOT, and CANNOT, guard the `egress-class=vpn` POD traffic that
-# ADR-0015 asks it to guard. It was measured, not reasoned about:
-#
-#   A diagnostic nft table (counters only) was installed on the gateway node
-#   alongside this one, counting every packet in `forward` and `postrouting` by
-#   (oifname, saddr). While a vpn-class pod egressed through the gateway:
-#       forward  saddr=pod CIDR ................  0 packets
-#       forward  saddr=PIA tunnel range ........  0 packets
-#       postrouting oif tun0 saddr=tunnel ......  0 packets
-#       postrouting oif WAN  saddr=pod CIDR ....  0 packets
-#       postrouting oif WAN  (any) ............. 45166 packets   <- hook IS live
-#   and, as a positive control, a curl from the NODE itself out tun0 counted
-#       postrouting oif tun0 saddr=tunnel ......  16 packets
-#
-#   So the hook is traversed and the matching works — but Cilium's egress-gateway
-#   datapath never presents the steered pod packets to netfilter at all. With
-#   `bpf.masquerade: true` the SNAT and the egress happen in eBPF, which
-#   bpf_redirect()s the skb straight to the egress device: POSTROUTING is not
-#   invoked for those packets, in either direction.
-#
-# WHAT ACTUALLY MAKES vpn-class FAIL CLOSED (also measured):
-#   Cilium caches the gateway's (egress IP, egress ifindex) when it resolves this
-#   policy. If the cached ifindex no longer exists — which is exactly what a tunnel
-#   restart produces, tun0 being re-created with a NEW ifindex — the eBPF redirect
-#   fails and the packet is DROPPED. Verified with tcpdump on BOTH the WAN and tun0
-#   during a tunnel restart and during a full stop: 0 packets left either device.
-#   There is no leak, but the mechanism is Cilium's, not this table's.
-#
-# WHAT THIS TABLE IS STILL GOOD FOR:
-#   NODE-ORIGINATED traffic sourced from a tunnel address (the 16-packet control
-#   above) — e.g. a host-side helper bound to tun0 — which DOES traverse netfilter.
-#   That is a real, if narrow, guarantee, and it is the only one claimed here.
-#   Every rule carries a `counter` so the claim is falsifiable at any moment:
-#     nft list table inet {{ vpn_egress_gateway_nft_table }}
-#   If `killswitch_drop` ever becomes non-zero, something (a Cilium datapath
-#   change, bpf.masquerade turned off) has put this traffic back into netfilter —
-#   read that as new information, not as reassurance.
-#
-# See platform/vpn-egress/README.md and the DECISIONS-LOG entry of 2026-09-12.
-# =============================================================================
+# ADR-0015 kill-switch (invariants 3 & 4) — SCOPED TO vpn-class ONLY, fails CLOSED.
 #
 # Own dedicated table so re-loading it never disturbs Cilium's or the node's own rules.
-# The base chain is policy ACCEPT: this table can ONLY ever drop traffic whose source is
-# in the set below; node traffic and internet-class traffic are untouched and keep
-# egressing the WAN (fail OPEN — invariant 4).
-
-# Idempotent reload. `table` + `flush table` clears the RULES only — a named set and its
-# elements are not chains, so the live tunnel address SURVIVES a reload and the guard is
-# not disarmed by an Ansible run. (A plain `nft -f` without this would append a second
-# copy of every rule on each load.)
-table inet {{ vpn_egress_gateway_nft_table }} { }
-flush table inet {{ vpn_egress_gateway_nft_table }}
+# The base chains are policy ACCEPT: this table can ONLY ever drop the narrow vpn-class
+# traffic matched below; node traffic and internet-class traffic are untouched and keep
+# egressing the WAN (fail OPEN — invariant 4). It matches on:
+#   * @{{ vpn_egress_gateway_nft_src_set }} — the LIVE tun0 address Cilium Egress Gateway SNATs
+#     vpn-class pods to (populated by the OpenVPN up-script; emptied by the down-script).
+#     This IS the egress-class=vpn traffic and nothing else.
+#   * meta mark {{ vpn_egress_gateway_fwmark }} — optional contract-hook: a tunnel-DOWN-independent
+#     backstop, active only if the CiliumEgressGatewayPolicy side stamps this fwmark.

 table inet {{ vpn_egress_gateway_nft_table }} {
-    # Every tunnel source address this gateway has had. The OpenVPN up-script ADDS the
-    # live address on each connect and never flushes (see vpn-egress-up.sh): an element
-    # can only ever tighten the guard, and retaining previous addresses means a packet
-    # still sourced from an older tunnel address is dropped rather than unmatched.
-    # Empty before the very first successful connect => the rules below match nothing,
-    # and vpn-class has no path at all (fail closed, by Cilium's drop — see above).
+    # Live tunnel source address(es). Empty whenever the tunnel is down => the two
+    # saddr rules below match nothing => vpn-class has no path at all (fail closed).
    set {{ vpn_egress_gateway_nft_src_set }} {
        type ipv4_addr
    }

    chain postrouting {
-        # oifname is resolved at postrouting, after any netfilter SNAT.
+        # oifname is resolved at postrouting; SNAT/masquerade has already applied here,
+        # so the Cilium-SNAT'd vpn-class packet's saddr is the tunnel address.
        type filter hook postrouting priority 100; policy accept;

-        # Tunnel UP: let tunnel-sourced traffic out tun0.
-        oifname "{{ vpn_egress_gateway_tun_iface }}" ip saddr @{{ vpn_egress_gateway_nft_src_set }} counter accept comment "killswitch_accept"
+        # Tunnel UP: let the steered vpn-class traffic out tun0.
+        oifname "{{ vpn_egress_gateway_tun_iface }}" ip saddr @{{ vpn_egress_gateway_nft_src_set }} accept

-        # FAIL CLOSED: tunnel-sourced traffic trying to leave ANY other interface (i.e.
-        # the WAN) is dropped — never leaks. Scoped by the set, so it can never match
-        # node or internet-class traffic. The counter is the proof-of-life for the whole
-        # table: see the header for why it reads 0 for POD traffic by construction.
-        ip saddr @{{ vpn_egress_gateway_nft_src_set }} oifname != "{{ vpn_egress_gateway_tun_iface }}" counter drop comment "killswitch_drop"
+        # FAIL CLOSED: a vpn-class (tunnel-sourced) packet trying to leave ANY other
+        # interface (i.e. the WAN) is dropped — never leaks. Scoped by the set, so it
+        # can never match node or internet-class traffic.
+        ip saddr @{{ vpn_egress_gateway_nft_src_set }} oifname != "{{ vpn_egress_gateway_tun_iface }}" drop

-        # REMOVED 2026-09-12 — the `meta mark 0x1c8 oifname != tun0 drop` rule. It was a
-        # "contract hook" for a fwmark that nothing stamps: Cilium OSS does not mark
-        # egress-gateway traffic with 0x1c8 (the only Cilium mark visible on the gateway
-        # node is 0x200/0xf00, which is the L7-proxy rule -> table 2004, unrelated), and
-        # the traffic does not reach netfilter anyway. A dead rule matching an arbitrary
-        # mark is a hazard, not a backstop: any future component that happens to use
-        # 0x1c8 would have had its egress silently dropped.
+        # Tunnel-down-independent backstop (contract hook, see role README): if vpn-class
+        # packets carry the agreed fwmark, drop any not leaving via tun0 even when the set
+        # is empty (tunnel fully down). Harmless if the Cilium side does not set the mark.
+        meta mark {{ vpn_egress_gateway_fwmark }} oifname != "{{ vpn_egress_gateway_tun_iface }}" drop
    }
 }
@@ -1,25 +0,0 @@
-# {{ ansible_managed }}
-# Publishes this node's CURRENT tun0 (address + ifindex) as a node label so Cilium
-# re-resolves the CiliumEgressGatewayPolicy's `interface: tun0` (ADR-0015). Without this,
-# Cilium keeps a cached (egress IP, egress ifindex) from the FIRST resolution and vpn-class
-# egress is silently blackholed after every tunnel restart or reboot — measured, see the
-# script header and the DECISIONS-LOG entry of 2026-09-12.
-#
-# Triggered by the OpenVPN up-script (`systemctl start --no-block`) on every connect, and
-# by vpn-egress-resteer.timer as the convergence backstop. Deliberately NOT ordered
-# Before/After the tunnel: it must never be able to block or fail the tunnel.
-[Unit]
-Description=Re-steer Cilium Egress Gateway onto the live tun0 — ADR-0015
-Documentation=file://{{ vpn_egress_gateway_scripts_dir }}/vpn-egress-resteer.sh
-After=network-online.target
-# The kill-switch owns the nft table this script re-registers the tunnel source into.
-After=vpn-egress-killswitch.service
-
-[Service]
-Type=oneshot
-ExecStart={{ vpn_egress_gateway_scripts_dir }}/vpn-egress-resteer.sh
-# Bounded: {{ vpn_egress_gateway_resteer_retries }} attempts x {{ vpn_egress_gateway_resteer_retry_seconds }}s waiting for the kube-API, plus slack.
-TimeoutStartSec={{ (vpn_egress_gateway_resteer_retries | int * vpn_egress_gateway_resteer_retry_seconds | int) + 60 }}
-# A failure here means "the kube-API was unreachable"; the timer retries. Do not let a
-# failed re-steer mark the whole gateway degraded.
-SuccessExitStatus=0 1
@@ -1,97 +0,0 @@
-#!/usr/bin/env bash
-# {{ ansible_managed }}
-# Re-steer the Cilium Egress Gateway onto THIS node's CURRENT tun0 (ADR-0015).
-#
-# WHY THIS EXISTS (measured on k3s-dev-3, 2026-09-12 — see DECISIONS-LOG):
-#   Cilium resolves a CiliumEgressGatewayPolicy's `interface: tun0` once, caching an
-#   (egress IP, egress ifindex) pair in its BPF egress map, and does NOT re-resolve when
-#   tun0 changes. Observed directly: after `systemctl restart vpn-egress-tunnel` the node
-#   had tun0 at ifindex 1987 while `cilium-dbg bpf egress list` still said 1986, and
-#   vpn-class pod egress was blackholed (tcpdump: 0 packets out the WAN, 0 out tun0 — no
-#   leak, but no service). `egress-gateway-reconciliation-trigger-interval: 1s` does NOT
-#   cover netlink link events; it only coalesces reconciliations that k8s object events
-#   already triggered.
-#
-#   Upstream remedy: "You can force a fresh selection by re-applying the Egress Gateway
-#   policy." Any k8s event that makes the gateway node re-process the policy does it. We
-#   use a NODE LABEL rather than touching the policy object, because:
-#     * the policy is Flux-managed GitOps — writing to it from a node invites drift
-#       fights and ownership churn; the node's labels are already this role's territory;
-#     * it was verified to work: setting an unrelated label on k3s-dev-3 moved the cached
-#       ifindex 1986 -> 1987 within 10s and vpn-class egress recovered (PIA IP returned);
-#     * the label VALUE is the uplink identity, so `kubectl get node -L ...` shows at a
-#       glance which tun0 Cilium was last told about. Diagnosis needs no exec into a pod.
-#
-# Idempotent: if the label already records the live uplink, this exits without touching
-# anything, so the timer can run as often as it likes.
-set -uo pipefail
-
-IFACE="{{ vpn_egress_gateway_tun_iface }}"
-TABLE_ID="{{ vpn_egress_gateway_route_table_id }}"
-RULE_PRIO="{{ vpn_egress_gateway_rule_priority }}"
-TUN_METRIC="{{ vpn_egress_gateway_tun_route_metric }}"
-NFT_TABLE="{{ vpn_egress_gateway_nft_table }}"
-NFT_SET="{{ vpn_egress_gateway_nft_src_set }}"
-NODE="{{ inventory_hostname }}"
-UPLINK_LABEL="{{ vpn_egress_gateway_uplink_label_key }}"
-RETRIES="{{ vpn_egress_gateway_resteer_retries }}"
-RETRY_SLEEP="{{ vpn_egress_gateway_resteer_retry_seconds }}"
-export KUBECONFIG="{{ vpn_egress_gateway_kubeconfig }}"
-
-log() { echo "vpn-egress-resteer: $*"; }
-
-# --- 1) what is the live uplink? -------------------------------------------------
-addr="$(ip -4 -o addr show dev "${IFACE}" 2>/dev/null | awk '{print $4}' | cut -d/ -f1 | head -1)"
-idx="$(cat "/sys/class/net/${IFACE}/ifindex" 2>/dev/null || true)"
-if [ -z "${addr}" ] || [ -z "${idx}" ]; then
-  # Tunnel down. Nothing to publish — and deliberately NOTHING is torn down here either:
-  # vpn-class is already fail-closed (Cilium drops when the cached ifindex is gone), and
-  # the kill-switch set is retained on purpose. The next successful connect re-runs this.
-  log "${IFACE} has no address yet (tunnel down) — nothing to re-steer"
-  exit 0
-fi
-# A k8s label value must be <=63 chars of [A-Za-z0-9] plus - _ . in the middle. An IPv4
-# address and an ifindex satisfy that; the separator is a dash.
-want="${addr}-${idx}"
-
-# --- 2) re-assert the host side (idempotent; repairs a kill-switch reload) --------
-# `flush table` in the .nft file keeps set elements, but a full ExecStop/ExecStart cycle
-# (or a hand-run `nft delete table`) does not, so re-register unconditionally.
-nft add element inet "${NFT_TABLE}" "${NFT_SET}" "{ ${addr} }" 2>/dev/null || \
-  log "WARNING could not add ${addr} to ${NFT_TABLE}/${NFT_SET} (is the kill-switch loaded?)"
-if ! grep -q "from ${addr} lookup" <<<"$(ip rule show)"; then
-  ip rule add from "${addr}" lookup "${TABLE_ID}" priority "${RULE_PRIO}" 2>/dev/null || true
-fi
-ip route replace default dev "${IFACE}" table "${TABLE_ID}" metric "${TUN_METRIC}" 2>/dev/null || true
-
-# --- 3) has Cilium already been told about THIS uplink? ---------------------------
-# The label is the record of what we last published. Retried because at boot the kube-API
-# is routinely not up yet when OpenVPN connects.
-i=0
-while :; do
-  i=$((i + 1))
-  # -o json + an exact-string match rather than jsonpath: the label key contains dots and
-  # a slash, which jsonpath needs escaped differently across kubectl versions, and getting
-  # that wrong would silently read "" and re-label on every timer tick.
-  node_json="$(k3s kubectl get node "${NODE}" -o json 2>/dev/null)"
-  if [ -n "${node_json}" ]; then
-    # Here-strings, NOT pipes: with `set -o pipefail`, `grep -q` exiting early on a match
-    # SIGPIPEs the writer and the pipeline reports 141 — i.e. a successful match would read
-    # as a failure and this script would re-label on every single run.
-    if grep -qF "\"${UPLINK_LABEL}\": \"${want}\"" <<<"${node_json}"; then
-      log "uplink unchanged (${want}) — Cilium already re-steered, nothing to do"
-      exit 0
-    fi
-    have="$(grep -F "\"${UPLINK_LABEL}\": " <<<"${node_json}" | head -1 | cut -d'"' -f4)"
-    log "uplink changed: label='${have:-<unset>}' live='${want}' — re-steering"
-    if k3s kubectl label node "${NODE}" "${UPLINK_LABEL}=${want}" --overwrite >/dev/null 2>&1; then
-      log "re-steered: ${NODE} ${UPLINK_LABEL}=${want}"
-      exit 0
-    fi
-  fi
-  if [ "${i}" -ge "${RETRIES}" ]; then
-    log "ERROR kube-API unreachable or label write failed after ${i} attempts — the timer will retry"
-    exit 1
-  fi
-  sleep "${RETRY_SLEEP}"
-done
@@ -1,19 +0,0 @@
-# {{ ansible_managed }}
-# Convergence backstop for the event-driven re-steer. The OpenVPN up-script fires
-# vpn-egress-resteer.service on every connect, but that call can be lost (kube-API not up
-# yet at boot and all in-script retries exhausted, an OpenVPN soft restart whose up-script
-# ran before k3s, a hand-run `nft delete table`). This timer makes the gateway converge on
-# its own instead of needing an operator. The script is a no-op when the published label
-# already matches the live tun0, so running it often is free.
-[Unit]
-Description=Periodic re-steer of the Cilium Egress Gateway onto the live tun0 — ADR-0015
-Documentation=file://{{ vpn_egress_gateway_scripts_dir }}/vpn-egress-resteer.sh
-
-[Timer]
-OnBootSec={{ vpn_egress_gateway_resteer_timer_boot }}
-OnUnitInactiveSec={{ vpn_egress_gateway_resteer_timer_interval }}
-AccuracySec=15s
-Unit=vpn-egress-resteer.service
-
-[Install]
-WantedBy=timers.target
@@ -35,34 +35,9 @@ ip route replace blackhole default table "${TABLE_ID}" metric "${BH_METRIC}"
 while ip rule del from "${SRC}" lookup "${TABLE_ID}" priority "${RULE_PRIO}" 2>/dev/null; do :; done
 ip rule add from "${SRC}" lookup "${TABLE_ID}" priority "${RULE_PRIO}"

-# Register the live tunnel source with the kill-switch set. ADDITIVE ON PURPOSE — the set
-# is NEVER flushed here (it used to be, see the DECISIONS-LOG entry of 2026-09-12).
-# `nft add element` is idempotent, and every address that has ever been this tunnel's
-# source can only ever TIGHTEN the `oifname != tun0 drop` rule. Flushing first meant that
-# between a reconnect and Cilium re-resolving its cached egress IP there was a moment
-# where a packet still sourced from the PREVIOUS address matched neither the ip-rule nor
-# the drop rule. (Measurement since showed that window cannot leak for POD traffic,
-# because that traffic never enters netfilter at all — see the kill-switch .nft header —
-# but node-originated tunnel traffic DOES, and additive costs nothing.)
+# Register the live tunnel source with the kill-switch set so the nft backstop can scope
+# to exactly this traffic. Flush any stale members first (address changes on reconnect).
+nft flush set inet "${NFT_TABLE}" "${NFT_SET}" 2>/dev/null || true
 nft add element inet "${NFT_TABLE}" "${NFT_SET}" "{ ${SRC} }"

-# THE LOAD-BEARING FIX (2026-09-12): tell Cilium that this gateway's uplink changed.
-#
-# Cilium resolves the CiliumEgressGatewayPolicy's `interface: tun0` ONCE, into a cached
-# (egress IP, egress ifindex) pair, and — measured on k3s-dev-3 — does NOT re-resolve it
-# when tun0 changes underneath, not even with `egress-gateway-reconciliation-trigger-
-# interval: 1s`. Upstream says as much: "it does not automatically respond to a change in
-# the gateway node's network configuration... You can force a fresh selection by
-# re-applying the Egress Gateway policy."
-#
-# Every tunnel restart re-creates tun0 with a NEW ifindex, so WITHOUT this the gateway is
-# silently dead after its first reconnect or reboot: vpn-class egress is blackholed (no
-# leak — the eBPF redirect to the dead ifindex just fails — but no service, and no alert).
-# Re-steering is done out-of-band by vpn-egress-resteer.service so this hook can never
-# block or fail the tunnel: OpenVPN's up-script must stay fast and must not depend on the
-# kube-API being reachable yet (at boot it usually is not — the resteer unit retries, and
-# its timer converges anyway).
-systemctl start --no-block vpn-egress-resteer.service || \
-  echo "vpn-egress-up: WARNING could not trigger vpn-egress-resteer.service (timer will converge)" >&2
-
 echo "vpn-egress-up: tun=${IFACE} src=${SRC} table=${TABLE_ID} — split-tunnel routing active"