diff --git a/.standup-dns/dns-before.txt b/.standup-dns/dns-before.txt new file mode 100644 index 0000000..3e5af65 --- /dev/null +++ b/.standup-dns/dns-before.txt @@ -0,0 +1,57 @@ +A|*.keithjackson.dev|136.51.115.252 +A|auth.home.keithjackson.dev|192.168.1.53 +A|brickbreak.keithjackson.dev|20.236.242.14 +A|crafty-mc.home.keithjackson.dev|192.168.1.55 +A|crafty.home.keithjackson.dev|192.168.1.53 +A|dev.brickbreak.keithjackson.dev|20.236.242.14 +A|gitea.home.keithjackson.dev|192.168.1.53 +A|grafana.home.keithjackson.dev|192.168.1.53 +A|headlamp.home.keithjackson.dev|192.168.1.53 +A|home.keithjackson.dev|192.168.1.53 +A|jackett.home.keithjackson.dev|192.168.1.53 +A|keycloak.home.keithjackson.dev|192.168.1.53 +A|nextcloud.home.keithjackson.dev|192.168.1.53 +A|pihole.home.keithjackson.dev|192.168.1.53 +A|plex.home.keithjackson.dev|192.168.1.53 +A|qbittorrent.home.keithjackson.dev|192.168.1.53 +A|radarr.home.keithjackson.dev|192.168.1.53 +A|seerr.home.keithjackson.dev|192.168.1.53 +A|semaphore.home.keithjackson.dev|192.168.1.53 +A|sonarr.home.keithjackson.dev|192.168.1.53 +A|unifi.home.keithjackson.dev|192.168.1.54 +TXT|a-auth.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/oauth2-proxy/oauth2-proxy +TXT|a-crafty-mc.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/crafty/crafty-l4-tcp +TXT|a-crafty.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/crafty/crafty +TXT|a-gitea.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/gitea/gitea-3000 +TXT|a-grafana.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/monitoring/grafana +TXT|a-headlamp.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/headlamp/headlamp +TXT|a-home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/homepage/homepage +TXT|a-jackett.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/jackett/jackett-9117 +TXT|a-keycloak.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/keycloak/keycloak +TXT|a-nextcloud.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/nextcloud/nextcloud-80 +TXT|a-pihole.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/pihole/pihole-80 +TXT|a-plex.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/plex/plex-32400 +TXT|a-qbittorrent.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/qbittorrent/qbittorrent-8080 +TXT|a-radarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/radarr/radarr-7878 +TXT|a-seerr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/seerr/seerr-5055 +TXT|a-semaphore.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/semaphore/semaphore-3000 +TXT|a-sonarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/sonarr/sonarr-8989 +TXT|a-unifi.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/unifi/unifi-l4-tcp +TXT|auth.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/oauth2-proxy/oauth2-proxy +TXT|crafty-mc.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/crafty/crafty-l4-tcp +TXT|crafty.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/crafty/crafty +TXT|gitea.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/gitea/gitea-3000 +TXT|grafana.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/monitoring/grafana +TXT|headlamp.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/headlamp/headlamp +TXT|home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/homepage/homepage +TXT|jackett.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/jackett/jackett-9117 +TXT|keycloak.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/keycloak/keycloak +TXT|nextcloud.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/nextcloud/nextcloud-80 +TXT|pihole.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/pihole/pihole-80 +TXT|plex.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/plex/plex-32400 +TXT|qbittorrent.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/qbittorrent/qbittorrent-8080 +TXT|radarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/radarr/radarr-7878 +TXT|seerr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/seerr/seerr-5055 +TXT|semaphore.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/semaphore/semaphore-3000 +TXT|sonarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/sonarr/sonarr-8989 +TXT|unifi.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/unifi/unifi-l4-tcp diff --git a/.standup-dns/dns-post-l2.txt b/.standup-dns/dns-post-l2.txt new file mode 100644 index 0000000..4ead883 --- /dev/null +++ b/.standup-dns/dns-post-l2.txt @@ -0,0 +1,95 @@ +A|*.keithjackson.dev|136.51.115.252 +A|auth.home.keithjackson.dev|192.168.1.53 +A|auth.prod.home.keithjackson.dev|192.168.2.20 +A|brickbreak.keithjackson.dev|20.236.242.14 +A|conformance-dns.home.keithjackson.dev|192.168.1.56 +A|crafty-mc.home.keithjackson.dev|192.168.1.55 +A|crafty-mc.prod.home.keithjackson.dev|192.168.2.22 +A|crafty.home.keithjackson.dev|192.168.1.53 +A|crafty.prod.home.keithjackson.dev|192.168.2.20 +A|dev.brickbreak.keithjackson.dev|20.236.242.14 +A|gitea.home.keithjackson.dev|192.168.1.53 +A|gitea.prod.home.keithjackson.dev|192.168.2.20 +A|grafana.home.keithjackson.dev|192.168.1.53 +A|grafana.prod.home.keithjackson.dev|192.168.2.20 +A|headlamp.home.keithjackson.dev|192.168.1.53 +A|headlamp.prod.home.keithjackson.dev|192.168.2.20 +A|home.keithjackson.dev|192.168.1.53 +A|jackett.home.keithjackson.dev|192.168.1.53 +A|jackett.prod.home.keithjackson.dev|192.168.2.20 +A|keycloak.home.keithjackson.dev|192.168.1.53 +A|keycloak.prod.home.keithjackson.dev|192.168.2.20 +A|nextcloud.home.keithjackson.dev|192.168.1.53 +A|nextcloud.prod.home.keithjackson.dev|192.168.2.20 +A|pihole.home.keithjackson.dev|192.168.1.53 +A|pihole.prod.home.keithjackson.dev|192.168.2.20 +A|plex.home.keithjackson.dev|192.168.1.53 +A|plex.prod.home.keithjackson.dev|192.168.2.20 +A|prod.home.keithjackson.dev|192.168.2.20 +A|qbittorrent.home.keithjackson.dev|192.168.1.53 +A|qbittorrent.prod.home.keithjackson.dev|192.168.2.20 +A|radarr.home.keithjackson.dev|192.168.1.53 +A|radarr.prod.home.keithjackson.dev|192.168.2.20 +A|seerr.home.keithjackson.dev|192.168.1.53 +A|seerr.prod.home.keithjackson.dev|192.168.2.20 +A|semaphore.home.keithjackson.dev|192.168.1.53 +A|semaphore.prod.home.keithjackson.dev|192.168.2.20 +A|sonarr.home.keithjackson.dev|192.168.1.53 +A|sonarr.prod.home.keithjackson.dev|192.168.2.20 +A|unifi.home.keithjackson.dev|192.168.1.54 +A|unifi.prod.home.keithjackson.dev|192.168.2.21 +TXT|a-auth.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/oauth2-proxy/oauth2-proxy +TXT|a-auth.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/oauth2-proxy/oauth2-proxy +TXT|a-conformance-dns.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/chainsaw-growing-grouse/dns-probe-svc +TXT|a-crafty-mc.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/crafty/crafty-l4-tcp +TXT|a-crafty-mc.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=service/crafty/crafty-l4-tcp +TXT|a-crafty.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/crafty/crafty +TXT|a-crafty.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/crafty/crafty +TXT|a-gitea.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/gitea/gitea-3000 +TXT|a-gitea.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/gitea/gitea-3000 +TXT|a-grafana.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/monitoring/grafana +TXT|a-grafana.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/monitoring/grafana +TXT|a-headlamp.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/headlamp/headlamp +TXT|a-headlamp.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/headlamp/headlamp +TXT|a-home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/homepage/homepage +TXT|a-jackett.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/jackett/jackett-9117 +TXT|a-jackett.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/jackett/jackett-9117 +TXT|a-keycloak.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/keycloak/keycloak +TXT|a-keycloak.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/keycloak/keycloak +TXT|a-nextcloud.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/nextcloud/nextcloud-80 +TXT|a-nextcloud.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/nextcloud/nextcloud-80 +TXT|a-pihole.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/pihole/pihole-80 +TXT|a-pihole.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/pihole/pihole-80 +TXT|a-plex.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/plex/plex-32400 +TXT|a-plex.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/plex/plex-32400 +TXT|a-prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/homepage/homepage +TXT|a-qbittorrent.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/qbittorrent/qbittorrent-8080 +TXT|a-qbittorrent.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/qbittorrent/qbittorrent-8080 +TXT|a-radarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/radarr/radarr-7878 +TXT|a-radarr.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/radarr/radarr-7878 +TXT|a-seerr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/seerr/seerr-5055 +TXT|a-seerr.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/seerr/seerr-5055 +TXT|a-semaphore.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/semaphore/semaphore-3000 +TXT|a-semaphore.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/semaphore/semaphore-3000 +TXT|a-sonarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/sonarr/sonarr-8989 +TXT|a-sonarr.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/sonarr/sonarr-8989 +TXT|a-unifi.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/unifi/unifi-l4-tcp +TXT|a-unifi.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=service/unifi/unifi-l4-tcp +TXT|auth.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/oauth2-proxy/oauth2-proxy +TXT|crafty-mc.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/crafty/crafty-l4-tcp +TXT|crafty.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/crafty/crafty +TXT|gitea.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/gitea/gitea-3000 +TXT|grafana.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/monitoring/grafana +TXT|headlamp.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/headlamp/headlamp +TXT|home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/homepage/homepage +TXT|jackett.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/jackett/jackett-9117 +TXT|keycloak.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/keycloak/keycloak +TXT|nextcloud.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/nextcloud/nextcloud-80 +TXT|pihole.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/pihole/pihole-80 +TXT|plex.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/plex/plex-32400 +TXT|qbittorrent.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/qbittorrent/qbittorrent-8080 +TXT|radarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/radarr/radarr-7878 +TXT|seerr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/seerr/seerr-5055 +TXT|semaphore.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/semaphore/semaphore-3000 +TXT|sonarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/sonarr/sonarr-8989 +TXT|unifi.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/unifi/unifi-l4-tcp diff --git a/.standup-dns/dns-verify.txt b/.standup-dns/dns-verify.txt new file mode 100644 index 0000000..ffa208e --- /dev/null +++ b/.standup-dns/dns-verify.txt @@ -0,0 +1,93 @@ +A|*.keithjackson.dev|136.51.115.252 +A|auth.home.keithjackson.dev|192.168.1.53 +A|auth.prod.home.keithjackson.dev|192.168.2.20 +A|brickbreak.keithjackson.dev|20.236.242.14 +A|crafty-mc.home.keithjackson.dev|192.168.1.55 +A|crafty-mc.prod.home.keithjackson.dev|192.168.2.22 +A|crafty.home.keithjackson.dev|192.168.1.53 +A|crafty.prod.home.keithjackson.dev|192.168.2.20 +A|dev.brickbreak.keithjackson.dev|20.236.242.14 +A|gitea.home.keithjackson.dev|192.168.1.53 +A|gitea.prod.home.keithjackson.dev|192.168.2.20 +A|grafana.home.keithjackson.dev|192.168.1.53 +A|grafana.prod.home.keithjackson.dev|192.168.2.20 +A|headlamp.home.keithjackson.dev|192.168.1.53 +A|headlamp.prod.home.keithjackson.dev|192.168.2.20 +A|home.keithjackson.dev|192.168.1.53 +A|jackett.home.keithjackson.dev|192.168.1.53 +A|jackett.prod.home.keithjackson.dev|192.168.2.20 +A|keycloak.home.keithjackson.dev|192.168.1.53 +A|keycloak.prod.home.keithjackson.dev|192.168.2.20 +A|nextcloud.home.keithjackson.dev|192.168.1.53 +A|nextcloud.prod.home.keithjackson.dev|192.168.2.20 +A|pihole.home.keithjackson.dev|192.168.1.53 +A|pihole.prod.home.keithjackson.dev|192.168.2.20 +A|plex.home.keithjackson.dev|192.168.1.53 +A|plex.prod.home.keithjackson.dev|192.168.2.20 +A|prod.home.keithjackson.dev|192.168.2.20 +A|qbittorrent.home.keithjackson.dev|192.168.1.53 +A|qbittorrent.prod.home.keithjackson.dev|192.168.2.20 +A|radarr.home.keithjackson.dev|192.168.1.53 +A|radarr.prod.home.keithjackson.dev|192.168.2.20 +A|seerr.home.keithjackson.dev|192.168.1.53 +A|seerr.prod.home.keithjackson.dev|192.168.2.20 +A|semaphore.home.keithjackson.dev|192.168.1.53 +A|semaphore.prod.home.keithjackson.dev|192.168.2.20 +A|sonarr.home.keithjackson.dev|192.168.1.53 +A|sonarr.prod.home.keithjackson.dev|192.168.2.20 +A|unifi.home.keithjackson.dev|192.168.1.54 +A|unifi.prod.home.keithjackson.dev|192.168.2.21 +TXT|a-auth.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/oauth2-proxy/oauth2-proxy +TXT|a-auth.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/oauth2-proxy/oauth2-proxy +TXT|a-crafty-mc.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/crafty/crafty-l4-tcp +TXT|a-crafty-mc.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=service/crafty/crafty-l4-tcp +TXT|a-crafty.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/crafty/crafty +TXT|a-crafty.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/crafty/crafty +TXT|a-gitea.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/gitea/gitea-3000 +TXT|a-gitea.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/gitea/gitea-3000 +TXT|a-grafana.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/monitoring/grafana +TXT|a-grafana.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/monitoring/grafana +TXT|a-headlamp.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/headlamp/headlamp +TXT|a-headlamp.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/headlamp/headlamp +TXT|a-home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/homepage/homepage +TXT|a-jackett.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/jackett/jackett-9117 +TXT|a-jackett.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/jackett/jackett-9117 +TXT|a-keycloak.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/keycloak/keycloak +TXT|a-keycloak.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/keycloak/keycloak +TXT|a-nextcloud.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/nextcloud/nextcloud-80 +TXT|a-nextcloud.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/nextcloud/nextcloud-80 +TXT|a-pihole.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/pihole/pihole-80 +TXT|a-pihole.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/pihole/pihole-80 +TXT|a-plex.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/plex/plex-32400 +TXT|a-plex.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/plex/plex-32400 +TXT|a-prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/homepage/homepage +TXT|a-qbittorrent.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/qbittorrent/qbittorrent-8080 +TXT|a-qbittorrent.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/qbittorrent/qbittorrent-8080 +TXT|a-radarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/radarr/radarr-7878 +TXT|a-radarr.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/radarr/radarr-7878 +TXT|a-seerr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/seerr/seerr-5055 +TXT|a-seerr.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/seerr/seerr-5055 +TXT|a-semaphore.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/semaphore/semaphore-3000 +TXT|a-semaphore.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/semaphore/semaphore-3000 +TXT|a-sonarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/sonarr/sonarr-8989 +TXT|a-sonarr.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/sonarr/sonarr-8989 +TXT|a-unifi.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/unifi/unifi-l4-tcp +TXT|a-unifi.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=service/unifi/unifi-l4-tcp +TXT|auth.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/oauth2-proxy/oauth2-proxy +TXT|crafty-mc.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/crafty/crafty-l4-tcp +TXT|crafty.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/crafty/crafty +TXT|gitea.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/gitea/gitea-3000 +TXT|grafana.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/monitoring/grafana +TXT|headlamp.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/headlamp/headlamp +TXT|home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/homepage/homepage +TXT|jackett.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/jackett/jackett-9117 +TXT|keycloak.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/keycloak/keycloak +TXT|nextcloud.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/nextcloud/nextcloud-80 +TXT|pihole.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/pihole/pihole-80 +TXT|plex.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/plex/plex-32400 +TXT|qbittorrent.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/qbittorrent/qbittorrent-8080 +TXT|radarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/radarr/radarr-7878 +TXT|seerr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/seerr/seerr-5055 +TXT|semaphore.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/semaphore/semaphore-3000 +TXT|sonarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/sonarr/sonarr-8989 +TXT|unifi.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/unifi/unifi-l4-tcp diff --git a/DECISIONS-LOG.md b/DECISIONS-LOG.md index f9aedc0..0bd477c 100644 --- a/DECISIONS-LOG.md +++ b/DECISIONS-LOG.md @@ -1847,3 +1847,110 @@ down in `platform/headlamp/README.md`: moving one without the other 401s every r `headlamp-oidc-credentials.sops.yaml.example` used to *teach* the broken pattern — it listed `OIDC_ISSUER_URL` and `OIDC_CALLBACK_URL` as keys to fill in. It now says, in the file, that adding either key back re-breaks prod. +## 2026-09-07 — k3s-prod-1 NotReady: etcd starved of disk I/O by Longhorn (branch `fix/etcd-io-priority`) + +**06:05:21Z, `k3s-prod-1` went NotReady** (`Kubelet stopped posting node status`) with the host +up, pingable and SSH-able. The embedded etcd could not complete a WAL `fdatasync`: + +``` +slow fdatasync took=3.56s / 6.71s / 4.95s / 3.40s (etcd expects 1s) +apply request took too long took=8.30s (expects 100ms) +etcdserver: too many requests +Failed to check local etcd status for learner management: context deadline exceeded +``` + +No OOM, no MCE, no hung task — **this is not the §0b memattr bug and not memory**. It is the §2 +shared-device risk, which §2 recorded as "accepted to be MEASURED, not assumed". The measurement +came back, in production, 2 d 7 h after bring-up. `/proc/pressure/io` **full avg300**: node 1 +**11.36**, node 2 (NVMe) 0.75, node 3 (WD Blue SATA) 1.29. Node 1's drive is a 2011 +`C400-MTFDDAC512MAM` with 61,440 reallocated sectors and no power-loss protection. §2's premise — +"every prod node is SSD or NVMe, where rebuild I/O costs latency rather than collapsing it" — was +right about node 2 and wrong about node 1. §2 now records this under "THE MEASUREMENT CAME BACK". + +**The exit condition is OPEN and the fix is hardware: replace node 1's drive, or give etcd its own +device (`node_baseline_etcd_source: device`, the path the role still supports). That is the +operator's.** Three software mitigations shipped meanwhile, none of which is the fix. + +### 1. The outage was invisible to the monitoring stack — fixed first + +There were alerts for **CPU** PSI and memory and **none for I/O**, so the first signal was a node +already out of the cluster. Added `NodeIOPressureHigh` (full IO PSI > 5% for 10m) and +`NodeIOPressureCritical` (> 10% for 5m), thresholds taken from the numbers above against a ~1% +steady state, plus an I/O row on the node-pressure dashboard. Verified live that node-exporter +already exports `node_pressure_io_{stalled,waiting}_seconds_total` on all three nodes. + +**Gap left open for the operator:** etcd's own `etcd_disk_wal_fsync_duration_seconds` — the metric +§2's exit condition literally asks for — is **not collected**. `kubeEtcd` is deliberately disabled +in `platform/monitoring/kube-prometheus-stack.yaml` for k3s. Enabling it (k3s +`--etcd-expose-metrics` + a scrape target) would make §2 directly measurable instead of inferred +from PSI. Not done here: it is a k3s server-flag change requiring a rolling restart, which is not +something to bundle into an incident fix. + +### 2. Longhorn was already throttled as far as Longhorn goes + +`concurrentReplicaRebuildPerNodeLimit: 1` was **in force during the outage and it held** — the 18 +volumes rebuilt one at a time and the node stalled anyway. Recorded plainly because it would be +easy to "fix" this incident by setting a knob that was already set. For the v1 data engine there is +no further rebuild lever: `replicaRebuildingBandwidthLimit` is v2/SPDK only (live Setting reads +`{"v2":"0"}`, no v1 key) and `rebuildConcurrentSyncLimit` is already at its floor of 1. + +So what was tightened is every *other* Longhorn I/O source, each defaulting to 5 concurrent +operations per node: `concurrentVolumeBackupRestorePerNodeLimit` 5→1 (an R-STATE-03 restore of 18 +volumes would otherwise run five parallel full-volume writes onto the disk etcd fsyncs to), +`snapshotHeavyTaskConcurrentLimit` 5→1, plus pins on `replicaReplenishmentWaitInterval`, +`backup`/`restoreConcurrentLimit` and `concurrentAutomaticEngineUpgradePerNodeLimit`. + +**And one setting in that file was inert.** `orphanAutoDeletion: true` has been silently dropped +since the 1.11→1.12 upgrade — Longhorn 1.9 renamed it `orphanResourceAutoDeletion` (a list, not a +bool) and the chart drops unknown `defaultSettings` keys with no error. Confirmed from both ends: +absent from chart v1.12.1's `templates/default-setting.yaml`, and absent from the rendered +`longhorn-default-setting` ConfigMap in the live cluster. Removed, deliberately not replaced. That +is the **third** time this repo has shipped config that looked applied and was not. + +### 3. Block-layer arbitration — and four mechanisms that are inert on these nodes + +`roles/etcd_io_priority` + `playbooks/etcd-io-priority.yml`: BFQ on the shared disk, `nr_requests` +64→32, `IOWeight=10000` on `system.slice` **and** `k3s.service`, then every value read back out of +the kernel and asserted. Enabled for `[k3s_prod]`; **applies nothing until the operator runs it**. + +The investigation's main value is the negative results, all checked on the live nodes: + +- **`IOSchedulingClass=realtime` on `k3s.service` is a footgun, not a fix.** `mq-deadline` does + honour the priority class, so it looks correct — but ioprio is inherited across `fork` and kept + across `exec` (`block/blk-ioc.c` `__copy_io` copies `ioc->ioprio`), and **containerd is a child + of the k3s server process** (verified: containerd pid 84553, ppid 84333 = `k3s-server`; and + `ionice -c2 -n0 -- sh -c 'ionice -p $$; sh -c "ionice -p \$\$"'` reports best-effort prio 0 at + both levels). Realtime for k3s would be realtime for every container including Longhorn. +- **`IOSchedulingPriority` (level within a class) is a no-op** — `mq-deadline` buckets by class only. +- **`io.latency` is unavailable** — `CONFIG_BLK_CGROUP_IOLATENCY` is not built in Debian trixie's + 7.1.8 kernel (only `IOCOST` is), and no `io.latency` exists under `/sys/fs/cgroup`. +- **A bare `IOWeight=` drop-in is decoration.** systemd writes it to `io.weight` (blk-iocost — + compiled in but disabled; `io.cost.qos` is empty) and `io.bfq.weight` (BFQ — not loaded). Neither + reader exists on the stock nodes. And `IOWeight=1000` would map to `io.bfq.weight` ≈ 101 against + BFQ's unweighted default of 100 — hence 10000, and an asserted floor of 500. + +**What none of it fixes, stated so it is not forgotten:** etcd and Longhorn still share one ext4 +filesystem, and no cgroup or scheduler control arbitrates the **jbd2 journal** that etcd's +`fdatasync` waits on and Longhorn's writes dirty. Software raises the load level at which this +happens; it does not remove it. + +**Unproven.** The role fails loudly if its settings do not *land*; it cannot tell you they helped. +Apply to `k3s-prod-3` first (WD Blue, PSI 1.29 — same code path, far lower stakes), watch +`NodeIOPressureHigh` for a day, then node 1. + +### Doc correction found on the way + +`infra/ansible/README.md` still claimed `k3s-prod-2`/`-3` "stay commented out" of the inventory. +They have been live in `[k3s_prod_secondary]` since 2026-09-03. Corrected, because that sentence is +exactly the kind used to reason about blast radius: **`-e k3s_hosts=k3s_prod` targets three hosts, +not one.** Confirmed by `--list-hosts` (dev default → 3 dev hosts, where the new role is a no-op; +`-e k3s_hosts=k3s_prod` → 3 prod hosts; `--limit k3s-prod-3` → 1). + +### Still requires the operator + +1. **The hardware fix.** Replace node 1's C400, or add an M.2 per node and set + `node_baseline_etcd_source: device`. Everything above is a delay, not a repair. +2. **Run `playbooks/etcd-io-priority.yml`** — node 3 first, `--limit`, `--list-hosts` confirmed. + Nothing was run against either cluster for this change. +3. **Decide on `kubeEtcd` metrics**, so §2's exit condition can be measured with etcd's own fsync + histograms rather than inferred from node PSI. diff --git a/docs/plans/proxmox-to-n5-air-migration.md b/docs/plans/proxmox-to-n5-air-migration.md new file mode 100644 index 0000000..99968ca --- /dev/null +++ b/docs/plans/proxmox-to-n5-air-migration.md @@ -0,0 +1,319 @@ +# Decommission Proxmox `192.168.2.102` → TrueNAS SCALE on the N5 Air + +**Status:** plan. **Drafted** 2026-09-06, **revised** 2026-09-06 after the operator settled the +target architecture. + +| | | +|---|---| +| **N5 Air** | TrueNAS SCALE. **Storage only** — no VMs, no hypervisor. | +| **Compute** | k3s **prod** cluster (`192.168.2.10/.11/.12`). Plex, sonarr, radarr, jackett, qbittorrent, seerr and nextcloud are already modelled in `apps/wave-0/`. | +| **Dev k3s** | Does not move to the NAS. | +| **Proxmox host** | **Decommissioned.** | +| **Drives** | Physically relocated. None purchased. | + +All figures measured on the live host, not inferred. + +--- + +## Already done — 2026-09-06 + +**Reclaimed 3.56 TB.** `Storage/plex@pre-cleanup-20260828` destroyed after confirming no holds and +no clones. Async free drained in ~90 seconds. + +| | before | after | +|---|---|---| +| `Storage` allocated | 12.3 T (75%) | **8.76 T (53%)** | +| `Storage` free | 4.04 T | **7.60 T** | +| `Storage/plex` used | 11.9 T | 8.33 T (0 B in snapshots) | + +That free space is not a nicety — **it is what lets this whole migration happen with no network +transfer at all.** See §3. + +--- + +## 1. The plan changed shape + +The first draft carried the VMs across on their own zvols and put Proxmox on the NAS. With no +hypervisor on the far side, that inverts: + +> **Every zvol is now dead weight, and every service's data has to be drained onto a plain dataset +> before the Proxmox box dies.** + +Dead-weight zvols, once their contents are extracted: + +| zvol | size | after extraction | +|---|---|---| +| `Storage:vm-101/102/105-disk-0,-disk-1` (dev k3s) | 585 G | delete | +| `Storage:vm-112-disk-0` (docker-services) | 65 G | delete | +| `Storage:vm-103-disk-0` (nextcloud boot) | 81 G | delete | +| `Storage2:vm-103-disk-0` (**nextcloud data**) | 4.06 T | extract, then delete | + +The good news is the shape of the drain. Everything that must be rescued is either already on +`Storage`, or can be copied onto `Storage` **locally, inside the same chassis, onto the very drives +that are moving.** With 7.60 T free there is room. **Nothing crosses the network.** + +--- + +## 2. Findings + +### 2a. "The path lies" — two confirmed instances, assume a third + +**Games — 422 GB, not on the Storage pool.** Challenged and re-verified; the test is decisive: + +``` +findmnt -T /mnt/Storage/Games → TARGET / SOURCE rpool/ROOT/pve-1 +findmnt -T /mnt/Storage/plex → TARGET /mnt/Storage/plex SOURCE Storage/plex +``` + +`Games` is a plain directory on the Proxmox **root** filesystem that merely sits inside the Storage +mount tree. The pool accounting closes on it exactly: + +``` +rpool ALLOC 540 G = ROOT 431 G + data 105 G + var-lib-vz 3.75 G + ROOT 431 G = Games 422 G + /usr 3.5 G + /var 5 G + /boot 0.4 G +``` + +The two Microns are a **mirror**, so the pool is ~888 G usable, not ~1.9 T — 422 G is 48% of it, and +`zpool list` agrees at 60% capacity overall. Surprising, but the numbers do match. + +**Gitea — 13 MB in an anonymous Docker volume, not where its compose file points.** +`/root/docker-compose.yml` in LXC 106 bind-mounts `/opt/gitea/data:/var/lib/data`. That path holds +**512 bytes**. The real Gitea state is in an unnamed Docker volume: + +``` +.../volumes/8a1d2616…/_data/git/repositories/ 12 M home-network.git, home-network-uds.git, jakman85/… +.../volumes/8a1d2616…/_data/data/gitea.db 11 M SQLite, mtime today 19:00 — live +``` + +Copying `/opt/gitea` would migrate an empty directory and silently lose all three repositories. + +**Rule for the rest of the drain:** for every service, verify where the bytes actually are before +trusting the path in a compose file, an export list or a mount table. Two for two so far. + +### 2b. Retiring pfSense is easier than rehoming it + +`pfsense` is VM 100 and **is** `192.168.1.1`. Decommissioning the host takes the entire +`192.168.1.0/24` with it: + +| | | +|---|---| +| `.1` pfSense · `.4` iDRAC · `.9` the Proxmox host itself | die with the box | +| `.5` nextcloud · `.6` plex · `.7` gitea · `.18` docker-services/Pi-hole | data drains, service moves to k3s | +| `.8` unifi · `.20` monitoring · `.21` auth · `.10` semaphore | rehome or retire (§2e) | +| `.49` kube-vip VIP · `.50/.51/.52` dev k3s | dev cluster is not moving; retire with the box | + +**But pfSense exists only to contain that lab LAN.** Prod k3s is already on `192.168.2.0/24` — +pfSense's *WAN* side — and the NAS will be too. Once the lab is gone, nothing lives behind pfSense. +**Retire it; do not rehome it.** That is only clean because everything else is leaving `1.0/24` +anyway, which is exactly what this migration does. + +**A documented blocker dissolves with it.** `infra/PROD-STANDUP.md` records that prod k3s cannot +reach the S3 backup endpoint at `192.168.1.9:9000` because prod sits on pfSense's WAN side and the +crossing rule does not exist. With the NAS on `192.168.2.0/24`, prod reaches its storage directly +and no pfSense rule is ever needed. + +**Three things to close before pfSense goes:** + +1. The SOHO router NATs inbound **443** to `192.168.2.102` for the legacy stack. It must be + repointed at prod's Gateway (`192.168.2.20`). `infra/PROD-PROVISIONING.md` §10a already flags + that the apex flip and the legacy cutover are the same event — **this migration is that event.** +2. **VLANs 5 and 123** (minecraft; the arr VM, which reaches Plex media as `10.123.1.10`) are + pfSense-side constructs. They disappear; the k3s apps need no VLAN. +3. The **OpenVPN split-tunnel egress gateway** currently runs on `k3s-dev-3` (ADR-0015). It is + already modelled for prod in `platform/vpn-egress/`, but confirm prod's gateway is live before + the dev node goes away. + +### 2c. Nextcloud is the critical path + +Its user data is **inside a filesystem, inside LVM, inside a 4 TB zvol, inside a VM**: + +``` +Storage2/vm-103-disk-0 → zd224 → PV /dev/zd224 → VG vg_storage → LV lv_storage (<4.00 T) +``` + +With no hypervisor on the far side this must be copied out **at the file level, while the VM is +still running** — it gates the decommission and it is the longest single task in the plan. + +**One number is missing and I could not get it.** VM 103's QEMU guest agent is not running +(`qm agent 103 ping` → *"QEMU guest agent is not running"*) and its SSH does not accept the +`home_network` key. The zvol shows `referenced 3.94 T`, but it is attached **without `discard=on`** +and is thick-provisioned (`refreservation 4.06 T`), so deletions inside the guest were never +returned to ZFS — 3.94 T is an **upper bound**, possibly far above the truth. + +**Run this inside the Nextcloud VM and the last open question closes:** + +```bash +df -h / # or wherever lv_storage is mounted +lsblk -f /dev/mapper/vg_storage-lv_storage +``` + +Separately: `Storage/nextcloud` is 270 G, and **267 G of it is a directory called `config_backup`** +(`config` itself is 103 M). Almost certainly a stale dump — likely another easy reclaim, but confirm +before deleting. + +### 2d. Storage2 — and the two oldest drives — can probably be retired + +After the drain and the zvol deletions, `Storage` would hold roughly: + +| | | +|---|---| +| plex | 8.33 T | +| nextcloud data | ≤ 3.94 T (real number pending §2c) | +| Games | 0.42 T | +| docker-volumes, other_containers, k8s-volumes, gitea | ~0.02 T | +| **total** | **≤ 12.7 T of 16.4 T (≤ 77%)** | + +If the real Nextcloud figure comes in under ~2 T — which the missing-TRIM situation makes plausible +— everything fits on the 18 TB mirror at a comfortable ~65%, and **`Storage2` can be retired +entirely.** That would remove the two **6.0-year-old** Seagates, by a wide margin the most +failure-prone hardware in the estate, and free two bays. This is the single best safety outcome +available, and it costs nothing. + +### 2e. Services that simply end + +Decommissioning kills these; none has a home in the target architecture unless you give it one: + +| guest | note | +|---|---| +| `docker-services` Pi-hole (`192.168.1.18`) | **LAN DNS.** Confirm what `192.168.2.0/24` uses for DNS before the box dies. | +| `unifi` (VM 121) | Network controller — needs a home or a cloud key. | +| `auth` (VM 122) | Superseded by Keycloak on prod k3s. | +| `monitoring` (VM 120) | Superseded by the k3s platform monitoring stack. | +| `semaphore` (LXC 110) | 432 MB resident of an 8 GB reservation; low value. | +| `minecraft` (LXC 108) | Operator call. Its 3.5 G subvol is on `Storage` and rides along. | +| **versitygw S3** on `192.168.1.9:9000` | **Velero + CNPG backup target.** Dies with the box. TrueNAS must take this role (MinIO app) or prod loses its backup destination — which `infra/PROD-STANDUP.md` already records as uncovered. | + +### 2f. Still true from the first draft + +- **No VM backups exist.** `/etc/pve/jobs.cfg` empty, no PBS, newest dump log April 2025. +- **`ata-WDC_…3FGHAB2T` has 36 ZFS read errors** on pristine SMART (0 reallocated / pending / + uncorrectable, `PASSED`) — points at cable or controller, not platter. Re-verify after the move. +- Drive ages: WD 18 TB pair **4.75 y**, Seagate 8 TB pair **6.0 y**. + +--- + +## 3. TrueNAS SCALE specifics + +**ZFS compatibility: verified, not assumed.** The only non-default features *active* on either pool +are `block_cloning`, `head_errlog`, `vdev_zaps_v2`, `zilsaxattr`, `zstd_compress`, `log_spacemap`, +`project_quota`, `spacemap_v2` — all OpenZFS **2.2** era or older. Nothing 2.3- or 2.4-only is +active, even though the pools were last written by `zfs-2.4.3-pve1`. Any TrueNAS SCALE from 24.10 +onward imports them cleanly. + +**Mountpoints mostly already match TrueNAS convention** — a lucky accident worth knowing: + +| dataset | current mountpoint | TrueNAS wants | | +|---|---|---|---| +| `Storage/plex`, `/nextcloud`, `/docker-volumes`, `/other_containers`, `/k8s-volumes`, `/k3s-backups` | `/mnt/Storage/…` | `/mnt/Storage/…` | already correct | +| `Storage` (pool root) | `/Storage` | `/mnt/Storage` | fix on import | +| `Storage2` (pool root) | `/Storage2` | `/mnt/Storage2` | fix on import | +| `Storage/subvol-108-disk-0` | `/Storage/subvol-108-disk-0` | — | rename or delete | + +**Clear the ZFS `sharenfs` properties.** `Storage/plex`, `Storage/nextcloud` and +`Storage/docker-volumes` carry exports as ZFS properties; `Storage/k8s-volumes` is in +`/etc/exports`. TrueNAS generates exports from its own middleware database and will fight +properties it did not set. Wipe them and recreate the shares in the UI. + +**Fix the export targets while you are there.** Today's lists are stale: `/mnt/k3s` → +`192.168.1.25/.26/.27` and `k8s-volumes` → `192.168.1.40/.41/.42` are hosts that do not answer ARP. +The new exports target **`192.168.2.10/.11/.12`**. + +--- + +## 4. Sequence + +### Phase A — drain, locally, no downtime, no network + +Everything below is a copy from somewhere else in the chassis onto `Storage`, which is moving. + +1. **Games** — `rpool` → new dataset. + ```bash + zfs create Storage/games + rsync -aHAX --info=progress2 /mnt/Storage/Games/ /mnt/Storage/games/ + ``` +2. **Gitea** — copy the *Docker volume*, not `/opt/gitea` (§2a). Stop the container first so + `gitea.db` is quiesced. + ```bash + pct exec 106 -- docker stop gitea + zfs create Storage/gitea + rsync -aHAX /rpool/data/subvol-106-disk-0/var/lib/docker/volumes/8a1d2616*/_data/ /mnt/Storage/gitea/ + ``` +3. **Nextcloud** — the long one. Copy `lv_storage`'s contents to a new dataset at file level from + inside the running VM, over its existing NFS mount of the host. Verify §2c's size first so you + know whether it fits (it will, in 7.60 T free). +4. **Confirm `config_backup`** (267 G) is disposable; drop it if so. +5. **Verify each service's real data location before copying it** — the §2a rule. + +### Phase B — prove the compute side before the storage moves + +Bring the `apps/wave-0` manifests up on prod k3s while the old box is still alive, so the cutover is +a storage swap against known-good workloads rather than two migrations at once. + +Wrinkle: prod cannot reach `192.168.1.9` today (§2b). Either add a temporary pfSense rule for the +test, or accept that app bring-up happens after the NAS is live on `192.168.2.0/24` — in which case +stage the app configs now and keep the window short. + +### Phase C — cutover + +1. Stop all guests; reverse `startup:` order, pfSense last. +2. Export cleanly — **never just cut power:** + ```bash + zpool export Storage && zpool export Storage2 + zpool status # both must be gone before power-off + ``` +3. **Label every drive** with pool and serial before it leaves the chassis + (`3FGJ6M9T`/`3FGHAB2T` = Storage; `WKD1S0TR`/`WKD1S10M` = Storage2). +4. Move the four HDDs — cold, upright. The 6-year-old Seagates are the fragile ones. +5. Install TrueNAS SCALE on the shipped 64 GB M.2 (Slot A). The five bays take the four HDDs with + one spare; the Micron SSDs are **not needed** in the target — leave them in the old chassis. +6. **Import read-only first**, verify, then import for real: + ```bash + zpool import # both pools listed + zpool import -o readonly=on Storage + zfs list # verify datasets, sizes + zpool export Storage && zpool import Storage + ``` + +### Phase D — repoint, reclaim, retire + +1. Recreate NFS shares in the TrueNAS UI targeting `192.168.2.10/.11/.12`; clear the old `sharenfs` + properties. +2. Point the `apps/wave-0` PVs at the NAS; bring Plex, the arr stack and Nextcloud up on prod k3s. +3. Stand up MinIO on TrueNAS to replace versitygw, and re-enable Velero + CNPG backups (§2e). +4. **Only once everything is verified from k3s:** delete the dead zvols (§1) — ~731 G on `Storage`, + 4.06 T on `Storage2`. +5. Re-evaluate §2d. If it fits, evacuate `Storage2`, retire the 6-year-old Seagates, keep two bays + free. +6. Repoint the SOHO 443 NAT, then power off the Proxmox host. +7. `zpool status` — confirm the 36 read errors did **not** follow `3FGHAB2T` into the new chassis. + Full scrub on the new hardware. + +--- + +## 5. Rollback, and the one-way door + +Until Phase D step 4, the old chassis is intact: `rpool`, the Proxmox install, both Microns and +every `/etc/pve` config untouched. Put the four HDDs back and it boots into the previous world. + +> ### Do not run `zpool upgrade` +> +> `zpool status Storage2` already nags — *"Some supported and requested features are not enabled… +> action: Enable all features using 'zpool upgrade'"* — and TrueNAS will offer it in the UI after +> import. It is the only irreversible step here: run it and the pools can never go back to the +> Proxmox host. Leave it until the NAS has been serving k3s for weeks. + +Note the asymmetry that makes this safe: TrueNAS's OpenZFS **reads** these pools happily, because +newer ZFS always reads older pools. The reverse is not true. + +--- + +## 6. Open items + +1. **The Nextcloud number** (§2c) — one `df -h` inside VM 103. It decides whether `Storage2` and the + two oldest drives can be retired. +2. **Is `config_backup` (267 G) disposable?** +3. **What serves DNS on `192.168.2.0/24`** once Pi-hole dies? +4. **UniFi controller** — rehome, cloud key, or drop? +5. **Minecraft** — keep or retire? +6. **Gitea** — its whole state is 3 repos and an 11 MB SQLite DB, and prod k3s already runs its own + Gitea. Is the legacy one worth migrating at all, or does it just end? diff --git a/infra/PROD-PROVISIONING.md b/infra/PROD-PROVISIONING.md index 2e7a9e4..682e980 100644 --- a/infra/PROD-PROVISIONING.md +++ b/infra/PROD-PROVISIONING.md @@ -779,6 +779,74 @@ disk each, already fully partitioned, no free slot verified populated. physically) and flip `node_baseline_etcd_source` back to `device` — the role still supports that path unchanged, and it is the dev shape. +#### THE MEASUREMENT CAME BACK — 2026-09-07. It does not hold on node 1. + +Not under a deliberate test: in production, 2 days 7 hours after the cluster came up. +**06:05:21Z `k3s-prod-1` went NotReady** (`Kubelet stopped posting node status`) while the +host stayed up, pingable and SSH-able. From the k3s journal: + +``` +slow fdatasync took=3.56s / 6.71s / 4.95s / 3.40s (etcd expects 1s) +apply request took too long took=8.30s (expects 100ms) +etcdserver: too many requests +Failed to check local etcd status for learner management: context deadline exceeded +``` + +No OOM, no MCE, no hung task in the kernel log — **this is not §0b and not memory**. Recovery +required a manual `systemctl restart k3s`; 18 Longhorn volumes then rebuilt, and I/O pressure +only fell once the rebuilds finished. The failure is **self-reinforcing**: the node dropping +out degrades volumes → Longhorn rebuilds → more write load on the disk etcd needs → etcd +stalls further. + +`/proc/pressure/io` **full avg300**, all three nodes, during the incident: + +| node | device | drive | IO PSI full avg300 | +|---|---|---|---| +| k3s-prod-1 | `/dev/sda5` | C400-MTFDDAC512MAM (2011 consumer SATA SSD, no PLP) | **11.36** | +| k3s-prod-2 | `/dev/nvme0n1p5` | Samsung PM9A1 NVMe | 0.75 | +| k3s-prod-3 | `/dev/sda5` | WDC WDS500G2B0A (WD Blue SATA) | 1.29 | + +The prediction in the paragraph above — "every prod node is SSD or NVMe end-to-end, where +concurrent rebuild I/O costs latency rather than collapsing it" — was **right about node 2 and +wrong about node 1**. A 2011 consumer SATA SSD with 61,440 reallocated sectors, 31% of its +endurance consumed and no power-loss protection is not the "SSD" that reasoning assumed. §0's +"three nodes, three different drive models" is the fault line, exactly where §0 said it would +be. + +**The exit condition is therefore OPEN and the second bullet applies.** Replacing node 1's +drive, or giving etcd its own device, is the fix, and it is the operator's. + +Pending that, three SOFTWARE mitigations shipped on 2026-09-07 (branch +`fix/etcd-io-priority`). **None of them is the fix**, and each says so where it lives: + +1. **Observability first** — `NodeIOPressureHigh` / `NodeIOPressureCritical` in + `platform/monitoring/alerts.yaml`, plus an I/O row on the node-pressure dashboard. The + stack alerted on CPU and memory PSI and had **no I/O alert at all**, so the first signal + of this outage was a node already out of the cluster. Thresholds (5% / 10% full IO PSI) + come from the numbers in the table above. Note the etcd histograms this section's first + bullet asks for are **not available**: kube-prometheus-stack's `kubeEtcd` scrape job is + deliberately disabled for k3s (`platform/monitoring/kube-prometheus-stack.yaml`), so PSI + is the signal that actually exists. Enabling the etcd scrape would make that bullet + directly measurable and is worth doing. +2. **Longhorn concurrency ceilings** — `platform/longhorn/helmrelease.yaml`. Honestly: + `concurrentReplicaRebuildPerNodeLimit: 1` was **already in force during this outage** and + held; the 18 rebuilds ran one at a time and the node stalled anyway. Longhorn offers no + further rebuild lever for the v1 data engine (`replicaRebuildingBandwidthLimit` is + v2/SPDK only; `rebuildConcurrentSyncLimit` is already at its floor of 1). What was + tightened is every *other* Longhorn I/O source — restore, snapshot purge, backup, engine + upgrade — each of which defaulted to 5 concurrent operations per node and each of which + could reproduce this on its own. +3. **Block-layer arbitration** — `roles/etcd_io_priority`, enabled for `[k3s_prod]`, applied + only when the operator runs `playbooks/etcd-io-priority.yml`. BFQ + `IOWeight` so etcd's + cgroup outweighs `kubepods.slice`. Its README is explicit that most of the obvious knobs + here (`IOSchedulingClass`, `io.latency`, bare `IOWeight`) are **inert on these nodes**, + and that even this one cannot arbitrate the ext4 journal both tenants share. + +The last point is the one that matters for this section: **no cgroup or scheduler control +separates two tenants on one filesystem.** etcd's `fdatasync` waits on a jbd2 journal commit, +jbd2 is a kernel thread in the root cgroup, and Longhorn's writes dirty the same journal. The +software mitigations raise the load level at which this happens. They do not remove it. + **~~Known, separate consequence — `/var` is only 12 GiB.~~ RESOLVED 2026-09-04 by moving k3s's data dir.** This was recorded here on 2026-09-03 as a capacity item "to watch or repartition". Re-measuring it against dev turned it from a watch item into a blocker: the diff --git a/infra/ansible/README.md b/infra/ansible/README.md index fffd3da..2a9d130 100644 --- a/infra/ansible/README.md +++ b/infra/ansible/README.md @@ -27,6 +27,7 @@ ansible-playbook playbooks/site.yml # core bring-up, in order (baseline # — or step by step — ansible-playbook playbooks/node-baseline.yml # 0. packages, kernel modules, Longhorn disk, swap-off ansible-playbook playbooks/k3s.yml # 1. HA k3s (primary --cluster-init, then joins) +ansible-playbook playbooks/etcd-io-priority.yml # 1b. shared etcd/Longhorn device I/O weights (PROD ONLY; a no-op on dev) ansible-playbook playbooks/cilium.yml # 2. Cilium CNI (cluster is NotReady until this) ansible-playbook playbooks/verify.yml # 3. gate #1: 3 nodes Ready, Hubble up, no kube-proxy ansible-playbook playbooks/flux-bootstrap.yml # 4. hand off to bootstrap/ (SOPS then Flux) @@ -71,6 +72,7 @@ should return a PIA exit IP (not the home ISP). |---|---| | `k3s.yml` / `k3s_server` | k3s server install with `--flannel-backend=none --disable-kube-proxy --disable-network-policy --disable=traefik,servicelb`; primary `--cluster-init`, secondaries join via the primary's node-token (fetched, never pre-stored) | | `cilium.yml` / `cilium` | Gateway API CRDs, then Helm `cilium` with `kubeProxyReplacement`, `k8sServiceHost/Port`, `gatewayAPI`, `egressGateway`, `hubble`, `l2announcements` | +| `etcd-io-priority.yml` / `etcd_io_priority` | **prod only** (master switch defaults false; `group_vars/k3s_prod` sets it true). Switches the disk carrying `k3s_data_dir` to **BFQ**, drops `nr_requests` to 32, and gives `system.slice` + `k3s.service` `IOWeight=10000` so the embedded etcd outweighs `kubepods.slice` — then reads every value back out of the kernel and **fails** if it did not land. Mitigation for the 2026-09-07 k3s-prod-1 etcd WAL-fsync starvation. Restarts nothing. NVMe-backed nodes are skipped, not decorated. See `roles/etcd_io_priority/README.md` for the mechanisms in this space that are **inert** here and must not be "fixed" back in | | `verify.yml` / `cluster_verify` | waits nodes Ready + Cilium/Hubble rollout; asserts kube-proxy absent | | `flux-bootstrap.yml` / `flux_bootstrap` | calls `bootstrap/` (SOPS age key, then `flux bootstrap --path=clusters/dev`) | @@ -120,10 +122,20 @@ for PM reconciliation. ## Targeting prod (R-ENV-02: same playbooks, same roles, scoped overrides) `inventory/hosts.ini` carries a `[k3s_prod_primary]` / `[k3s_prod_secondary]` / -`[k3s_prod:children]` section for the three planned bare-metal nodes (`k3s-prod-1` at -`192.168.2.10` exists and is reachable today; `k3s-prod-2`/`-3` at `.11`/`.12` are addressed but -not yet racked, so their inventory lines stay commented out), with its own `group_vars/k3s_prod/` -(vaulted `ansible_become_password` + non-secret overrides). +`[k3s_prod:children]` section for the three bare-metal nodes — `k3s-prod-1` `192.168.2.10`, +`k3s-prod-2` `.11`, `k3s-prod-3` `.12` — with its own `group_vars/k3s_prod/` (vaulted +`ansible_become_password` + non-secret overrides). + +**All three are live in the inventory.** This paragraph used to say `k3s-prod-2`/`-3` "are +addressed but not yet racked, so their inventory lines stay commented out"; that stopped being +true on 2026-09-03 when node 3 came online, and the group has been complete since. Corrected +2026-09-07 because the stale sentence is the kind that gets used to reason about blast radius: +`-e k3s_hosts=k3s_prod` targets **three** hosts, not one. Confirm it, every time, before running +anything: + +```bash +ansible-playbook playbooks/.yml -e k3s_hosts=k3s_prod --limit k3s-prod-3 --list-hosts +``` **Nothing here is forked for prod.** Every node-host play takes its target group from a variable that defaults to the dev group, so the dev commands above are unchanged, and prod is the same @@ -153,6 +165,11 @@ ansible-playbook playbooks/kube-vip.yml -e k3s_hosts=k3s_prod ansible-playbook playbooks/verify.yml -e k3s_hosts=k3s_prod ansible-playbook playbooks/flux-bootstrap.yml -e k3s_hosts=k3s_prod # see caveat below +# prod — I/O arbitration for the shared etcd/Longhorn disk. ONE NODE AT A TIME, node 3 first +# (WD Blue, IO PSI 1.29 during the incident) before node 1 (the 2011 C400, 11.36). +ansible-playbook playbooks/etcd-io-priority.yml -e k3s_hosts=k3s_prod --limit k3s-prod-3 --list-hosts +ansible-playbook playbooks/etcd-io-priority.yml -e k3s_hosts=k3s_prod --limit k3s-prod-3 + # prod — k3s.yml needs both halves of the primary/secondary split ansible-playbook playbooks/k3s.yml \ -e k3s_primary_hosts=k3s_prod_primary -e k3s_secondary_hosts=k3s_prod_secondary