Compare commits
4
Commits
09265e0381
...
863af01062
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
863af01062 | ||
|
|
f7a9229d81 | ||
|
|
a819c50ffe | ||
|
|
efc6db6c1f |
No files matched your search
@@ -0,0 +1,57 @@
|
||||
A|*.keithjackson.dev|136.51.115.252
|
||||
A|auth.home.keithjackson.dev|192.168.1.53
|
||||
A|brickbreak.keithjackson.dev|20.236.242.14
|
||||
A|crafty-mc.home.keithjackson.dev|192.168.1.55
|
||||
A|crafty.home.keithjackson.dev|192.168.1.53
|
||||
A|dev.brickbreak.keithjackson.dev|20.236.242.14
|
||||
A|gitea.home.keithjackson.dev|192.168.1.53
|
||||
A|grafana.home.keithjackson.dev|192.168.1.53
|
||||
A|headlamp.home.keithjackson.dev|192.168.1.53
|
||||
A|home.keithjackson.dev|192.168.1.53
|
||||
A|jackett.home.keithjackson.dev|192.168.1.53
|
||||
A|keycloak.home.keithjackson.dev|192.168.1.53
|
||||
A|nextcloud.home.keithjackson.dev|192.168.1.53
|
||||
A|pihole.home.keithjackson.dev|192.168.1.53
|
||||
A|plex.home.keithjackson.dev|192.168.1.53
|
||||
A|qbittorrent.home.keithjackson.dev|192.168.1.53
|
||||
A|radarr.home.keithjackson.dev|192.168.1.53
|
||||
A|seerr.home.keithjackson.dev|192.168.1.53
|
||||
A|semaphore.home.keithjackson.dev|192.168.1.53
|
||||
A|sonarr.home.keithjackson.dev|192.168.1.53
|
||||
A|unifi.home.keithjackson.dev|192.168.1.54
|
||||
TXT|a-auth.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/oauth2-proxy/oauth2-proxy
|
||||
TXT|a-crafty-mc.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/crafty/crafty-l4-tcp
|
||||
TXT|a-crafty.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/crafty/crafty
|
||||
TXT|a-gitea.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/gitea/gitea-3000
|
||||
TXT|a-grafana.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/monitoring/grafana
|
||||
TXT|a-headlamp.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/headlamp/headlamp
|
||||
TXT|a-home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/homepage/homepage
|
||||
TXT|a-jackett.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/jackett/jackett-9117
|
||||
TXT|a-keycloak.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/keycloak/keycloak
|
||||
TXT|a-nextcloud.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/nextcloud/nextcloud-80
|
||||
TXT|a-pihole.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/pihole/pihole-80
|
||||
TXT|a-plex.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/plex/plex-32400
|
||||
TXT|a-qbittorrent.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/qbittorrent/qbittorrent-8080
|
||||
TXT|a-radarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/radarr/radarr-7878
|
||||
TXT|a-seerr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/seerr/seerr-5055
|
||||
TXT|a-semaphore.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/semaphore/semaphore-3000
|
||||
TXT|a-sonarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/sonarr/sonarr-8989
|
||||
TXT|a-unifi.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/unifi/unifi-l4-tcp
|
||||
TXT|auth.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/oauth2-proxy/oauth2-proxy
|
||||
TXT|crafty-mc.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/crafty/crafty-l4-tcp
|
||||
TXT|crafty.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/crafty/crafty
|
||||
TXT|gitea.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/gitea/gitea-3000
|
||||
TXT|grafana.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/monitoring/grafana
|
||||
TXT|headlamp.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/headlamp/headlamp
|
||||
TXT|home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/homepage/homepage
|
||||
TXT|jackett.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/jackett/jackett-9117
|
||||
TXT|keycloak.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/keycloak/keycloak
|
||||
TXT|nextcloud.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/nextcloud/nextcloud-80
|
||||
TXT|pihole.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/pihole/pihole-80
|
||||
TXT|plex.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/plex/plex-32400
|
||||
TXT|qbittorrent.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/qbittorrent/qbittorrent-8080
|
||||
TXT|radarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/radarr/radarr-7878
|
||||
TXT|seerr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/seerr/seerr-5055
|
||||
TXT|semaphore.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/semaphore/semaphore-3000
|
||||
TXT|sonarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/sonarr/sonarr-8989
|
||||
TXT|unifi.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/unifi/unifi-l4-tcp
|
||||
@@ -0,0 +1,95 @@
|
||||
A|*.keithjackson.dev|136.51.115.252
|
||||
A|auth.home.keithjackson.dev|192.168.1.53
|
||||
A|auth.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|brickbreak.keithjackson.dev|20.236.242.14
|
||||
A|conformance-dns.home.keithjackson.dev|192.168.1.56
|
||||
A|crafty-mc.home.keithjackson.dev|192.168.1.55
|
||||
A|crafty-mc.prod.home.keithjackson.dev|192.168.2.22
|
||||
A|crafty.home.keithjackson.dev|192.168.1.53
|
||||
A|crafty.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|dev.brickbreak.keithjackson.dev|20.236.242.14
|
||||
A|gitea.home.keithjackson.dev|192.168.1.53
|
||||
A|gitea.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|grafana.home.keithjackson.dev|192.168.1.53
|
||||
A|grafana.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|headlamp.home.keithjackson.dev|192.168.1.53
|
||||
A|headlamp.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|home.keithjackson.dev|192.168.1.53
|
||||
A|jackett.home.keithjackson.dev|192.168.1.53
|
||||
A|jackett.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|keycloak.home.keithjackson.dev|192.168.1.53
|
||||
A|keycloak.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|nextcloud.home.keithjackson.dev|192.168.1.53
|
||||
A|nextcloud.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|pihole.home.keithjackson.dev|192.168.1.53
|
||||
A|pihole.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|plex.home.keithjackson.dev|192.168.1.53
|
||||
A|plex.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|prod.home.keithjackson.dev|192.168.2.20
|
||||
A|qbittorrent.home.keithjackson.dev|192.168.1.53
|
||||
A|qbittorrent.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|radarr.home.keithjackson.dev|192.168.1.53
|
||||
A|radarr.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|seerr.home.keithjackson.dev|192.168.1.53
|
||||
A|seerr.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|semaphore.home.keithjackson.dev|192.168.1.53
|
||||
A|semaphore.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|sonarr.home.keithjackson.dev|192.168.1.53
|
||||
A|sonarr.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|unifi.home.keithjackson.dev|192.168.1.54
|
||||
A|unifi.prod.home.keithjackson.dev|192.168.2.21
|
||||
TXT|a-auth.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/oauth2-proxy/oauth2-proxy
|
||||
TXT|a-auth.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/oauth2-proxy/oauth2-proxy
|
||||
TXT|a-conformance-dns.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/chainsaw-growing-grouse/dns-probe-svc
|
||||
TXT|a-crafty-mc.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/crafty/crafty-l4-tcp
|
||||
TXT|a-crafty-mc.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=service/crafty/crafty-l4-tcp
|
||||
TXT|a-crafty.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/crafty/crafty
|
||||
TXT|a-crafty.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/crafty/crafty
|
||||
TXT|a-gitea.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/gitea/gitea-3000
|
||||
TXT|a-gitea.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/gitea/gitea-3000
|
||||
TXT|a-grafana.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/monitoring/grafana
|
||||
TXT|a-grafana.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/monitoring/grafana
|
||||
TXT|a-headlamp.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/headlamp/headlamp
|
||||
TXT|a-headlamp.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/headlamp/headlamp
|
||||
TXT|a-home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/homepage/homepage
|
||||
TXT|a-jackett.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/jackett/jackett-9117
|
||||
TXT|a-jackett.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/jackett/jackett-9117
|
||||
TXT|a-keycloak.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/keycloak/keycloak
|
||||
TXT|a-keycloak.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/keycloak/keycloak
|
||||
TXT|a-nextcloud.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/nextcloud/nextcloud-80
|
||||
TXT|a-nextcloud.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/nextcloud/nextcloud-80
|
||||
TXT|a-pihole.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/pihole/pihole-80
|
||||
TXT|a-pihole.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/pihole/pihole-80
|
||||
TXT|a-plex.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/plex/plex-32400
|
||||
TXT|a-plex.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/plex/plex-32400
|
||||
TXT|a-prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/homepage/homepage
|
||||
TXT|a-qbittorrent.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/qbittorrent/qbittorrent-8080
|
||||
TXT|a-qbittorrent.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/qbittorrent/qbittorrent-8080
|
||||
TXT|a-radarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/radarr/radarr-7878
|
||||
TXT|a-radarr.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/radarr/radarr-7878
|
||||
TXT|a-seerr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/seerr/seerr-5055
|
||||
TXT|a-seerr.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/seerr/seerr-5055
|
||||
TXT|a-semaphore.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/semaphore/semaphore-3000
|
||||
TXT|a-semaphore.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/semaphore/semaphore-3000
|
||||
TXT|a-sonarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/sonarr/sonarr-8989
|
||||
TXT|a-sonarr.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/sonarr/sonarr-8989
|
||||
TXT|a-unifi.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/unifi/unifi-l4-tcp
|
||||
TXT|a-unifi.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=service/unifi/unifi-l4-tcp
|
||||
TXT|auth.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/oauth2-proxy/oauth2-proxy
|
||||
TXT|crafty-mc.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/crafty/crafty-l4-tcp
|
||||
TXT|crafty.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/crafty/crafty
|
||||
TXT|gitea.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/gitea/gitea-3000
|
||||
TXT|grafana.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/monitoring/grafana
|
||||
TXT|headlamp.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/headlamp/headlamp
|
||||
TXT|home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/homepage/homepage
|
||||
TXT|jackett.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/jackett/jackett-9117
|
||||
TXT|keycloak.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/keycloak/keycloak
|
||||
TXT|nextcloud.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/nextcloud/nextcloud-80
|
||||
TXT|pihole.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/pihole/pihole-80
|
||||
TXT|plex.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/plex/plex-32400
|
||||
TXT|qbittorrent.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/qbittorrent/qbittorrent-8080
|
||||
TXT|radarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/radarr/radarr-7878
|
||||
TXT|seerr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/seerr/seerr-5055
|
||||
TXT|semaphore.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/semaphore/semaphore-3000
|
||||
TXT|sonarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/sonarr/sonarr-8989
|
||||
TXT|unifi.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/unifi/unifi-l4-tcp
|
||||
@@ -0,0 +1,93 @@
|
||||
A|*.keithjackson.dev|136.51.115.252
|
||||
A|auth.home.keithjackson.dev|192.168.1.53
|
||||
A|auth.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|brickbreak.keithjackson.dev|20.236.242.14
|
||||
A|crafty-mc.home.keithjackson.dev|192.168.1.55
|
||||
A|crafty-mc.prod.home.keithjackson.dev|192.168.2.22
|
||||
A|crafty.home.keithjackson.dev|192.168.1.53
|
||||
A|crafty.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|dev.brickbreak.keithjackson.dev|20.236.242.14
|
||||
A|gitea.home.keithjackson.dev|192.168.1.53
|
||||
A|gitea.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|grafana.home.keithjackson.dev|192.168.1.53
|
||||
A|grafana.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|headlamp.home.keithjackson.dev|192.168.1.53
|
||||
A|headlamp.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|home.keithjackson.dev|192.168.1.53
|
||||
A|jackett.home.keithjackson.dev|192.168.1.53
|
||||
A|jackett.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|keycloak.home.keithjackson.dev|192.168.1.53
|
||||
A|keycloak.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|nextcloud.home.keithjackson.dev|192.168.1.53
|
||||
A|nextcloud.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|pihole.home.keithjackson.dev|192.168.1.53
|
||||
A|pihole.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|plex.home.keithjackson.dev|192.168.1.53
|
||||
A|plex.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|prod.home.keithjackson.dev|192.168.2.20
|
||||
A|qbittorrent.home.keithjackson.dev|192.168.1.53
|
||||
A|qbittorrent.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|radarr.home.keithjackson.dev|192.168.1.53
|
||||
A|radarr.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|seerr.home.keithjackson.dev|192.168.1.53
|
||||
A|seerr.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|semaphore.home.keithjackson.dev|192.168.1.53
|
||||
A|semaphore.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|sonarr.home.keithjackson.dev|192.168.1.53
|
||||
A|sonarr.prod.home.keithjackson.dev|192.168.2.20
|
||||
A|unifi.home.keithjackson.dev|192.168.1.54
|
||||
A|unifi.prod.home.keithjackson.dev|192.168.2.21
|
||||
TXT|a-auth.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/oauth2-proxy/oauth2-proxy
|
||||
TXT|a-auth.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/oauth2-proxy/oauth2-proxy
|
||||
TXT|a-crafty-mc.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/crafty/crafty-l4-tcp
|
||||
TXT|a-crafty-mc.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=service/crafty/crafty-l4-tcp
|
||||
TXT|a-crafty.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/crafty/crafty
|
||||
TXT|a-crafty.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/crafty/crafty
|
||||
TXT|a-gitea.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/gitea/gitea-3000
|
||||
TXT|a-gitea.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/gitea/gitea-3000
|
||||
TXT|a-grafana.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/monitoring/grafana
|
||||
TXT|a-grafana.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/monitoring/grafana
|
||||
TXT|a-headlamp.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/headlamp/headlamp
|
||||
TXT|a-headlamp.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/headlamp/headlamp
|
||||
TXT|a-home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/homepage/homepage
|
||||
TXT|a-jackett.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/jackett/jackett-9117
|
||||
TXT|a-jackett.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/jackett/jackett-9117
|
||||
TXT|a-keycloak.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/keycloak/keycloak
|
||||
TXT|a-keycloak.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/keycloak/keycloak
|
||||
TXT|a-nextcloud.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/nextcloud/nextcloud-80
|
||||
TXT|a-nextcloud.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/nextcloud/nextcloud-80
|
||||
TXT|a-pihole.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/pihole/pihole-80
|
||||
TXT|a-pihole.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/pihole/pihole-80
|
||||
TXT|a-plex.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/plex/plex-32400
|
||||
TXT|a-plex.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/plex/plex-32400
|
||||
TXT|a-prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/homepage/homepage
|
||||
TXT|a-qbittorrent.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/qbittorrent/qbittorrent-8080
|
||||
TXT|a-qbittorrent.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/qbittorrent/qbittorrent-8080
|
||||
TXT|a-radarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/radarr/radarr-7878
|
||||
TXT|a-radarr.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/radarr/radarr-7878
|
||||
TXT|a-seerr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/seerr/seerr-5055
|
||||
TXT|a-seerr.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/seerr/seerr-5055
|
||||
TXT|a-semaphore.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/semaphore/semaphore-3000
|
||||
TXT|a-semaphore.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/semaphore/semaphore-3000
|
||||
TXT|a-sonarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/sonarr/sonarr-8989
|
||||
TXT|a-sonarr.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=httproute/sonarr/sonarr-8989
|
||||
TXT|a-unifi.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/unifi/unifi-l4-tcp
|
||||
TXT|a-unifi.prod.home.keithjackson.dev|heritage=external-dns,external-dns/owner=prod,external-dns/resource=service/unifi/unifi-l4-tcp
|
||||
TXT|auth.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/oauth2-proxy/oauth2-proxy
|
||||
TXT|crafty-mc.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/crafty/crafty-l4-tcp
|
||||
TXT|crafty.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/crafty/crafty
|
||||
TXT|gitea.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/gitea/gitea-3000
|
||||
TXT|grafana.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/monitoring/grafana
|
||||
TXT|headlamp.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/headlamp/headlamp
|
||||
TXT|home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/homepage/homepage
|
||||
TXT|jackett.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/jackett/jackett-9117
|
||||
TXT|keycloak.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/keycloak/keycloak
|
||||
TXT|nextcloud.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/nextcloud/nextcloud-80
|
||||
TXT|pihole.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/pihole/pihole-80
|
||||
TXT|plex.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/plex/plex-32400
|
||||
TXT|qbittorrent.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/qbittorrent/qbittorrent-8080
|
||||
TXT|radarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/radarr/radarr-7878
|
||||
TXT|seerr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/seerr/seerr-5055
|
||||
TXT|semaphore.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/semaphore/semaphore-3000
|
||||
TXT|sonarr.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=httproute/sonarr/sonarr-8989
|
||||
TXT|unifi.home.keithjackson.dev|heritage=external-dns,external-dns/owner=default,external-dns/resource=service/unifi/unifi-l4-tcp
|
||||
@@ -1847,3 +1847,110 @@ down in `platform/headlamp/README.md`: moving one without the other 401s every r
|
||||
`headlamp-oidc-credentials.sops.yaml.example` used to *teach* the broken pattern — it listed
|
||||
`OIDC_ISSUER_URL` and `OIDC_CALLBACK_URL` as keys to fill in. It now says, in the file, that adding
|
||||
either key back re-breaks prod.
|
||||
## 2026-09-07 — k3s-prod-1 NotReady: etcd starved of disk I/O by Longhorn (branch `fix/etcd-io-priority`)
|
||||
|
||||
**06:05:21Z, `k3s-prod-1` went NotReady** (`Kubelet stopped posting node status`) with the host
|
||||
up, pingable and SSH-able. The embedded etcd could not complete a WAL `fdatasync`:
|
||||
|
||||
```
|
||||
slow fdatasync took=3.56s / 6.71s / 4.95s / 3.40s (etcd expects 1s)
|
||||
apply request took too long took=8.30s (expects 100ms)
|
||||
etcdserver: too many requests
|
||||
Failed to check local etcd status for learner management: context deadline exceeded
|
||||
```
|
||||
|
||||
No OOM, no MCE, no hung task — **this is not the §0b memattr bug and not memory**. It is the §2
|
||||
shared-device risk, which §2 recorded as "accepted to be MEASURED, not assumed". The measurement
|
||||
came back, in production, 2 d 7 h after bring-up. `/proc/pressure/io` **full avg300**: node 1
|
||||
**11.36**, node 2 (NVMe) 0.75, node 3 (WD Blue SATA) 1.29. Node 1's drive is a 2011
|
||||
`C400-MTFDDAC512MAM` with 61,440 reallocated sectors and no power-loss protection. §2's premise —
|
||||
"every prod node is SSD or NVMe, where rebuild I/O costs latency rather than collapsing it" — was
|
||||
right about node 2 and wrong about node 1. §2 now records this under "THE MEASUREMENT CAME BACK".
|
||||
|
||||
**The exit condition is OPEN and the fix is hardware: replace node 1's drive, or give etcd its own
|
||||
device (`node_baseline_etcd_source: device`, the path the role still supports). That is the
|
||||
operator's.** Three software mitigations shipped meanwhile, none of which is the fix.
|
||||
|
||||
### 1. The outage was invisible to the monitoring stack — fixed first
|
||||
|
||||
There were alerts for **CPU** PSI and memory and **none for I/O**, so the first signal was a node
|
||||
already out of the cluster. Added `NodeIOPressureHigh` (full IO PSI > 5% for 10m) and
|
||||
`NodeIOPressureCritical` (> 10% for 5m), thresholds taken from the numbers above against a ~1%
|
||||
steady state, plus an I/O row on the node-pressure dashboard. Verified live that node-exporter
|
||||
already exports `node_pressure_io_{stalled,waiting}_seconds_total` on all three nodes.
|
||||
|
||||
**Gap left open for the operator:** etcd's own `etcd_disk_wal_fsync_duration_seconds` — the metric
|
||||
§2's exit condition literally asks for — is **not collected**. `kubeEtcd` is deliberately disabled
|
||||
in `platform/monitoring/kube-prometheus-stack.yaml` for k3s. Enabling it (k3s
|
||||
`--etcd-expose-metrics` + a scrape target) would make §2 directly measurable instead of inferred
|
||||
from PSI. Not done here: it is a k3s server-flag change requiring a rolling restart, which is not
|
||||
something to bundle into an incident fix.
|
||||
|
||||
### 2. Longhorn was already throttled as far as Longhorn goes
|
||||
|
||||
`concurrentReplicaRebuildPerNodeLimit: 1` was **in force during the outage and it held** — the 18
|
||||
volumes rebuilt one at a time and the node stalled anyway. Recorded plainly because it would be
|
||||
easy to "fix" this incident by setting a knob that was already set. For the v1 data engine there is
|
||||
no further rebuild lever: `replicaRebuildingBandwidthLimit` is v2/SPDK only (live Setting reads
|
||||
`{"v2":"0"}`, no v1 key) and `rebuildConcurrentSyncLimit` is already at its floor of 1.
|
||||
|
||||
So what was tightened is every *other* Longhorn I/O source, each defaulting to 5 concurrent
|
||||
operations per node: `concurrentVolumeBackupRestorePerNodeLimit` 5→1 (an R-STATE-03 restore of 18
|
||||
volumes would otherwise run five parallel full-volume writes onto the disk etcd fsyncs to),
|
||||
`snapshotHeavyTaskConcurrentLimit` 5→1, plus pins on `replicaReplenishmentWaitInterval`,
|
||||
`backup`/`restoreConcurrentLimit` and `concurrentAutomaticEngineUpgradePerNodeLimit`.
|
||||
|
||||
**And one setting in that file was inert.** `orphanAutoDeletion: true` has been silently dropped
|
||||
since the 1.11→1.12 upgrade — Longhorn 1.9 renamed it `orphanResourceAutoDeletion` (a list, not a
|
||||
bool) and the chart drops unknown `defaultSettings` keys with no error. Confirmed from both ends:
|
||||
absent from chart v1.12.1's `templates/default-setting.yaml`, and absent from the rendered
|
||||
`longhorn-default-setting` ConfigMap in the live cluster. Removed, deliberately not replaced. That
|
||||
is the **third** time this repo has shipped config that looked applied and was not.
|
||||
|
||||
### 3. Block-layer arbitration — and four mechanisms that are inert on these nodes
|
||||
|
||||
`roles/etcd_io_priority` + `playbooks/etcd-io-priority.yml`: BFQ on the shared disk, `nr_requests`
|
||||
64→32, `IOWeight=10000` on `system.slice` **and** `k3s.service`, then every value read back out of
|
||||
the kernel and asserted. Enabled for `[k3s_prod]`; **applies nothing until the operator runs it**.
|
||||
|
||||
The investigation's main value is the negative results, all checked on the live nodes:
|
||||
|
||||
- **`IOSchedulingClass=realtime` on `k3s.service` is a footgun, not a fix.** `mq-deadline` does
|
||||
honour the priority class, so it looks correct — but ioprio is inherited across `fork` and kept
|
||||
across `exec` (`block/blk-ioc.c` `__copy_io` copies `ioc->ioprio`), and **containerd is a child
|
||||
of the k3s server process** (verified: containerd pid 84553, ppid 84333 = `k3s-server`; and
|
||||
`ionice -c2 -n0 -- sh -c 'ionice -p $$; sh -c "ionice -p \$\$"'` reports best-effort prio 0 at
|
||||
both levels). Realtime for k3s would be realtime for every container including Longhorn.
|
||||
- **`IOSchedulingPriority` (level within a class) is a no-op** — `mq-deadline` buckets by class only.
|
||||
- **`io.latency` is unavailable** — `CONFIG_BLK_CGROUP_IOLATENCY` is not built in Debian trixie's
|
||||
7.1.8 kernel (only `IOCOST` is), and no `io.latency` exists under `/sys/fs/cgroup`.
|
||||
- **A bare `IOWeight=` drop-in is decoration.** systemd writes it to `io.weight` (blk-iocost —
|
||||
compiled in but disabled; `io.cost.qos` is empty) and `io.bfq.weight` (BFQ — not loaded). Neither
|
||||
reader exists on the stock nodes. And `IOWeight=1000` would map to `io.bfq.weight` ≈ 101 against
|
||||
BFQ's unweighted default of 100 — hence 10000, and an asserted floor of 500.
|
||||
|
||||
**What none of it fixes, stated so it is not forgotten:** etcd and Longhorn still share one ext4
|
||||
filesystem, and no cgroup or scheduler control arbitrates the **jbd2 journal** that etcd's
|
||||
`fdatasync` waits on and Longhorn's writes dirty. Software raises the load level at which this
|
||||
happens; it does not remove it.
|
||||
|
||||
**Unproven.** The role fails loudly if its settings do not *land*; it cannot tell you they helped.
|
||||
Apply to `k3s-prod-3` first (WD Blue, PSI 1.29 — same code path, far lower stakes), watch
|
||||
`NodeIOPressureHigh` for a day, then node 1.
|
||||
|
||||
### Doc correction found on the way
|
||||
|
||||
`infra/ansible/README.md` still claimed `k3s-prod-2`/`-3` "stay commented out" of the inventory.
|
||||
They have been live in `[k3s_prod_secondary]` since 2026-09-03. Corrected, because that sentence is
|
||||
exactly the kind used to reason about blast radius: **`-e k3s_hosts=k3s_prod` targets three hosts,
|
||||
not one.** Confirmed by `--list-hosts` (dev default → 3 dev hosts, where the new role is a no-op;
|
||||
`-e k3s_hosts=k3s_prod` → 3 prod hosts; `--limit k3s-prod-3` → 1).
|
||||
|
||||
### Still requires the operator
|
||||
|
||||
1. **The hardware fix.** Replace node 1's C400, or add an M.2 per node and set
|
||||
`node_baseline_etcd_source: device`. Everything above is a delay, not a repair.
|
||||
2. **Run `playbooks/etcd-io-priority.yml`** — node 3 first, `--limit`, `--list-hosts` confirmed.
|
||||
Nothing was run against either cluster for this change.
|
||||
3. **Decide on `kubeEtcd` metrics**, so §2's exit condition can be measured with etcd's own fsync
|
||||
histograms rather than inferred from node PSI.
|
||||
@@ -0,0 +1,319 @@
|
||||
# Decommission Proxmox `192.168.2.102` → TrueNAS SCALE on the N5 Air
|
||||
|
||||
**Status:** plan. **Drafted** 2026-09-06, **revised** 2026-09-06 after the operator settled the
|
||||
target architecture.
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| **N5 Air** | TrueNAS SCALE. **Storage only** — no VMs, no hypervisor. |
|
||||
| **Compute** | k3s **prod** cluster (`192.168.2.10/.11/.12`). Plex, sonarr, radarr, jackett, qbittorrent, seerr and nextcloud are already modelled in `apps/wave-0/`. |
|
||||
| **Dev k3s** | Does not move to the NAS. |
|
||||
| **Proxmox host** | **Decommissioned.** |
|
||||
| **Drives** | Physically relocated. None purchased. |
|
||||
|
||||
All figures measured on the live host, not inferred.
|
||||
|
||||
---
|
||||
|
||||
## Already done — 2026-09-06
|
||||
|
||||
**Reclaimed 3.56 TB.** `Storage/plex@pre-cleanup-20260828` destroyed after confirming no holds and
|
||||
no clones. Async free drained in ~90 seconds.
|
||||
|
||||
| | before | after |
|
||||
|---|---|---|
|
||||
| `Storage` allocated | 12.3 T (75%) | **8.76 T (53%)** |
|
||||
| `Storage` free | 4.04 T | **7.60 T** |
|
||||
| `Storage/plex` used | 11.9 T | 8.33 T (0 B in snapshots) |
|
||||
|
||||
That free space is not a nicety — **it is what lets this whole migration happen with no network
|
||||
transfer at all.** See §3.
|
||||
|
||||
---
|
||||
|
||||
## 1. The plan changed shape
|
||||
|
||||
The first draft carried the VMs across on their own zvols and put Proxmox on the NAS. With no
|
||||
hypervisor on the far side, that inverts:
|
||||
|
||||
> **Every zvol is now dead weight, and every service's data has to be drained onto a plain dataset
|
||||
> before the Proxmox box dies.**
|
||||
|
||||
Dead-weight zvols, once their contents are extracted:
|
||||
|
||||
| zvol | size | after extraction |
|
||||
|---|---|---|
|
||||
| `Storage:vm-101/102/105-disk-0,-disk-1` (dev k3s) | 585 G | delete |
|
||||
| `Storage:vm-112-disk-0` (docker-services) | 65 G | delete |
|
||||
| `Storage:vm-103-disk-0` (nextcloud boot) | 81 G | delete |
|
||||
| `Storage2:vm-103-disk-0` (**nextcloud data**) | 4.06 T | extract, then delete |
|
||||
|
||||
The good news is the shape of the drain. Everything that must be rescued is either already on
|
||||
`Storage`, or can be copied onto `Storage` **locally, inside the same chassis, onto the very drives
|
||||
that are moving.** With 7.60 T free there is room. **Nothing crosses the network.**
|
||||
|
||||
---
|
||||
|
||||
## 2. Findings
|
||||
|
||||
### 2a. "The path lies" — two confirmed instances, assume a third
|
||||
|
||||
**Games — 422 GB, not on the Storage pool.** Challenged and re-verified; the test is decisive:
|
||||
|
||||
```
|
||||
findmnt -T /mnt/Storage/Games → TARGET / SOURCE rpool/ROOT/pve-1
|
||||
findmnt -T /mnt/Storage/plex → TARGET /mnt/Storage/plex SOURCE Storage/plex
|
||||
```
|
||||
|
||||
`Games` is a plain directory on the Proxmox **root** filesystem that merely sits inside the Storage
|
||||
mount tree. The pool accounting closes on it exactly:
|
||||
|
||||
```
|
||||
rpool ALLOC 540 G = ROOT 431 G + data 105 G + var-lib-vz 3.75 G
|
||||
ROOT 431 G = Games 422 G + /usr 3.5 G + /var 5 G + /boot 0.4 G
|
||||
```
|
||||
|
||||
The two Microns are a **mirror**, so the pool is ~888 G usable, not ~1.9 T — 422 G is 48% of it, and
|
||||
`zpool list` agrees at 60% capacity overall. Surprising, but the numbers do match.
|
||||
|
||||
**Gitea — 13 MB in an anonymous Docker volume, not where its compose file points.**
|
||||
`/root/docker-compose.yml` in LXC 106 bind-mounts `/opt/gitea/data:/var/lib/data`. That path holds
|
||||
**512 bytes**. The real Gitea state is in an unnamed Docker volume:
|
||||
|
||||
```
|
||||
.../volumes/8a1d2616…/_data/git/repositories/ 12 M home-network.git, home-network-uds.git, jakman85/…
|
||||
.../volumes/8a1d2616…/_data/data/gitea.db 11 M SQLite, mtime today 19:00 — live
|
||||
```
|
||||
|
||||
Copying `/opt/gitea` would migrate an empty directory and silently lose all three repositories.
|
||||
|
||||
**Rule for the rest of the drain:** for every service, verify where the bytes actually are before
|
||||
trusting the path in a compose file, an export list or a mount table. Two for two so far.
|
||||
|
||||
### 2b. Retiring pfSense is easier than rehoming it
|
||||
|
||||
`pfsense` is VM 100 and **is** `192.168.1.1`. Decommissioning the host takes the entire
|
||||
`192.168.1.0/24` with it:
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| `.1` pfSense · `.4` iDRAC · `.9` the Proxmox host itself | die with the box |
|
||||
| `.5` nextcloud · `.6` plex · `.7` gitea · `.18` docker-services/Pi-hole | data drains, service moves to k3s |
|
||||
| `.8` unifi · `.20` monitoring · `.21` auth · `.10` semaphore | rehome or retire (§2e) |
|
||||
| `.49` kube-vip VIP · `.50/.51/.52` dev k3s | dev cluster is not moving; retire with the box |
|
||||
|
||||
**But pfSense exists only to contain that lab LAN.** Prod k3s is already on `192.168.2.0/24` —
|
||||
pfSense's *WAN* side — and the NAS will be too. Once the lab is gone, nothing lives behind pfSense.
|
||||
**Retire it; do not rehome it.** That is only clean because everything else is leaving `1.0/24`
|
||||
anyway, which is exactly what this migration does.
|
||||
|
||||
**A documented blocker dissolves with it.** `infra/PROD-STANDUP.md` records that prod k3s cannot
|
||||
reach the S3 backup endpoint at `192.168.1.9:9000` because prod sits on pfSense's WAN side and the
|
||||
crossing rule does not exist. With the NAS on `192.168.2.0/24`, prod reaches its storage directly
|
||||
and no pfSense rule is ever needed.
|
||||
|
||||
**Three things to close before pfSense goes:**
|
||||
|
||||
1. The SOHO router NATs inbound **443** to `192.168.2.102` for the legacy stack. It must be
|
||||
repointed at prod's Gateway (`192.168.2.20`). `infra/PROD-PROVISIONING.md` §10a already flags
|
||||
that the apex flip and the legacy cutover are the same event — **this migration is that event.**
|
||||
2. **VLANs 5 and 123** (minecraft; the arr VM, which reaches Plex media as `10.123.1.10`) are
|
||||
pfSense-side constructs. They disappear; the k3s apps need no VLAN.
|
||||
3. The **OpenVPN split-tunnel egress gateway** currently runs on `k3s-dev-3` (ADR-0015). It is
|
||||
already modelled for prod in `platform/vpn-egress/`, but confirm prod's gateway is live before
|
||||
the dev node goes away.
|
||||
|
||||
### 2c. Nextcloud is the critical path
|
||||
|
||||
Its user data is **inside a filesystem, inside LVM, inside a 4 TB zvol, inside a VM**:
|
||||
|
||||
```
|
||||
Storage2/vm-103-disk-0 → zd224 → PV /dev/zd224 → VG vg_storage → LV lv_storage (<4.00 T)
|
||||
```
|
||||
|
||||
With no hypervisor on the far side this must be copied out **at the file level, while the VM is
|
||||
still running** — it gates the decommission and it is the longest single task in the plan.
|
||||
|
||||
**One number is missing and I could not get it.** VM 103's QEMU guest agent is not running
|
||||
(`qm agent 103 ping` → *"QEMU guest agent is not running"*) and its SSH does not accept the
|
||||
`home_network` key. The zvol shows `referenced 3.94 T`, but it is attached **without `discard=on`**
|
||||
and is thick-provisioned (`refreservation 4.06 T`), so deletions inside the guest were never
|
||||
returned to ZFS — 3.94 T is an **upper bound**, possibly far above the truth.
|
||||
|
||||
**Run this inside the Nextcloud VM and the last open question closes:**
|
||||
|
||||
```bash
|
||||
df -h / # or wherever lv_storage is mounted
|
||||
lsblk -f /dev/mapper/vg_storage-lv_storage
|
||||
```
|
||||
|
||||
Separately: `Storage/nextcloud` is 270 G, and **267 G of it is a directory called `config_backup`**
|
||||
(`config` itself is 103 M). Almost certainly a stale dump — likely another easy reclaim, but confirm
|
||||
before deleting.
|
||||
|
||||
### 2d. Storage2 — and the two oldest drives — can probably be retired
|
||||
|
||||
After the drain and the zvol deletions, `Storage` would hold roughly:
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| plex | 8.33 T |
|
||||
| nextcloud data | ≤ 3.94 T (real number pending §2c) |
|
||||
| Games | 0.42 T |
|
||||
| docker-volumes, other_containers, k8s-volumes, gitea | ~0.02 T |
|
||||
| **total** | **≤ 12.7 T of 16.4 T (≤ 77%)** |
|
||||
|
||||
If the real Nextcloud figure comes in under ~2 T — which the missing-TRIM situation makes plausible
|
||||
— everything fits on the 18 TB mirror at a comfortable ~65%, and **`Storage2` can be retired
|
||||
entirely.** That would remove the two **6.0-year-old** Seagates, by a wide margin the most
|
||||
failure-prone hardware in the estate, and free two bays. This is the single best safety outcome
|
||||
available, and it costs nothing.
|
||||
|
||||
### 2e. Services that simply end
|
||||
|
||||
Decommissioning kills these; none has a home in the target architecture unless you give it one:
|
||||
|
||||
| guest | note |
|
||||
|---|---|
|
||||
| `docker-services` Pi-hole (`192.168.1.18`) | **LAN DNS.** Confirm what `192.168.2.0/24` uses for DNS before the box dies. |
|
||||
| `unifi` (VM 121) | Network controller — needs a home or a cloud key. |
|
||||
| `auth` (VM 122) | Superseded by Keycloak on prod k3s. |
|
||||
| `monitoring` (VM 120) | Superseded by the k3s platform monitoring stack. |
|
||||
| `semaphore` (LXC 110) | 432 MB resident of an 8 GB reservation; low value. |
|
||||
| `minecraft` (LXC 108) | Operator call. Its 3.5 G subvol is on `Storage` and rides along. |
|
||||
| **versitygw S3** on `192.168.1.9:9000` | **Velero + CNPG backup target.** Dies with the box. TrueNAS must take this role (MinIO app) or prod loses its backup destination — which `infra/PROD-STANDUP.md` already records as uncovered. |
|
||||
|
||||
### 2f. Still true from the first draft
|
||||
|
||||
- **No VM backups exist.** `/etc/pve/jobs.cfg` empty, no PBS, newest dump log April 2025.
|
||||
- **`ata-WDC_…3FGHAB2T` has 36 ZFS read errors** on pristine SMART (0 reallocated / pending /
|
||||
uncorrectable, `PASSED`) — points at cable or controller, not platter. Re-verify after the move.
|
||||
- Drive ages: WD 18 TB pair **4.75 y**, Seagate 8 TB pair **6.0 y**.
|
||||
|
||||
---
|
||||
|
||||
## 3. TrueNAS SCALE specifics
|
||||
|
||||
**ZFS compatibility: verified, not assumed.** The only non-default features *active* on either pool
|
||||
are `block_cloning`, `head_errlog`, `vdev_zaps_v2`, `zilsaxattr`, `zstd_compress`, `log_spacemap`,
|
||||
`project_quota`, `spacemap_v2` — all OpenZFS **2.2** era or older. Nothing 2.3- or 2.4-only is
|
||||
active, even though the pools were last written by `zfs-2.4.3-pve1`. Any TrueNAS SCALE from 24.10
|
||||
onward imports them cleanly.
|
||||
|
||||
**Mountpoints mostly already match TrueNAS convention** — a lucky accident worth knowing:
|
||||
|
||||
| dataset | current mountpoint | TrueNAS wants | |
|
||||
|---|---|---|---|
|
||||
| `Storage/plex`, `/nextcloud`, `/docker-volumes`, `/other_containers`, `/k8s-volumes`, `/k3s-backups` | `/mnt/Storage/…` | `/mnt/Storage/…` | already correct |
|
||||
| `Storage` (pool root) | `/Storage` | `/mnt/Storage` | fix on import |
|
||||
| `Storage2` (pool root) | `/Storage2` | `/mnt/Storage2` | fix on import |
|
||||
| `Storage/subvol-108-disk-0` | `/Storage/subvol-108-disk-0` | — | rename or delete |
|
||||
|
||||
**Clear the ZFS `sharenfs` properties.** `Storage/plex`, `Storage/nextcloud` and
|
||||
`Storage/docker-volumes` carry exports as ZFS properties; `Storage/k8s-volumes` is in
|
||||
`/etc/exports`. TrueNAS generates exports from its own middleware database and will fight
|
||||
properties it did not set. Wipe them and recreate the shares in the UI.
|
||||
|
||||
**Fix the export targets while you are there.** Today's lists are stale: `/mnt/k3s` →
|
||||
`192.168.1.25/.26/.27` and `k8s-volumes` → `192.168.1.40/.41/.42` are hosts that do not answer ARP.
|
||||
The new exports target **`192.168.2.10/.11/.12`**.
|
||||
|
||||
---
|
||||
|
||||
## 4. Sequence
|
||||
|
||||
### Phase A — drain, locally, no downtime, no network
|
||||
|
||||
Everything below is a copy from somewhere else in the chassis onto `Storage`, which is moving.
|
||||
|
||||
1. **Games** — `rpool` → new dataset.
|
||||
```bash
|
||||
zfs create Storage/games
|
||||
rsync -aHAX --info=progress2 /mnt/Storage/Games/ /mnt/Storage/games/
|
||||
```
|
||||
2. **Gitea** — copy the *Docker volume*, not `/opt/gitea` (§2a). Stop the container first so
|
||||
`gitea.db` is quiesced.
|
||||
```bash
|
||||
pct exec 106 -- docker stop gitea
|
||||
zfs create Storage/gitea
|
||||
rsync -aHAX /rpool/data/subvol-106-disk-0/var/lib/docker/volumes/8a1d2616*/_data/ /mnt/Storage/gitea/
|
||||
```
|
||||
3. **Nextcloud** — the long one. Copy `lv_storage`'s contents to a new dataset at file level from
|
||||
inside the running VM, over its existing NFS mount of the host. Verify §2c's size first so you
|
||||
know whether it fits (it will, in 7.60 T free).
|
||||
4. **Confirm `config_backup`** (267 G) is disposable; drop it if so.
|
||||
5. **Verify each service's real data location before copying it** — the §2a rule.
|
||||
|
||||
### Phase B — prove the compute side before the storage moves
|
||||
|
||||
Bring the `apps/wave-0` manifests up on prod k3s while the old box is still alive, so the cutover is
|
||||
a storage swap against known-good workloads rather than two migrations at once.
|
||||
|
||||
Wrinkle: prod cannot reach `192.168.1.9` today (§2b). Either add a temporary pfSense rule for the
|
||||
test, or accept that app bring-up happens after the NAS is live on `192.168.2.0/24` — in which case
|
||||
stage the app configs now and keep the window short.
|
||||
|
||||
### Phase C — cutover
|
||||
|
||||
1. Stop all guests; reverse `startup:` order, pfSense last.
|
||||
2. Export cleanly — **never just cut power:**
|
||||
```bash
|
||||
zpool export Storage && zpool export Storage2
|
||||
zpool status # both must be gone before power-off
|
||||
```
|
||||
3. **Label every drive** with pool and serial before it leaves the chassis
|
||||
(`3FGJ6M9T`/`3FGHAB2T` = Storage; `WKD1S0TR`/`WKD1S10M` = Storage2).
|
||||
4. Move the four HDDs — cold, upright. The 6-year-old Seagates are the fragile ones.
|
||||
5. Install TrueNAS SCALE on the shipped 64 GB M.2 (Slot A). The five bays take the four HDDs with
|
||||
one spare; the Micron SSDs are **not needed** in the target — leave them in the old chassis.
|
||||
6. **Import read-only first**, verify, then import for real:
|
||||
```bash
|
||||
zpool import # both pools listed
|
||||
zpool import -o readonly=on Storage
|
||||
zfs list # verify datasets, sizes
|
||||
zpool export Storage && zpool import Storage
|
||||
```
|
||||
|
||||
### Phase D — repoint, reclaim, retire
|
||||
|
||||
1. Recreate NFS shares in the TrueNAS UI targeting `192.168.2.10/.11/.12`; clear the old `sharenfs`
|
||||
properties.
|
||||
2. Point the `apps/wave-0` PVs at the NAS; bring Plex, the arr stack and Nextcloud up on prod k3s.
|
||||
3. Stand up MinIO on TrueNAS to replace versitygw, and re-enable Velero + CNPG backups (§2e).
|
||||
4. **Only once everything is verified from k3s:** delete the dead zvols (§1) — ~731 G on `Storage`,
|
||||
4.06 T on `Storage2`.
|
||||
5. Re-evaluate §2d. If it fits, evacuate `Storage2`, retire the 6-year-old Seagates, keep two bays
|
||||
free.
|
||||
6. Repoint the SOHO 443 NAT, then power off the Proxmox host.
|
||||
7. `zpool status` — confirm the 36 read errors did **not** follow `3FGHAB2T` into the new chassis.
|
||||
Full scrub on the new hardware.
|
||||
|
||||
---
|
||||
|
||||
## 5. Rollback, and the one-way door
|
||||
|
||||
Until Phase D step 4, the old chassis is intact: `rpool`, the Proxmox install, both Microns and
|
||||
every `/etc/pve` config untouched. Put the four HDDs back and it boots into the previous world.
|
||||
|
||||
> ### Do not run `zpool upgrade`
|
||||
>
|
||||
> `zpool status Storage2` already nags — *"Some supported and requested features are not enabled…
|
||||
> action: Enable all features using 'zpool upgrade'"* — and TrueNAS will offer it in the UI after
|
||||
> import. It is the only irreversible step here: run it and the pools can never go back to the
|
||||
> Proxmox host. Leave it until the NAS has been serving k3s for weeks.
|
||||
|
||||
Note the asymmetry that makes this safe: TrueNAS's OpenZFS **reads** these pools happily, because
|
||||
newer ZFS always reads older pools. The reverse is not true.
|
||||
|
||||
---
|
||||
|
||||
## 6. Open items
|
||||
|
||||
1. **The Nextcloud number** (§2c) — one `df -h` inside VM 103. It decides whether `Storage2` and the
|
||||
two oldest drives can be retired.
|
||||
2. **Is `config_backup` (267 G) disposable?**
|
||||
3. **What serves DNS on `192.168.2.0/24`** once Pi-hole dies?
|
||||
4. **UniFi controller** — rehome, cloud key, or drop?
|
||||
5. **Minecraft** — keep or retire?
|
||||
6. **Gitea** — its whole state is 3 repos and an 11 MB SQLite DB, and prod k3s already runs its own
|
||||
Gitea. Is the legacy one worth migrating at all, or does it just end?
|
||||
@@ -779,6 +779,74 @@ disk each, already fully partitioned, no free slot verified populated.
|
||||
physically) and flip `node_baseline_etcd_source` back to `device` — the role still
|
||||
supports that path unchanged, and it is the dev shape.
|
||||
|
||||
#### THE MEASUREMENT CAME BACK — 2026-09-07. It does not hold on node 1.
|
||||
|
||||
Not under a deliberate test: in production, 2 days 7 hours after the cluster came up.
|
||||
**06:05:21Z `k3s-prod-1` went NotReady** (`Kubelet stopped posting node status`) while the
|
||||
host stayed up, pingable and SSH-able. From the k3s journal:
|
||||
|
||||
```
|
||||
slow fdatasync took=3.56s / 6.71s / 4.95s / 3.40s (etcd expects 1s)
|
||||
apply request took too long took=8.30s (expects 100ms)
|
||||
etcdserver: too many requests
|
||||
Failed to check local etcd status for learner management: context deadline exceeded
|
||||
```
|
||||
|
||||
No OOM, no MCE, no hung task in the kernel log — **this is not §0b and not memory**. Recovery
|
||||
required a manual `systemctl restart k3s`; 18 Longhorn volumes then rebuilt, and I/O pressure
|
||||
only fell once the rebuilds finished. The failure is **self-reinforcing**: the node dropping
|
||||
out degrades volumes → Longhorn rebuilds → more write load on the disk etcd needs → etcd
|
||||
stalls further.
|
||||
|
||||
`/proc/pressure/io` **full avg300**, all three nodes, during the incident:
|
||||
|
||||
| node | device | drive | IO PSI full avg300 |
|
||||
|---|---|---|---|
|
||||
| k3s-prod-1 | `/dev/sda5` | C400-MTFDDAC512MAM (2011 consumer SATA SSD, no PLP) | **11.36** |
|
||||
| k3s-prod-2 | `/dev/nvme0n1p5` | Samsung PM9A1 NVMe | 0.75 |
|
||||
| k3s-prod-3 | `/dev/sda5` | WDC WDS500G2B0A (WD Blue SATA) | 1.29 |
|
||||
|
||||
The prediction in the paragraph above — "every prod node is SSD or NVMe end-to-end, where
|
||||
concurrent rebuild I/O costs latency rather than collapsing it" — was **right about node 2 and
|
||||
wrong about node 1**. A 2011 consumer SATA SSD with 61,440 reallocated sectors, 31% of its
|
||||
endurance consumed and no power-loss protection is not the "SSD" that reasoning assumed. §0's
|
||||
"three nodes, three different drive models" is the fault line, exactly where §0 said it would
|
||||
be.
|
||||
|
||||
**The exit condition is therefore OPEN and the second bullet applies.** Replacing node 1's
|
||||
drive, or giving etcd its own device, is the fix, and it is the operator's.
|
||||
|
||||
Pending that, three SOFTWARE mitigations shipped on 2026-09-07 (branch
|
||||
`fix/etcd-io-priority`). **None of them is the fix**, and each says so where it lives:
|
||||
|
||||
1. **Observability first** — `NodeIOPressureHigh` / `NodeIOPressureCritical` in
|
||||
`platform/monitoring/alerts.yaml`, plus an I/O row on the node-pressure dashboard. The
|
||||
stack alerted on CPU and memory PSI and had **no I/O alert at all**, so the first signal
|
||||
of this outage was a node already out of the cluster. Thresholds (5% / 10% full IO PSI)
|
||||
come from the numbers in the table above. Note the etcd histograms this section's first
|
||||
bullet asks for are **not available**: kube-prometheus-stack's `kubeEtcd` scrape job is
|
||||
deliberately disabled for k3s (`platform/monitoring/kube-prometheus-stack.yaml`), so PSI
|
||||
is the signal that actually exists. Enabling the etcd scrape would make that bullet
|
||||
directly measurable and is worth doing.
|
||||
2. **Longhorn concurrency ceilings** — `platform/longhorn/helmrelease.yaml`. Honestly:
|
||||
`concurrentReplicaRebuildPerNodeLimit: 1` was **already in force during this outage** and
|
||||
held; the 18 rebuilds ran one at a time and the node stalled anyway. Longhorn offers no
|
||||
further rebuild lever for the v1 data engine (`replicaRebuildingBandwidthLimit` is
|
||||
v2/SPDK only; `rebuildConcurrentSyncLimit` is already at its floor of 1). What was
|
||||
tightened is every *other* Longhorn I/O source — restore, snapshot purge, backup, engine
|
||||
upgrade — each of which defaulted to 5 concurrent operations per node and each of which
|
||||
could reproduce this on its own.
|
||||
3. **Block-layer arbitration** — `roles/etcd_io_priority`, enabled for `[k3s_prod]`, applied
|
||||
only when the operator runs `playbooks/etcd-io-priority.yml`. BFQ + `IOWeight` so etcd's
|
||||
cgroup outweighs `kubepods.slice`. Its README is explicit that most of the obvious knobs
|
||||
here (`IOSchedulingClass`, `io.latency`, bare `IOWeight`) are **inert on these nodes**,
|
||||
and that even this one cannot arbitrate the ext4 journal both tenants share.
|
||||
|
||||
The last point is the one that matters for this section: **no cgroup or scheduler control
|
||||
separates two tenants on one filesystem.** etcd's `fdatasync` waits on a jbd2 journal commit,
|
||||
jbd2 is a kernel thread in the root cgroup, and Longhorn's writes dirty the same journal. The
|
||||
software mitigations raise the load level at which this happens. They do not remove it.
|
||||
|
||||
**~~Known, separate consequence — `/var` is only 12 GiB.~~ RESOLVED 2026-09-04 by moving
|
||||
k3s's data dir.** This was recorded here on 2026-09-03 as a capacity item "to watch or
|
||||
repartition". Re-measuring it against dev turned it from a watch item into a blocker: the
|
||||
|
||||
+21
-4
@@ -27,6 +27,7 @@ ansible-playbook playbooks/site.yml # core bring-up, in order (baseline
|
||||
# — or step by step —
|
||||
ansible-playbook playbooks/node-baseline.yml # 0. packages, kernel modules, Longhorn disk, swap-off
|
||||
ansible-playbook playbooks/k3s.yml # 1. HA k3s (primary --cluster-init, then joins)
|
||||
ansible-playbook playbooks/etcd-io-priority.yml # 1b. shared etcd/Longhorn device I/O weights (PROD ONLY; a no-op on dev)
|
||||
ansible-playbook playbooks/cilium.yml # 2. Cilium CNI (cluster is NotReady until this)
|
||||
ansible-playbook playbooks/verify.yml # 3. gate #1: 3 nodes Ready, Hubble up, no kube-proxy
|
||||
ansible-playbook playbooks/flux-bootstrap.yml # 4. hand off to bootstrap/ (SOPS then Flux)
|
||||
@@ -71,6 +72,7 @@ should return a PIA exit IP (not the home ISP).
|
||||
|---|---|
|
||||
| `k3s.yml` / `k3s_server` | k3s server install with `--flannel-backend=none --disable-kube-proxy --disable-network-policy --disable=traefik,servicelb`; primary `--cluster-init`, secondaries join via the primary's node-token (fetched, never pre-stored) |
|
||||
| `cilium.yml` / `cilium` | Gateway API CRDs, then Helm `cilium` with `kubeProxyReplacement`, `k8sServiceHost/Port`, `gatewayAPI`, `egressGateway`, `hubble`, `l2announcements` |
|
||||
| `etcd-io-priority.yml` / `etcd_io_priority` | **prod only** (master switch defaults false; `group_vars/k3s_prod` sets it true). Switches the disk carrying `k3s_data_dir` to **BFQ**, drops `nr_requests` to 32, and gives `system.slice` + `k3s.service` `IOWeight=10000` so the embedded etcd outweighs `kubepods.slice` — then reads every value back out of the kernel and **fails** if it did not land. Mitigation for the 2026-09-07 k3s-prod-1 etcd WAL-fsync starvation. Restarts nothing. NVMe-backed nodes are skipped, not decorated. See `roles/etcd_io_priority/README.md` for the mechanisms in this space that are **inert** here and must not be "fixed" back in |
|
||||
| `verify.yml` / `cluster_verify` | waits nodes Ready + Cilium/Hubble rollout; asserts kube-proxy absent |
|
||||
| `flux-bootstrap.yml` / `flux_bootstrap` | calls `bootstrap/` (SOPS age key, then `flux bootstrap --path=clusters/dev`) |
|
||||
|
||||
@@ -120,10 +122,20 @@ for PM reconciliation.
|
||||
## Targeting prod (R-ENV-02: same playbooks, same roles, scoped overrides)
|
||||
|
||||
`inventory/hosts.ini` carries a `[k3s_prod_primary]` / `[k3s_prod_secondary]` /
|
||||
`[k3s_prod:children]` section for the three planned bare-metal nodes (`k3s-prod-1` at
|
||||
`192.168.2.10` exists and is reachable today; `k3s-prod-2`/`-3` at `.11`/`.12` are addressed but
|
||||
not yet racked, so their inventory lines stay commented out), with its own `group_vars/k3s_prod/`
|
||||
(vaulted `ansible_become_password` + non-secret overrides).
|
||||
`[k3s_prod:children]` section for the three bare-metal nodes — `k3s-prod-1` `192.168.2.10`,
|
||||
`k3s-prod-2` `.11`, `k3s-prod-3` `.12` — with its own `group_vars/k3s_prod/` (vaulted
|
||||
`ansible_become_password` + non-secret overrides).
|
||||
|
||||
**All three are live in the inventory.** This paragraph used to say `k3s-prod-2`/`-3` "are
|
||||
addressed but not yet racked, so their inventory lines stay commented out"; that stopped being
|
||||
true on 2026-09-03 when node 3 came online, and the group has been complete since. Corrected
|
||||
2026-09-07 because the stale sentence is the kind that gets used to reason about blast radius:
|
||||
`-e k3s_hosts=k3s_prod` targets **three** hosts, not one. Confirm it, every time, before running
|
||||
anything:
|
||||
|
||||
```bash
|
||||
ansible-playbook playbooks/<play>.yml -e k3s_hosts=k3s_prod --limit k3s-prod-3 --list-hosts
|
||||
```
|
||||
|
||||
**Nothing here is forked for prod.** Every node-host play takes its target group from a variable
|
||||
that defaults to the dev group, so the dev commands above are unchanged, and prod is the same
|
||||
@@ -153,6 +165,11 @@ ansible-playbook playbooks/kube-vip.yml -e k3s_hosts=k3s_prod
|
||||
ansible-playbook playbooks/verify.yml -e k3s_hosts=k3s_prod
|
||||
ansible-playbook playbooks/flux-bootstrap.yml -e k3s_hosts=k3s_prod # see caveat below
|
||||
|
||||
# prod — I/O arbitration for the shared etcd/Longhorn disk. ONE NODE AT A TIME, node 3 first
|
||||
# (WD Blue, IO PSI 1.29 during the incident) before node 1 (the 2011 C400, 11.36).
|
||||
ansible-playbook playbooks/etcd-io-priority.yml -e k3s_hosts=k3s_prod --limit k3s-prod-3 --list-hosts
|
||||
ansible-playbook playbooks/etcd-io-priority.yml -e k3s_hosts=k3s_prod --limit k3s-prod-3
|
||||
|
||||
# prod — k3s.yml needs both halves of the primary/secondary split
|
||||
ansible-playbook playbooks/k3s.yml \
|
||||
-e k3s_primary_hosts=k3s_prod_primary -e k3s_secondary_hosts=k3s_prod_secondary
|
||||
|
||||
@@ -357,3 +357,29 @@ k3s_base_domain: "prod.home.keithjackson.dev"
|
||||
# from the prod nodes — it is outside every cluster and can simply be powered off;
|
||||
# - the `sops-age` Secret is installed out of band on prod (bootstrap/sops).
|
||||
flux_bootstrap_env_path: "clusters/prod"
|
||||
|
||||
# --- shared etcd/Longhorn device I/O arbitration (2026-09-07 incident) --------------
|
||||
# TURNS ON roles/etcd_io_priority FOR PROD ONLY. Role default is false, and dev must stay
|
||||
# that way: dev has a DEDICATED etcd device (/dev/vdc mounted at server/db,
|
||||
# docs/plans/ssd-etcd-isolation.md) so it has nothing to arbitrate, and pulling a LIVE
|
||||
# cluster onto a different I/O scheduler as a side effect of sharing a role would be a real
|
||||
# change for no benefit. Same seam as node_baseline_backports_kernel above.
|
||||
#
|
||||
# WHY PROD NEEDS IT. The §2 ruling put /data/k3s (etcd WAL) and /data/longhorn (every
|
||||
# replica) on ONE partition per node and recorded the contention as "accepted to be
|
||||
# MEASURED, not assumed". It has now been measured, the hard way: 2026-09-07 06:05:21Z
|
||||
# k3s-prod-1 went NotReady with the host still up, because etcd could not finish a WAL
|
||||
# fdatasync (3.4-6.7 s against a 1 s expectation) while Longhorn wrote to the same disk.
|
||||
# /proc/pressure/io FULL avg300 was 11.36 there against 0.75 (node 2, NVMe) and 1.29
|
||||
# (node 3). See PROD-PROVISIONING.md §2 "THE MEASUREMENT CAME BACK".
|
||||
#
|
||||
# WHAT ENABLING THIS DOES *NOT* DO: nothing, until the operator runs
|
||||
# playbooks/etcd-io-priority.yml. It is not applied by any other play except through
|
||||
# site.yml, and site.yml is not run against prod casually. Apply to k3s-prod-3 (the WD Blue,
|
||||
# IO PSI 1.29) FIRST and watch NodeIOPressureHigh for a day — the role's own README explains
|
||||
# why, what it cannot fix, and how to roll it back in three commands.
|
||||
#
|
||||
# k3s-prod-2 is NVMe-backed; the role detects that and deliberately installs nothing there
|
||||
# rather than leaving an inert drop-in behind, so this group-wide `true` is still correct
|
||||
# for it.
|
||||
etcd_io_priority_enabled: true
|
||||
@@ -0,0 +1,38 @@
|
||||
---
|
||||
# Give the k3s server process — and therefore the EMBEDDED ETCD inside it — a larger share of
|
||||
# the block device it shares with Longhorn. Mitigates the 2026-09-07 k3s-prod-1 outage, where
|
||||
# etcd's WAL fdatasync took 3.4-6.7 s (expects 1 s) and the node went NotReady while the host
|
||||
# stayed up. See roles/etcd_io_priority/README.md for the mechanism, the measurements, and the
|
||||
# list of things in this space that do NOT work on these nodes.
|
||||
#
|
||||
# NO-OP BY DEFAULT. The role's master switch (etcd_io_priority_enabled) is false in
|
||||
# defaults/main.yml and true only in inventory/group_vars/k3s_prod/vars.yml, so running this
|
||||
# with no extra vars against dev changes nothing on any node — dev already has a dedicated
|
||||
# etcd device and has nothing to arbitrate.
|
||||
#
|
||||
# TARGET (R-ENV-02): same `k3s_hosts` seam every other whole-cluster play uses.
|
||||
# ansible-playbook playbooks/etcd-io-priority.yml # dev, a no-op
|
||||
# ansible-playbook playbooks/etcd-io-priority.yml -e k3s_hosts=k3s_prod # prod
|
||||
#
|
||||
# RUN IT ON ONE NODE FIRST, AND CONFIRM THE TARGET BEFORE YOU RUN IT AT ALL. A prior mistake in
|
||||
# this repo silently targeted dev's three nodes; --list-hosts is how that is caught:
|
||||
# ansible-playbook playbooks/etcd-io-priority.yml -e k3s_hosts=k3s_prod \
|
||||
# --limit k3s-prod-3 --list-hosts
|
||||
# k3s-prod-3 first, deliberately: it is the WD Blue SATA node (IO PSI 1.29 during the incident),
|
||||
# so it exercises the identical code path as the failing node 1 (11.36) at a fraction of the
|
||||
# stakes. Watch NodeIOPressureHigh / the "Nodes — CPU / Memory / I-O Pressure" dashboard for a
|
||||
# day before touching node 1.
|
||||
#
|
||||
# ROLLBACK is immediate and needs no playbook — see the header of
|
||||
# /usr/local/sbin/k3s-etcd-io-priority on the node.
|
||||
|
||||
- name: I/O priority for the shared etcd + Longhorn device
|
||||
hosts: "{{ k3s_hosts | default('k3s_servers') }}"
|
||||
become: true
|
||||
# serial: 1 — one node at a time. Nothing here restarts k3s, but changing a live node's I/O
|
||||
# scheduler is a real change to a cluster member, and the whole point is that a bad outcome
|
||||
# on one node must never be simultaneous across all three.
|
||||
serial: 1
|
||||
gather_facts: true
|
||||
roles:
|
||||
- role: etcd_io_priority
|
||||
@@ -23,6 +23,17 @@
|
||||
- name: Import - install k3s HA control plane
|
||||
ansible.builtin.import_playbook: k3s.yml
|
||||
|
||||
# ADDED 2026-09-07, after the k3s-prod-1 etcd I/O-starvation outage. It must come AFTER
|
||||
# k3s.yml because it writes an IOWeight drop-in for k3s.service and verifies the value in
|
||||
# that unit's LIVE cgroup — on a cold build the cgroup does not exist until k3s runs, and
|
||||
# the role reports that rather than asserting on it. It restarts nothing.
|
||||
#
|
||||
# A NO-OP FOR DEV, by design: roles/etcd_io_priority defaults its master switch to false and
|
||||
# only inventory/group_vars/k3s_prod/vars.yml sets it true. Dev has a dedicated etcd device
|
||||
# and must not be dragged onto a different I/O scheduler by a shared role.
|
||||
- name: Import - I/O arbitration for the shared etcd + Longhorn device (prod only)
|
||||
ansible.builtin.import_playbook: etcd-io-priority.yml
|
||||
|
||||
# ADDED 2026-09-06, AND ITS POSITION IS LOAD-BEARING: kube-vip MUST come up between k3s and
|
||||
# Cilium. Cilium's k8sServiceHost is now the control-plane VIP (k3s_apiserver_endpoint =
|
||||
# kube_vip_address) rather than the founder's own node IP, so installing Cilium before the
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
# roles/etcd_io_priority
|
||||
|
||||
Give the k3s server process — and therefore the **embedded etcd** it hosts — a larger share of
|
||||
the block device it shares with Longhorn.
|
||||
|
||||
## The incident
|
||||
|
||||
2026-09-07 06:05:21Z, `k3s-prod-1` went NotReady (`Kubelet stopped posting node status`) while
|
||||
the host stayed up, pingable and SSH-able. From the k3s journal:
|
||||
|
||||
```
|
||||
slow fdatasync took=3.56s / 6.71s / 4.95s / 3.40s (etcd expects 1s)
|
||||
apply request took too long took=8.30s (expects 100ms)
|
||||
etcdserver: too many requests
|
||||
Failed to check local etcd status for learner management: context deadline exceeded
|
||||
```
|
||||
|
||||
No OOM, no MCE, no hung task. All three prod nodes put `/data/k3s` (etcd's WAL **and** the
|
||||
containerd image store) and `/data/longhorn` (every replica) in two directories on **one
|
||||
partition**:
|
||||
|
||||
| node | device | drive | `/proc/pressure/io` full avg300 |
|
||||
|---|---|---|---|
|
||||
| k3s-prod-1 | `/dev/sda5` | C400-MTFDDAC512MAM — 2011 consumer SATA SSD, no PLP, 61,440 reallocated sectors | **11.36** |
|
||||
| k3s-prod-2 | `/dev/nvme0n1p5` | Samsung PM9A1 NVMe | 0.75 |
|
||||
| k3s-prod-3 | `/dev/sda5` | WDC WDS500G2B0A (WD Blue SATA) | 1.29 |
|
||||
|
||||
Recovery took a manual `systemctl restart k3s`; 18 Longhorn volumes then rebuilt onto the same
|
||||
disk etcd was trying to fsync on. The failure is self-reinforcing.
|
||||
|
||||
## What this role does
|
||||
|
||||
Two things, both instantly reversible, neither requiring a k3s restart:
|
||||
|
||||
1. **`k3s-etcd-io-priority.service`** (oneshot, ordered `Before=k3s.service`) switches the
|
||||
shared device to the **BFQ** I/O scheduler and lowers `nr_requests` from 64 to 32.
|
||||
2. **`IOWeight=10000` drop-ins** on `system.slice` and `k3s.service`, which BFQ turns into
|
||||
`io.bfq.weight=1000` — a 10:1 share against the default-weighted `kubepods.slice` where
|
||||
Longhorn's instance-managers live.
|
||||
|
||||
Then it **reads every one of those values back out of the kernel and asserts on them**. If the
|
||||
scheduler did not switch, or the weight did not land, the play fails. A mitigation that looks
|
||||
applied and is not is worse than none.
|
||||
|
||||
## Why not the obvious things — all checked on the live nodes, all inert
|
||||
|
||||
**`IOSchedulingClass=realtime` on `k3s.service`.** The scheduler here (`mq-deadline`, and BFQ
|
||||
too) *does* honour the I/O priority class, so this looks like the answer. It is not: I/O
|
||||
priority is inherited across `fork` and preserved across `exec` — `block/blk-ioc.c`'s
|
||||
`__copy_io()` copies `ioc->ioprio` into the child — and **containerd is a child of the k3s
|
||||
server process**. Verified on k3s-prod-1: `ps -eo pid,ppid,comm` shows containerd's PPID is the
|
||||
`k3s-server` PID (84553 → 84333), and
|
||||
|
||||
```
|
||||
$ ionice -c2 -n0 -- sh -c 'ionice -p $$; sh -c "ionice -p \$\$"'
|
||||
best-effort: prio 0
|
||||
best-effort: prio 0
|
||||
```
|
||||
|
||||
confirms inheritance through two levels of fork+exec on that node. Putting k3s in the realtime
|
||||
class therefore puts **every container on the node** in the realtime class, Longhorn included —
|
||||
no relative gain, and it demotes the rest of `system.slice` (sshd, journald) for nothing.
|
||||
Getting this to work would need containerd out of k3s's process tree, i.e. an external
|
||||
containerd via `--container-runtime-endpoint`. That is an architecture change, not a tweak.
|
||||
|
||||
**`IOSchedulingPriority` (the level within a class).** `mq-deadline` buckets requests by
|
||||
priority **class** only — `IOPRIO_CLASS_RT/BE/IDLE` map to three `dd_per_prio` queues and the
|
||||
level inside a class is not read. So best-effort 0 versus best-effort 4 is a no-op here.
|
||||
|
||||
**`io.latency` (blk-iolatency).** The ideal mechanism for this problem — scheduler-independent,
|
||||
protects one cgroup's latency target by throttling its siblings. **Not available:** Debian
|
||||
trixie's kernel does not build it. `grep CONFIG_BLK_CGROUP /boot/config-7.1.8+deb13-amd64`
|
||||
returns `CONFIG_BLK_CGROUP_IOCOST=y` and no `IOLATENCY`, and there is no `io.latency` file
|
||||
anywhere under `/sys/fs/cgroup`.
|
||||
|
||||
**`IOWeight=` on the stock configuration.** systemd writes `IOWeight=` to both `io.weight`
|
||||
(blk-iocost) and `io.bfq.weight` (BFQ). On these nodes as shipped, **neither reader exists**:
|
||||
the scheduler is `mq-deadline`, which has no cgroup awareness, and blk-iocost is compiled in but
|
||||
disabled (`/sys/fs/cgroup/io.cost.qos` is empty). Adding a drop-in without switching the
|
||||
scheduler would be pure decoration. That is why the scheduler change and the weights are one
|
||||
role and one verification, not two independent knobs.
|
||||
|
||||
**`IOWeight=1000` instead of `10000`.** systemd rescales `IOWeight`'s 1..10000 range onto
|
||||
`io.bfq.weight`'s 1..1000, so 1000 maps to roughly 101 — BFQ's default for an unweighted cgroup
|
||||
is 100. Only the top of the range separates anything. The role asserts the resulting
|
||||
`io.bfq.weight` is ≥ `etcd_io_priority_min_bfq_weight` (500) rather than trusting that mapping.
|
||||
|
||||
**A udev rule instead of a unit.** `ATTR{queue/scheduler}="bfq"` on the *boot* disk fires its
|
||||
`add` event in the initramfs, where `/etc/udev/rules.d` is absent and `bfq.ko` is not loaded —
|
||||
it silently does nothing and the disk comes up on `mq-deadline` with no error anywhere.
|
||||
|
||||
**A drop-in on `kubepods.slice`** (to lower Longhorn instead of raising etcd). `kubepods.slice`
|
||||
is a **transient** unit the kubelet creates through the systemd API (`systemctl cat
|
||||
kubepods.slice` shows `/run/systemd/transient/kubepods.slice`), and whether systemd merges an
|
||||
`/etc` drop-in into a transient unit was not verified on these nodes. Raising `system.slice`
|
||||
gets the same ratio through a unit systemd definitely owns.
|
||||
|
||||
## What this role does NOT fix
|
||||
|
||||
- **It does not separate etcd from Longhorn.** Both still write to one ext4 filesystem on one
|
||||
partition. Weights arbitrate the *device queue*; they cannot arbitrate the **shared ext4
|
||||
journal**. `fdatasync` on the WAL waits on a jbd2 commit, jbd2 is a kernel thread in the root
|
||||
cgroup, and Longhorn's replica writes dirty the same journal. That serialization point is
|
||||
invisible to every cgroup control in this file and is a hard limit of co-tenancy on one
|
||||
filesystem.
|
||||
- **It is unproven on this hardware.** BFQ is the right mechanism on paper and it is what
|
||||
systemd's `IOWeight` was built for, but nothing here has been measured on a C400 under a real
|
||||
Longhorn rebuild. The role fails loudly if the settings do not *land*; it cannot tell you they
|
||||
*helped*. Apply to `k3s-prod-3` first and watch the `NodeIOPressureHigh` alert and the
|
||||
"Nodes — CPU / Memory / I-O Pressure" dashboard before touching node 1.
|
||||
- **BFQ has costs.** It disables writeback throttling (`wbt_lat_usec` is 2000 on sda today),
|
||||
spends more CPU per request, and gives up some peak throughput. On a device that is already
|
||||
the bottleneck this is the right trade; it is still a trade.
|
||||
- **k3s-prod-2 gets nothing.** Its data dir is on NVMe, BFQ is a poor fit for NVMe, and its IO
|
||||
PSI was 0.75 during the incident. The role detects this and installs nothing rather than
|
||||
leaving inert files behind.
|
||||
- **The real fix is hardware and is the operator's**: replace the C400, or give etcd its own
|
||||
device and flip `node_baseline_etcd_source` back to `device` — the path
|
||||
`roles/node_baseline` still supports unchanged, and the exit condition already written into
|
||||
`infra/PROD-PROVISIONING.md` §2.
|
||||
|
||||
## Running it
|
||||
|
||||
```bash
|
||||
cd infra/ansible
|
||||
|
||||
# ALWAYS confirm the target first — a prior mistake in this repo silently hit dev.
|
||||
ansible-playbook playbooks/etcd-io-priority.yml -e k3s_hosts=k3s_prod \
|
||||
--limit k3s-prod-3 --list-hosts
|
||||
|
||||
ansible-playbook playbooks/etcd-io-priority.yml -e k3s_hosts=k3s_prod --limit k3s-prod-3
|
||||
```
|
||||
|
||||
Rollback, per node, immediate:
|
||||
|
||||
```bash
|
||||
systemctl disable --now k3s-etcd-io-priority.service
|
||||
printf 'mq-deadline\n' > /sys/block/sda/queue/scheduler
|
||||
printf '64\n' > /sys/block/sda/queue/nr_requests
|
||||
rm -f /etc/systemd/system/{system.slice,k3s.service}.d/10-etcd-io-weight.conf
|
||||
systemctl daemon-reload
|
||||
```
|
||||
@@ -0,0 +1,84 @@
|
||||
---
|
||||
# etcd_io_priority — give the k3s process (and therefore the embedded etcd it hosts) a
|
||||
# larger share of the block device it SHARES with Longhorn, on nodes where those two
|
||||
# tenants sit on one partition.
|
||||
#
|
||||
# READ roles/etcd_io_priority/README.md BEFORE CHANGING ANY OF THESE. Two of the obvious
|
||||
# knobs in this space are INERT on these nodes and are deliberately absent:
|
||||
# * IOSchedulingClass= / IOSchedulingPriority= on k3s.service. I/O priority is inherited
|
||||
# across fork and preserved across exec (block/blk-ioc.c __copy_io copies
|
||||
# ioc->ioprio), and containerd is a CHILD of the k3s server process — verified live on
|
||||
# k3s-prod-1: `ps -eo pid,ppid,comm` shows containerd's PPID is the k3s-server PID, and
|
||||
# `ionice -c2 -n0 -- sh -c 'ionice -p $$; sh -c "ionice -p \$\$"'` reports best-effort
|
||||
# prio 0 at BOTH levels. So raising k3s to the realtime I/O class raises every container
|
||||
# on the node with it, Longhorn's instance-managers included, and buys nothing against
|
||||
# the thing we are trying to beat.
|
||||
# * io.latency (blk-iolatency), which WOULD be the ideal scheduler-independent protection.
|
||||
# Not available: CONFIG_BLK_CGROUP_IOLATENCY is absent from Debian trixie's kernel
|
||||
# (`grep CONFIG_BLK_CGROUP /boot/config-7.1.8+deb13-amd64` returns IOCOST but not
|
||||
# IOLATENCY), and there is no /sys/fs/cgroup/**/io.latency on any prod node.
|
||||
# What is left, and what this role does, is cgroup-proportional weighting — which requires
|
||||
# a scheduler that implements it.
|
||||
|
||||
# MASTER SWITCH. Default FALSE so the LIVE dev cluster is untouched by a shared role:
|
||||
# dev already has a DEDICATED etcd device (/dev/vdc mounted at server/db,
|
||||
# docs/plans/ssd-etcd-isolation.md), so it has nothing to arbitrate and no reason to take a
|
||||
# scheduler change. Set true for the [k3s_prod] group in
|
||||
# inventory/group_vars/k3s_prod/vars.yml — the same seam node_baseline_backports_kernel,
|
||||
# kube_vip_address and k3s_data_dir use.
|
||||
etcd_io_priority_enabled: false
|
||||
|
||||
# THE I/O SCHEDULER, AND WHY IT HAS TO CHANGE FOR ANY OF THIS TO WORK.
|
||||
# Measured on k3s-prod-1: `cat /sys/block/sda/queue/scheduler` -> `none [mq-deadline]`.
|
||||
# mq-deadline honours the I/O priority CLASS (RT/BE/IDLE) but knows nothing about cgroups,
|
||||
# and cgroup v2's io.weight is implemented by blk-iocost — which is compiled in
|
||||
# (CONFIG_BLK_CGROUP_IOCOST=y) but DISABLED: /sys/fs/cgroup/io.cost.qos is empty. So on the
|
||||
# nodes as they stand today, systemd's IOWeight= writes a number nothing reads.
|
||||
# BFQ is the one scheduler here that implements cgroup weights (CONFIG_IOSCHED_BFQ=m,
|
||||
# CONFIG_BFQ_GROUP_IOSCHED=y — module present at
|
||||
# /lib/modules/7.1.8+deb13-amd64/kernel/block/bfq.ko.xz). Under BFQ, systemd's IOWeight=
|
||||
# lands in io.bfq.weight and becomes real.
|
||||
#
|
||||
# THE COSTS, STATED: BFQ turns OFF writeback throttling (wbt_lat_usec currently 2000 on
|
||||
# sda), costs some peak throughput and some CPU, and is a poor fit for NVMe — which is why
|
||||
# the role SKIPS the whole scheduler+weight path on an NVMe-backed node (k3s-prod-2), rather
|
||||
# than pretending to help there. Set to "" to install nothing but still run the report.
|
||||
etcd_io_priority_scheduler: bfq
|
||||
|
||||
# Block-layer queue depth for the shared device. sda's current value is 64 (blk-mq derives
|
||||
# it from the SATA NCQ depth). A shallower queue bounds how many Longhorn writes can already
|
||||
# be sitting in front of an etcd WAL flush at the DEVICE, below any scheduler's reach — the
|
||||
# one lever in this file that does not depend on cgroups or on the scheduler honouring
|
||||
# anything. Costs sequential throughput. Set to "" to leave nr_requests alone.
|
||||
etcd_io_priority_nr_requests: 32
|
||||
|
||||
# systemd IOWeight= for system.slice (which contains k3s.service, hence etcd) and for
|
||||
# k3s.service itself. THE VALUE IS 10000 — systemd's maximum — AND THAT IS NOT COSMETIC.
|
||||
# systemd rescales IOWeight (1..10000) onto io.bfq.weight (1..1000) linearly, so IOWeight=1000
|
||||
# maps to roughly 101 — indistinguishable from BFQ's own default of 100 for an unweighted
|
||||
# cgroup. Only near the top of the range does the weight actually separate: 10000 -> 1000,
|
||||
# a 10:1 share against the default-weighted kubepods.slice. A "reasonable-looking" 500 or
|
||||
# 1000 here would be a change that reads as applied and does nothing.
|
||||
etcd_io_priority_weight: 10000
|
||||
|
||||
# FAIL-LOUD FLOOR for the verification step. The role reads io.bfq.weight back out of the
|
||||
# live cgroups and asserts it is at least this. It exists because the rescaling above is
|
||||
# systemd's business, not ours: if a systemd version maps IOWeight differently, this role
|
||||
# must FAIL rather than leave a drop-in that looks applied. 500 = 5x an unweighted sibling.
|
||||
etcd_io_priority_min_bfq_weight: 500
|
||||
|
||||
# The k3s data dir whose backing device is the one to protect. Defers to the CLUSTER-WIDE
|
||||
# k3s_data_dir (group_vars/k3s_prod sets /data/k3s; the roles/k3s_server default restates
|
||||
# k3s's own /var/lib/rancher/k3s for dev) — NOT a second independent variable, for the same
|
||||
# reason node_baseline_k3s_data_dir defers to it.
|
||||
etcd_io_priority_k3s_data_dir: "{{ k3s_data_dir | default('/var/lib/rancher/k3s') }}"
|
||||
|
||||
# The systemd units that get an IOWeight drop-in, and the section each one takes.
|
||||
# BOTH ENTRIES ARE LOAD-BEARING — see tasks/main.yml "THE CGROUP WEIGHTS" for why weighting
|
||||
# k3s.service alone would arbitrate k3s against sshd and not against Longhorn at all.
|
||||
# system.slice must come first so the parent is weighted before its child.
|
||||
etcd_io_priority_weighted_units:
|
||||
- unit: system.slice
|
||||
section: Slice
|
||||
- unit: k3s.service
|
||||
section: Service
|
||||
@@ -0,0 +1,18 @@
|
||||
---
|
||||
# Re-run the oneshot so a changed script/unit takes effect NOW rather than at the next boot.
|
||||
# It only writes two sysfs files; there is nothing to drain and nothing to restart.
|
||||
- name: Apply k3s etcd I/O settings
|
||||
ansible.builtin.systemd_service:
|
||||
name: k3s-etcd-io-priority.service
|
||||
state: restarted
|
||||
daemon_reload: true
|
||||
|
||||
# daemon-reload re-realizes cgroup attributes for units that are already running, which is how
|
||||
# a changed IOWeight= reaches a LIVE k3s.service without restarting it. It is deliberately NOT
|
||||
# a `systemctl restart k3s`: restarting the k3s server to apply a performance tweak is exactly
|
||||
# the disruption this role exists to prevent, and on the primary it bounces etcd. If the reload
|
||||
# turns out not to be enough, the role's verification step FAILS and tells the operator to do a
|
||||
# rolling restart deliberately — it never quietly leaves the weight unapplied.
|
||||
- name: Reload systemd for I/O weights
|
||||
ansible.builtin.systemd_service:
|
||||
daemon_reload: true
|
||||
@@ -0,0 +1,238 @@
|
||||
---
|
||||
# etcd_io_priority — see README.md in this directory for the incident, the mechanism, and the
|
||||
# honest list of what this does NOT achieve. Everything runs under one `when` on the master
|
||||
# switch, so a run against dev (default false) touches nothing at all.
|
||||
#
|
||||
# TAG: `io-priority`. Every task carries it, so this role can be run alone against a live
|
||||
# cluster without dragging in any other role's work:
|
||||
# ansible-playbook playbooks/etcd-io-priority.yml -e k3s_hosts=k3s_prod --limit k3s-prod-3
|
||||
#
|
||||
# ALWAYS --limit AND ALWAYS CHECK --list-hosts FIRST. This role changes the I/O scheduler of a
|
||||
# live node's boot disk; it is safe and instantly reversible, but it is a real change and the
|
||||
# blast radius must be one node at a time (playbooks/etcd-io-priority.yml sets serial: 1 too).
|
||||
|
||||
- name: I/O priority for the shared etcd device
|
||||
when: etcd_io_priority_enabled | bool
|
||||
tags: [io-priority]
|
||||
block:
|
||||
|
||||
# -------------------------------------------------------------------------------------
|
||||
# DISCOVER THE DEVICE. Never hardcoded, and never taken from inventory: k3s-prod-1 and -3
|
||||
# boot from SATA (/dev/sda5) while k3s-prod-2 boots from NVMe (/dev/nvme0n1p5), and a
|
||||
# per-host var freezes today's hardware into inventory where it goes stale silently the
|
||||
# first time a disk is swapped. Inventory declares the INTENT ("the disk under k3s's data
|
||||
# dir"); the node answers what that is. Same reasoning as roles/node_baseline's disk
|
||||
# discovery.
|
||||
# -------------------------------------------------------------------------------------
|
||||
- name: Find the filesystem source backing the k3s data dir
|
||||
ansible.builtin.command:
|
||||
cmd: "findmnt --noheadings --output SOURCE --target {{ etcd_io_priority_k3s_data_dir }}"
|
||||
register: _eio_findmnt
|
||||
changed_when: false
|
||||
check_mode: false
|
||||
|
||||
- name: Find the whole-disk parent of that filesystem source
|
||||
ansible.builtin.command:
|
||||
cmd: "lsblk --noheadings --output PKNAME {{ _eio_findmnt.stdout | trim }}"
|
||||
register: _eio_pkname
|
||||
changed_when: false
|
||||
check_mode: false
|
||||
|
||||
# PKNAME is empty when the source IS already a whole disk (no partition), so fall back to
|
||||
# its basename rather than producing an empty device name.
|
||||
- name: Resolve the block device to tune
|
||||
ansible.builtin.set_fact:
|
||||
_eio_device: >-
|
||||
{{ (_eio_pkname.stdout_lines | map('trim') | reject('equalto', '') | list
|
||||
+ [_eio_findmnt.stdout | trim | basename]) | first }}
|
||||
|
||||
# -------------------------------------------------------------------------------------
|
||||
# NVMe IS DELIBERATELY LEFT ALONE, and this is the whole reason the role discovers the
|
||||
# device instead of assuming it. BFQ is a bad fit for NVMe (single software queue, real
|
||||
# throughput cost) and k3s-prod-2 — the NVMe node — measured /proc/pressure/io full avg300
|
||||
# of 0.75 during the 2026-09-07 incident against k3s-prod-1's 11.36. It does not have this
|
||||
# problem. Installing an inert weight drop-in there would be exactly the false confidence
|
||||
# this role exists to avoid, so nothing is written on such a node.
|
||||
# -------------------------------------------------------------------------------------
|
||||
- name: Decide whether a scheduler change applies to this device
|
||||
ansible.builtin.set_fact:
|
||||
_eio_scheduler: "{{ '' if _eio_device is match('nvme') else etcd_io_priority_scheduler }}"
|
||||
|
||||
- name: Report that this node is intentionally skipped (NVMe-backed)
|
||||
ansible.builtin.debug:
|
||||
msg: >-
|
||||
{{ inventory_hostname }}: k3s data dir {{ etcd_io_priority_k3s_data_dir }} is on
|
||||
/dev/{{ _eio_device }} (NVMe). BFQ and its cgroup weights are not applied to NVMe —
|
||||
nothing installed here. This is intentional, not a failure.
|
||||
when: not _eio_scheduler
|
||||
|
||||
# -------------------------------------------------------------------------------------
|
||||
# THE BLOCK-LAYER SETTINGS (scheduler + queue depth).
|
||||
# -------------------------------------------------------------------------------------
|
||||
- name: Install the block-layer settings script
|
||||
ansible.builtin.template:
|
||||
src: k3s-etcd-io-priority.sh.j2
|
||||
dest: /usr/local/sbin/k3s-etcd-io-priority
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0755"
|
||||
when: _eio_scheduler | length > 0
|
||||
notify: Apply k3s etcd I/O settings
|
||||
|
||||
- name: Install the boot-time unit that applies them
|
||||
ansible.builtin.template:
|
||||
src: k3s-etcd-io-priority.service.j2
|
||||
dest: /etc/systemd/system/k3s-etcd-io-priority.service
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
when: _eio_scheduler | length > 0
|
||||
notify: Apply k3s etcd I/O settings
|
||||
|
||||
- name: Enable and start the block-layer settings unit
|
||||
ansible.builtin.systemd_service:
|
||||
name: k3s-etcd-io-priority.service
|
||||
enabled: true
|
||||
state: started
|
||||
daemon_reload: true
|
||||
when: _eio_scheduler | length > 0
|
||||
|
||||
# -------------------------------------------------------------------------------------
|
||||
# THE CGROUP WEIGHTS.
|
||||
#
|
||||
# BOTH LEVELS ARE REQUIRED AND NEITHER IS SUFFICIENT. cgroup v2 weights are hierarchical:
|
||||
# k3s.service's weight only decides its share WITHIN system.slice, and system.slice's
|
||||
# weight is what competes with kubepods.slice (where Longhorn's instance-managers live).
|
||||
# Weighting only k3s.service — the intuitive thing to do — would arbitrate k3s against
|
||||
# sshd and journald and would not touch Longhorn at all.
|
||||
#
|
||||
# kubepods.slice is deliberately NOT given a drop-in of its own. It is a TRANSIENT unit
|
||||
# created by the kubelet through the systemd API (`systemctl cat kubepods.slice` on
|
||||
# k3s-prod-1 shows /run/systemd/transient/kubepods.slice), and whether systemd merges an
|
||||
# /etc drop-in into a transient unit is not something this change verified on these nodes.
|
||||
# Raising system.slice achieves the same ratio through a unit systemd definitely owns.
|
||||
# -------------------------------------------------------------------------------------
|
||||
- name: Ensure the systemd drop-in directories exist
|
||||
ansible.builtin.file:
|
||||
path: "/etc/systemd/system/{{ item.unit }}.d"
|
||||
state: directory
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0755"
|
||||
loop: "{{ etcd_io_priority_weighted_units }}"
|
||||
loop_control:
|
||||
label: "{{ item.unit }}"
|
||||
when: _eio_scheduler | length > 0
|
||||
|
||||
- name: Write the IOWeight drop-ins
|
||||
ansible.builtin.template:
|
||||
src: io-weight.conf.j2
|
||||
dest: "/etc/systemd/system/{{ item.unit }}.d/10-etcd-io-weight.conf"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
vars:
|
||||
_eio_dropin_unit: "{{ item.unit }}"
|
||||
_eio_dropin_section: "{{ item.section }}"
|
||||
loop: "{{ etcd_io_priority_weighted_units }}"
|
||||
loop_control:
|
||||
label: "{{ item.unit }}"
|
||||
when: _eio_scheduler | length > 0
|
||||
notify: Reload systemd for I/O weights
|
||||
|
||||
# Handlers must run BEFORE the verification below, or the verification reads the state from
|
||||
# before this run's changes and reports a stale pass.
|
||||
- name: Apply pending I/O changes before verifying them
|
||||
ansible.builtin.meta: flush_handlers
|
||||
|
||||
# -------------------------------------------------------------------------------------
|
||||
# VERIFY — the point of the role, not a formality.
|
||||
#
|
||||
# This repo has repeatedly shipped configuration that looked applied and was not (the
|
||||
# Keycloak reconciler that could not read its own Secret; Longhorn's orphanAutoDeletion,
|
||||
# silently dropped by the chart for two minor versions). Every mechanism here has a
|
||||
# plausible silent-failure mode: the scheduler module may not load; a systemd version may
|
||||
# rescale IOWeight differently; daemon-reload may not re-realize a running unit's cgroup.
|
||||
# So each is READ BACK OUT OF THE KERNEL and asserted. A failure here means the mitigation
|
||||
# is not in place — which is strictly better than believing that it is.
|
||||
# -------------------------------------------------------------------------------------
|
||||
- name: Read back the live block-layer state
|
||||
ansible.builtin.command:
|
||||
cmd: "cat /sys/block/{{ _eio_device }}/queue/scheduler /sys/block/{{ _eio_device }}/queue/nr_requests"
|
||||
register: _eio_queue_state
|
||||
changed_when: false
|
||||
check_mode: false
|
||||
when: _eio_scheduler | length > 0
|
||||
|
||||
- name: VERIFY the scheduler and queue depth actually took effect
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- "'[' ~ _eio_scheduler ~ ']' in _eio_queue_state.stdout_lines[0]"
|
||||
- "etcd_io_priority_nr_requests | string | length == 0
|
||||
or _eio_queue_state.stdout_lines[1] | trim | int == etcd_io_priority_nr_requests | int"
|
||||
fail_msg: >-
|
||||
/dev/{{ _eio_device }} did not take the requested settings. Wanted scheduler
|
||||
'{{ _eio_scheduler }}' and nr_requests '{{ etcd_io_priority_nr_requests }}'; the kernel
|
||||
reports {{ _eio_queue_state.stdout_lines | join(' / ') }}. The cgroup weights below are
|
||||
MEANINGLESS without the scheduler, so this is a hard failure, not a warning. Check
|
||||
`journalctl -u k3s-etcd-io-priority` and that bfq.ko exists for this kernel.
|
||||
success_msg: >-
|
||||
/dev/{{ _eio_device }}: scheduler {{ _eio_scheduler }}, nr_requests
|
||||
{{ _eio_queue_state.stdout_lines[1] | trim }}.
|
||||
when: _eio_scheduler | length > 0
|
||||
|
||||
# k3s.service's cgroup does not exist until k3s is running, so a cold-build run legitimately
|
||||
# has nothing to read there. Reported, not asserted away.
|
||||
- name: Check whether k3s.service has a live cgroup to verify
|
||||
ansible.builtin.stat:
|
||||
path: /sys/fs/cgroup/system.slice/k3s.service
|
||||
register: _eio_k3s_cgroup
|
||||
when: _eio_scheduler | length > 0
|
||||
|
||||
- name: Report that k3s is not running, so its own cgroup weight cannot be verified yet
|
||||
ansible.builtin.debug:
|
||||
msg: >-
|
||||
{{ inventory_hostname }}: /sys/fs/cgroup/system.slice/k3s.service does not exist — k3s
|
||||
is not running on this node, so its drop-in is written but UNVERIFIED. Re-run
|
||||
playbooks/etcd-io-priority.yml once k3s is up.
|
||||
when:
|
||||
- _eio_scheduler | length > 0
|
||||
- not (_eio_k3s_cgroup.stat.exists | default(false))
|
||||
|
||||
- name: Read back the live cgroup I/O weights
|
||||
ansible.builtin.command:
|
||||
cmd: "cat /sys/fs/cgroup/{{ item }}/io.bfq.weight"
|
||||
register: _eio_bfq_weights
|
||||
changed_when: false
|
||||
check_mode: false
|
||||
failed_when: false
|
||||
# `| default(false)`: Ansible templates `loop:` BEFORE evaluating `when:`, so on an NVMe
|
||||
# node — where the stat task above is skipped and _eio_k3s_cgroup has no `.stat` at all —
|
||||
# a bare `_eio_k3s_cgroup.stat.exists` raises even though this task will not run.
|
||||
loop: >-
|
||||
{{ ['system.slice']
|
||||
+ (['system.slice/k3s.service']
|
||||
if (_eio_k3s_cgroup.stat.exists | default(false)) else []) }}
|
||||
when: _eio_scheduler | length > 0
|
||||
|
||||
- name: VERIFY every live cgroup weight actually carries the boost
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- item.rc == 0
|
||||
- item.stdout | trim | int >= etcd_io_priority_min_bfq_weight | int
|
||||
fail_msg: >-
|
||||
/sys/fs/cgroup/{{ item.item }}/io.bfq.weight reads
|
||||
'{{ item.stdout | default('<unreadable>') | trim }}' (rc={{ item.rc }}), below the
|
||||
required floor of {{ etcd_io_priority_min_bfq_weight }}. IOWeight={{ etcd_io_priority_weight }}
|
||||
did NOT reach the kernel. Most likely causes, in order: (1) systemd rescaled the value
|
||||
differently from what defaults/main.yml assumes — compare
|
||||
`systemctl show {{ item.item | basename }} -p IOWeight` against the file; (2) the cgroup
|
||||
was not re-realized by daemon-reload, in which case a ROLLING `systemctl restart k3s`
|
||||
(one node at a time — etcd quorum is 2/3) applies it; (3) BFQ is not active on the
|
||||
device, which the previous assert should already have caught. The drop-in on disk is
|
||||
NOT the mitigation — this file is.
|
||||
success_msg: "{{ item.item }} io.bfq.weight = {{ item.stdout | default('') | trim }}"
|
||||
loop: "{{ _eio_bfq_weights.results | default([]) }}"
|
||||
loop_control:
|
||||
label: "{{ item.item | default('') }}"
|
||||
when: _eio_scheduler | length > 0
|
||||
@@ -0,0 +1,21 @@
|
||||
# {{ ansible_managed }}
|
||||
#
|
||||
# Proportional share of {{ _eio_device }} for {{ _eio_dropin_unit }}, which is where the k3s server
|
||||
# process — and therefore the EMBEDDED ETCD it hosts — does its WAL writes. Its competitor is
|
||||
# kubepods.slice (Longhorn's instance-managers), which stays at the default weight.
|
||||
#
|
||||
# THIS ONLY WORKS BECAUSE THE SCHEDULER IS {{ _eio_scheduler | upper }}. systemd writes IOWeight= to both
|
||||
# io.weight (blk-iocost, which is compiled in but DISABLED here — /sys/fs/cgroup/io.cost.qos is
|
||||
# empty) and io.bfq.weight (BFQ). Under the stock mq-deadline this file is a no-op; the sibling
|
||||
# k3s-etcd-io-priority.service is what makes it mean something. roles/etcd_io_priority verifies
|
||||
# io.bfq.weight after writing this and FAILS the play if the value did not land.
|
||||
#
|
||||
# NOT IOSchedulingClass=. I/O priority is inherited across fork and kept across exec, and
|
||||
# containerd is a child of the k3s server process — so realtime here would be realtime for every
|
||||
# container on the node, Longhorn included. See roles/etcd_io_priority/README.md.
|
||||
#
|
||||
# {{ etcd_io_priority_weight }} is systemd's MAXIMUM and that is deliberate: systemd rescales
|
||||
# 1..10000 onto io.bfq.weight's 1..1000, so IOWeight=1000 would map to ~101 — BFQ's unweighted
|
||||
# default is 100. Only the top of the range actually separates this cgroup from its siblings.
|
||||
[{{ _eio_dropin_section }}]
|
||||
IOWeight={{ etcd_io_priority_weight }}
|
||||
@@ -0,0 +1,24 @@
|
||||
# {{ ansible_managed }}
|
||||
#
|
||||
# WHY A ONESHOT UNIT AND NOT A UDEV RULE. The obvious form of this is
|
||||
# `ACTION=="add|change", KERNEL=="{{ _eio_device }}", ATTR{queue/scheduler}="{{ _eio_scheduler }}"`,
|
||||
# and it is unreliable for the ONE device that matters: {{ _eio_device }} is the boot disk, so its
|
||||
# `add` event fires in the initramfs, where /etc/udev/rules.d is not present and bfq.ko is not
|
||||
# loaded — the ATTR assignment silently does nothing and the disk comes up on mq-deadline with
|
||||
# no error anywhere. A unit ordered Before=k3s.service runs after modules are loadable, fails
|
||||
# LOUDLY if the scheduler is unavailable, and is visible in `systemctl status`.
|
||||
[Unit]
|
||||
Description=Block-layer I/O settings for the shared etcd + Longhorn device ({{ _eio_device }})
|
||||
Documentation=file:///etc/systemd/system/k3s-etcd-io-priority.service
|
||||
DefaultDependencies=no
|
||||
After=systemd-modules-load.service
|
||||
Before=k3s.service shutdown.target
|
||||
Conflicts=shutdown.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
ExecStart=/usr/local/sbin/k3s-etcd-io-priority
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,38 @@
|
||||
#!/bin/sh
|
||||
# {{ ansible_managed }}
|
||||
#
|
||||
# Block-layer settings for the device that carries BOTH etcd's WAL ({{ etcd_io_priority_k3s_data_dir }})
|
||||
# and every Longhorn replica on this node. Run at boot by k3s-etcd-io-priority.service, and
|
||||
# safe to run by hand at any time — it is idempotent and takes effect immediately.
|
||||
#
|
||||
# Rollback (immediate, no reboot):
|
||||
# printf 'mq-deadline\n' > /sys/block/{{ _eio_device }}/queue/scheduler
|
||||
# printf '64\n' > /sys/block/{{ _eio_device }}/queue/nr_requests
|
||||
# and `systemctl disable --now k3s-etcd-io-priority.service` so it does not come back.
|
||||
set -eu
|
||||
|
||||
DEV="{{ _eio_device }}"
|
||||
QUEUE="/sys/block/${DEV}/queue"
|
||||
|
||||
if [ ! -d "${QUEUE}" ]; then
|
||||
echo "k3s-etcd-io-priority: /sys/block/${DEV} does not exist — device renamed?" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
{% if _eio_scheduler %}
|
||||
# BFQ is a module here (CONFIG_IOSCHED_BFQ=m); loading it is what makes it selectable at all.
|
||||
modprobe -q {{ _eio_scheduler }} 2>/dev/null || true
|
||||
|
||||
if ! grep -qw '{{ _eio_scheduler }}' "${QUEUE}/scheduler"; then
|
||||
echo "k3s-etcd-io-priority: scheduler '{{ _eio_scheduler }}' not available for ${DEV}: $(cat "${QUEUE}/scheduler")" >&2
|
||||
exit 1
|
||||
fi
|
||||
printf '%s\n' '{{ _eio_scheduler }}' > "${QUEUE}/scheduler"
|
||||
{% endif %}
|
||||
|
||||
{% if etcd_io_priority_nr_requests %}
|
||||
# ORDER MATTERS: switching the elevator resets nr_requests, so this must come second.
|
||||
printf '%s\n' '{{ etcd_io_priority_nr_requests }}' > "${QUEUE}/nr_requests"
|
||||
{% endif %}
|
||||
|
||||
exit 0
|
||||
@@ -87,6 +87,49 @@ spec:
|
||||
summary: "Load per core high on {{ $labels.instance }}"
|
||||
description: "5-min load average per core above 1.0 for 15m on {{ $labels.instance }} — the node's run queue exceeds its core count."
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/node-cpu"
|
||||
# --- I/O PRESSURE (added after the 2026-09-07 k3s-prod-1 NotReady incident) ---------
|
||||
# THE METRIC THAT WOULD HAVE CALLED THE 2026-09-07 OUTAGE AND DID NOT EXIST.
|
||||
# At 06:05:21Z k3s-prod-1 went NotReady ("Kubelet stopped posting node status") with the
|
||||
# host up, pingable and SSH-able. Root cause was disk I/O starvation of the embedded etcd:
|
||||
# slow fdatasync took=3.56s/6.71s/4.95s/3.40s (etcd expects 1s)
|
||||
# apply request took too long took=8.30s (expects 100ms)
|
||||
# etcdserver: too many requests
|
||||
# `/proc/pressure/io` FULL avg300 read 11.36 on k3s-prod-1 at the time, against 0.75 on
|
||||
# k3s-prod-2 (NVMe) and 1.29 on k3s-prod-3. The stack already alerted on CPU-PSI and
|
||||
# memory but had NO I/O alert at all, so the first signal the operator got was a node
|
||||
# already out of the cluster.
|
||||
#
|
||||
# WHY "full" AND NOT "some": node_pressure_io_stalled_seconds_total is PSI *full* — time
|
||||
# in which EVERY runnable task was blocked on I/O, i.e. the machine did no useful work.
|
||||
# `some` (node_pressure_io_waiting_seconds_total) is noisy on a storage node; `full` is
|
||||
# the one that tracked the outage.
|
||||
#
|
||||
# THRESHOLDS ARE MEASURED, NOT GUESSED. rate(...[5m]) is a 0-1 fraction, so the incident's
|
||||
# 11.36% is 0.1136 and the healthy steady state measured on the same node is ~0.01:
|
||||
# warning 0.05 for 10m — 5x steady state, well below the outage, leaves room to act
|
||||
# critical 0.10 for 5m — the level k3s-prod-1 actually sat at while etcd was stalling
|
||||
# Deliberately NOT scoped to prod: dev shares this file and its k3s VMs are HDD-backed,
|
||||
# where the same alert is if anything more likely to earn its keep.
|
||||
- alert: NodeIOPressureHigh
|
||||
expr: rate(node_pressure_io_stalled_seconds_total[5m]) > 0.05
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Disk I/O pressure high on {{ $labels.instance }}"
|
||||
description: "I/O PSI (full) above 5% for 10m on {{ $labels.instance }} — every task on the node is spending real time blocked on disk. On a node whose etcd WAL shares a partition with Longhorn replicas this is the leading indicator of etcd fsync stalls. Check `cat /proc/pressure/io`, `iostat -x 1`, and whether Longhorn is rebuilding replicas."
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/node-io"
|
||||
- alert: NodeIOPressureCritical
|
||||
# 0.10 is the level k3s-prod-1 held while etcd logged multi-second fdatasync and then
|
||||
# dropped out of the cluster. At this point etcd is already at risk; do not wait 10m.
|
||||
expr: rate(node_pressure_io_stalled_seconds_total[5m]) > 0.10
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Disk I/O starvation on {{ $labels.instance }}"
|
||||
description: "I/O PSI (full) above 10% for 5m on {{ $labels.instance }} — the level at which k3s-prod-1 stalled etcd's WAL fsync and went NotReady on 2026-09-07 while the host stayed up. Expect etcd 'slow fdatasync' / 'apply request took too long' in `journalctl -u k3s`. Stop or throttle whatever is writing (Longhorn rebuild, backup, restore) before the node drops."
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/node-io"
|
||||
|
||||
# =====================================================================================
|
||||
# Storage — Longhorn
|
||||
|
||||
@@ -1,9 +1,17 @@
|
||||
---
|
||||
# Grafana dashboard for node CPU/memory PRESSURE (docs/10 — platform-health capacity view). A plain
|
||||
# Grafana dashboard for node CPU/memory/IO PRESSURE (docs/10 — platform-health capacity view). A plain
|
||||
# ConfigMap labeled `grafana_dashboard` that the kube-prometheus-stack Grafana sidecar auto-loads
|
||||
# (searchNamespace ALL). Surfaces PSI + load-per-core, which averaged CPU% (the default node
|
||||
# dashboards / kubectl top) hides — this is the view that reveals "spikes cause blips of delay".
|
||||
# Pairs with the NodeCPUPressureHigh / NodeCPULoadHigh alerts in alerts.yaml.
|
||||
# Pairs with the NodeCPUPressureHigh / NodeCPULoadHigh / NodeIOPressureHigh alerts in alerts.yaml.
|
||||
#
|
||||
# THE I/O ROW WAS ADDED AFTER 2026-09-07. k3s-prod-1 went NotReady that morning with the host
|
||||
# still up: its embedded etcd could not complete WAL fdatasync (3.5-6.7 s against a 1 s
|
||||
# expectation) because Longhorn replicas and etcd share one partition on a 2011 consumer SATA
|
||||
# SSD. `/proc/pressure/io` FULL avg300 was 11.36 on that node vs 0.75 (NVMe node 2) and 1.29
|
||||
# (node 3) — and this dashboard, like the alerts, plotted only CPU and memory. I/O PSI *full*
|
||||
# (every task blocked on disk) is the series that tracked the outage; `some` is too noisy on a
|
||||
# storage node to read. Both are plotted, `full` first.
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
@@ -14,7 +22,7 @@ metadata:
|
||||
data:
|
||||
node-pressure.json: |
|
||||
{
|
||||
"title": "Nodes — CPU / Memory Pressure",
|
||||
"title": "Nodes — CPU / Memory / I-O Pressure",
|
||||
"uid": "node-pressure",
|
||||
"schemaVersion": 39,
|
||||
"timezone": "browser",
|
||||
@@ -102,6 +110,43 @@ data:
|
||||
"targets": [
|
||||
{"expr": "rate(node_pressure_memory_waiting_seconds_total[5m]) * 100", "legendFormat": "{{instance}}"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"type": "stat",
|
||||
"title": "Max I/O pressure (PSI full, now)",
|
||||
"description": "Fraction of time EVERY task on the node was blocked on disk. Steady state on these nodes is ~1%. k3s-prod-1 sat at 11.4% while its etcd stalled and the node went NotReady on 2026-09-07.",
|
||||
"gridPos": {"h": 6, "w": 6, "x": 18, "y": 0},
|
||||
"fieldConfig": {"defaults": {"unit": "percent", "thresholds": {"steps": [
|
||||
{"color": "green", "value": null}, {"color": "yellow", "value": 5}, {"color": "red", "value": 10}
|
||||
]}}},
|
||||
"targets": [
|
||||
{"expr": "max(rate(node_pressure_io_stalled_seconds_total[5m])) * 100", "legendFormat": "max IO PSI full"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"type": "timeseries",
|
||||
"title": "I/O pressure (PSI full) per node — etcd stall risk",
|
||||
"description": "Thresholds match the NodeIOPressureHigh (5%) and NodeIOPressureCritical (10%) alerts. A node held above 10% is the shape that starved etcd's WAL fsync on 2026-09-07.",
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 22},
|
||||
"fieldConfig": {"defaults": {"unit": "percent", "custom": {"drawStyle": "line", "fillOpacity": 10}, "thresholds": {"mode": "absolute", "steps": [
|
||||
{"color": "green", "value": null}, {"color": "yellow", "value": 5}, {"color": "red", "value": 10}
|
||||
]}}},
|
||||
"targets": [
|
||||
{"expr": "rate(node_pressure_io_stalled_seconds_total[5m]) * 100", "legendFormat": "{{instance}}"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"type": "timeseries",
|
||||
"title": "I/O pressure (PSI some) per node",
|
||||
"description": "Time at least one task was blocked on disk. Always higher than `full`; useful for spotting which node carries the write load, not for alerting.",
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 22},
|
||||
"fieldConfig": {"defaults": {"unit": "percent", "custom": {"drawStyle": "line", "fillOpacity": 10}}},
|
||||
"targets": [
|
||||
{"expr": "rate(node_pressure_io_waiting_seconds_total[5m]) * 100", "legendFormat": "{{instance}}"}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in new issue
Block a user