# promtool rule-unit tests for VpnIngressNodeAnnouncesL2VIP + VpnIngressNodeLabelMissing # (platform/monitoring/alerts.yaml) # # Run with: platform/monitoring/tests/run.sh # (the merge gate runs the same harness via # conformance/tests/l0-static/alert-rules-unit-tested) # # WHAT THESE RULES CLAIM, AND WHY IT NEEDS PINNING IN BOTH DIRECTIONS # MEASURED on prod 2026-10-08: a node cannot reach a LoadBalancer VIP whose Cilium L2 announcement # lease it holds ITSELF -- the announcer does not answer its own ARP request, so on the lease holder # the neighbour entry settles at `FAILED` and packets to that VIP die before leaving the node. # platform/vpn-ingress runs WireGuard on `hostNetwork` and MASQUERADEs remote clients to the node's # own address, so a VPN client takes that node's LAN path and inherits the failure. With the # router's udp/51820 forward on k3s-prod-1, which held the `shared` Gateway lease, all 16 Gateway # hostnames were dead over the VPN -- while PiHole DNS kept answering, because its VIP has two # Services and k3s-prod-2 held the other lease. The tunnel handshook, names resolved, every app # hung, and nothing reported it for ~2 days. # # platform-config/cilium/l2-announcement-policy.yaml now excludes the node labelled # `homelab.keithjackson.dev/vpn-ingress=true` from announcing, so VpnIngressNodeAnnouncesL2VIP # should be permanently unsatisfiable. THAT IS EXACTLY WHY IT NEEDS A TEST: an alert expected never # to fire is indistinguishable, in production, from an alert that CANNOT fire. Nothing would ever # reveal a mistyped label, a broken `label_replace`, or a join on the wrong label. Test 1 proves it # fires on the real failure; test 2 proves it is quiet in the fixed state, so it is not a rule that # fires on anything; test 3 pins the `for: 15m` against routine lease churn. # # THE JOIN IS THE FRAGILE PART AND IS TESTED DELIBERATELY. `kube_lease_owner` names the holder in # `lease_holder`; `kube_node_labels` names it `node`. The rule bridges them with `label_replace` # and joins `on (node) group_left ()`. Fixtures therefore carry BOTH series shapes with real label # sets, because a join that silently matches nothing returns no series -- which looks exactly like # "healthy" and is the failure mode this whole group exists to prevent. # # Fixture values are prod's real shape (measured 2026-10-08): leases # `cilium-l2announce-gateway-cilium-gateway-shared` (k3s-prod-1) and # `cilium-l2announce-pihole-pihole-dns-tcp` (k3s-prod-2), in namespace kube-system, with # k3s-prod-1 the vpn-ingress node. rule_files: - generated.rules.yaml evaluation_interval: 1m tests: # ============================================================================================= # 1. THE BUG, EXACTLY AS IT HAPPENED -- the vpn-ingress node holds the Gateway lease: FIRES. # ============================================================================================= - interval: 1m name: vpn-ingress node holds the shared Gateway L2 lease - VpnIngressNodeAnnouncesL2VIP fires input_series: # The collision. A lease holder keeps renewing, so this series does not lapse on its own -- # which is why the condition never self-heals and why a 15m `for:` is safe. - series: 'kube_lease_owner{job="kube-state-metrics",namespace="kube-system",lease="cilium-l2announce-gateway-cilium-gateway-shared",lease_holder="k3s-prod-1"}' values: '1+0x40' # A second lease held by a DIFFERENT node, present throughout. This is the discriminator that # stops the rule from being "any l2announce lease exists anywhere": if the join were dropped, # this series alone would fire the alert, and test 2 would fail. - series: 'kube_lease_owner{job="kube-state-metrics",namespace="kube-system",lease="cilium-l2announce-pihole-pihole-dns-tcp",lease_holder="k3s-prod-2"}' values: '1+0x40' # k3s-prod-1 is the vpn-ingress node. Only this node carries the label. - series: 'kube_node_labels{job="kube-state-metrics",node="k3s-prod-1",label_homelab_keithjackson_dev_vpn_ingress="true"}' values: '1+0x40' # The other two nodes publish kube_node_labels WITHOUT the vpn-ingress label. Present so the # join is exercised against non-matching nodes rather than against nothing at all. - series: 'kube_node_labels{job="kube-state-metrics",node="k3s-prod-2"}' values: '1+0x40' - series: 'kube_node_labels{job="kube-state-metrics",node="k3s-prod-3"}' values: '1+0x40' alert_rule_test: # Held 15m, so nothing before then. - eval_time: 14m alertname: VpnIngressNodeAnnouncesL2VIP exp_alerts: [] - eval_time: 20m alertname: VpnIngressNodeAnnouncesL2VIP exp_alerts: # EXACTLY ONE alert: the pihole lease on prod-2 must NOT produce a second one. - exp_labels: severity: critical lease: cilium-l2announce-gateway-cilium-gateway-shared exp_annotations: summary: "VPN ingress node is announcing a LoadBalancer VIP (cilium-l2announce-gateway-cilium-gateway-shared)" description: >- The node labelled homelab.keithjackson.dev/vpn-ingress=true terminates remote-access WireGuard, and it now holds the Cilium L2 announcement lease `cilium-l2announce-gateway-cilium-gateway-shared`. A node cannot reach a VIP it announces itself, so every remote VPN client has just lost all access to that VIP's Service -- silently, with DNS still resolving. Check that platform-config/cilium/l2-announcement-policy.yaml still excludes that label and that the label is still on the node the router forwards udp/51820 to. runbook_url: "https://${BASE_DOMAIN}/runbooks/vpn-ingress-l2-collision" # The label IS present, so the watchdog must stay quiet here. - eval_time: 40m alertname: VpnIngressNodeLabelMissing exp_alerts: [] # ============================================================================================= # 2. THE FIXED STATE -- every lease is held by a node that is NOT the vpn-ingress node: QUIET. # This is what l2-announcement-policy.yaml's exclusion produces, and the assertion that keeps # test 1 meaningful rather than an alert that fires on anything. # ============================================================================================= - interval: 1m name: all l2 leases held by other nodes - VpnIngressNodeAnnouncesL2VIP stays quiet input_series: - series: 'kube_lease_owner{job="kube-state-metrics",namespace="kube-system",lease="cilium-l2announce-gateway-cilium-gateway-shared",lease_holder="k3s-prod-2"}' values: '1+0x40' - series: 'kube_lease_owner{job="kube-state-metrics",namespace="kube-system",lease="cilium-l2announce-plex-plex-l4-tcp",lease_holder="k3s-prod-3"}' values: '1+0x40' # A lease the vpn-ingress node DOES legitimately hold, in a different family -- proves the # `cilium-l2announce-.*` matcher is doing work. Every node holds operator/controller leases; # only the ANNOUNCEMENT ones matter here, and a matcher broad enough to catch this would page # on a healthy cluster forever. - series: 'kube_lease_owner{job="kube-state-metrics",namespace="kube-system",lease="cilium-operator-resource-lock",lease_holder="k3s-prod-1"}' values: '1+0x40' - series: 'kube_node_labels{job="kube-state-metrics",node="k3s-prod-1",label_homelab_keithjackson_dev_vpn_ingress="true"}' values: '1+0x40' - series: 'kube_node_labels{job="kube-state-metrics",node="k3s-prod-2"}' values: '1+0x40' - series: 'kube_node_labels{job="kube-state-metrics",node="k3s-prod-3"}' values: '1+0x40' alert_rule_test: - eval_time: 40m alertname: VpnIngressNodeAnnouncesL2VIP exp_alerts: [] - eval_time: 40m alertname: VpnIngressNodeLabelMissing exp_alerts: [] # ============================================================================================= # 3. A LEGITIMATE, TRANSIENT LEASE MOVE -- the vpn-ingress node holds it for 11m during a node # reboot or a Cilium roll, then Cilium re-elects away: QUIET. This pins the `for: 15m` as # deliberate. Without it, every rolling restart of the cilium DaemonSet would page, and a paging # alert that cries wolf on routine maintenance gets muted -- which is how the real collision # would go unseen again. # ============================================================================================= - interval: 1m name: transient lease move shorter than for 15m - VpnIngressNodeAnnouncesL2VIP stays quiet input_series: # Held by the vpn-ingress node for 11 samples, then the series goes absent (the lease moved, # so kube-state-metrics retires this label set and starts a new one). - series: 'kube_lease_owner{job="kube-state-metrics",namespace="kube-system",lease="cilium-l2announce-gateway-cilium-gateway-shared",lease_holder="k3s-prod-1"}' values: '1+0x10 _x30' # Re-elected onto prod-3 from t=11m. A label VALUE change retires one series and starts # another, which is why this is a second series and not a changing value. - series: 'kube_lease_owner{job="kube-state-metrics",namespace="kube-system",lease="cilium-l2announce-gateway-cilium-gateway-shared",lease_holder="k3s-prod-3"}' values: '_x11 1+0x29' - series: 'kube_node_labels{job="kube-state-metrics",node="k3s-prod-1",label_homelab_keithjackson_dev_vpn_ingress="true"}' values: '1+0x40' alert_rule_test: # Inside the transient window: pending at most, never firing. - eval_time: 10m alertname: VpnIngressNodeAnnouncesL2VIP exp_alerts: [] # Well past it, with the lease settled on another node. - eval_time: 30m alertname: VpnIngressNodeAnnouncesL2VIP exp_alerts: [] # ============================================================================================= # 4. THE WATCHDOG -- no node carries the label at all. The policy's `NotIn` then excludes # NOTHING (a missing key matches), so the 2026-10-08 failure is fully re-armed, AND # VpnIngressNodeAnnouncesL2VIP is structurally INERT: its join matches nothing and returns no # series, which is indistinguishable from healthy. Both halves are asserted here, because an # inert critical alert is the exact shape this repo has been burned by before. # ============================================================================================= - interval: 1m name: vpn-ingress label absent everywhere - watchdog fires and the critical alert is inert input_series: # A lease held by the node that IS (in reality) the forwarded one -- but with no label # anywhere, nothing can tell. This is the dangerous state. - series: 'kube_lease_owner{job="kube-state-metrics",namespace="kube-system",lease="cilium-l2announce-gateway-cilium-gateway-shared",lease_holder="k3s-prod-1"}' values: '1+0x60' - series: 'kube_node_labels{job="kube-state-metrics",node="k3s-prod-1"}' values: '1+0x60' - series: 'kube_node_labels{job="kube-state-metrics",node="k3s-prod-2"}' values: '1+0x60' - series: 'kube_node_labels{job="kube-state-metrics",node="k3s-prod-3"}' values: '1+0x60' alert_rule_test: # The critical alert CANNOT fire -- this is the inertness being pinned, not an aside. - eval_time: 45m alertname: VpnIngressNodeAnnouncesL2VIP exp_alerts: [] # ... which is precisely why the watchdog must. - eval_time: 45m alertname: VpnIngressNodeLabelMissing exp_alerts: - exp_labels: severity: warning # absent() propagates the matcher's own labels onto the series it returns, so the # alert inherits this one. Asserted rather than worked around: it is harmless, and # pinning it means a future change to the matcher cannot alter the alert's identity # (and therefore its Alertmanager grouping) unnoticed. label_homelab_keithjackson_dev_vpn_ingress: "true" exp_annotations: summary: "No node carries the vpn-ingress label -- VPN VIP protection is OFF" description: >- No node publishes homelab.keithjackson.dev/vpn-ingress=true, so l2-announcement-policy.yaml's `NotIn` exclusion matches every node and excludes none: any node may now announce a LoadBalancer VIP, including the one terminating remote-access WireGuard. `VpnIngressNodeAnnouncesL2VIP` is INERT while this fires, so the warning matters more than its severity suggests. Re-run the node_baseline role on the forwarded node, and confirm the label is in kube-state-metrics' metricLabelsAllowlist. runbook_url: "https://${BASE_DOMAIN}/runbooks/vpn-ingress-l2-collision"