# promtool rule-unit tests for VPNEgressSteeringStale + VPNEgressUplinkLabelMissing
# (platform/monitoring/alerts.yaml)
#
# Run with: platform/monitoring/tests/run.sh
#           (the merge gate runs the same harness via
#            conformance/tests/l0-static/alert-rules-unit-tested)
#
# WHAT THESE RULES CLAIM, AND WHY IT NEEDS PINNING IN BOTH DIRECTIONS
# `egress-class=vpn` pods leave through one gateway node's tun0. Cilium resolves that interface to an
# IFINDEX and caches it; an OpenVPN reconnect destroys and re-creates tun0 with a NEW ifindex, and the
# gateway node's up-script publishes `homelab.keithjackson.dev/egress-uplink=<addr>-<ifindex>` so that
# node write forces Cilium to re-resolve. If the label does not follow the tunnel, the cache points at a
# destroyed interface and every vpn-class pod is dropped — while tun0 itself stays up, so VPNTunnelDown
# says nothing. That shape ran ~80 minutes undetected on the dev gateway.
#
# VPNEgressSteeringStale detects it by comparing HOW MANY TIMES tun0's ifindex changed in the last hour
# against HOW MANY DISTINCT `egress-uplink` label values appeared in the same hour. The subtlety worth
# testing is that a label VALUE change is not a sample change: it RETIRES one series and STARTS another,
# so "the label followed the tunnel" shows up as two overlapping series in the window, not as a
# changing value. Tests 1 and 2 are the same tunnel reconnect with and without that second series, so
# they isolate exactly that mechanism — anything that broke the series-counting half would make test 2
# fire, and anything that broke the tun0 half would make test 1 quiet.
#
# VPNEgressUplinkLabelMissing is the WATCHDOG for the rule above: with no uplink label anywhere, the
# staleness comparison has nothing to compare and is structurally INERT — present, evaluating, and
# incapable of firing. Tests 3 and 4 pin that, because an inert critical alert is the failure mode this
# whole group exists to prevent.
#
# Fixture values are prod's real shape: gateway k3s-prod-3, tun0 ifindex 139 with address
# 10.150.192.86 (live label value `10.150.192.86-139`), reconnecting to ifindex 140.

rule_files:
  - generated.rules.yaml

evaluation_interval: 1m

tests:
  # =============================================================================================
  # 1. THE BUG — tun0 was re-created, the label did NOT follow. Steering is stale: FIRES.
  # =============================================================================================
  - interval: 1m
    name: tunnel reconnect with no label update - VPNEgressSteeringStale fires
    input_series:
      # tun0 re-created at t=41m: ifindex 139 -> 140. One change inside the trailing hour.
      - series: 'node_network_iface_id{job="node-exporter",instance="192.168.2.12:9100",device="tun0"}'
        values: '139+0x40 140+0x40'
      # tun0 is UP and passing traffic the whole time — this is what makes the failure deceptive, and
      # it is asserted below: VPNTunnelDown stays QUIET while egress is actually broken.
      - series: 'node_network_receive_bytes_total{job="node-exporter",instance="192.168.2.12:9100",device="tun0"}'
        values: '0+120000x80'
      # The node label NEVER changes, so only ONE distinct uplink value exists in the window.
      - series: 'kube_node_labels{job="kube-state-metrics",node="k3s-prod-3",label_homelab_keithjackson_dev_egress_gateway="true",label_homelab_keithjackson_dev_egress_uplink="10.150.192.86-139"}'
        values: '1+0x80'
    alert_rule_test:
      # 1 tunnel change > (1 distinct label value - 1) = 0, held for 15m.
      - eval_time: 60m
        alertname: VPNEgressSteeringStale
        exp_alerts:
          - exp_labels:
              severity: critical
            exp_annotations:
              summary: "VPN egress steering is stale — vpn-class pods have no egress"
              description: >-
                The gateway node's tun0 was re-created without the `egress-uplink` label following it, so
                Cilium's cached egress ifindex points at a destroyed interface: egress-class=vpn pods are
                being dropped, silently, while tun0 still looks healthy. Compare `kubectl get nodes -L
                homelab.keithjackson.dev/egress-uplink` against `cilium-dbg bpf egress list` on the
                gateway node; re-applying the CiliumEgressGatewayPolicy (or setting any node label)
                forces a fresh resolution.
              runbook_url: "https://${BASE_DOMAIN}/runbooks/vpn-egress-stale"
      # Not yet: at 45m the condition has been true only ~4m, under the 15m `for`.
      - eval_time: 45m
        alertname: VPNEgressSteeringStale
        exp_alerts: []
      # No gateway label AND no tun0 history, so VPNTunnelDown's `or` guard is false: a cluster with
      # no VPN gateway must not be told its tunnel is down forever.
      - eval_time: 45m
        alertname: VPNTunnelDown
        exp_alerts: []
      # The watchdog must stay quiet — the label IS present here.
      - eval_time: 60m
        alertname: VPNEgressUplinkLabelMissing
        exp_alerts: []
      # THE HEADLINE CLAIM, asserted rather than asserted-in-prose: the tunnel alert says NOTHING
      # while vpn-class egress is completely broken. If VPNTunnelDown covered this failure, the
      # staleness rule would not need to exist.
      - eval_time: 60m
        alertname: VPNTunnelDown
        exp_alerts: []

  # =============================================================================================
  # 2. THE HEALTHY RECONNECT — same tunnel change, but the label followed it. STAYS QUIET.
  #    This is the half that proves the rule is not simply "any reconnect alerts".
  # =============================================================================================
  - interval: 1m
    name: tunnel reconnect WITH the label following - stays quiet
    input_series:
      - series: 'node_network_iface_id{job="node-exporter",instance="192.168.2.12:9100",device="tun0"}'
        values: '139+0x40 140+0x40'
      - series: 'node_network_receive_bytes_total{job="node-exporter",instance="192.168.2.12:9100",device="tun0"}'
        values: '0+120000x80'
      # A label VALUE change retires one series and starts another. Both are inside the trailing
      # hour, so the window holds TWO distinct uplink values: 2 - 1 = 1, which is NOT < 1 change.
      - series: 'kube_node_labels{job="kube-state-metrics",node="k3s-prod-3",label_homelab_keithjackson_dev_egress_gateway="true",label_homelab_keithjackson_dev_egress_uplink="10.150.192.86-139"}'
        values: '1+0x40 _x40'
      - series: 'kube_node_labels{job="kube-state-metrics",node="k3s-prod-3",label_homelab_keithjackson_dev_egress_gateway="true",label_homelab_keithjackson_dev_egress_uplink="10.150.192.86-140"}'
        values: '_x41 1+0x39'
    alert_rule_test:
      - eval_time: 60m
        alertname: VPNEgressSteeringStale
        exp_alerts: []
      - eval_time: 75m
        alertname: VPNEgressSteeringStale
        exp_alerts: []
      - eval_time: 60m
        alertname: VPNEgressUplinkLabelMissing
        exp_alerts: []
      - eval_time: 60m
        alertname: VPNTunnelDown
        exp_alerts: []

  # =============================================================================================
  # 3. THE INERT-ALERT CASE — a gateway node publishing NO uplink label at all.
  #    VPNEgressSteeringStale cannot fire in this state, which is exactly why this warning exists.
  # =============================================================================================
  - interval: 1m
    name: gateway node with no egress-uplink label - VPNEgressUplinkLabelMissing fires
    input_series:
      - series: 'kube_node_labels{job="kube-state-metrics",node="k3s-prod-3",label_homelab_keithjackson_dev_egress_gateway="true"}'
        values: '1+0x60'
      # tun0 is reconnecting the whole time. The staleness rule STILL cannot fire, because it has no
      # label series to compare against — that is the inertness being alerted on.
      - series: 'node_network_iface_id{job="node-exporter",instance="192.168.2.12:9100",device="tun0"}'
        values: '139+0x30 140+0x30'
      - series: 'node_network_receive_bytes_total{job="node-exporter",instance="192.168.2.12:9100",device="tun0"}'
        values: '0+120000x60'
    alert_rule_test:
      - eval_time: 35m
        alertname: VPNEgressUplinkLabelMissing
        exp_alerts:
          - exp_labels:
              severity: warning
            exp_annotations:
              summary: "Egress gateway node publishes no egress-uplink label"
              description: >-
                A node carries `homelab.keithjackson.dev/egress-gateway=true` but no
                `homelab.keithjackson.dev/egress-uplink` label is visible in kube-state-metrics. Without
                it nothing can tell whether Cilium's egress steering followed the last tunnel reconnect,
                and `VPNEgressSteeringStale` is inert. Check the label on the node, then the
                `vpn-egress-resteer` service/timer on the gateway node, then the kube-state-metrics
                metricLabelsAllowlist.
              runbook_url: "https://${BASE_DOMAIN}/runbooks/vpn-egress-stale"
      # Under the 30m `for`.
      - eval_time: 20m
        alertname: VPNEgressUplinkLabelMissing
        exp_alerts: []
      # And the proof of inertness: the critical rule is silent despite a real reconnect.
      - eval_time: 55m
        alertname: VPNEgressSteeringStale
        exp_alerts: []
      - eval_time: 55m
        alertname: VPNTunnelDown
        exp_alerts: []

  # =============================================================================================
  # 4. NO GATEWAY NODE AT ALL — neither rule may fire. Both are guarded by an `and on()` on the
  #    egress-gateway label so they stay silent on a cluster with no VPN gateway, rather than
  #    alerting forever about an absent feature.
  # =============================================================================================
  - interval: 1m
    name: no egress-gateway node - both rules silent
    input_series:
      - series: 'kube_node_labels{job="kube-state-metrics",node="k3s-prod-1"}'
        values: '1+0x60'
      - series: 'node_network_iface_id{job="node-exporter",instance="192.168.2.10:9100",device="eth0"}'
        values: '2+0x60'
    alert_rule_test:
      - eval_time: 45m
        alertname: VPNEgressUplinkLabelMissing
        exp_alerts: []
      - eval_time: 45m
        alertname: VPNEgressSteeringStale
        exp_alerts: []
      # No gateway label AND no tun0 history, so VPNTunnelDown's `or` guard is false: a cluster with
      # no VPN gateway must not be told its tunnel is down forever.
      - eval_time: 45m
        alertname: VPNTunnelDown
        exp_alerts: []