home-network-k3s-impl/platform/oauth2-proxy/auth-visibility-alerts.promtool-test.yaml

---
# promtool unit tests for OAuth2ProxyGateUnobserved (platform/oauth2-proxy/auth-visibility-alerts.yaml).
#
# WHY. The rule is an `unless` between two aggregations over DIFFERENT metric families, and the
# failure modes of that shape are both silent: a label that does not survive `count by` makes the
# right-hand side match nothing (the alert fires for the scraped namespace too, i.e. permanently),
# and a right-hand side that over-matches makes it fire for nobody (i.e. inert). Neither is
# visible by reading the expression, and neither is visible on a live cluster where the answer
# happens to look plausible. So all four states are pinned: fires for an unscraped namespace,
# stays quiet for a scraped one, honours `for: 1h`, and STOPS once a scrape appears.
#
# HOW TO RUN. The one repo-wide harness, by the companion convention:
#   ./infra/devcontainer/dev.sh run bash -c 'platform/monitoring/tests/run.sh platform/oauth2-proxy/auth-visibility-alerts.promtool-test.yaml'
# promtool is pinned in infra/devcontainer/versions.env; never run it off the host PATH.
#
# THE SERIES BELOW ARE REAL SHAPES read off live prod Prometheus on 2026-09-30 before being
# copied here — kube_pod_container_info as kube-state-metrics emits it (job="kube-state-metrics",
# its own instance/pod labels, image/image_id/container_id) and `up` as the oauth2-proxy
# ServiceMonitor emits it (endpoint="metrics", port 44180). The point of using the real shapes is
# that `count by (namespace)` has to discard all of them; a test with tidy two-label series would
# pass while the real expression collapsed differently.
#
# ASSERTIONS USE `ALERTS` RATHER THAN exp_alerts: the description annotation is long prose and
# comparing it verbatim would couple this file to every wording change while proving nothing the
# ALERTS series does not already prove.
rule_files:
  - auth-visibility-alerts.rules.yaml
evaluation_interval: 1m

tests:
  # ---------------------------------------------------------------------------------------------
  # 1. THE STATE PROD IS ACTUALLY IN (2026-09-30): jackett runs a gate sidecar with no scrape;
  #    the shared oauth2-proxy namespace runs two that ARE scraped. Asserts the alert fires for
  #    jackett and NOT for oauth2-proxy, and that `for: 1h` is honoured (pending at 59m).
  - interval: 1m
    name: an unscraped gate namespace fires after 1h; the scraped one never does
    input_series:
      - series: 'kube_pod_container_info{container="oauth2-proxy",container_id="containerd://937ae6fc",endpoint="http",image="quay.io/oauth2-proxy/oauth2-proxy:v7.8.1",image_id="quay.io/oauth2-proxy/oauth2-proxy@sha256:d62e2d81",image_spec="quay.io/oauth2-proxy/oauth2-proxy:v7.8.1",instance="10.0.1.213:8080",job="kube-state-metrics",namespace="jackett",pod="jackett-c5bfcbfb8-c75f9",service="kube-prometheus-stack-kube-state-metrics",uid="5b57d48f-e7d3-4050-9cd1-16aaaa052996"}'
        values: '1x130'
      # The shared instance: kube-state-metrics sees both replicas...
      - series: 'kube_pod_container_info{container="oauth2-proxy",container_id="containerd://aa11bb22",endpoint="http",image="quay.io/oauth2-proxy/oauth2-proxy:v7.8.1",image_id="quay.io/oauth2-proxy/oauth2-proxy@sha256:d62e2d81",image_spec="quay.io/oauth2-proxy/oauth2-proxy:v7.8.1",instance="10.0.1.213:8080",job="kube-state-metrics",namespace="oauth2-proxy",pod="oauth2-proxy-7568c78795-cqz27",service="kube-prometheus-stack-kube-state-metrics",uid="1111aaaa-0000-0000-0000-000000000001"}'
        values: '1x130'
      - series: 'kube_pod_container_info{container="oauth2-proxy",container_id="containerd://cc33dd44",endpoint="http",image="quay.io/oauth2-proxy/oauth2-proxy:v7.8.1",image_id="quay.io/oauth2-proxy/oauth2-proxy@sha256:d62e2d81",image_spec="quay.io/oauth2-proxy/oauth2-proxy:v7.8.1",instance="10.0.1.213:8080",job="kube-state-metrics",namespace="oauth2-proxy",pod="oauth2-proxy-7568c78795-bscxg",service="kube-prometheus-stack-kube-state-metrics",uid="1111aaaa-0000-0000-0000-000000000002"}'
        values: '1x130'
      # ...and both replicas are genuinely scraped, so `up` carries container="oauth2-proxy" there.
      - series: 'up{container="oauth2-proxy",endpoint="metrics",instance="10.0.1.189:44180",job="oauth2-proxy",namespace="oauth2-proxy",pod="oauth2-proxy-7568c78795-cqz27",service="oauth2-proxy"}'
        values: '1x130'
      - series: 'up{container="oauth2-proxy",endpoint="metrics",instance="10.0.2.47:44180",job="oauth2-proxy",namespace="oauth2-proxy",pod="oauth2-proxy-7568c78795-bscxg",service="oauth2-proxy"}'
        values: '1x130'
    promql_expr_test:
      # Not yet firing at 59m — `for: 1h` has not elapsed. Guards against someone "simplifying"
      # the hold away and turning every rollout into a page.
      - expr: 'ALERTS{alertname="OAuth2ProxyGateUnobserved",alertstate="firing"}'
        eval_time: 59m
        exp_samples: []
      - expr: 'ALERTS{alertname="OAuth2ProxyGateUnobserved",alertstate="pending",namespace="jackett"}'
        eval_time: 59m
        exp_samples:
          - labels: 'ALERTS{alertname="OAuth2ProxyGateUnobserved",alertstate="pending",namespace="jackett",severity="warning"}'
            value: 1
      # Firing for jackett past the hold...
      - expr: 'ALERTS{alertname="OAuth2ProxyGateUnobserved",alertstate="firing"}'
        eval_time: 65m
        exp_samples:
          - labels: 'ALERTS{alertname="OAuth2ProxyGateUnobserved",alertstate="firing",namespace="jackett",severity="warning"}'
            value: 1
      # ...and never for the namespace that IS scraped. This is the assertion that catches an
      # `unless` whose right-hand side stopped matching: without it, a broken rule that fires for
      # every gate namespace would pass the test above.
      - expr: 'ALERTS{alertname="OAuth2ProxyGateUnobserved",namespace="oauth2-proxy"}'
        eval_time: 65m
        exp_samples: []
      # The count must be 1 per namespace-with-a-gate, not 1 overall: a `count by (namespace)`
      # that lost its `by` would collapse to a single unlabelled series.
      - expr: 'count by (namespace) (kube_pod_container_info{container="oauth2-proxy"}) unless count by (namespace) (up{container="oauth2-proxy"})'
        eval_time: 65m
        exp_samples:
          - labels: '{namespace="jackett"}'
            value: 1

  # ---------------------------------------------------------------------------------------------
  # 2. THE FIX LANDING. jackett's sidecar gains --metrics-address and a ServiceMonitor 70m in, so
  #    `up` starts carrying container="oauth2-proxy" for namespace="jackett" under a DIFFERENT job
  #    name than the shared proxy's. The alert must stop. This is what makes the rule a gap
  #    tracker rather than a permanent nag, and it pins that the match is on `container`, not on
  #    a hardcoded job="oauth2-proxy" (which would keep firing forever after a correct fix).
  - interval: 1m
    name: the alert clears once the sidecar is actually scraped
    input_series:
      - series: 'kube_pod_container_info{container="oauth2-proxy",container_id="containerd://937ae6fc",endpoint="http",image="quay.io/oauth2-proxy/oauth2-proxy:v7.8.1",image_id="quay.io/oauth2-proxy/oauth2-proxy@sha256:d62e2d81",image_spec="quay.io/oauth2-proxy/oauth2-proxy:v7.8.1",instance="10.0.1.213:8080",job="kube-state-metrics",namespace="jackett",pod="jackett-c5bfcbfb8-c75f9",service="kube-prometheus-stack-kube-state-metrics",uid="5b57d48f-e7d3-4050-9cd1-16aaaa052996"}'
        values: '1x130'
      # Absent for the first 70 minutes, then present. `_x` is "no sample"; the scrape did not
      # exist yet.
      - series: 'up{container="oauth2-proxy",endpoint="metrics",instance="10.0.1.213:44180",job="jackett-auth",namespace="jackett",pod="jackett-c5bfcbfb8-c75f9",service="jackett-auth-metrics"}'
        values: '_x70 1x60'
    promql_expr_test:
      - expr: 'ALERTS{alertname="OAuth2ProxyGateUnobserved",alertstate="firing",namespace="jackett"}'
        eval_time: 65m
        exp_samples:
          - labels: 'ALERTS{alertname="OAuth2ProxyGateUnobserved",alertstate="firing",namespace="jackett",severity="warning"}'
            value: 1
      - expr: 'ALERTS{alertname="OAuth2ProxyGateUnobserved",namespace="jackett"}'
        eval_time: 80m
        exp_samples: []

  # ---------------------------------------------------------------------------------------------
  # 3. THE REGRESSION GUARD FOR `== 1` — the scenario that exposed a real bug on 2026-10-01.
  #
  #    A target is DISCOVERED but every scrape FAILS: `up` exists with value 0. This is not
  #    hypothetical — it is the exact state podmonitor.yaml creates today, because the app
  #    namespaces' default-deny drops the scrape (measured; see podmonitor.yaml).
  #
  #    With the original `unless count by (namespace) (up{container="oauth2-proxy"})`, the `up`
  #    SERIES EXISTS, so `count by (namespace)` returns 1, `unless` removes the namespace, and the
  #    alert GOES QUIET while nothing is being scraped at all — a green signal standing in for an
  #    absent one, which is the precise disease this whole rule was written to cure. Deleting the
  #    `== 1` makes THIS test fail and every other test in this file still pass, which is the
  #    property that makes it worth having.
  - interval: 1m
    name: a discovered-but-FAILING target does not count as observed (up==0 must still fire)
    input_series:
      - series: 'kube_pod_container_info{container="oauth2-proxy",container_id="containerd://937ae6fc",endpoint="http",image="quay.io/oauth2-proxy/oauth2-proxy:v7.8.1",image_id="quay.io/oauth2-proxy/oauth2-proxy@sha256:d62e2d81",image_spec="quay.io/oauth2-proxy/oauth2-proxy:v7.8.1",instance="10.0.1.213:8080",job="kube-state-metrics",namespace="jackett",pod="jackett-6877fcd9f5-ht9d9",service="kube-prometheus-stack-kube-state-metrics",uid="5b57d48f-e7d3-4050-9cd1-16aaaa052996"}'
        values: '1x130'
      # The PodMonitor target exists and never succeeds: NetworkPolicy drops the packets, so the
      # scrape times out and `up` is published as 0 rather than going absent.
      - series: 'up{container="oauth2-proxy",endpoint="metrics",instance="10.0.0.189:44180",job="oauth2-proxy-sidecars",namespace="jackett",pod="jackett-6877fcd9f5-ht9d9"}'
        values: '0x130'
    promql_expr_test:
      - expr: 'ALERTS{alertname="OAuth2ProxyGateUnobserved",alertstate="firing",namespace="jackett"}'
        eval_time: 65m
        exp_samples:
          - labels: 'ALERTS{alertname="OAuth2ProxyGateUnobserved",alertstate="firing",namespace="jackett",severity="warning"}'
            value: 1
      # And the inner clause itself yields NOTHING while the scrape is failing — which is the
      # mechanism the alert depends on, asserted directly so a future reader can see why the
      # `== 1` matters without re-deriving it. (Scenario 4 below covers the other direction:
      # a succeeding scrape really does clear the alert, so `== 1` has not made the rule
      # impossible to satisfy.)
      - expr: 'count by (namespace) (up{container="oauth2-proxy"} == 1)'
        eval_time: 65m
        exp_samples: []

  # ---------------------------------------------------------------------------------------------
  # 4. A FLAPPING target: up goes 1 -> 0 and stays there. The alert must come BACK, not latch
  #    cleared. Covers the ordering bug where a once-observed namespace is permanently excluded.
  - interval: 1m
    name: a target that starts working and then fails re-fires
    input_series:
      - series: 'kube_pod_container_info{container="oauth2-proxy",container_id="containerd://937ae6fc",endpoint="http",image="quay.io/oauth2-proxy/oauth2-proxy:v7.8.1",image_id="quay.io/oauth2-proxy/oauth2-proxy@sha256:d62e2d81",image_spec="quay.io/oauth2-proxy/oauth2-proxy:v7.8.1",instance="10.0.1.213:8080",job="kube-state-metrics",namespace="radarr",pod="radarr-7d55cbffb9-42hmh",service="kube-prometheus-stack-kube-state-metrics",uid="3333cccc-0000-0000-0000-000000000001"}'
        values: '1x250'
      # Healthy for the first 60m, then every scrape fails for the rest.
      - series: 'up{container="oauth2-proxy",endpoint="metrics",instance="10.0.1.77:44180",job="oauth2-proxy-sidecars",namespace="radarr",pod="radarr-7d55cbffb9-42hmh"}'
        values: '1x60 0x180'
    promql_expr_test:
      # Quiet while it was working.
      - expr: 'ALERTS{alertname="OAuth2ProxyGateUnobserved",namespace="radarr"}'
        eval_time: 55m
        exp_samples: []
      # Firing again an hour after it broke.
      - expr: 'ALERTS{alertname="OAuth2ProxyGateUnobserved",alertstate="firing",namespace="radarr"}'
        eval_time: 125m
        exp_samples:
          - labels: 'ALERTS{alertname="OAuth2ProxyGateUnobserved",alertstate="firing",namespace="radarr",severity="warning"}'
            value: 1

  # ---------------------------------------------------------------------------------------------
  # 5. NO GATE, NO ALERT. A namespace with no oauth2-proxy container must never appear, or the
  #    rule would page for every app on the platform the day someone widened the matcher.
  - interval: 1m
    name: namespaces without a gate sidecar are never reported
    input_series:
      - series: 'kube_pod_container_info{container="plex",container_id="containerd://feed0001",endpoint="http",image="ghcr.io/example/plex:1",image_id="ghcr.io/example/plex@sha256:feed0001",image_spec="ghcr.io/example/plex:1",instance="10.0.1.213:8080",job="kube-state-metrics",namespace="plex",pod="plex-0",service="kube-prometheus-stack-kube-state-metrics",uid="2222bbbb-0000-0000-0000-000000000001"}'
        values: '1x130'
    promql_expr_test:
      - expr: 'ALERTS{alertname="OAuth2ProxyGateUnobserved"}'
        eval_time: 65m
        exp_samples: []