diff --git a/catalog/render.py b/catalog/render.py index 894d6f0..eb15ab3 100644 --- a/catalog/render.py +++ b/catalog/render.py @@ -156,11 +156,17 @@ OAUTH2_COOKIE_REFRESH = "2m30s" # ready endpoints. The flag is confirmed present in the PINNED v7.8.1 binary (`--help`), # which matters because the shared proxy is on v7.15.3 and could not have vouched for it. # -# THIS IS THE PREREQUISITE ONLY, AND IT IS NOT YET SCRAPED. A container port is not a -# Service port: these sidecars' Services expose the app port and :4180, and each namespace's -# default-deny baseline has `ingress: []`, so Prometheus can neither resolve nor reach this -# port until a Service port (or PodMonitor), a monitoring-ingress allow and a ServiceMonitor -# are added. Do not read this flag as "metrics are working". +# THE SCRAPE WORKS AS OF 2026-10-05 — but it took THREE independent pieces, and this flag is +# only the first. A container port is not a Service port, and each namespace's default-deny +# baseline denies all ingress, so the flag alone bought nothing for 63.7h: +# 1. this flag + a named `metrics` container port (PR #28) +# 2. a monitor that targets POD ports, not Service ports — platform/oauth2-proxy/podmonitor.yaml +# is a PodMonitor for exactly that reason (no Service exposes 44180) +# 3. an ingress allowance from ns monitoring (PR #49, r_monitoring_ingress_policy below) +# Verified on prod 2026-10-05: all 6 targets up == 1, scraping in 1-6ms. +# STILL do not read this flag alone as "metrics are working" — 1 without 3 is precisely the state +# that produced 5 x TargetDown for 63.7h. If you add a sidecar metrics port anywhere else, check +# that r_monitoring_ingress_policy covers it. OAUTH2_PROXY_METRICS_PORT = 44180 # --- Hardware capability: `hardware.gpu: intel-quicksync` ------------------------------ @@ -1362,8 +1368,9 @@ def _oauth2_sidecar_container(s: dict) -> dict: f"--cookie-refresh={OAUTH2_COOKIE_REFRESH}", "--pass-access-token=true", "--set-xauthrequest=true", - # Start the /metrics listener. Prerequisite only — not yet reachable by Prometheus; - # see OAUTH2_PROXY_METRICS_PORT above for exactly what is still missing. + # Start the /metrics listener. Scraped as of 2026-10-05, but only because + # r_monitoring_ingress_policy also opens this port to ns monitoring — the flag alone was + # inert for 63.7h. See OAUTH2_PROXY_METRICS_PORT above for the three pieces required. f"--metrics-address=0.0.0.0:{OAUTH2_PROXY_METRICS_PORT}", ] secret = f"{s['name']}-oauth2-proxy" diff --git a/decisions/2026-10-04-oauth2-sidecar-metrics-scrape-netpol.md b/decisions/2026-10-04-oauth2-sidecar-metrics-scrape-netpol.md index 5d7a83c..b217b88 100644 --- a/decisions/2026-10-04-oauth2-sidecar-metrics-scrape-netpol.md +++ b/decisions/2026-10-04-oauth2-sidecar-metrics-scrape-netpol.md @@ -4,6 +4,11 @@ branch: fix/oauth2-sidecar-metrics-scrape-netpol title: The forward-auth gate sidecars' /metrics was NetworkPolicy-blocked — the renderer now opens 44180 to ns monitoring --- +> **Added 2026-10-05, after merge:** the top item under *What I did NOT verify* below — "that the +> scrape now succeeds" — **has since been verified on prod**. All 6 targets read `up == 1`. The +> original text is left exactly as written; the measurements are in +> [[2026-10-05-oauth2-sidecar-scrape-verified]]. + ## What changed `catalog/render.py`'s `r_monitoring_ingress_policy` now emits its CiliumNetworkPolicy for a diff --git a/decisions/2026-10-05-oauth2-sidecar-scrape-verified.md b/decisions/2026-10-05-oauth2-sidecar-scrape-verified.md new file mode 100644 index 0000000..d7db097 --- /dev/null +++ b/decisions/2026-10-05-oauth2-sidecar-scrape-verified.md @@ -0,0 +1,90 @@ +--- +date: 2026-10-05 +branch: docs/oauth2-sidecar-scrape-verified +title: The gate-sidecar scrape is verified working on prod — and four places in the tree still said it wasn't +--- + +## What changed + +Documentation only. No manifest, no rendered output, no behaviour. + +PR #49 merged as `21c88b0` and the fix works, but four places in the tree still asserted the +*pre-fix* state as current — in the present tense, in capitals, in the files a reader would consult +first. This is the stale-documentation failure mode this repo has been burned by twice already +(`git push origin main` in the runbooks, "append to DECISIONS-LOG.md" in the old handoffs), so: + +- **`platform/oauth2-proxy/podmonitor.yaml`** — the banner read *"READ THIS BEFORE BELIEVING THE + TARGETS ARE UP: THE SCRAPE IS CURRENTLY DENIED BY NETWORKPOLICY"* and *"SO THE TARGETS BELOW WILL + READ `up == 0` UNTIL AN INGRESS ALLOWANCE LANDS"*. Both now false, and the first one actively + instructs the reader to disbelieve a correct `up == 1`. Replaced with a `STATUS: THE SCRAPE WORKS` + block carrying the post-fix numbers; the diagnosis is kept below it under an explicit `HISTORY` + heading, because it is why the fix is shaped the way it is. Added the drop fingerprint to + recognise if it regresses — `lastScrapeDuration` pinned at *exactly* the `scrapeTimeout` with + `context deadline exceeded`, versus 1–6 ms when healthy. +- **`catalog/render.py:159`** — *"THIS IS THE PREREQUISITE ONLY, AND IT IS NOT YET SCRAPED … Do not + read this flag as 'metrics are working'."* Rewritten to enumerate the three pieces a working + scrape needs and which PR landed each. The warning is **kept, deliberately**: the flag alone + really is insufficient, and "1 without 3" is exactly what caused the 63.7h outage. Only the + "not yet scraped" claim was false. +- **`catalog/render.py:1365`** — the sidecar arg comment, same correction, one line. +- **`platform/oauth2-proxy/README.md`** — its two observability subsections are *dated* + ("measured 2026-09-30", "Update 2026-10-01"), so they are honest records rather than lies. But a + reader hits the stale body long before any correction, so a `> SUPERSEDED` pointer now sits + directly under the heading, and an `### Update 2026-10-05` section at the end carries the + before/after table, the TSDB series counts and the alert outcome. + +Two substantive errors fixed while there, not just tense: + +1. The README's *"To close the gap for an app, three things are needed"* listed **a Service port + and a ServiceMonitor**. Wrong — a PodMonitor targets pod ports directly, which is the whole + reason no Service exposes 44180 and the renderer needed no Service change. Following that + paragraph would have sent someone to edit `catalog/render.py`'s Service rendering for no reason. +2. `podmonitor.yaml` ended by deferring the fix to *"the catalog/renderer, which is in flight on + another branch"* — that branch does not exist and never did; the work happened in PR #49. It now + names `r_monitoring_ingress_policy` directly, and a new `CRAFTY` section explains why crafty is + the one namespace this file cannot help. + +## What I verified + +- **The fix itself, on prod 2026-10-05, read-only** (this is the evidence the docs now quote): + Flux applied `21c88b0` through the real path — `apps-wave-0` `lastAppliedRevision` at 21:37:01Z, + `apps-wave-1` at 21:37:46Z, in dependency order — and all five `-allow-monitoring-scrape` + CNPs exist, each `ports=['44180']`, `from=[['monitoring']]`. +- **All 6 targets `up == 1`**, `health=up`, `lastError=''`, `lastScrapeDuration` 0.001–0.006s — + against 10.0s and `context deadline exceeded` before. +- **`oauth2_proxy_*` is in the TSDB for these namespaces for the first time:** 5 metric names, + 148 series; `sum by (namespace) (oauth2_proxy_requests_total)` = qbittorrent 35072, radarr 23560, + sonarr 1284. (Queried with `count by (__name__)` plus a matcher, not label-values.) +- **Alerts:** all 5 `TargetDown` and 5 of 6 `OAuth2ProxyGateUnobserved` resolved; one series + remains, `OAuth2ProxyGateUnobserved{namespace="crafty"}`, firing. +- **The rendered output is byte-identical.** `catalog/render.py` changed only in comments, so this + branch touches no `apps/` file and no golden. Confirmed by `render-golden` and `l1` both passing + with no regeneration, and by `git status` showing nothing under `apps/`. +- **This branch touches no sensitive path**, unlike PR #49: `platform/oauth2-proxy/**`, + `catalog/**` and `decisions/**` are none of them in `infra/gitea/approval-policy.json`. So it + should merge on a green gate with no human approval — the opposite of the PR it documents, and + the reason the two were kept apart. +- `render.py --selftest`: PASS. +- Toolbox gate: see the handoff's own rule — reported below rather than claimed here. + +## What I did NOT verify + +- **I did not re-measure the 63.7h figure or the ~22 emails/day rate**; both are carried forward + from the 2026-10-04 measurements, now quoted inside the docs as historical. The 15-day + `query_range` that established "one episode, not flapping" was run once, on 2026-10-04. +- **`oauth2_proxy_requests_total` is still absent for jackett and pihole.** I assert in the README + that this is because the counter is not created until the gate serves its first request. That is + the documented behaviour of a Prometheus counter with no initialisation, and it is consistent + with the other three namespaces having traffic — but I did not force a request through jackett's + gate and watch the series appear. If it is still absent after real use, that is a finding. +- **The auth-churn rule is still unwritten.** The README now says it is "writable rather than + decorative"; that is a claim about the metrics being present, not a rule anyone has authored or + unit-tested. The thing the 2026-09-30 incident actually needed does not exist yet. +- **crafty remains unobserved.** I read its container spec on 2026-10-05 and confirmed no + `--metrics-address`; I did not attempt the fix, and nothing here changes its alert. +- **I did not audit the whole tree for other stale "metrics are not scraped" claims** beyond the + four sites a targeted grep found (`not yet reachable`, `UNTIL AN INGRESS`, `up == 0`, + `not reachable by Prometheus` and neighbours). `runbooks/oauth2-proxy-auth-failures.md` is + described in the README as "the by-hand substitute meanwhile" and I left it alone — the by-hand + log procedure still works and is not wrong, but whether it should now be rewritten against the + new metrics is unexamined. diff --git a/platform/oauth2-proxy/README.md b/platform/oauth2-proxy/README.md index 66a0ec6..b76c27d 100644 --- a/platform/oauth2-proxy/README.md +++ b/platform/oauth2-proxy/README.md @@ -91,6 +91,10 @@ This proves the R-IDP-02 property (unauthenticated cannot reach the upstream) on ## Observability: what this component can and cannot see (measured 2026-09-30) +> **SUPERSEDED — read "Update 2026-10-05" at the end of this section first.** The gate sidecars +> **are** scraped now and the TSDB **does** hold `oauth2_proxy_*` for them. The two subsections +> below are kept as the dated record of how this was diagnosed, not as current state. + `servicemonitor.yaml` scrapes **this namespace's** deployment and nothing else. On prod that is two targets, `job="oauth2-proxy"`, both `up=1`. The complete set of oauth2-proxy-specific series in the TSDB is: @@ -157,3 +161,55 @@ present `up` series. A discovered-but-failing target publishes `up` with value 0 would have cleared `OAuth2ProxyGateUnobserved` the moment the PodMonitor landed — reporting the gates as observed while nothing was being scraped. The unit test pins that scenario; deleting the `== 1` fails it and nothing else. + +### Update 2026-10-05: the scrape works + +PR #49 (merged `21c88b0`) added the missing third piece. `catalog/render.py`'s +`r_monitoring_ingress_policy` now also fires for the gate sidecar's metrics port, keyed on +`_needs_oauth2_sidecar`, rendering `-allow-monitoring-scrape` per app: ns `monitoring` → that +app's `endpointSelector` → TCP 44180 only. Five objects, one each for jackett, pihole, qbittorrent, +radarr, sonarr. + +Measured on prod 2026-10-05 after Flux applied it (`apps-wave-0` 21:37:01Z, `apps-wave-1` +21:37:46Z): + +| | before | after | +|---|---|---| +| target health (6) | `down` | `up` | +| `lastError` | `context deadline exceeded` | `''` | +| `lastScrapeDuration` | **10.0s** — exactly the `scrapeTimeout` | **0.001–0.006s** | +| `up` | `0` × 6 | `1` × 6 | + + count by (__name__) ({__name__=~"oauth2_proxy_.*", + namespace=~"jackett|radarr|sonarr|qbittorrent|pihole"}) + oauth2_proxy_requests_in_flight => 6 + oauth2_proxy_requests_total => 16 + oauth2_proxy_response_duration_seconds_bucket => 108 + oauth2_proxy_response_duration_seconds_sum => 9 + oauth2_proxy_response_duration_seconds_count => 9 + + sum by (namespace) (oauth2_proxy_requests_total) + qbittorrent => 35072 radarr => 23560 sonarr => 1284 + +`oauth2_proxy_requests_total` is absent for jackett and pihole simply because that counter is not +created until the gate has served its first request. + +**So the auth-churn rule is now writable rather than decorative** — sustained `code="302"` as a +share of `oauth2_proxy_requests_total`, held across a window longer than the session lifetime. That +is the rule the 2026-09-30 incident above could not be caught by. It is still unwritten. + +**Alert outcome:** all 5 `TargetDown` resolved, and 5 of 6 `OAuth2ProxyGateUnobserved` with them. +At `repeat_interval: 12h` with `group_by: [alertname, namespace]`, those 11 groups had been +re-notifying at roughly **22 emails/day** since 2026-10-02; that drops to 2/day. + +**`crafty` still fires, and that is correct.** It is a hand-maintained Deployment under +`platform/crafty`, outside the renderer, whose `oauth2-proxy` container has **no +`--metrics-address` at all** — no listener, so no port to open and nothing for the PodMonitor's +pod-label selector to match. Closing it needs the flag and a `metrics` containerPort on crafty's +own manifest first, then a hand-written allowance beside it. Open follow-up work. + +**The paragraph above headed "To close the gap for an app" is superseded**: a *Service* port and a +ServiceMonitor are **not** required. A PodMonitor targets pod ports directly, which is why no +Service exposes 44180 and why the renderer needed no Service change. The three things actually +needed are `--metrics-address` + named container port, a PodMonitor, and the monitoring-ingress +allowance. diff --git a/platform/oauth2-proxy/podmonitor.yaml b/platform/oauth2-proxy/podmonitor.yaml index f3e2247..320ac35 100644 --- a/platform/oauth2-proxy/podmonitor.yaml +++ b/platform/oauth2-proxy/podmonitor.yaml @@ -27,10 +27,28 @@ # created by the apps-wave-0/1 tiers, which this tier does not and must not depend on. # # ============================================================================================ -# READ THIS BEFORE BELIEVING THE TARGETS ARE UP: THE SCRAPE IS CURRENTLY DENIED BY NETWORKPOLICY +# STATUS: THE SCRAPE WORKS. Verified on prod 2026-10-05 (PR #49 merged as 21c88b0). # ============================================================================================ -# This PodMonitor is NECESSARY AND NOT SUFFICIENT. The endpoint serves, and the path from ns -# monitoring is blocked. Both halves measured on prod 2026-10-01, read-only: +# All 6 targets in pool podMonitor/oauth2-proxy/oauth2-proxy-sidecars/0 read `up == 1`, +# health=up, lastError='', lastScrapeDuration 0.001-0.006s. The TSDB holds oauth2_proxy_* for +# these namespaces for the first time: 5 metric names / 148 series, with live counters +# (oauth2_proxy_requests_total 35072 qbittorrent, 23560 radarr, 1284 sonarr). All 5 TargetDown +# and 5 of 6 OAuth2ProxyGateUnobserved resolved; only namespace="crafty" still fires, and that +# is correct — see CRAFTY below. +# +# The missing piece was an ingress allowance, and it now ships WITH EACH APP, rendered by +# catalog/render.py's r_monitoring_ingress_policy as `-allow-monitoring-scrape`: +# ns monitoring -> the app's endpointSelector -> TCP 44180 only. See WHY NOT IN THIS DIRECTORY +# below for why it is rendered per-app instead of shipped from this tier. +# +# Keep the history below. It is why the fix is shaped the way it is, and `up == 1` in +# auth-visibility-alerts.yaml is load-bearing precisely because of it. +# +# -------------------------------------------------------------------------------------------- +# HISTORY: 2026-10-02 -> 2026-10-05, THE SCRAPE WAS DENIED BY NETWORKPOLICY FOR 63.7h +# -------------------------------------------------------------------------------------------- +# This PodMonitor was NECESSARY AND NOT SUFFICIENT: the endpoint served, and the path from ns +# monitoring was blocked. Both halves measured on prod 2026-10-01, read-only: # # THE ENDPOINT IS HEALTHY — port-forward straight to the jackett pod's 44180: # GET /metrics -> http=200 bytes=10208 @@ -54,19 +72,24 @@ # `jackett-allow-gateway-ingress`: `fromEntities: [ingress]` to port 4180 — the Gateway's Envoy # and nothing else. Prometheus matches no clause and its packets are dropped, not refused. # -# SO THE TARGETS BELOW WILL READ `up == 0` UNTIL AN INGRESS ALLOWANCE LANDS. That is a known, -# measured, documented state — NOT a regression introduced here, and NOT something to silence. -# It is deliberately left visible: `OAuth2ProxyGateUnobserved` in auth-visibility-alerts.yaml +# THE TARGETS READ `up == 0` FROM 2026-10-02 05:09 UTC UNTIL THE ALLOWANCE LANDED — one +# continuous 63.7h episode (a 15d query_range shows 1 episode per namespace, not flapping), +# starting 11 minutes after this file was created, i.e. TargetDown's `for: 10m`. It was never +# silenced, and that was the right call: `OAuth2ProxyGateUnobserved` in auth-visibility-alerts.yaml # requires `up == 1`, precisely so a discovered-but-failing target does not read as "observed" # (see that file's header — this exact trap is why the `== 1` is there). # +# The signature to recognise if it ever returns: lastScrapeDuration pinned at EXACTLY the +# scrapeTimeout (10.0s) with `context deadline exceeded`. That is a DROP, not a refusal — a +# refused connection fails in milliseconds. Post-fix the same targets scrape in 1-6ms. +# # THIS IS THE SAME BUG LONGHORN HAD, and the platform has a documented fix idiom for it: # platform/longhorn/networkpolicy-metrics.yaml, whose header records the identical fingerprint # measured on 2026-09-13 — ServiceMonitor in place, 3 targets DISCOVERED, `up` 0/0/0, TSDB empty, # `lastError: context deadline exceeded` (a drop, not a refusal). The fix there is a narrow # additive `networking.k8s.io/v1` NetworkPolicy: ns monitoring -> the pod label, that one port. # -# WHY THAT POLICY IS NOT IN THIS DIRECTORY. It would have to be five objects, one per app +# WHY NOT IN THIS DIRECTORY (the reasoning the fix followed). It would have to be five objects, one per app # namespace, and those namespaces are created by apps-wave-0/1. Measured Flux graph, 2026-10-01: # oauth2-proxy dependsOn=['platform-config'] <- this tier # apps-wave-0 dependsOn=['platform-config'] <- SIBLING, no ordering between them @@ -76,8 +99,19 @@ # running cluster and break on a rebuild — the "works now, broken on restore" shape this repo # keeps paying for. Longhorn's precedent is safe only because longhorn-system is created by the # same tier that ships its policy. The allowance therefore belongs WITH THE APPS, rendered next to -# the rest of each app's network policy — i.e. in the catalog/renderer, which is in flight on -# another branch. Tracked in the handoff; it is one narrow rule per app, not a redesign. +# the rest of each app's network policy — i.e. in the catalog/renderer. THAT IS WHERE IT LANDED: +# `r_monitoring_ingress_policy`, keyed on `_needs_oauth2_sidecar`, one narrow rule per app. +# +# -------------------------------------------------------------------------------------------- +# CRAFTY: STILL UNOBSERVED, AND THIS FILE CANNOT HELP IT +# -------------------------------------------------------------------------------------------- +# `OAuth2ProxyGateUnobserved{namespace="crafty"}` still fires after the fix, correctly. crafty is +# a hand-maintained Deployment under platform/crafty, OUTSIDE the renderer, and its oauth2-proxy +# container has NO `--metrics-address` at all (measured 2026-10-05) — so its metrics listener +# never starts and there is no port to open. The selector below therefore does not match it, and +# the per-app allowance is never rendered for it. Closing crafty's gap needs the flag plus a +# `metrics` containerPort added to its own manifest FIRST; the allowance is the second half, and +# it would have to be written by hand there rather than rendered. Open follow-up work. apiVersion: monitoring.coreos.com/v1 kind: PodMonitor metadata: