From 81f627167d0269bd9e9b9949a4aba5aa42476ca4 Mon Sep 17 00:00:00 2001 From: Keith Jackson Date: Mon, 5 Oct 2026 21:18:33 -0500 Subject: [PATCH 1/2] =?UTF-8?q?docs(obs):=20record=20that=20the=20gate-sid?= =?UTF-8?q?ecar=20scrape=20works=20=E2=80=94=20four=20places=20still=20sai?= =?UTF-8?q?d=20it=20did=20not?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #49 (merged 21c88b0) fixed the blocked scrape and it is verified working on prod 2026-10-05: all 6 targets up == 1, health=up, lastError='', lastScrapeDuration 0.001-0.006s against 10.0s and "context deadline exceeded" before. oauth2_proxy_* is in the TSDB for these namespaces for the first time (5 metric names, 148 series). Four places in the tree still asserted the pre-fix state as current, in the present tense, in the files a reader consults first: platform/oauth2-proxy/podmonitor.yaml "READ THIS BEFORE BELIEVING THE TARGETS ARE UP: THE SCRAPE IS CURRENTLY DENIED BY NETWORKPOLICY" and "SO THE TARGETS BELOW WILL READ up == 0 UNTIL AN INGRESS ALLOWANCE LANDS". The first actively instructs the reader to disbelieve a correct up == 1. Now a STATUS block with the post-fix numbers, with the diagnosis kept below it under an explicit HISTORY heading because it is why the fix is shaped the way it is. catalog/render.py:159 "THIS IS THE PREREQUISITE ONLY, AND IT IS NOT YET SCRAPED". catalog/render.py:1365 the same claim on the sidecar arg. platform/oauth2-proxy/README.md its subsections are dated and so are honest records, but a reader hits the stale body long before any correction. A SUPERSEDED pointer now sits under the heading and an Update 2026-10-05 section carries the before/after. The "not scraped" claims are corrected; the "a flag alone is not enough" warning is KEPT deliberately, because that part is still true and "flag without ingress allowance" is exactly what caused the 63.7h outage. Two substantive errors fixed while there, not just tense. The README's "three things are needed" listed a Service port and a ServiceMonitor — wrong, a PodMonitor targets pod ports directly, which is why no Service exposes 44180 and the renderer needed no Service change; following it would have sent someone to edit Service rendering for nothing. And podmonitor.yaml deferred the fix to "the catalog/renderer, in flight on another branch" — no such branch existed; it now names r_monitoring_ingress_policy, with a new CRAFTY section explaining the one namespace this file cannot help. Comments only in render.py, so the rendered output is byte-identical: no apps/ file and no golden changes, and therefore no sensitive path and no human approval needed — the opposite of the PR this documents. Co-Authored-By: Claude Opus 5 (1M context) --- catalog/render.py | 21 +++-- ...04-oauth2-sidecar-metrics-scrape-netpol.md | 5 ++ ...26-10-05-oauth2-sidecar-scrape-verified.md | 90 +++++++++++++++++++ platform/oauth2-proxy/README.md | 56 ++++++++++++ platform/oauth2-proxy/podmonitor.yaml | 52 +++++++++-- 5 files changed, 208 insertions(+), 16 deletions(-) create mode 100644 decisions/2026-10-05-oauth2-sidecar-scrape-verified.md diff --git a/catalog/render.py b/catalog/render.py index 894d6f0..eb15ab3 100644 --- a/catalog/render.py +++ b/catalog/render.py @@ -156,11 +156,17 @@ OAUTH2_COOKIE_REFRESH = "2m30s" # ready endpoints. The flag is confirmed present in the PINNED v7.8.1 binary (`--help`), # which matters because the shared proxy is on v7.15.3 and could not have vouched for it. # -# THIS IS THE PREREQUISITE ONLY, AND IT IS NOT YET SCRAPED. A container port is not a -# Service port: these sidecars' Services expose the app port and :4180, and each namespace's -# default-deny baseline has `ingress: []`, so Prometheus can neither resolve nor reach this -# port until a Service port (or PodMonitor), a monitoring-ingress allow and a ServiceMonitor -# are added. Do not read this flag as "metrics are working". +# THE SCRAPE WORKS AS OF 2026-10-05 — but it took THREE independent pieces, and this flag is +# only the first. A container port is not a Service port, and each namespace's default-deny +# baseline denies all ingress, so the flag alone bought nothing for 63.7h: +# 1. this flag + a named `metrics` container port (PR #28) +# 2. a monitor that targets POD ports, not Service ports — platform/oauth2-proxy/podmonitor.yaml +# is a PodMonitor for exactly that reason (no Service exposes 44180) +# 3. an ingress allowance from ns monitoring (PR #49, r_monitoring_ingress_policy below) +# Verified on prod 2026-10-05: all 6 targets up == 1, scraping in 1-6ms. +# STILL do not read this flag alone as "metrics are working" — 1 without 3 is precisely the state +# that produced 5 x TargetDown for 63.7h. If you add a sidecar metrics port anywhere else, check +# that r_monitoring_ingress_policy covers it. OAUTH2_PROXY_METRICS_PORT = 44180 # --- Hardware capability: `hardware.gpu: intel-quicksync` ------------------------------ @@ -1362,8 +1368,9 @@ def _oauth2_sidecar_container(s: dict) -> dict: f"--cookie-refresh={OAUTH2_COOKIE_REFRESH}", "--pass-access-token=true", "--set-xauthrequest=true", - # Start the /metrics listener. Prerequisite only — not yet reachable by Prometheus; - # see OAUTH2_PROXY_METRICS_PORT above for exactly what is still missing. + # Start the /metrics listener. Scraped as of 2026-10-05, but only because + # r_monitoring_ingress_policy also opens this port to ns monitoring — the flag alone was + # inert for 63.7h. See OAUTH2_PROXY_METRICS_PORT above for the three pieces required. f"--metrics-address=0.0.0.0:{OAUTH2_PROXY_METRICS_PORT}", ] secret = f"{s['name']}-oauth2-proxy" diff --git a/decisions/2026-10-04-oauth2-sidecar-metrics-scrape-netpol.md b/decisions/2026-10-04-oauth2-sidecar-metrics-scrape-netpol.md index 5d7a83c..b217b88 100644 --- a/decisions/2026-10-04-oauth2-sidecar-metrics-scrape-netpol.md +++ b/decisions/2026-10-04-oauth2-sidecar-metrics-scrape-netpol.md @@ -4,6 +4,11 @@ branch: fix/oauth2-sidecar-metrics-scrape-netpol title: The forward-auth gate sidecars' /metrics was NetworkPolicy-blocked — the renderer now opens 44180 to ns monitoring --- +> **Added 2026-10-05, after merge:** the top item under *What I did NOT verify* below — "that the +> scrape now succeeds" — **has since been verified on prod**. All 6 targets read `up == 1`. The +> original text is left exactly as written; the measurements are in +> [[2026-10-05-oauth2-sidecar-scrape-verified]]. + ## What changed `catalog/render.py`'s `r_monitoring_ingress_policy` now emits its CiliumNetworkPolicy for a diff --git a/decisions/2026-10-05-oauth2-sidecar-scrape-verified.md b/decisions/2026-10-05-oauth2-sidecar-scrape-verified.md new file mode 100644 index 0000000..d7db097 --- /dev/null +++ b/decisions/2026-10-05-oauth2-sidecar-scrape-verified.md @@ -0,0 +1,90 @@ +--- +date: 2026-10-05 +branch: docs/oauth2-sidecar-scrape-verified +title: The gate-sidecar scrape is verified working on prod — and four places in the tree still said it wasn't +--- + +## What changed + +Documentation only. No manifest, no rendered output, no behaviour. + +PR #49 merged as `21c88b0` and the fix works, but four places in the tree still asserted the +*pre-fix* state as current — in the present tense, in capitals, in the files a reader would consult +first. This is the stale-documentation failure mode this repo has been burned by twice already +(`git push origin main` in the runbooks, "append to DECISIONS-LOG.md" in the old handoffs), so: + +- **`platform/oauth2-proxy/podmonitor.yaml`** — the banner read *"READ THIS BEFORE BELIEVING THE + TARGETS ARE UP: THE SCRAPE IS CURRENTLY DENIED BY NETWORKPOLICY"* and *"SO THE TARGETS BELOW WILL + READ `up == 0` UNTIL AN INGRESS ALLOWANCE LANDS"*. Both now false, and the first one actively + instructs the reader to disbelieve a correct `up == 1`. Replaced with a `STATUS: THE SCRAPE WORKS` + block carrying the post-fix numbers; the diagnosis is kept below it under an explicit `HISTORY` + heading, because it is why the fix is shaped the way it is. Added the drop fingerprint to + recognise if it regresses — `lastScrapeDuration` pinned at *exactly* the `scrapeTimeout` with + `context deadline exceeded`, versus 1–6 ms when healthy. +- **`catalog/render.py:159`** — *"THIS IS THE PREREQUISITE ONLY, AND IT IS NOT YET SCRAPED … Do not + read this flag as 'metrics are working'."* Rewritten to enumerate the three pieces a working + scrape needs and which PR landed each. The warning is **kept, deliberately**: the flag alone + really is insufficient, and "1 without 3" is exactly what caused the 63.7h outage. Only the + "not yet scraped" claim was false. +- **`catalog/render.py:1365`** — the sidecar arg comment, same correction, one line. +- **`platform/oauth2-proxy/README.md`** — its two observability subsections are *dated* + ("measured 2026-09-30", "Update 2026-10-01"), so they are honest records rather than lies. But a + reader hits the stale body long before any correction, so a `> SUPERSEDED` pointer now sits + directly under the heading, and an `### Update 2026-10-05` section at the end carries the + before/after table, the TSDB series counts and the alert outcome. + +Two substantive errors fixed while there, not just tense: + +1. The README's *"To close the gap for an app, three things are needed"* listed **a Service port + and a ServiceMonitor**. Wrong — a PodMonitor targets pod ports directly, which is the whole + reason no Service exposes 44180 and the renderer needed no Service change. Following that + paragraph would have sent someone to edit `catalog/render.py`'s Service rendering for no reason. +2. `podmonitor.yaml` ended by deferring the fix to *"the catalog/renderer, which is in flight on + another branch"* — that branch does not exist and never did; the work happened in PR #49. It now + names `r_monitoring_ingress_policy` directly, and a new `CRAFTY` section explains why crafty is + the one namespace this file cannot help. + +## What I verified + +- **The fix itself, on prod 2026-10-05, read-only** (this is the evidence the docs now quote): + Flux applied `21c88b0` through the real path — `apps-wave-0` `lastAppliedRevision` at 21:37:01Z, + `apps-wave-1` at 21:37:46Z, in dependency order — and all five `-allow-monitoring-scrape` + CNPs exist, each `ports=['44180']`, `from=[['monitoring']]`. +- **All 6 targets `up == 1`**, `health=up`, `lastError=''`, `lastScrapeDuration` 0.001–0.006s — + against 10.0s and `context deadline exceeded` before. +- **`oauth2_proxy_*` is in the TSDB for these namespaces for the first time:** 5 metric names, + 148 series; `sum by (namespace) (oauth2_proxy_requests_total)` = qbittorrent 35072, radarr 23560, + sonarr 1284. (Queried with `count by (__name__)` plus a matcher, not label-values.) +- **Alerts:** all 5 `TargetDown` and 5 of 6 `OAuth2ProxyGateUnobserved` resolved; one series + remains, `OAuth2ProxyGateUnobserved{namespace="crafty"}`, firing. +- **The rendered output is byte-identical.** `catalog/render.py` changed only in comments, so this + branch touches no `apps/` file and no golden. Confirmed by `render-golden` and `l1` both passing + with no regeneration, and by `git status` showing nothing under `apps/`. +- **This branch touches no sensitive path**, unlike PR #49: `platform/oauth2-proxy/**`, + `catalog/**` and `decisions/**` are none of them in `infra/gitea/approval-policy.json`. So it + should merge on a green gate with no human approval — the opposite of the PR it documents, and + the reason the two were kept apart. +- `render.py --selftest`: PASS. +- Toolbox gate: see the handoff's own rule — reported below rather than claimed here. + +## What I did NOT verify + +- **I did not re-measure the 63.7h figure or the ~22 emails/day rate**; both are carried forward + from the 2026-10-04 measurements, now quoted inside the docs as historical. The 15-day + `query_range` that established "one episode, not flapping" was run once, on 2026-10-04. +- **`oauth2_proxy_requests_total` is still absent for jackett and pihole.** I assert in the README + that this is because the counter is not created until the gate serves its first request. That is + the documented behaviour of a Prometheus counter with no initialisation, and it is consistent + with the other three namespaces having traffic — but I did not force a request through jackett's + gate and watch the series appear. If it is still absent after real use, that is a finding. +- **The auth-churn rule is still unwritten.** The README now says it is "writable rather than + decorative"; that is a claim about the metrics being present, not a rule anyone has authored or + unit-tested. The thing the 2026-09-30 incident actually needed does not exist yet. +- **crafty remains unobserved.** I read its container spec on 2026-10-05 and confirmed no + `--metrics-address`; I did not attempt the fix, and nothing here changes its alert. +- **I did not audit the whole tree for other stale "metrics are not scraped" claims** beyond the + four sites a targeted grep found (`not yet reachable`, `UNTIL AN INGRESS`, `up == 0`, + `not reachable by Prometheus` and neighbours). `runbooks/oauth2-proxy-auth-failures.md` is + described in the README as "the by-hand substitute meanwhile" and I left it alone — the by-hand + log procedure still works and is not wrong, but whether it should now be rewritten against the + new metrics is unexamined. diff --git a/platform/oauth2-proxy/README.md b/platform/oauth2-proxy/README.md index 66a0ec6..b76c27d 100644 --- a/platform/oauth2-proxy/README.md +++ b/platform/oauth2-proxy/README.md @@ -91,6 +91,10 @@ This proves the R-IDP-02 property (unauthenticated cannot reach the upstream) on ## Observability: what this component can and cannot see (measured 2026-09-30) +> **SUPERSEDED — read "Update 2026-10-05" at the end of this section first.** The gate sidecars +> **are** scraped now and the TSDB **does** hold `oauth2_proxy_*` for them. The two subsections +> below are kept as the dated record of how this was diagnosed, not as current state. + `servicemonitor.yaml` scrapes **this namespace's** deployment and nothing else. On prod that is two targets, `job="oauth2-proxy"`, both `up=1`. The complete set of oauth2-proxy-specific series in the TSDB is: @@ -157,3 +161,55 @@ present `up` series. A discovered-but-failing target publishes `up` with value 0 would have cleared `OAuth2ProxyGateUnobserved` the moment the PodMonitor landed — reporting the gates as observed while nothing was being scraped. The unit test pins that scenario; deleting the `== 1` fails it and nothing else. + +### Update 2026-10-05: the scrape works + +PR #49 (merged `21c88b0`) added the missing third piece. `catalog/render.py`'s +`r_monitoring_ingress_policy` now also fires for the gate sidecar's metrics port, keyed on +`_needs_oauth2_sidecar`, rendering `-allow-monitoring-scrape` per app: ns `monitoring` → that +app's `endpointSelector` → TCP 44180 only. Five objects, one each for jackett, pihole, qbittorrent, +radarr, sonarr. + +Measured on prod 2026-10-05 after Flux applied it (`apps-wave-0` 21:37:01Z, `apps-wave-1` +21:37:46Z): + +| | before | after | +|---|---|---| +| target health (6) | `down` | `up` | +| `lastError` | `context deadline exceeded` | `''` | +| `lastScrapeDuration` | **10.0s** — exactly the `scrapeTimeout` | **0.001–0.006s** | +| `up` | `0` × 6 | `1` × 6 | + + count by (__name__) ({__name__=~"oauth2_proxy_.*", + namespace=~"jackett|radarr|sonarr|qbittorrent|pihole"}) + oauth2_proxy_requests_in_flight => 6 + oauth2_proxy_requests_total => 16 + oauth2_proxy_response_duration_seconds_bucket => 108 + oauth2_proxy_response_duration_seconds_sum => 9 + oauth2_proxy_response_duration_seconds_count => 9 + + sum by (namespace) (oauth2_proxy_requests_total) + qbittorrent => 35072 radarr => 23560 sonarr => 1284 + +`oauth2_proxy_requests_total` is absent for jackett and pihole simply because that counter is not +created until the gate has served its first request. + +**So the auth-churn rule is now writable rather than decorative** — sustained `code="302"` as a +share of `oauth2_proxy_requests_total`, held across a window longer than the session lifetime. That +is the rule the 2026-09-30 incident above could not be caught by. It is still unwritten. + +**Alert outcome:** all 5 `TargetDown` resolved, and 5 of 6 `OAuth2ProxyGateUnobserved` with them. +At `repeat_interval: 12h` with `group_by: [alertname, namespace]`, those 11 groups had been +re-notifying at roughly **22 emails/day** since 2026-10-02; that drops to 2/day. + +**`crafty` still fires, and that is correct.** It is a hand-maintained Deployment under +`platform/crafty`, outside the renderer, whose `oauth2-proxy` container has **no +`--metrics-address` at all** — no listener, so no port to open and nothing for the PodMonitor's +pod-label selector to match. Closing it needs the flag and a `metrics` containerPort on crafty's +own manifest first, then a hand-written allowance beside it. Open follow-up work. + +**The paragraph above headed "To close the gap for an app" is superseded**: a *Service* port and a +ServiceMonitor are **not** required. A PodMonitor targets pod ports directly, which is why no +Service exposes 44180 and why the renderer needed no Service change. The three things actually +needed are `--metrics-address` + named container port, a PodMonitor, and the monitoring-ingress +allowance. diff --git a/platform/oauth2-proxy/podmonitor.yaml b/platform/oauth2-proxy/podmonitor.yaml index f3e2247..320ac35 100644 --- a/platform/oauth2-proxy/podmonitor.yaml +++ b/platform/oauth2-proxy/podmonitor.yaml @@ -27,10 +27,28 @@ # created by the apps-wave-0/1 tiers, which this tier does not and must not depend on. # # ============================================================================================ -# READ THIS BEFORE BELIEVING THE TARGETS ARE UP: THE SCRAPE IS CURRENTLY DENIED BY NETWORKPOLICY +# STATUS: THE SCRAPE WORKS. Verified on prod 2026-10-05 (PR #49 merged as 21c88b0). # ============================================================================================ -# This PodMonitor is NECESSARY AND NOT SUFFICIENT. The endpoint serves, and the path from ns -# monitoring is blocked. Both halves measured on prod 2026-10-01, read-only: +# All 6 targets in pool podMonitor/oauth2-proxy/oauth2-proxy-sidecars/0 read `up == 1`, +# health=up, lastError='', lastScrapeDuration 0.001-0.006s. The TSDB holds oauth2_proxy_* for +# these namespaces for the first time: 5 metric names / 148 series, with live counters +# (oauth2_proxy_requests_total 35072 qbittorrent, 23560 radarr, 1284 sonarr). All 5 TargetDown +# and 5 of 6 OAuth2ProxyGateUnobserved resolved; only namespace="crafty" still fires, and that +# is correct — see CRAFTY below. +# +# The missing piece was an ingress allowance, and it now ships WITH EACH APP, rendered by +# catalog/render.py's r_monitoring_ingress_policy as `-allow-monitoring-scrape`: +# ns monitoring -> the app's endpointSelector -> TCP 44180 only. See WHY NOT IN THIS DIRECTORY +# below for why it is rendered per-app instead of shipped from this tier. +# +# Keep the history below. It is why the fix is shaped the way it is, and `up == 1` in +# auth-visibility-alerts.yaml is load-bearing precisely because of it. +# +# -------------------------------------------------------------------------------------------- +# HISTORY: 2026-10-02 -> 2026-10-05, THE SCRAPE WAS DENIED BY NETWORKPOLICY FOR 63.7h +# -------------------------------------------------------------------------------------------- +# This PodMonitor was NECESSARY AND NOT SUFFICIENT: the endpoint served, and the path from ns +# monitoring was blocked. Both halves measured on prod 2026-10-01, read-only: # # THE ENDPOINT IS HEALTHY — port-forward straight to the jackett pod's 44180: # GET /metrics -> http=200 bytes=10208 @@ -54,19 +72,24 @@ # `jackett-allow-gateway-ingress`: `fromEntities: [ingress]` to port 4180 — the Gateway's Envoy # and nothing else. Prometheus matches no clause and its packets are dropped, not refused. # -# SO THE TARGETS BELOW WILL READ `up == 0` UNTIL AN INGRESS ALLOWANCE LANDS. That is a known, -# measured, documented state — NOT a regression introduced here, and NOT something to silence. -# It is deliberately left visible: `OAuth2ProxyGateUnobserved` in auth-visibility-alerts.yaml +# THE TARGETS READ `up == 0` FROM 2026-10-02 05:09 UTC UNTIL THE ALLOWANCE LANDED — one +# continuous 63.7h episode (a 15d query_range shows 1 episode per namespace, not flapping), +# starting 11 minutes after this file was created, i.e. TargetDown's `for: 10m`. It was never +# silenced, and that was the right call: `OAuth2ProxyGateUnobserved` in auth-visibility-alerts.yaml # requires `up == 1`, precisely so a discovered-but-failing target does not read as "observed" # (see that file's header — this exact trap is why the `== 1` is there). # +# The signature to recognise if it ever returns: lastScrapeDuration pinned at EXACTLY the +# scrapeTimeout (10.0s) with `context deadline exceeded`. That is a DROP, not a refusal — a +# refused connection fails in milliseconds. Post-fix the same targets scrape in 1-6ms. +# # THIS IS THE SAME BUG LONGHORN HAD, and the platform has a documented fix idiom for it: # platform/longhorn/networkpolicy-metrics.yaml, whose header records the identical fingerprint # measured on 2026-09-13 — ServiceMonitor in place, 3 targets DISCOVERED, `up` 0/0/0, TSDB empty, # `lastError: context deadline exceeded` (a drop, not a refusal). The fix there is a narrow # additive `networking.k8s.io/v1` NetworkPolicy: ns monitoring -> the pod label, that one port. # -# WHY THAT POLICY IS NOT IN THIS DIRECTORY. It would have to be five objects, one per app +# WHY NOT IN THIS DIRECTORY (the reasoning the fix followed). It would have to be five objects, one per app # namespace, and those namespaces are created by apps-wave-0/1. Measured Flux graph, 2026-10-01: # oauth2-proxy dependsOn=['platform-config'] <- this tier # apps-wave-0 dependsOn=['platform-config'] <- SIBLING, no ordering between them @@ -76,8 +99,19 @@ # running cluster and break on a rebuild — the "works now, broken on restore" shape this repo # keeps paying for. Longhorn's precedent is safe only because longhorn-system is created by the # same tier that ships its policy. The allowance therefore belongs WITH THE APPS, rendered next to -# the rest of each app's network policy — i.e. in the catalog/renderer, which is in flight on -# another branch. Tracked in the handoff; it is one narrow rule per app, not a redesign. +# the rest of each app's network policy — i.e. in the catalog/renderer. THAT IS WHERE IT LANDED: +# `r_monitoring_ingress_policy`, keyed on `_needs_oauth2_sidecar`, one narrow rule per app. +# +# -------------------------------------------------------------------------------------------- +# CRAFTY: STILL UNOBSERVED, AND THIS FILE CANNOT HELP IT +# -------------------------------------------------------------------------------------------- +# `OAuth2ProxyGateUnobserved{namespace="crafty"}` still fires after the fix, correctly. crafty is +# a hand-maintained Deployment under platform/crafty, OUTSIDE the renderer, and its oauth2-proxy +# container has NO `--metrics-address` at all (measured 2026-10-05) — so its metrics listener +# never starts and there is no port to open. The selector below therefore does not match it, and +# the per-app allowance is never rendered for it. Closing crafty's gap needs the flag plus a +# `metrics` containerPort added to its own manifest FIRST; the allowance is the second half, and +# it would have to be written by hand there rather than rendered. Open follow-up work. apiVersion: monitoring.coreos.com/v1 kind: PodMonitor metadata: From d0db63911aeb51ca538cf5373c0518e76a63241e Mon Sep 17 00:00:00 2001 From: Keith Jackson Date: Mon, 5 Oct 2026 21:19:13 -0500 Subject: [PATCH 2/2] feat(cilium): roll the agents when cilium-config changes (merged config was silently inert) A merged Cilium config change did nothing, and everything reported success. The Hubble flow exporter (#50, merged 2026-10-04) never took effect: HelmRelease cilium Ready=True, "Helm upgrade succeeded ... cilium.v6" cilium-config hubble-export-file-path: stdout + the allowlist, PRESENT cilium agent pods all three still PRE-MERGE, startTime 2026-10-04T07:11Z exported flow lines 0, 0, 0 Flux every Kustomization Ready=True Cause: the chart's rollOutCiliumPods defaults to false, so Helm rewrites the ConfigMap and leaves the DaemonSet's pod template untouched -- and cilium-agent reads that ConfigMap only at STARTUP. Merged, believed delivered, green everywhere, doing nothing. The same class of failure this repo already guards against elsewhere: a green signal over something that never ran. Confirmed by contrast rather than reasoned: the cgroup fix (#42) DID apply -- both the DaemonSet and the running pod mount /sys/fs/cgroup -- because it changed the POD TEMPLATE, which rolls the DaemonSet on its own. That also explains why the agents are dated 2026-10-04T07:11Z: that was #42's rollout, and nothing has rolled them since. The inertness is specific to ConfigMap-only changes. Set in all three value files to hold the [parity] contract. Cost, and it is the point: every future Cilium config change now rolls the agents instead of waiting for a manual restart. Strictly more honest -- it applies when merged or fails visibly -- but config edits become disruptive by default. maxUnavailable: 1 bounds it to one node at a time. The lease-holding node briefly loses the Gateway, PiHole-DNS and Plex VIPs as it cycles, so merge at a quiet moment. Merging this IS one rollout (the ConfigMap checksum annotation changes the pod template) -- which is what finally activates the Hubble exporter. UNVERIFIED: unreconciled; `filePath: stdout` still unproven and only answerable after the agents restart (check for flow lines immediately); and the rollout's actual impact on LAN DNS is unmeasured. Co-Authored-By: Claude Opus 5 (1M context) --- ...-10-05-cilium-roll-out-on-config-change.md | 88 +++++++++++++++++++ .../cilium/templates/cilium-values.yaml.j2 | 6 ++ platform/cilium/helmrelease.yaml | 31 +++++++ platform/cilium/values.yaml | 9 ++ 4 files changed, 134 insertions(+) create mode 100644 decisions/2026-10-05-cilium-roll-out-on-config-change.md diff --git a/decisions/2026-10-05-cilium-roll-out-on-config-change.md b/decisions/2026-10-05-cilium-roll-out-on-config-change.md new file mode 100644 index 0000000..9b3b25e --- /dev/null +++ b/decisions/2026-10-05-cilium-roll-out-on-config-change.md @@ -0,0 +1,88 @@ +--- +date: 2026-10-05 +branch: feat/cilium-roll-out-on-config-change +title: Cilium config changes were silently inert — roll the agents when cilium-config changes, which also finally activates the Hubble exporter +--- + +## What changed + +**A merged Cilium config change did nothing, and everything reported success.** The Hubble flow +exporter (PR #50, merged 2026-10-04) wrote its settings into `cilium-config` and never took effect. + +| Measured 2026-10-05, over a day after the merge | | +|---|---| +| HelmRelease `cilium` | `Ready=True`, `UpgradeSucceeded`, *"Helm upgrade succeeded for release kube-system/cilium.v6"* | +| `cilium-config` keys | `hubble-export-file-path: stdout`, `hubble-export-allowlist: {"destination_port":["32400"]} {"source_port":["32400"]}` — **all present** | +| cilium agent pods | **all three still the PRE-MERGE pods**, `startTime 2026-10-04T07:11Z` | +| exported flow lines in agent stdout | **0, 0, 0** | +| Flux Kustomizations | all `Ready=True` | + +Cause: the chart's `rollOutCiliumPods` defaults to **`false`** (cilium 1.20.1 `values.yaml`: *"Roll +out cilium agent pods automatically when configmap is updated"*), so Helm rewrites the ConfigMap and +leaves the DaemonSet's pod template untouched — and **cilium-agent reads that ConfigMap only at +startup**. The feature was merged, believed delivered, green everywhere, and doing nothing at all. + +That is the same class of failure this repo already guards against elsewhere: a green signal sitting +over something that never ran. + +- **`platform/cilium/helmrelease.yaml`** (authoritative): `rollOutCiliumPods: true`. +- **`platform/cilium/values.yaml`** (reference mirror) and + **`infra/ansible/roles/cilium/templates/cilium-values.yaml.j2`** (the `[parity]` source of truth): + the same, so a bootstrap render does not differ from what Flux reconciles. + +**The cost is real and is the point.** Every future Cilium config change now triggers a cluster-wide +agent rollout instead of waiting for someone to remember a manual restart. That is strictly more +honest — the change applies when merged, or it fails visibly — but it makes config edits disruptive +by default, so they want a quiet moment. Mitigation already in place: +`updateStrategy.rollingUpdate.maxUnavailable: 1` means the DaemonSet will not proceed past an unready +agent, so a bad render costs one node rather than the cluster. + +**Merging this is itself one rollout**, because adding the ConfigMap checksum annotation changes the +pod template — and that rollout is what finally activates the Hubble exporter. + +## What I verified + +- **Every row of the table above**, read from the live HelmRelease status, the live `cilium-config` + ConfigMap, the agent pods' `startTime`, and a grep for `"flow"` in each agent's stdout. +- **The chart value name, default and semantics**, quoted from cilium **v1.20.1**'s own + `values.yaml` — the version actually running here (`quay.io/cilium/cilium:v1.20.1`), not a guess + from another release: `rollOutCiliumPods: false` with the comment *"Roll out cilium agent pods + automatically when configmap is updated."* +- **All three value files carry the key** and both YAML files still parse, with + `spec.values.rollOutCiliumPods` resolving to `true`. +- **The blast-radius mitigation is genuinely present**, not assumed: + `updateStrategy.rollingUpdate.maxUnavailable: 1` is already set in the HelmRelease, with a long + existing comment explaining that a CNI rollout has no healthy-peer fallback. +- **THE MECHANISM IS CONFIRMED BY CONTRAST, not just reasoned.** The cgroup fix (PR #42, + 2026-10-03) DID take effect: both the DaemonSet and the running pod mount + `cilium-cgroup hostPath=/sys/fs/cgroup`, matching what the HelmRelease declares. The difference is + exactly the one claimed above — #42 changed the POD TEMPLATE (a volume hostPath), which rolls the + DaemonSet on its own, while #50 changed ONLY the ConfigMap, which does not. It also explains the + agents' `startTime 2026-10-04T07:11Z`: that was #42's rollout, and nothing has rolled them since. + So the inertness is specific to ConfigMap-only changes, which is precisely what this flag fixes. +- **Flux is healthy before this change** — every Kustomization `Ready=True`, so this lands on a clean + graph rather than into an existing cascade. +- **Toolbox gate**: recorded in the PR. + +## What I did NOT verify + +- **THE OUTCOME, ENTIRELY. Unreconciled.** No agent has rolled, no flow has been exported. This + change is an argument that the rollout will happen, not evidence that it did. +- **Whether `filePath: stdout` actually works** — still the open question from #50, and it cannot be + answered until the agents restart. If the literal is not special-cased, the agent may create a + FILE named `stdout` and nothing reaches Loki: a silent no-op, not a crash. Fallback documented in + #50: keep the chart's default path and add a `loki.source.file` target to Alloy with a hostPath + mount. **Check for flow lines in the agent logs immediately after the rollout.** +- **THE ROLLOUT'S IMPACT ON LAN DNS IS UNMEASURED AND IS THE MAIN RISK.** The node holding the L2 + lease for the Gateway VIP, **PiHole's DNS VIP** and Plex's VIP loses them briefly as it cycles. + `maxUnavailable: 1` bounds it to one node at a time, and BPF maps persist across an agent restart, + but I have not measured how long a VIP is unreachable during a lease move, nor whether the two + PiHole replicas cover the gap. Merge it at a quiet moment. +- **Whether enabling this is the operator's preferred trade.** The alternative — leaving it `false` + and restarting by hand — keeps config edits non-disruptive at the cost of them being silently + inert. I judged silent inertness the worse failure, but it is a judgement, not a measurement. +- **Whether anything ELSE in `cilium-config` is currently inert.** I checked the one other recent + Cilium change and it is fine (see below), but I did not audit every key against the running + agents' actual state. +- **`hubble.relay.rollOutPods` / `hubble.ui.rollOutPods`** are left at their `false` defaults. Both + have the same silently-inert property for their own ConfigMaps; not addressed here. diff --git a/infra/ansible/roles/cilium/templates/cilium-values.yaml.j2 b/infra/ansible/roles/cilium/templates/cilium-values.yaml.j2 index 01f7f17..24be12f 100644 --- a/infra/ansible/roles/cilium/templates/cilium-values.yaml.j2 +++ b/infra/ansible/roles/cilium/templates/cilium-values.yaml.j2 @@ -39,6 +39,12 @@ bpf: masquerade: true # Flow observability / isolation audit (ADR-0002). +# Roll the agents when cilium-config changes. Kept EQUAL to the Flux values +# (platform/cilium/helmrelease.yaml), which carry the full rationale: the chart default of false +# leaves the pod template untouched on a config change, and cilium-agent reads that ConfigMap only +# at startup, so the change is silently inert until someone restarts the DaemonSet by hand. +rollOutCiliumPods: true + hubble: enabled: true relay: diff --git a/platform/cilium/helmrelease.yaml b/platform/cilium/helmrelease.yaml index c1f2b65..c61df80 100644 --- a/platform/cilium/helmrelease.yaml +++ b/platform/cilium/helmrelease.yaml @@ -188,6 +188,37 @@ spec: enabled: false hostRoot: /sys/fs/cgroup # --- Agent rollout: ONE node at a time (blast-radius bound) — 2026-09-01 incident --- + # --- Roll the agents when their ConfigMap changes (2026-10-05) --- + # [parity] MUST equal Ansible: rollOutCiliumPods: true + # + # WHY: a merged Cilium config change DID NOTHING. The chart default is `false` + # (cilium 1.20.1 values.yaml: "Roll out cilium agent pods automatically when configmap is + # updated. rollOutCiliumPods: false"), so Helm rewrites `cilium-config` and leaves the + # DaemonSet's pod template untouched — and cilium-agent reads that ConfigMap only at STARTUP. + # The result is a silently inert change: Flux green, HelmRelease `UpgradeSucceeded`, the new + # keys visibly present in the ConfigMap, and the running agents still on the old settings + # until a human remembers to restart them. + # + # MEASURED, and it is why this exists: the Hubble flow exporter merged 2026-10-04 wrote + # `hubble-export-file-path: stdout` and its allowlist into `cilium-config`, the HelmRelease + # reported `Helm upgrade succeeded ... cilium.v6`, and over a DAY later all three agents were + # still the pre-merge pods (startTime 2026-10-04T07:11Z) with ZERO exported flow lines. The + # feature was merged, believed delivered, and doing nothing at all. That is the same class of + # failure this repo already documents elsewhere — a green signal over something that never ran. + # + # THE COST IS REAL AND IS THE POINT: every future Cilium config change now triggers a + # cluster-wide agent rollout instead of waiting for a manual restart. That is strictly more + # honest — the change applies when it is merged, or it fails visibly — but it means config + # edits are disruptive by default, so they want a quiet moment. One node at a time: + # `updateStrategy.rollingUpdate.maxUnavailable: 1` below means the DaemonSet will not proceed + # past an unready agent, so a bad render costs one node rather than the cluster. Note that the + # node holding the L2 lease for the Gateway, PiHole-DNS and Plex VIPs loses them briefly as it + # cycles, so a rollout can blip LAN DNS. + # + # ENABLING THIS IS ITSELF ONE ROLLOUT: adding the ConfigMap checksum annotation changes the pod + # template, so merging this rolls the agents once — which is what finally activates the Hubble + # exporter that has been inert since 2026-10-04. + rollOutCiliumPods: true # [parity] MUST equal Ansible: updateStrategy.rollingUpdate.maxUnavailable: 1. # The chart's own default is 2 (cilium 1.20.1 values.yaml: updateStrategy.rollingUpdate. # maxUnavailable: 2) — NOT a local choice, and on a 3-node cluster that lets a single bad diff --git a/platform/cilium/values.yaml b/platform/cilium/values.yaml index b37d11e..dba78a3 100644 --- a/platform/cilium/values.yaml +++ b/platform/cilium/values.yaml @@ -32,6 +32,15 @@ enableIPv4Masquerade: true bpf: masquerade: true # [parity] flow observability / isolation audit +# [parity] roll the agents when cilium-config changes. The chart default (false) rewrites the +# ConfigMap but leaves the pod template alone, and cilium-agent only reads it at STARTUP — so a +# merged config change is SILENTLY INERT. Measured 2026-10-04..05: the Hubble exporter keys were +# live in cilium-config and the HelmRelease said UpgradeSucceeded, while all three agents were +# still pre-merge pods exporting nothing for over a day. Cost: config edits now roll the agents +# (one node at a time via maxUnavailable: 1), which can briefly blip the VIPs on the lease-holding +# node. Full rationale in helmrelease.yaml. +rollOutCiliumPods: true + hubble: enabled: true relay: