From 09f219fe7fcf0280f3a45f52fbd329e3d4c651e4 Mon Sep 17 00:00:00 2001 From: Keith Jackson Date: Sun, 8 Nov 2026 12:39:56 -0600 Subject: [PATCH] fix(cloudflared): auditable Service+ServiceMonitor scrape; placeholder-only template value servicemonitor-ports-resolve could not audit a PodMonitor with no in-repo Service, and no-plaintext-secrets requires a template's secret value to be a bare placeholder. Co-Authored-By: Claude Opus 5.5 --- .../check.sh | 9 ++-- decisions/2026-11-08-cloudflare-tunnel.md | 2 +- platform/cloudflared/README.md | 2 +- .../cloudflared/alerts.promtool-test.yaml | 2 +- platform/cloudflared/alerts.yaml | 2 +- .../cloudflared-credentials.sops.yaml.example | 5 ++- platform/cloudflared/kustomization.yaml | 2 +- platform/cloudflared/networkpolicy.yaml | 2 +- platform/cloudflared/podmonitor.yaml | 22 --------- platform/cloudflared/servicemonitor.yaml | 45 +++++++++++++++++++ 10 files changed, 59 insertions(+), 34 deletions(-) delete mode 100644 platform/cloudflared/podmonitor.yaml create mode 100644 platform/cloudflared/servicemonitor.yaml diff --git a/conformance/tests/l0-static/alert-metrics-have-scrape-targets/check.sh b/conformance/tests/l0-static/alert-metrics-have-scrape-targets/check.sh index 4e446eb..741eb40 100755 --- a/conformance/tests/l0-static/alert-metrics-have-scrape-targets/check.sh +++ b/conformance/tests/l0-static/alert-metrics-have-scrape-targets/check.sh @@ -413,13 +413,14 @@ SCRAPE_OWNERS: list[dict] = [ }, # ── ADDED 2026-11-08 (feat/cloudflare-tunnel). A NEW OWNER, NOT A RELAXED ASSERTION. ─────── # Purely additive, the same shape as the kured entry: it registers the cloudflared_* family - # and the PodMonitor that scrapes it, and makes the check STRICTER for that component. + # and the Service+ServiceMonitor that scrapes it, and makes the check STRICTER for that component. { - "owner": "cloudflared: the PodMonitor on the `metrics` (:2000) container port", + "owner": "cloudflared: headless metrics Service + ServiceMonitor on port `metrics` (:2000)", "prefixes": ("cloudflared_",), - "file": "platform/cloudflared/podmonitor.yaml", + "file": "platform/cloudflared/servicemonitor.yaml", "require": [ - r"^kind:\s*PodMonitor\s*$", + r"^kind:\s*ServiceMonitor\s*$", + r"^kind:\s*Service\s*$", r"^\s*-\s*port:\s*metrics\s*$", ], "forbid": [], diff --git a/decisions/2026-11-08-cloudflare-tunnel.md b/decisions/2026-11-08-cloudflare-tunnel.md index 252abd4..7f91178 100644 --- a/decisions/2026-11-08-cloudflare-tunnel.md +++ b/decisions/2026-11-08-cloudflare-tunnel.md @@ -52,7 +52,7 @@ different agent, moves DNS. - The egress CNP allows world tcp/udp 7844 and tcp 443, plus the `ingress` entity on tcp 443. It also allows **Gateway backends by `io.cilium.k8s.namespace.labels.gateway-access=shared`, minus the LAN-only namespaces.** - - `podmonitor.yaml` and `alerts.yaml` (`CloudflaredTunnelDown`, `CloudflaredReplicasDegraded`) + - `servicemonitor.yaml` (headless metrics Service + ServiceMonitor) and `alerts.yaml` (`CloudflaredTunnelDown`, `CloudflaredReplicasDegraded`) ship with `alerts.promtool-test.yaml`, and `README.md` documents the component. - `clusters/base/cloudflared.yaml` (new) is its own Flux Kustomization: `dependsOn: platform-config`, **`wait: false`**, prune, and postBuild `cluster-vars`. It is listed in `clusters/base/kustomization.yaml`. diff --git a/platform/cloudflared/README.md b/platform/cloudflared/README.md index e587b1c..8bad7a1 100644 --- a/platform/cloudflared/README.md +++ b/platform/cloudflared/README.md @@ -15,7 +15,7 @@ outbound-only from the cluster (no router port forward, no hairpin, no split-DNS | `cloudflared-credentials.sops.yaml` | SOPS-encrypted tunnel credentials JSON. Ships with `REPLACE-ME`; the operator fills it. | | `deployment.yaml` | 2 replicas, hard spread across nodes, non-root, read-only root fs, pinned digest; PDB `minAvailable: 1`. | | `networkpolicy.yaml` | default-deny (single owner — not an app namespace) + Prometheus scrape allow + egress CNP: world 7844 tcp/udp and 443, the Gateway (`ingress` entity), and Gateway backends by `gateway-access=shared` minus the LAN-only namespaces. | -| `podmonitor.yaml`, `alerts.yaml` | Scrape of :2000 and the two alerts, both written to fire (not go quiet) when their series vanish. | +| `servicemonitor.yaml`, `alerts.yaml` | Headless metrics Service + ServiceMonitor scraping :2000, and the two alerts, both written to fire (not go quiet) when their series vanish. | **Phase 1 only.** This component changes no live traffic: Cloudflare sends a hostname through the tunnel only when its DNS record is a CNAME to `.cfargotunnel.com`, and nothing here creates diff --git a/platform/cloudflared/alerts.promtool-test.yaml b/platform/cloudflared/alerts.promtool-test.yaml index d915763..bdf1b3d 100644 --- a/platform/cloudflared/alerts.promtool-test.yaml +++ b/platform/cloudflared/alerts.promtool-test.yaml @@ -33,7 +33,7 @@ tests: name: nothing scraped at all - TunnelDown still fires (no self-silencing on absence) input_series: # kube-state-metrics says both pods are Available, but no cloudflared series exists — the - # broken-PodMonitor / dropped-scrape case. The rule must not read this as healthy. + # broken-ServiceMonitor / dropped-scrape case. The rule must not read this as healthy. - series: 'kube_deployment_status_replicas_available{namespace="cloudflared",deployment="cloudflared"}' values: '2x60' promql_expr_test: diff --git a/platform/cloudflared/alerts.yaml b/platform/cloudflared/alerts.yaml index 1a79691..53d204c 100644 --- a/platform/cloudflared/alerts.yaml +++ b/platform/cloudflared/alerts.yaml @@ -6,7 +6,7 @@ # # * CloudflaredTunnelDown reads `cloudflared_tunnel_ha_connections` — the per-replica gauge of # live edge connections — and falls back to `vector(0)` when NO such series exists. A dead -# tunnel, zero pods, a broken PodMonitor, or a NetworkPolicy that drops the scrape all make the +# tunnel, zero pods, a broken ServiceMonitor, or a NetworkPolicy that drops the scrape all make the # series absent, and all of them FIRE rather than read as healthy. # * CloudflaredReplicasDegraded reads kube-state-metrics, which does not depend on cloudflared # being scrapable at all, and has the same `vector(0)` fallback. diff --git a/platform/cloudflared/cloudflared-credentials.sops.yaml.example b/platform/cloudflared/cloudflared-credentials.sops.yaml.example index bee5329..d224a0f 100644 --- a/platform/cloudflared/cloudflared-credentials.sops.yaml.example +++ b/platform/cloudflared/cloudflared-credentials.sops.yaml.example @@ -29,6 +29,7 @@ metadata: namespace: cloudflared type: Opaque stringData: - # One line of JSON, exactly as `cloudflared tunnel create` wrote it (compacted with `jq -c`). + # One line of JSON, exactly as `cloudflared tunnel create` wrote it (compacted with `jq -c`): + # {"AccountTag":"","TunnelSecret":"","TunnelID":""} # TunnelID MUST equal the `tunnel:` UUID in config.yaml. - credentials.json: '{"AccountTag":"REPLACE-ME","TunnelSecret":"REPLACE-ME","TunnelID":"REPLACE-ME"}' + credentials.json: REPLACE-ME diff --git a/platform/cloudflared/kustomization.yaml b/platform/cloudflared/kustomization.yaml index b1d910b..a1de2b2 100644 --- a/platform/cloudflared/kustomization.yaml +++ b/platform/cloudflared/kustomization.yaml @@ -15,7 +15,7 @@ resources: # Namespace default-deny + the Prometheus scrape allow + the egress CiliumNetworkPolicy. Keep # them in ONE file: a baseline without its scrape allow silently empties the TSDB. - networkpolicy.yaml - - podmonitor.yaml + - servicemonitor.yaml # headless metrics Service + ServiceMonitor (:2000) - alerts.yaml # The tunnel credentials JSON — the ONE secret here. SOPS-encrypted, decrypted per-env by the # `cloudflared` Flux Kustomization (both overlays patch it), referenced by name only. Its diff --git a/platform/cloudflared/networkpolicy.yaml b/platform/cloudflared/networkpolicy.yaml index 330182a..a4297fd 100644 --- a/platform/cloudflared/networkpolicy.yaml +++ b/platform/cloudflared/networkpolicy.yaml @@ -31,7 +31,7 @@ spec: port: 53 --- # The ONLY ingress: Prometheus scraping :2000. Shipped in the same file as the baseline on purpose -# (the Longhorn/kured lesson): a deny-all without this allow leaves the PodMonitor's targets +# (the Longhorn/kured lesson): a deny-all without this allow leaves the ServiceMonitor's targets # DISCOVERED and every scrape timing out, which is the state the CloudflaredTunnelDown rule is # written to fire on rather than go quiet in. apiVersion: networking.k8s.io/v1 diff --git a/platform/cloudflared/podmonitor.yaml b/platform/cloudflared/podmonitor.yaml deleted file mode 100644 index 082d7e9..0000000 --- a/platform/cloudflared/podmonitor.yaml +++ /dev/null @@ -1,22 +0,0 @@ ---- -# Scrapes cloudflared's own metrics server (:2000, `metrics:` in config.yaml) on every replica. -# kube-prometheus-stack runs with podMonitorSelectorNilUsesHelmValues: false, so Prometheus selects -# every PodMonitor in every namespace; the `release` label is kept for consistency with the -# platform's other monitors. The ingress allow that lets this scrape through the namespace's -# default-deny is cloudflared-allow-prometheus-scrape in networkpolicy.yaml. -apiVersion: monitoring.coreos.com/v1 -kind: PodMonitor -metadata: - name: cloudflared - namespace: cloudflared - labels: - app.kubernetes.io/name: cloudflared - release: kube-prometheus-stack -spec: - selector: - matchLabels: - app.kubernetes.io/name: cloudflared - podMetricsEndpoints: - - port: metrics - path: /metrics - interval: 30s diff --git a/platform/cloudflared/servicemonitor.yaml b/platform/cloudflared/servicemonitor.yaml new file mode 100644 index 0000000..5b23990 --- /dev/null +++ b/platform/cloudflared/servicemonitor.yaml @@ -0,0 +1,45 @@ +--- +# Scrape of cloudflared's own metrics server (:2000, `metrics:` in config.yaml) on every replica. +# +# A Service + ServiceMonitor in ONE file rather than a PodMonitor, on purpose: the repo's L0 checks +# (servicemonitor-ports-resolve, alert-metrics-have-scrape-targets assertion 4) can then prove the +# monitor's port is a port this Service actually defines. A PodMonitor would have to be declared as +# an unauditable exception. The Service is headless and carries only the metrics port — it is not a +# way into the tunnel, and nothing routes to it but Prometheus (networkpolicy.yaml admits only the +# monitoring namespace on :2000). +# +# kube-prometheus-stack runs with serviceMonitorSelectorNilUsesHelmValues: false, so this is +# selected from any namespace; the `release` label is kept for consistency. +apiVersion: v1 +kind: Service +metadata: + name: cloudflared-metrics + namespace: cloudflared + labels: + app.kubernetes.io/name: cloudflared +spec: + clusterIP: None + selector: + app.kubernetes.io/name: cloudflared + ports: + - name: metrics + port: 2000 + targetPort: metrics + protocol: TCP +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: cloudflared + namespace: cloudflared + labels: + app.kubernetes.io/name: cloudflared + release: kube-prometheus-stack +spec: + selector: + matchLabels: + app.kubernetes.io/name: cloudflared + endpoints: + - port: metrics + path: /metrics + interval: 30s