#!/usr/bin/env bash # ============================================================================= # L0 static check (offline, no cluster) — alert-metrics-have-scrape-targets # Serves: R-OBS-04 ("The platform alerts on its own health (nodes, storage, GitOps, certs, # VPN)." — spec docs/10-observability-and-alerting.md) # # WHY THIS EXISTS — AN ALERT ON AN UNSCRAPED METRIC IS AN ABSENT ALERT THAT READS AS A PRESENT ONE # ----------------------------------------------------------------------------------------------- # `longhorn_volume_robustness == 2` evaluated against a TSDB that holds no longhorn_* series # returns no vector. The rule is therefore permanently quiet, permanently "healthy", and # INDISTINGUISHABLE from a correct rule on a healthy cluster. It passes review (the PromQL is # right). It passes `promtool test rules` (the unit test supplies the series itself, so a unit # test proves nothing about whether anything scrapes them). It shows up on no dashboard, because # nobody looks for a panel that was never built. # # Measured on prod 2026-09-13, BEFORE the change that added this check — 13 rules were inert: # count(longhorn_volume_robustness) => (empty) LonghornVolumeDegraded/Faulted/ # CapacityHigh, LonghornNodeStorageHigh # count({__name__=~"certmanager.*"}) => (empty) CertificateExpiringSoon/ # ExpiryImminent/NotReady # count(hubble_drop_total) => (empty) HubbleDeniedFlowSpike # count by (job) (up) => 20 jobs, none Longhorn, cert-manager or Hubble # Storage health was unmonitored on the cluster whose prod-1 had just had a failing SSD; TLS # expiry was unmonitored with the live `wildcard-home` wildcard 89 days out; there was no # policy-drop visibility at all. The gotk_* family (5 more rules) was the same defect, fixed # separately in a79693d. # # WHAT MAKES THIS CLASS DIFFERENT from the two failure modes alert-rules-unit-tested catches: a # permanently-firing rule is loud, and a rule that behaves wrongly against synthetic series is # caught by promtool. This one is caught by NEITHER, because the missing thing is not in the rule # file at all — it is a `serviceMonitor.enabled` that was never set in a completely different # component's HelmRelease. So the audit has to cross from the alert to the scrape config, which is # what the owner table below does. # # GIVEN every alerting rule in every PrometheusRule in the repo, # WHEN each rule's PromQL is parsed for the metric names it selects, # THEN (1) every metric name is CLAIMED by exactly one declared owner in SCRAPE_OWNERS, and the # owner's `require` regexes still match its file — so deleting the enablement from # platform/longhorn/helmrelease.yaml FAILS here instead of silently re-inerting four # storage alerts; # (2) each owner's `forbid` regexes do NOT match, for the cases where the enablement can be # defeated without being deleted (cert-manager's chart `fail`s if podmonitor and # servicemonitor are both enabled; an `enabled: false` inside the exporter's own block); # (3) a metric with no owner FAILS unless declared in KNOWN_UNWIRED_METRICS (shrink-only); # (4) `up{job="X"}` is audited separately — `up` itself always exists, so (1) is blind to # this — by requiring that where this repo authors a Service and a ServiceMonitor in the # SAME file, the monitor's endpoint port is a port that Service actually defines; # violations must be declared in KNOWN_UNWIRED_JOBS (shrink-only); # AND (5) the check is LOAD-BEARING: this test declares `conformance.home.k3s/scope: whole-repo` # so ci/l0.sh unions it into every gate run, and ci/gate.sh blocks on an l0 failure. # # WHY THE REGISTERS ARE RATCHETS (the decision and its reasons) — the same three options as # alert-rules-unit-tested faced: # * FAIL outright on all 13: correct in principle, but 12 of them are app SLI rules that need # Prometheus exporters for Plex, Pi-hole, Jackett, qBittorrent, Sonarr, Radarr and Seerr — # none of which speak Prometheus natively. The gate would be red for every change until that # work lands, so it would get disabled, and a disabled gate protects nothing. # * merely REPORT: that is precisely how these rules stayed inert across phases 4-6. # * RATCHET (chosen): each gap is declared here, printed with counts on every run, and CANNOT # GROW. A new alert on an unclaimed metric FAILS. A new same-file Service+ServiceMonitor pair # with a mismatched port FAILS. A STALE entry — the gap was closed, or the alert deleted — # also FAILS, so neither register can rot into fiction. # # WHAT THIS DOES *NOT* PROVE (honest gaps — this is a STATIC check): # - It does not prove a target is UP, that an endpoint answers, that the chart still honours the # values key, or that a NetworkPolicy is sufficient. Those were established by live prod reads # recorded in the commit message: with the enablements applied, longhorn_volume_robustness # returned 44 series, {__name__=~"certmanager.*"} returned 104, and all six new targets read # up==1. A live probe needs a cluster, which L0 by definition does not have. # - It cannot audit a ServiceMonitor whose Service is created by a Helm chart rather than by this # repo: the port name lives outside the tree. That is why (4) is scoped to same-file # Service+ServiceMonitor pairs — the apps/* ServiceDefinition pattern — and why the # platform/* monitors (oauth2-proxy, kubevirt, keycloak, flux) are out of its reach. Their # liveness is covered by the fact that their metrics have owners under (1). # - IT PROVES THE SCRAPE TARGET, NOT THE SERIES NAME. An owner claims a metric by PREFIX, so a # metric in a scraped family whose exact name the exporter does not emit still passes. This is # a real residual hole and it was live in this repo: `kubevirt_vmi_phase_count{phase="Failed"}` # (KubeVirtVMLaunchFailing) sat inside the scraped `kubevirt_` family on an UP target, and the # series simply does not exist on the deployed KubeVirt — verified on prod 2026-09-13, where the # only kubevirt_vmi_* names are `info` and `number_of_outdated`, and the only phase-bearing # names are the phase_transition_time histograms. Corrected in the same change to # `kubevirt_vmi_info{phase="failed"}` (note the LOWERCASE value KubeVirt writes: a capital-F # matcher selects nothing and is just as inert). Catching that class needs the exporter's own # metric list, which is a live read, not a static one — it belongs to a future L1/L3 check that # diffs every alert metric against `/api/v1/label/__name__/values`. The same limit applies to # every LABEL VALUE an alert matches on. # - The metric-name parser is a PromQL approximation, not a parser: it strips string literals, # label-matcher bodies and aggregation label lists, then treats remaining bare identifiers as # metric selectors. A metric referenced ONLY through a `{__name__=~"..."}` regex selector would # be missed. No rule in this repo does that, and assertion (0) fails if the parser stops # finding metrics at all. # - It does not judge thresholds, and does not prove an alert reaches a human (R-OBS-02 and the # L3 alerting test own that). # - It does not read ../home-network-k3s. The spec is read-only (R-TEST-06). # # Exit 0 = all assertions pass. Exit 1 = a failure. Exit 3 (honest SKIP, and because this test # declares the whole-repo scope the runner turns it into rc 3 = UNWIRED, never green) only if the # repo contains no PrometheusRule at all — i.e. there is genuinely nothing to audit. # ============================================================================= set -euo pipefail DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT="$(cd "${DIR}/../../../.." && pwd)" # repo root PYTHON="${PYTHON:-python3}" # Honest SKIP (never green for a scope-declaring test) if there is nothing to audit at all. if ! grep -rl 'kind: PrometheusRule' "${ROOT}/platform" "${ROOT}/apps" >/dev/null 2>&1; then echo "check SKIPPED: no PrometheusRule found under platform/ or apps/ — no alert rules to audit" >&2 exit 3 fi cd "${ROOT}" "${PYTHON}" - "${ROOT}" <<'PY' """Static audit: every alert rule's metrics are claimed by a scrape target this repo configures. PyYAML is used deliberately rather than the line-based parsing the neighbouring checks favour: alert `expr:` values are multi-line block scalars, and a line-based reader of PromQL would be the kind of approximate parser that silently stops matching — the exact failure class this check exists to prevent. Same dependency as catalog/render.py and dev-nodes-provisioned-as-code. """ from __future__ import annotations import re import sys from pathlib import Path try: import yaml except ImportError: # pragma: no cover print("check FAIL: PyYAML is required (same dependency as catalog/render.py)", file=sys.stderr) raise SystemExit(1) ROOT = Path(sys.argv[1]) sys.dont_write_bytecode = True # keep __pycache__ out of the tracked tree FAILURES: list[str] = [] CHECKS = 0 def check(cond, msg): global CHECKS CHECKS += 1 if not cond: FAILURES.append(msg) def read(p: Path) -> str: try: return p.read_text() except (OSError, UnicodeDecodeError): return "" # ============================================================================================= # THE DECLARED SCRAPE-OWNER TABLE (assertions 1 and 2). # # Each entry claims a set of metric names (exact names and/or prefixes) and names the file whose # declaration is what makes those metrics exist in the TSDB, plus the regexes that prove it. This # is the crossing from "an alert queries X" to "something in this repo scrapes X", and it is the # part that makes the check a REGRESSION guard rather than a one-time audit: every `require` # regex is re-evaluated against the working tree on every gate run. # # `require` — ALL must match the file. The enablement itself. # `forbid` — NONE may match. For enablements that can be defeated without being deleted. # # Longest prefix wins, so kube_velero_schedule_ is resolved before kube_. # ============================================================================================= KPS = "platform/monitoring/kube-prometheus-stack.yaml" SCRAPE_OWNERS: list[dict] = [ { "owner": "kube-prometheus-stack: bundled node-exporter DaemonSet + its ServiceMonitor", "prefixes": ("node_",), "file": KPS, # The subchart block must be present and configured here (it is where the exporter's # resources live). The chart default is enabled:true, so the regression to guard is a # DISABLE, not a missing enable. "require": [r"^\s*prometheus-node-exporter:\s*$"], "forbid": [r"prometheus-node-exporter:\s*\n(?:\s*#[^\n]*\n)*\s*enabled:\s*false"], "why": "node_* powers all seven platform-health.nodes rules including the two IO-pressure " "rules that exist because of prod-1's etcd WAL starvation", }, { "owner": "kube-prometheus-stack: the kubelet ServiceMonitor (cAdvisor + volume stats)", # container_* is cAdvisor, which the kubelet ServiceMonitor scrapes on the SAME target as # kubelet_* via its /metrics/cadvisor path (this entry's title always said so; the prefix # was added 2026-09-14 when ContainerIOStallHigh became the first rule to query it). # Verified on prod: container_pressure_io_stalled_seconds_total, 405 series across the # three nodes, job="kubelet", metrics_path="/metrics/cadvisor". "prefixes": ("kubelet_", "container_"), "file": KPS, # Chart default kubelet.enabled:true. There is no explicit enable to require, so the only # honest assertion is that it has not been turned off the way kubeProxy/kubeEtcd were. "require": [r"kubeProxy:"], # sentinel: the exporter-toggle block is still authored here "forbid": [r"\n\s*kubelet:\s*\n(?:\s*#[^\n]*\n)*\s*enabled:\s*false"], "why": "kubelet_volume_stats_* powers MonitoringVolumeFillingUp and all three " "pvc-capacity-alerts rules; container_pressure_io_stalled_seconds_total is the " "only signal on this cluster that can name WHICH workload an I/O stall is hitting", }, { "owner": "kube-prometheus-stack: the kube-apiserver ServiceMonitor (chart default, NOT " "disabled the way kubeEtcd/kubeProxy are)", # DELIBERATELY EXACT NAMES, NOT AN `etcd_` PREFIX. On this cluster the only etcd-shaped # metrics that exist are the APISERVER'S CLIENT-SIDE view of its own local member. etcd's # server-side disk metrics (etcd_disk_wal_fsync_duration_seconds, # etcd_disk_backend_commit_duration_seconds) are NOT scraped — kubeEtcd.enabled:false for # k3s' embedded etcd — and verified absent on prod 2026-09-14: a query for # {__name__=~"etcd_disk_.*|etcd_server_.*|etcd_mvcc_.*"} returns nothing at all. A blanket # `etcd_` claim would let a future rule on those names pass this check while being inert, # which is precisely the hole this test exists to close. Widen only with a real scrape. "exact": ( "etcd_request_duration_seconds_bucket", "etcd_request_duration_seconds_count", ), "file": KPS, "require": [r"kubeProxy:"], # sentinel: the exporter-toggle block is still authored here "forbid": [r"\n\s*kubeApiServer:\s*\n(?:\s*#[^\n]*\n)*\s*enabled:\s*false"], "why": "etcd_request_duration_seconds_* is the ONLY per-node view of the etcd storage path " "this cluster has, and NodeEtcdRequestsSlow is the only alert that can see the " "condition that took k3s-prod-1 NotReady on 2026-09-07 before the node drops", }, { "owner": "kube-prometheus-stack: kube-state-metrics custom-resource-state for Velero Schedules", "prefixes": ("kube_velero_schedule_",), "file": KPS, "require": [ r"customResourceState:\s*\n\s*enabled:\s*true", r"metricNamePrefix:\s*kube_velero_schedule", # The RBAC is MANDATORY, not cosmetic: without velero.io read access KSM emits the # custom-resource-state metrics as an EMPTY SET and logs a forbidden error — silent, # and the inert-alert failure mode all over again (measured; see the block's header). r'apiGroups:\s*\["velero\.io"\]', r'resources:\s*\["schedules"\]', ], "forbid": [r"customResourceState:\s*\n\s*enabled:\s*false"], "why": "kube_velero_schedule_* is the only thing that can see a Schedule that has NEVER " "backed up (VeleroScheduleNeverBackedUp / SchedulePaused / NoVolumeDataBackup); " "Velero itself publishes nothing until a backup is attempted", }, { "owner": "kube-prometheus-stack: kube-state-metrics custom-resource-state for Flux revisions", "prefixes": ("kube_flux_",), "file": KPS, "require": [ r"customResourceState:\s*\n\s*enabled:\s*true", r"metricNamePrefix:\s*kube_flux_kustomization", r"metricNamePrefix:\s*kube_flux_gitrepository", # The RBAC is MANDATORY, not cosmetic — the same measured failure mode as velero.io # above: without read access to these two API groups KSM publishes the # custom-resource-state metrics as an EMPTY SET and logs a forbidden error, which would # leave FluxRolloutInProgress/FluxRolloutStuck inert while looking configured. r'apiGroups:\s*\["kustomize\.toolkit\.fluxcd\.io"\]', r'apiGroups:\s*\["source\.toolkit\.fluxcd\.io"\]', # A config-only edit does not restart KSM (the chart sets no checksum annotation over # the ConfigMap), so the pod annotation that forces the roll is part of the claim: drop # it and a future change to the flux collector set ships green and never takes effect. r"homelab\.keithjackson\.dev/crs-config-rollout", ], "forbid": [r"customResourceState:\s*\n\s*enabled:\s*false"], "why": "kube_flux_* is the ONLY revision-carrying signal in the TSDB — gotk_* has no " "revision label anywhere (measured 2026-09-29) and the Flux event stream emits " "ReconciliationSucceeded on every interval tick whether or not anything changed, so " "without these metrics nothing can tell a deploy starting from an idle cluster", }, { "owner": "kube-prometheus-stack: bundled kube-state-metrics", "prefixes": ("kube_",), "file": KPS, "require": [r"^\s*kube-state-metrics:\s*$"], "forbid": [r"kube-state-metrics:\s*\n(?:\s*#[^\n]*\n)*\s*enabled:\s*false"], # kube_node_labels is a SPECIAL CASE and is checked on its own below: KSM exposes no # custom node labels by default, so the metric does not exist unless metricLabelsAllowlist # names the label. The prefix claim alone would pass while VPNTunnelDown's join matched # nothing. "why": "kube_* powers the Keycloak, KubeVirt-crashloop, CNPG-colocation and Velero-schedule " "rules", }, { "owner": "kube-state-metrics metricLabelsAllowlist (the egress-gateway node label)", "exact": ("kube_node_labels",), "file": KPS, "require": [ r"metricLabelsAllowlist:", # The label must be IN the allowlist, not BE the whole allowlist. This pattern used to # end in a literal `]`, asserting the list held exactly this one label — so adding a # second one (`egress-uplink`, for the steering-staleness alerts) made this check FAIL # while the property it guards stayed perfectly satisfied. A guard that breaks when you # correctly extend the thing it guards teaches people to edit the guard, which is how it # stops guarding. `[,\\]]` accepts either the end of the list or a further entry. r"nodes=\[homelab\.keithjackson\.dev/egress-gateway[,\]]", ], "forbid": [], "why": "KSM exposes NO custom labels by default. Without the allowlist entry " "kube_node_labels carries no egress-gateway label and VPNTunnelDown's join silently " "matches nothing — a rule that is present, evaluating, and structurally inert", }, { "owner": "CNPG: enablePodMonitor on the Cluster CRs (the operator renders one PodMonitor each)", "prefixes": ("cnpg_",), "file": "platform/keycloak/postgres-cluster.yaml", "require": [r"^\s*enablePodMonitor:\s*true\s*$"], "forbid": [], "why": "cnpg_* powers all six platform-health.cnpg rules, including the WAL-archive backlog " "and no-recoverability-point rules that guard R-STATE-03", }, { "owner": "Flux controllers: platform/monitoring/flux-podmonitor.yaml (merged a79693d)", "prefixes": ("gotk_",), "file": "platform/monitoring/flux-podmonitor.yaml", "require": [r"kind:\s*PodMonitor", r"port:\s*http-prom"], "forbid": [], "why": "gotk_* powers FluxReconciliationFailure, FluxResourceSuspended and the three " "automated-update rules — on a platform whose entire delivery model is Flux", }, { "owner": "cert-manager: prometheus.servicemonitor.enabled in the HelmRelease", "prefixes": ("certmanager_",), "file": "platform/cert-manager/helmrelease.yaml", "require": [r"prometheus:\s*\n\s*servicemonitor:\s*\n\s*enabled:\s*true"], # The chart calls `fail` when both monitor kinds are enabled, which would break the whole # HelmRelease rather than just the scrape. Measured against v1.21.2. "forbid": [r"podmonitor:\s*\n\s*enabled:\s*true"], "why": "certmanager_* powers CertificateExpiringSoon / ExpiryImminent / NotReady — the only " "warning that an ACME DNS-01 renewal of the live wildcard has quietly failed", }, { "owner": "Longhorn: metrics.serviceMonitor.enabled in the HelmRelease", "prefixes": ("longhorn_",), "file": "platform/longhorn/helmrelease.yaml", "require": [r"metrics:\s*\n\s*serviceMonitor:\s*\n\s*enabled:\s*true"], "forbid": [], "why": "longhorn_* powers the four storage rules on a cluster whose recurring failure mode " "is a disk", # The ServiceMonitor ALONE is not sufficient here — see the companion assertion below. "companion": { "file": "platform/longhorn/kustomization.yaml", "require": [r"^\s*-\s*networkpolicy-metrics\.yaml\s*$"], "why": "Longhorn 1.12.1 ships its own effectively-unconditional NetworkPolicies " "(guarded on networkPolicies.restrictInternalTraffic, default true — NOT on " "networkPolicies.enabled) whose longhorn-manager policy admits only other " "Longhorn components in the same namespace. MEASURED on prod: with the " "ServiceMonitor applied and no allowance, all three targets were DISCOVERED and " "every scrape failed with 'context deadline exceeded' after 10.0s — a drop, not " "a refusal — and the TSDB stayed empty. So without networkpolicy-metrics.yaml " "the four storage alerts are inert even though the monitor exists", }, }, { "owner": "KubeVirt/CDI: platform/kubevirt/servicemonitor.yaml", "prefixes": ("kubevirt_",), "file": "platform/kubevirt/servicemonitor.yaml", "require": [r"kind:\s*ServiceMonitor", r"port:\s*metrics"], "forbid": [], "why": "kubevirt_* powers the four platform-health.kubevirt rules (note the CDI rules use " "the kubevirt_cdi_* names, not bare cdi_*)", }, { "owner": "Velero: metrics.serviceMonitor.enabled in the HelmRelease", "prefixes": ("velero_",), "file": "platform/velero/helmrelease.yaml", "require": [r"metrics:\s*\n\s*serviceMonitor:\s*\n\s*enabled:\s*true"], "forbid": [], "why": "velero_* powers VeleroBackupFailure / PartialFailure / NoRecentBackup — the rules " "that failed to notice prod running with zero volume backups", }, # ── ADDED 2026-10-17 (feat/automated-upgrades). A NEW OWNER, NOT A RELAXED ASSERTION. ────── # This entry is purely additive: it registers a new metric family and its enablement, and it # changes nothing about how any existing owner is evaluated. It makes the check STRICTER for the # component it covers, not looser — it adds two `require` patterns plus a `companion` obligation # that platform/kured must keep satisfying, so a future edit that disables kured's Service, # ServiceMonitor or NetworkPolicy FAILS the gate instead of silently re-inerting three alerts. # # WHY THE CHECK HAD TO BE TOUCHED AT ALL, rather than the manifest being wrong: assertion (3) # fails any alert over a metric no owner claims, and the escape hatch (KNOWN_UNWIRED_METRICS) is # SHRINK-ONLY, so declaring `kured_*` as accepted debt is not permitted and would be the wrong # answer anyway — the scrape genuinely IS wired here. The only two honest options were to # register the owner (this) or to ship the reboot automation with no alerting at all, which for a # component whose every failure mode is silent would have been the worse engineering choice. # `conformance/**` is a sensitive approval path precisely so that this reasoning gets read. { "owner": "kured: service.create + metrics.create in the HelmRelease", "prefixes": ("kured_",), "file": "platform/kured/helmrelease.yaml", # BOTH keys are required and the chart defaults BOTH to false. `metrics.create` makes the # ServiceMonitor; `service.create` makes the Service it selects. Enable only the monitor and # it selects nothing, so the single kured metric is never scraped — and a PrometheusRule over # a series no target produces is INERT, which for this component means its own liveness # alarm is the thing that breaks. Matched as two separate anchored blocks rather than one # pattern because they are separate top-level values keys, not nested. "require": [ r"service:\s*\n\s*create:\s*true", r"metrics:\s*\n\s*create:\s*true", ], "forbid": [], "why": "kured_reboot_required is the ONLY metric kured exposes (one GaugeVec, " "cmd/kured/main.go:92-96) and it powers all three platform-health.node-reboots " "rules. Those rules are the only way to learn that OS reboot automation has stalled: " "kured stays 1/1 Running while rebooting nothing, so a missed scrape here is " "indistinguishable from a cluster that needed no reboots", # The ServiceMonitor ALONE is not sufficient — the same NetworkPolicy trap Longhorn hit. "companion": { "file": "platform/kured/kustomization.yaml", "require": [r"^\s*-\s*networkpolicy\.yaml\s*$"], "why": "platform/kured/networkpolicy.yaml ships a default-deny baseline (the kured " "namespace is a platform namespace, so Kyverno's app-namespace-scoped generator " "does not cover it) AND the ingress allow that lets Prometheus reach :8080 " "through it. Drop the networkpolicy from this kustomization and the deny-all " "goes too, so this currently fails SAFE — but reinstating the baseline without " "the allow reproduces Longhorn's measured failure exactly: targets DISCOVERED, " "every scrape 'context deadline exceeded', TSDB empty, all three rules quiet. " "Keeping the two in one file makes that pair inseparable", }, }, # ── ADDED 2026-11-09 (CVE visibility, ADR-0027). A NEW OWNER, NOT A RELAXED ASSERTION. ────── # Registers the vuln_scanner_* family and the Service+ServiceMonitor that scrapes it. { "owner": "vuln-scanner: Service + ServiceMonitor on port `metrics` (:9090)", "prefixes": ("vuln_scanner_",), "file": "platform/vuln-scanner/servicemonitor.yaml", "require": [ r"^kind:\s*ServiceMonitor\s*$", r"^\s*-\s*port:\s*metrics\s*$", ], "forbid": [], "why": "vuln_scanner_* powers the five VulnScan alerts, the only email path for scan " "failure, staleness and overdue critical fixes (R-VULN-06). Two of those rules fire " "on an absent metric, so a lost scrape is loud rather than inert", "companion": { "file": "platform/vuln-scanner/kustomization.yaml", "require": [r"^\s*-\s*networkpolicy\.yaml\s*$"], "why": "platform/vuln-scanner/networkpolicy.yaml ships the namespace's default-deny " "and the ingress allow for Prometheus on :9090 in one file, so a baseline " "without its allow cannot empty the TSDB", }, }, # ── ADDED 2026-11-08 (feat/cloudflare-tunnel). A NEW OWNER, NOT A RELAXED ASSERTION. ─────── # Purely additive, the same shape as the kured entry: it registers the cloudflared_* family # and the Service+ServiceMonitor that scrapes it, and makes the check STRICTER for that component. { "owner": "cloudflared: headless metrics Service + ServiceMonitor on port `metrics` (:2000)", "prefixes": ("cloudflared_",), "file": "platform/cloudflared/servicemonitor.yaml", "require": [ r"^kind:\s*ServiceMonitor\s*$", r"^kind:\s*Service\s*$", r"^\s*-\s*port:\s*metrics\s*$", ], "forbid": [], "why": "cloudflared_tunnel_ha_connections powers CloudflaredTunnelDown, the only signal " "that the tunnel has lost the Cloudflare edge. That rule is written to FIRE on an " "absent series, so a lost scrape is loud rather than inert — but it would then be " "permanently firing for a wiring reason, which teaches the operator to ignore it", "companion": { "file": "platform/cloudflared/kustomization.yaml", "require": [r"^\s*-\s*networkpolicy\.yaml\s*$"], "why": "platform/cloudflared/networkpolicy.yaml ships the namespace's default-deny " "(a platform namespace, outside Kyverno's generator) together with the ingress " "allow for Prometheus on :2000. Keeping them in one file, applied together, is " "what stops a baseline-without-allow from emptying the TSDB", }, }, ] # `up` is produced by EVERY scrape target, so no owner can meaningfully claim it and assertion (1) # is structurally blind to `up{job="X"}` naming a job nothing produces. That axis is assertion (4). UNIVERSAL = {"up"} # ============================================================================================= # REGISTER 1 — KNOWN_UNWIRED_METRICS. Shrink-only. A metric here is queried by an alert and # claimed by NOTHING, so that alert is inert and the platform is blind to what it describes. # Removing an entry means wiring the scrape. An entry that stops being referenced by any alert, # or that gains an owner, FAILS as stale. # ============================================================================================= KNOWN_UNWIRED_METRICS: dict[str, str] = { "hubble_drop_total": "HubbleDeniedFlowSpike — the platform has NO policy-drop / NetworkPolicy-regression " "visibility. Unlike Longhorn and cert-manager (whose charts already serve /metrics and " "only lacked a ServiceMonitor) Hubble publishes NOTHING to scrape until it is switched on " "in Cilium's own config: platform/cilium/helmrelease.yaml sets hubble.{enabled,relay,ui} " "but not hubble.metrics.enabled, and not prometheus.enabled for the agent. Turning them on " "OPENS PORT 9965 ON EVERY cilium-agent AND ROLLS THE CNI DaemonSet across all three nodes " "— the single most dangerous rollout on this cluster (the 2026-09-01 k8sServiceHost outage; " "cilium was moved to its own Flux Kustomization precisely to isolate that blast radius) — " "and requires a matching edit to roles/cilium/templates/cilium-values.yaml.j2 in the same " "commit or a rebuilt node's bootstrap CNI config silently disagrees with the reconciled " "one. Deliberately DEFERRED to its own change with a one-node canary; see the rule's " "header in platform/monitoring/alerts.yaml.", } # ============================================================================================= # REGISTER 2 — KNOWN_UNWIRED_JOBS. Shrink-only, and NOW EMPTY. # # Each entry was a `up{job="X"}` selector whose monitor named an endpoint port the Service it # selects did not define, so the monitor discovered ZERO targets, no `up{job="X"}` series existed, # and `up{job="X"} == 0` could never fire. The nastiest variant of the class: `up` exists in # abundance, so nothing about the metric name looks wrong, and `PrometheusTargetsMissing` cannot see # it either — a target that was never discovered has no up series to be 0. # # IT HELD ALL TWELVE apps/* SLI RULES, AND IT IS EMPTY BECAUSE THEY WERE FIXED, not because the gap # was waived (2026-09-13, branch fix/app-sli-rules-discover-no-targets). The register's own note # said closing it needed "a Prometheus exporter per app". That was the wrong conclusion from a true # premise: Plex, Pi-hole, Jackett, qBittorrent, Sonarr, Radarr, Seerr, Semaphore and Nextcloud # genuinely export no Prometheus metrics, so instead the `up` SLI was rewritten against a signal that # DOES exist for every workload — kube-state-metrics readiness of the service's own workload object # (`kube_deployment_status_replicas_available` / `kube_statefulset_status_replicas_ready`), and # `kubevirt_vmi_info{phase="running"}` for the VM. No exporter, no new scrape target; each rule is # unit-tested firing AND quiet (apps/wave-*/tests/app-slis.test.yaml) and each expression was # evaluated live on prod, where eleven of the twelve return no series on a healthy cluster and the # twelfth (vm-demo) is not deployed at all. # # The rendered ServiceMonitors went with them: a monitor for a workload that serves no metrics is # dead configuration that reads as coverage. So assertion (4) below now has NOTHING to audit under # apps/ — which is why the monitor-side axis moved into its own check, # tests/l0-static/servicemonitor-ports-resolve. THAT is where a ServiceMonitor naming a port no # Service defines fails, with no alert rule required, resolving by selector across the whole tree # rather than by same-file adjacency, and with its resolver proven on fixtures so it cannot pass for # lack of subjects. Keeping this register non-empty as "documentation" of a closed gap is exactly the # fiction the ratchet forbids: the stale-entry assertion below would fail, and rightly. # ============================================================================================= KNOWN_UNWIRED_JOBS: dict[str, str] = {} # ============================================================================================= # (0) PARSE EVERY ALERT RULE # ============================================================================================= # PromQL metric-name approximation. Documented as a gap in the header: string literals, # label-matcher bodies and aggregation/vector-matching label lists are stripped first, then every # remaining bare identifier that is not a PromQL keyword or function is treated as a metric # selector. Assertion (0b) fails loudly if this stops finding metrics at all, which is the way an # approximate parser silently stops guarding. PROMQL_KEYWORDS = set(""" abs absent absent_over_time ceil changes clamp clamp_max clamp_min day_of_month day_of_week day_of_year days_in_month delta deriv exp floor histogram_quantile histogram_count histogram_sum histogram_fraction histogram_stddev histogram_stdvar holt_winters hour idelta increase irate label_join label_replace ln log2 log10 minute month predict_linear rate resets round scalar sgn sort sort_desc sort_by_label sort_by_label_desc sqrt time timestamp vector year avg count count_values group max min quantile stddev stdvar sum topk bottomk limitk limit_ratio last_over_time avg_over_time min_over_time max_over_time sum_over_time count_over_time quantile_over_time stddev_over_time stdvar_over_time present_over_time mad_over_time double_exponential_smoothing info ts_of_max_over_time ts_of_min_over_time ts_of_last_over_time by without on ignoring group_left group_right offset bool and or unless if atan2 start end inf nan NaN Inf """.split()) IDENT_RE = re.compile(r"(? list[str]: e = re.sub(r'"(?:[^"\\]|\\.)*"', '""', expr) e = re.sub(r"'(?:[^'\\]|\\.)*'", "''", e) e = re.sub(r"`(?:[^`\\]|\\.)*`", "``", e) e = re.sub(r"\{[^{}]*\}", " ", e) # label matchers for _ in range(6): # nested aggregations e = re.sub(r"\b(?:by|without|on|ignoring|group_left|group_right)\s*\([^()]*\)", " ", e) out, seen = [], set() for m in IDENT_RE.finditer(e): n = m.group(1) if n in PROMQL_KEYWORDS or re.fullmatch(r"\d+[smhdwy]?", n) or n in seen: continue seen.add(n) out.append(n) return out def is_golden_fixture(p: Path) -> bool: """conformance/tests/**/golden/** holds EXPECTED-OUTPUT fixtures, not deployed manifests. conformance/tests/l1-rendering/egress-class-render/golden/*.expected.yaml each contain a rendered `-down` PrometheusRule. They are the assertion data of another test, never reconciled into a cluster, so auditing their scrape targets would be auditing a copy — the same "reference, not declaration" distinction ci/touched-requirements.sh draws for ci/tests/ fixture paths. The templates they are rendered FROM are audited through apps/ and catalog/. """ parts = p.parts return "conformance" in parts and "golden" in parts # NESTED WORKTREES: skip dot-DIRECTORIES (.git, .claude) while still matching dot-FILES such # as .sops.yaml. .claude/worktrees/agent-*/ holds full repo copies for isolated agent runs, so # a bare rglob from ROOT audited up to 18 duplicates of every file — which both invents FAILs # and, worse, can satisfy an "something declares X" check from a copy rather than real source. # Same fix as egress-paths-selected-by-rules. .claude/ is gitignored (.gitignore:43). rule_files = sorted( p for p in ROOT.rglob("*.yaml") if not any(part.startswith(".") for part in p.relative_to(ROOT).parent.parts) and not is_golden_fixture(p) and "kind: PrometheusRule" in read(p) ) check(len(rule_files) > 0, "no file declaring `kind: PrometheusRule` found — the parser or the repo layout changed; " "this check would silently audit nothing") # metric -> [ ":", ... ] ; and the same shape for up{job="X"} selectors refs: dict[str, list[str]] = {} job_refs: dict[str, list[str]] = {} n_rules = 0 for f in rule_files: rel = str(f.relative_to(ROOT)) try: docs = list(yaml.safe_load_all(read(f))) except yaml.YAMLError as exc: check(False, f"{rel} is not parseable YAML ({exc}); every alert in it is unaudited") continue for d in docs: if not isinstance(d, dict) or d.get("kind") != "PrometheusRule": continue for grp in (d.get("spec") or {}).get("groups") or []: for r in grp.get("rules") or []: if "alert" not in r: continue # recording rules: not this check's subject n_rules += 1 site = f"{rel}:{r['alert']}" expr = str(r.get("expr", "")) for n in metric_names(expr): refs.setdefault(n, []).append(site) for j in UP_JOB_RE.findall(expr): job_refs.setdefault(j, []).append(site) check(n_rules > 0, "no alerting rule parsed out of any PrometheusRule — the parse contract broke and this " "check would pass vacuously while every alert's metric went unaudited") check(len(refs) >= 20, f"only {len(refs)} distinct metric name(s) parsed out of {n_rules} alerting rule(s). The " f"PromQL approximation has stopped matching — which is exactly how an approximate parser " f"turns into a guard that passes while checking nothing. Fix metric_names() or replace it " f"with a real parser") # ============================================================================================= # (1)(2) RESOLVE EVERY METRIC TO AN OWNER, AND PROVE THE OWNER'S CLAIM AGAINST THE REPO # ============================================================================================= def resolve(metric: str) -> dict | None: """Longest-prefix / exact-name match against SCRAPE_OWNERS.""" best, best_len = None, -1 for o in SCRAPE_OWNERS: if metric in o.get("exact", ()): # exact names win outright return o for pfx in o.get("prefixes", ()): if metric.startswith(pfx) and len(pfx) > best_len: best, best_len = o, len(pfx) return best # Every owner's FILE must exist, whether or not a metric currently lands on it: an owner pointing # at a deleted file is a claim nothing backs. for o in SCRAPE_OWNERS: check((ROOT / o["file"]).is_file(), f"SCRAPE_OWNERS names {o['file']} as the scrape owner for " f"{o.get('prefixes') or o.get('exact')}, and that file does not exist in this repo — the " f"claim is unbacked and the alerts that depend on it are inert") claimed: dict[str, dict] = {} unowned: list[str] = [] for metric in sorted(refs): if metric in UNIVERSAL: continue o = resolve(metric) if o is None: if metric in KNOWN_UNWIRED_METRICS: continue # declared gap; ratcheted below unowned.append(metric) check(False, f"alert metric `{metric}` (queried by {', '.join(sorted(set(refs[metric])))}) is " f"claimed by NO entry in SCRAPE_OWNERS. Nothing in this repo configures a scrape " f"that produces it, so every rule above is STRUCTURALLY INERT: it evaluates to no " f"vector, stays silent forever, and is indistinguishable from a healthy result. " f"Either wire the scrape and add the owner entry, or — if it cannot be wired yet — " f"declare it in KNOWN_UNWIRED_METRICS with the reason, so the gap is reported " f"instead of silent") continue check(metric not in KNOWN_UNWIRED_METRICS, f"`{metric}` is listed in KNOWN_UNWIRED_METRICS but is now claimed by " f"{o['owner']} — the register is shrink-only: delete the entry so the next real gap is " f"the one that shows up") claimed[metric] = o # The owners that are actually load-bearing right now get their declarations verified. for o in SCRAPE_OWNERS: if o not in claimed.values(): continue # nothing queries it today; nothing to guard body = read(ROOT / o["file"]) for rx in o.get("require", []): check(re.search(rx, body, re.M) is not None, f"{o['file']} no longer matches /{rx}/, which is what makes {o['owner']} scrape. " f"Removing it re-inerts every alert on " f"{o.get('prefixes') or o.get('exact')} — {o['why']}") for rx in o.get("forbid", []): check(re.search(rx, body, re.M) is None, f"{o['file']} matches the FORBIDDEN pattern /{rx}/, which defeats {o['owner']} " f"even though the enablement is still authored. {o['why']}") comp = o.get("companion") if comp: cbody = read(ROOT / comp["file"]) check((ROOT / comp["file"]).is_file(), f"{comp['file']} is missing — {comp['why']}") for rx in comp.get("require", []): check(re.search(rx, cbody, re.M) is not None, f"{comp['file']} no longer matches /{rx}/. The monitor alone is NOT enough here: " f"{comp['why']}") # The Longhorn NetworkPolicy must actually grant the path it exists to grant, and no more: the # whole point of splitting it out instead of flipping networkPolicies.restrictInternalTraffic off. NP = ROOT / "platform/longhorn/networkpolicy-metrics.yaml" if any(m.startswith("longhorn_") for m in refs): np = read(NP) check(np != "", f"{NP.relative_to(ROOT)} is missing; the Longhorn scrape is dropped and its " f"four alerts are inert") check(re.search(r"kind:\s*NetworkPolicy", np) is not None, "platform/longhorn/networkpolicy-metrics.yaml is not a NetworkPolicy") check(re.search(r"app:\s*longhorn-manager", np) is not None, "the Longhorn metrics NetworkPolicy does not select app=longhorn-manager, so it does not " "open the pods that serve /metrics") check(re.search(r"kubernetes\.io/metadata\.name:\s*monitoring", np) is not None, "the Longhorn metrics NetworkPolicy does not admit namespace monitoring, so Prometheus " "still cannot reach longhorn-manager and the storage alerts stay inert") check(re.search(r"port:\s*9500", np) is not None, "the Longhorn metrics NetworkPolicy does not name port 9500 (longhorn-manager's `manager` " "port, which serves /metrics)") # LEAST PRIVILEGE (R-SEC-02): ingress only, one port, and NOT the webhook/recovery ports. check(re.search(r"policyTypes:\s*\n\s*-\s*Ingress\s*$", np, re.M) is not None and "Egress" not in np, "the Longhorn metrics NetworkPolicy is not Ingress-only — it must grant exactly the " "scrape path and nothing else (R-SEC-02)") # Comments are stripped first: the file's header deliberately NAMES 9502/9503 to record that # they stay closed, and a substring search over prose would fail on the documentation of the # very property it is asserting. np_spec = "\n".join(re.sub(r"#.*$", "", ln) for ln in np.splitlines()) for closed in ("9502", "9503"): check(closed not in np_spec, f"the Longhorn metrics NetworkPolicy names port {closed} (admission webhook / " f"recovery backend). Only 9500 serves metrics; opening the others to ns monitoring " f"is reach this observability path does not need (R-SEC-02)") # stale KNOWN_UNWIRED_METRICS entries for m in sorted(KNOWN_UNWIRED_METRICS): check(m in refs, f"KNOWN_UNWIRED_METRICS declares `{m}`, which NO alert rule in this repo queries any " f"more — a stale exemption hides the next real gap; delete the entry") # ============================================================================================= # (4) up{job="X"} — the axis assertion (1) is structurally blind to. # # Scoped to files where THIS REPO authors both the Service and the ServiceMonitor (the apps/* # ServiceDefinition pattern), because only then is the port name inside the tree. A monitor whose # Service comes from a Helm chart cannot be audited statically and is out of scope (header gap). # ============================================================================================= def yaml_docs(path: Path) -> list[dict]: try: return [d for d in yaml.safe_load_all(read(path)) if isinstance(d, dict)] except yaml.YAMLError: return [] # job name -> (relpath, monitor endpoint ports, ports the co-authored Services define) same_file_monitors: dict[str, tuple[str, set[str], set[str]]] = {} for p in sorted(ROOT.rglob("*.yaml")): if any(part.startswith(".") for part in p.relative_to(ROOT).parent.parts): continue txt = read(p) if "kind: ServiceMonitor" not in txt or "kind: Service" not in txt: continue docs = yaml_docs(p) monitors = [d for d in docs if d.get("kind") == "ServiceMonitor"] services = [d for d in docs if d.get("kind") == "Service"] if not monitors or not services: continue svc_ports: set[str] = set() for s in services: for port in (s.get("spec") or {}).get("ports") or []: if isinstance(port, dict) and port.get("name"): svc_ports.add(str(port["name"])) for m in monitors: ep_ports = {str(e["port"]) for e in (m.get("spec") or {}).get("endpoints") or [] if isinstance(e, dict) and e.get("port")} name = (m.get("metadata") or {}).get("name") if name: same_file_monitors[str(name)] = (str(p.relative_to(ROOT)), ep_ports, svc_ports) for job in sorted(job_refs): sites = ", ".join(sorted(set(job_refs[job]))) mon = same_file_monitors.get(job) if mon is None: # No same-file Service+ServiceMonitor pair is named for this job. It may well be produced # by a platform monitor against a chart Service (oauth2-proxy, kubevirt, velero, flux all # are), which this check cannot see — so this is NOT a failure, only unauditable. continue rel, ep_ports, svc_ports = mon missing = sorted(ep_ports - svc_ports) if missing: check(job in KNOWN_UNWIRED_JOBS, f"`up{{job=\"{job}\"}}` ({sites}) can never produce a series: {rel}'s ServiceMonitor " f"scrapes port(s) {missing}, and the Service(s) in that same file define only " f"{sorted(svc_ports) or ['(no named ports)']}. The monitor therefore discovers ZERO " f"targets, there is no `up{{job=\"{job}\"}}` to compare against 0, and the rule is " f"structurally inert — note PrometheusTargetsMissing cannot see this either, because " f"a target that was never discovered has no up series to be 0. Give the Service a " f"named port the workload actually serves metrics on (and an exporter if it has " f"none), or declare it in KNOWN_UNWIRED_JOBS") else: check(job not in KNOWN_UNWIRED_JOBS, f"`{job}` is listed in KNOWN_UNWIRED_JOBS but {rel}'s ServiceMonitor port(s) " f"{sorted(ep_ports)} now all exist on its Service — the register is shrink-only: " f"delete the entry") # stale KNOWN_UNWIRED_JOBS entries for job in sorted(KNOWN_UNWIRED_JOBS): check(job in job_refs, f"KNOWN_UNWIRED_JOBS declares `{job}`, which no alert rule selects via up{{job=...}} any " f"more — a stale exemption hides the next real gap; delete the entry") # ============================================================================================= # (5) THE CHECK IS LOAD-BEARING. Asserted against the DECLARATIVE mechanism, in both halves — # neither alone is sufficient. The per-test hand-written ci/l0.sh stage is deliberately NOT used # (whole-repo-invariants-wired FAILS the gate if one reappears); the property is DECLARED here and # HONOURED by ci/l0.sh's --include-scope union. # ============================================================================================= SCOPE_DECL_RE = r"^\s*conformance\.home\.k3s/scope:\s*[\"']?whole-repo" SELF = read(ROOT / "conformance" / "tests" / "l0-static" / "alert-metrics-have-scrape-targets" / "chainsaw-test.yaml") check(re.search(SCOPE_DECL_RE, SELF, re.M) is not None, "alert-metrics-have-scrape-targets/chainsaw-test.yaml does not declare " "conformance.home.k3s/scope: whole-repo, so the merge gate selects this check only when a " "change happens to touch R-OBS-04. ci/touched-requirements.sh maps apps/* to NOTHING and " "platform/monitoring/* to R-OBS-01/R-OBS-02 — so adding an alert under apps/ or flipping a " "chart's serviceMonitor.enabled back to false would be filtered out of exactly the change " "that re-inerts the alerts") L0 = read(ROOT / "ci" / "l0.sh") GATE = read(ROOT / "ci" / "gate.sh") check("--include-scope" in L0, "ci/l0.sh no longer passes --include-scope to the conformance runner, so the L0 stage is a " "purely requirement-scoped run again and the scope annotation above is inert decoration") check(re.search(r"^for s in .*\bl0\b.*; do", GATE, re.M) is not None and re.search(r"fail\).*BLOCK=1", GATE) is not None, "ci/gate.sh does not set the merge block when the l0 stage fails") # ======================================================================== report owners_used = {id(o): o for o in claimed.values()} print(f"Alert metric inventory: {n_rules} alerting rule(s) across {len(rule_files)} " f"PrometheusRule file(s) -> {len(refs)} distinct metric name(s)") print(f" claimed by a declared scrape owner: {len(claimed)}/{len(refs) - len(UNIVERSAL & set(refs))}" f" owners exercised: {len(owners_used)}/{len(SCRAPE_OWNERS)}") for o in SCRAPE_OWNERS: ms = sorted(m for m, oo in claimed.items() if oo is o) mark = " " if ms else "· " print(f" {mark}{o['file']:<46} {len(ms):>2} metric(s) {o['owner'][:64]}") print(f"\n DECLARED GAPS (shrink-only registers):") print(f" KNOWN_UNWIRED_METRICS: {len(KNOWN_UNWIRED_METRICS)} — " f"{', '.join(sorted(KNOWN_UNWIRED_METRICS))}") print(f" KNOWN_UNWIRED_JOBS: {len(KNOWN_UNWIRED_JOBS)} inert up{{job=...}} SLI rule(s) — " f"{', '.join(sorted(KNOWN_UNWIRED_JOBS))}") if unowned: print(f"\n UNCLAIMED (inert, undeclared): {', '.join(unowned)}", file=sys.stderr) print() if not FAILURES: print(f"check OK: alert-metrics-have-scrape-targets — {CHECKS} assertions (R-OBS-04)") raise SystemExit(0) print(f"CHECK FAIL (alert-metrics-have-scrape-targets): {len(FAILURES)} failed, " f"{CHECKS - len(FAILURES)} passed", file=sys.stderr) for f in FAILURES: print(f" FAIL {f}", file=sys.stderr) raise SystemExit(1) PY