diff --git a/apps/wave-0/gitea.yaml b/apps/wave-0/gitea.yaml index d79b97b..bf1f088 100644 --- a/apps/wave-0/gitea.yaml +++ b/apps/wave-0/gitea.yaml @@ -238,26 +238,6 @@ spec: - name: gitea port: 3000 --- -# ServiceMonitor (R-OBS-01) -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: gitea - namespace: gitea - labels: - release: kube-prometheus-stack - app.kubernetes.io/name: gitea -spec: - selector: - matchLabels: - app.kubernetes.io/name: gitea - namespaceSelector: - matchNames: - - gitea - endpoints: - - port: metrics - interval: 30s ---- # PrometheusRule SLI alerts (R-OBS-01) apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule @@ -272,11 +252,11 @@ spec: - name: gitea.slis rules: - alert: gitea-down - expr: up{job="gitea"} == 0 + expr: kube_deployment_status_replicas_available{namespace="gitea", deployment="gitea"} == 0 or absent(kube_deployment_status_replicas_available{namespace="gitea", deployment="gitea"}) for: 5m labels: severity: critical service: gitea annotations: summary: gitea is down - description: 'gitea has been unreachable for 5m (SLI: up).' + description: 'gitea: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available).' diff --git a/apps/wave-0/jackett.yaml b/apps/wave-0/jackett.yaml index 0f21bec..96a4c5d 100644 --- a/apps/wave-0/jackett.yaml +++ b/apps/wave-0/jackett.yaml @@ -284,26 +284,6 @@ spec: - name: jackett port: 4180 --- -# ServiceMonitor (R-OBS-01) -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: jackett - namespace: jackett - labels: - release: kube-prometheus-stack - app.kubernetes.io/name: jackett -spec: - selector: - matchLabels: - app.kubernetes.io/name: jackett - namespaceSelector: - matchNames: - - jackett - endpoints: - - port: metrics - interval: 30s ---- # PrometheusRule SLI alerts (R-OBS-01) apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule @@ -318,11 +298,11 @@ spec: - name: jackett.slis rules: - alert: jackett-down - expr: up{job="jackett"} == 0 + expr: kube_deployment_status_replicas_available{namespace="jackett", deployment="jackett"} == 0 or absent(kube_deployment_status_replicas_available{namespace="jackett", deployment="jackett"}) for: 5m labels: severity: critical service: jackett annotations: summary: jackett is down - description: 'jackett has been unreachable for 5m (SLI: up).' + description: 'jackett: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available).' diff --git a/apps/wave-0/pihole.yaml b/apps/wave-0/pihole.yaml index 680e52c..edf537e 100644 --- a/apps/wave-0/pihole.yaml +++ b/apps/wave-0/pihole.yaml @@ -239,26 +239,6 @@ spec: - name: pihole port: 4180 --- -# ServiceMonitor (R-OBS-01) -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: pihole - namespace: pihole - labels: - release: kube-prometheus-stack - app.kubernetes.io/name: pihole -spec: - selector: - matchLabels: - app.kubernetes.io/name: pihole - namespaceSelector: - matchNames: - - pihole - endpoints: - - port: metrics - interval: 30s ---- # PrometheusRule SLI alerts (R-OBS-01) apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule @@ -273,11 +253,11 @@ spec: - name: pihole.slis rules: - alert: pihole-down - expr: up{job="pihole"} == 0 + expr: kube_deployment_status_replicas_available{namespace="pihole", deployment="pihole"} == 0 or absent(kube_deployment_status_replicas_available{namespace="pihole", deployment="pihole"}) for: 5m labels: severity: critical service: pihole annotations: summary: pihole is down - description: 'pihole has been unreachable for 5m (SLI: up).' + description: 'pihole: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available).' diff --git a/apps/wave-0/plex.yaml b/apps/wave-0/plex.yaml index d8d663b..d189d25 100644 --- a/apps/wave-0/plex.yaml +++ b/apps/wave-0/plex.yaml @@ -244,26 +244,6 @@ spec: - name: plex port: 32400 --- -# ServiceMonitor (R-OBS-01) -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: plex - namespace: plex - labels: - release: kube-prometheus-stack - app.kubernetes.io/name: plex -spec: - selector: - matchLabels: - app.kubernetes.io/name: plex - namespaceSelector: - matchNames: - - plex - endpoints: - - port: metrics - interval: 30s ---- # PrometheusRule SLI alerts (R-OBS-01) apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule @@ -278,11 +258,11 @@ spec: - name: plex.slis rules: - alert: plex-down - expr: up{job="plex"} == 0 + expr: kube_deployment_status_replicas_available{namespace="plex", deployment="plex"} == 0 or absent(kube_deployment_status_replicas_available{namespace="plex", deployment="plex"}) for: 5m labels: severity: critical service: plex annotations: summary: plex is down - description: 'plex has been unreachable for 5m (SLI: up).' + description: 'plex: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available).' diff --git a/apps/wave-0/qbittorrent.yaml b/apps/wave-0/qbittorrent.yaml index 922274e..bf0940f 100644 --- a/apps/wave-0/qbittorrent.yaml +++ b/apps/wave-0/qbittorrent.yaml @@ -360,26 +360,6 @@ spec: - name: qbittorrent port: 4180 --- -# ServiceMonitor (R-OBS-01) -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: qbittorrent - namespace: qbittorrent - labels: - release: kube-prometheus-stack - app.kubernetes.io/name: qbittorrent -spec: - selector: - matchLabels: - app.kubernetes.io/name: qbittorrent - namespaceSelector: - matchNames: - - qbittorrent - endpoints: - - port: metrics - interval: 30s ---- # PrometheusRule SLI alerts (R-OBS-01) apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule @@ -394,11 +374,11 @@ spec: - name: qbittorrent.slis rules: - alert: qbittorrent-down - expr: up{job="qbittorrent"} == 0 + expr: kube_deployment_status_replicas_available{namespace="qbittorrent", deployment="qbittorrent"} == 0 or absent(kube_deployment_status_replicas_available{namespace="qbittorrent", deployment="qbittorrent"}) for: 5m labels: severity: critical service: qbittorrent annotations: summary: qbittorrent is down - description: 'qbittorrent has been unreachable for 5m (SLI: up).' + description: 'qbittorrent: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available).' diff --git a/apps/wave-0/tests/app-slis.test.yaml b/apps/wave-0/tests/app-slis.test.yaml new file mode 100644 index 0000000..c95052d --- /dev/null +++ b/apps/wave-0/tests/app-slis.test.yaml @@ -0,0 +1,267 @@ +# promtool rule-unit tests for the wave-0 per-app `-down` SLI rules. +# Manifests under test: every PrometheusRule in apps/wave-0/ — gitea, jackett, pihole, plex, +# qbittorrent (container/Deployment) and vm-demo (KubeVirt VMI). The harness discovers this file by +# the SUITE convention (`/tests/*.test.yaml`) and flattens those RENDERED manifests, so these +# tests run against the exact expressions Flux applies, never a copy. +# +# Run with: platform/monitoring/tests/run.sh apps/wave-0/tests/app-slis.test.yaml +# (the merge gate runs the same harness via +# conformance/tests/l0-static/alert-rules-unit-tested) +# +# WHY THESE EXIST. Until 2026-09-13 every one of these rules read `up{job=""} == 0` while the +# rendered ServiceMonitor named a Service port (`metrics`) that no rendered Service defined. The +# monitor discovered ZERO targets, so no `up{job=""}` series existed and the rule could not +# fire — on prod, `count by (job) (up)` listed 24 jobs and not one of the twelve appeared. +# `PrometheusTargetsMissing` could not see it either: a target that was never discovered has no `up` +# series to be 0. The rules now read kube-state-metrics / kubevirt series that DO exist for these +# workloads (verified on prod), and every rule below is pinned in BOTH directions: +# * QUIET on the healthy state -> it is not permanently firing; +# * FIRING when the workload has no ready replica -> it is not inert; +# * FIRING when the signal is ABSENT -> the failure mode that produced this whole defect class +# (a rule that goes quiet because its series vanished) is itself alerted on. +# +# CLOCK/`for` CONVENTION: sample interval 1m, series 21 samples long, evaluated at 10m — comfortably +# past every rule's `for: 5m` and well inside Prometheus' 5m lookback. +rule_files: + - generated.rules.yaml + +evaluation_interval: 1m + +tests: + # =========================================================================================== + # 1. HEALTHY WAVE-0 (what prod looks like right now) -> every rule SILENT. + # This is the "not permanently firing" half. It is also the test that catches a mis-spelled + # label: get `deployment=` wrong and the `== 0` half matches nothing while the `absent()` half + # matches everything, so this test goes red instead of the rule going quietly wrong. + # =========================================================================================== + - interval: 1m + name: wave-0 healthy - every app has a ready replica, the demo VM runs - all quiet + input_series: + - series: 'kube_deployment_status_replicas_available{namespace="gitea", deployment="gitea"}' + values: '1+0x20' + - series: 'kube_deployment_status_replicas_available{namespace="jackett", deployment="jackett"}' + values: '1+0x20' + - series: 'kube_deployment_status_replicas_available{namespace="pihole", deployment="pihole"}' + values: '1+0x20' + - series: 'kube_deployment_status_replicas_available{namespace="plex", deployment="plex"}' + values: '1+0x20' + - series: 'kube_deployment_status_replicas_available{namespace="qbittorrent", deployment="qbittorrent"}' + values: '1+0x20' + - series: 'kubevirt_vmi_info{namespace="vm-demo", name="vm-demo", phase="running", node="k3s-prod-1"}' + values: '1+0x20' + alert_rule_test: + - eval_time: 10m + alertname: gitea-down + exp_alerts: [] + - eval_time: 10m + alertname: jackett-down + exp_alerts: [] + - eval_time: 10m + alertname: pihole-down + exp_alerts: [] + - eval_time: 10m + alertname: plex-down + exp_alerts: [] + - eval_time: 10m + alertname: qbittorrent-down + exp_alerts: [] + - eval_time: 10m + alertname: vm-demo-down + exp_alerts: [] + + # =========================================================================================== + # 2. EACH APP DOWN IN TURN (0 available replicas) -> ONLY that app's rule fires. + # The per-app isolation matters: a rule that forgot its `namespace=`/`deployment=` matchers + # would light up every app at once and be useless for routing. + # =========================================================================================== + - interval: 1m + name: wave-0 plex has no available replica - only plex-down fires + input_series: + - series: 'kube_deployment_status_replicas_available{namespace="gitea", deployment="gitea"}' + values: '1+0x20' + - series: 'kube_deployment_status_replicas_available{namespace="jackett", deployment="jackett"}' + values: '1+0x20' + - series: 'kube_deployment_status_replicas_available{namespace="pihole", deployment="pihole"}' + values: '1+0x20' + - series: 'kube_deployment_status_replicas_available{namespace="plex", deployment="plex"}' + values: '0+0x20' + - series: 'kube_deployment_status_replicas_available{namespace="qbittorrent", deployment="qbittorrent"}' + values: '1+0x20' + - series: 'kubevirt_vmi_info{namespace="vm-demo", name="vm-demo", phase="running"}' + values: '1+0x20' + alert_rule_test: + - eval_time: 10m + alertname: plex-down + exp_alerts: + - exp_labels: + severity: critical + service: plex + namespace: plex + deployment: plex + exp_annotations: + summary: "plex is down" + description: "plex: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available)." + - eval_time: 10m + alertname: gitea-down + exp_alerts: [] + - eval_time: 10m + alertname: jackett-down + exp_alerts: [] + - eval_time: 10m + alertname: pihole-down + exp_alerts: [] + - eval_time: 10m + alertname: qbittorrent-down + exp_alerts: [] + - eval_time: 10m + alertname: vm-demo-down + exp_alerts: [] + # Before `for: 5m` has elapsed the rule must be PENDING, not firing — proof the 5m is real and + # a single scrape blip does not page anyone. + promql_expr_test: + - expr: 'kube_deployment_status_replicas_available{namespace="plex", deployment="plex"} == 0 or absent(kube_deployment_status_replicas_available{namespace="plex", deployment="plex"})' + eval_time: 2m + exp_samples: + - labels: 'kube_deployment_status_replicas_available{deployment="plex", namespace="plex"}' + value: 0 + + - interval: 1m + name: wave-0 gitea/jackett/pihole/qbittorrent each down - each own rule fires + input_series: + - series: 'kube_deployment_status_replicas_available{namespace="gitea", deployment="gitea"}' + values: '0+0x20' + - series: 'kube_deployment_status_replicas_available{namespace="jackett", deployment="jackett"}' + values: '0+0x20' + - series: 'kube_deployment_status_replicas_available{namespace="pihole", deployment="pihole"}' + values: '0+0x20' + - series: 'kube_deployment_status_replicas_available{namespace="plex", deployment="plex"}' + values: '1+0x20' + - series: 'kube_deployment_status_replicas_available{namespace="qbittorrent", deployment="qbittorrent"}' + values: '0+0x20' + - series: 'kubevirt_vmi_info{namespace="vm-demo", name="vm-demo", phase="running"}' + values: '1+0x20' + alert_rule_test: + - eval_time: 10m + alertname: gitea-down + exp_alerts: + - exp_labels: + severity: critical + service: gitea + namespace: gitea + deployment: gitea + exp_annotations: + summary: "gitea is down" + description: "gitea: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available)." + - eval_time: 10m + alertname: jackett-down + exp_alerts: + - exp_labels: + severity: critical + service: jackett + namespace: jackett + deployment: jackett + exp_annotations: + summary: "jackett is down" + description: "jackett: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available)." + - eval_time: 10m + alertname: pihole-down + exp_alerts: + - exp_labels: + severity: critical + service: pihole + namespace: pihole + deployment: pihole + exp_annotations: + summary: "pihole is down" + description: "pihole: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available)." + - eval_time: 10m + alertname: qbittorrent-down + exp_alerts: + - exp_labels: + severity: critical + service: qbittorrent + namespace: qbittorrent + deployment: qbittorrent + exp_annotations: + summary: "qbittorrent is down" + description: "qbittorrent: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available)." + - eval_time: 10m + alertname: plex-down + exp_alerts: [] + - eval_time: 10m + alertname: vm-demo-down + exp_alerts: [] + + # =========================================================================================== + # 3. THE SIGNAL VANISHES -> the rule FIRES. This is the defect class itself, asserted on. + # The old `up{job=...} == 0` rule had exactly this shape of hole and was silent for it. Here + # the Deployment series for plex is simply not in the input at all, and plex-down must still + # fire (via the `absent()` half), with the labels the selector's equality matchers carry. + # =========================================================================================== + - interval: 1m + name: wave-0 plex Deployment series ABSENT (never scraped / workload gone) - plex-down fires + input_series: + - series: 'kube_deployment_status_replicas_available{namespace="gitea", deployment="gitea"}' + values: '1+0x20' + - series: 'kubevirt_vmi_info{namespace="vm-demo", name="vm-demo", phase="running"}' + values: '1+0x20' + alert_rule_test: + - eval_time: 10m + alertname: plex-down + exp_alerts: + - exp_labels: + severity: critical + service: plex + namespace: plex + deployment: plex + exp_annotations: + summary: "plex is down" + description: "plex: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available)." + - eval_time: 10m + alertname: gitea-down + exp_alerts: [] + + # =========================================================================================== + # 4. THE VM: a VMI carries no replica count, so liveness is the EXISTENCE of a running-phase + # series. Two ways to be down, both asserted: the VMI is gone entirely, and the VMI exists in + # some OTHER phase. The second is the one a naive `kubevirt_vmi_info == 1` would miss, and it + # is why the phase value is matched — LOWERCASE `running`, the value KubeVirt actually writes + # (a capital-R matcher selects nothing and would be just as inert). + # =========================================================================================== + - interval: 1m + name: wave-0 demo VM has no running-phase VMI - vm-demo-down fires + input_series: + - series: 'kube_deployment_status_replicas_available{namespace="gitea", deployment="gitea"}' + values: '1+0x20' + alert_rule_test: + - eval_time: 10m + alertname: vm-demo-down + exp_alerts: + - exp_labels: + severity: critical + service: vm-demo + namespace: vm-demo + name: vm-demo + phase: running + exp_annotations: + summary: "vm-demo is down" + description: "vm-demo: no KubeVirt VMI in phase running for 5m (SLI: up, via kubevirt_vmi_info)." + + - interval: 1m + name: wave-0 demo VM exists but is Failed, not running - vm-demo-down still fires + input_series: + - series: 'kubevirt_vmi_info{namespace="vm-demo", name="vm-demo", phase="failed"}' + values: '1+0x20' + alert_rule_test: + - eval_time: 10m + alertname: vm-demo-down + exp_alerts: + - exp_labels: + severity: critical + service: vm-demo + namespace: vm-demo + name: vm-demo + phase: running + exp_annotations: + summary: "vm-demo is down" + description: "vm-demo: no KubeVirt VMI in phase running for 5m (SLI: up, via kubevirt_vmi_info)." diff --git a/apps/wave-0/vm-demo.yaml b/apps/wave-0/vm-demo.yaml index a147c77..c82b1b6 100644 --- a/apps/wave-0/vm-demo.yaml +++ b/apps/wave-0/vm-demo.yaml @@ -115,26 +115,6 @@ spec: - toEntities: - world --- -# ServiceMonitor (R-OBS-01) -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: vm-demo - namespace: vm-demo - labels: - release: kube-prometheus-stack - app.kubernetes.io/name: vm-demo -spec: - selector: - matchLabels: - app.kubernetes.io/name: vm-demo - namespaceSelector: - matchNames: - - vm-demo - endpoints: - - port: metrics - interval: 30s ---- # PrometheusRule SLI alerts (R-OBS-01) apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule @@ -149,11 +129,11 @@ spec: - name: vm-demo.slis rules: - alert: vm-demo-down - expr: up{job="vm-demo"} == 0 + expr: absent(kubevirt_vmi_info{namespace="vm-demo", name="vm-demo", phase="running"}) for: 5m labels: severity: critical service: vm-demo annotations: summary: vm-demo is down - description: 'vm-demo has been unreachable for 5m (SLI: up).' + description: 'vm-demo: no KubeVirt VMI in phase running for 5m (SLI: up, via kubevirt_vmi_info).' diff --git a/apps/wave-1/keycloak.yaml b/apps/wave-1/keycloak.yaml index 6c52ca5..a21e8ff 100644 --- a/apps/wave-1/keycloak.yaml +++ b/apps/wave-1/keycloak.yaml @@ -184,26 +184,6 @@ spec: - name: keycloak port: 8080 --- -# ServiceMonitor (R-OBS-01) -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: keycloak - namespace: keycloak - labels: - release: kube-prometheus-stack - app.kubernetes.io/name: keycloak -spec: - selector: - matchLabels: - app.kubernetes.io/name: keycloak - namespaceSelector: - matchNames: - - keycloak - endpoints: - - port: metrics - interval: 30s ---- # PrometheusRule SLI alerts (R-OBS-01) apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule @@ -218,11 +198,11 @@ spec: - name: keycloak.slis rules: - alert: keycloak-down - expr: up{job="keycloak"} == 0 + expr: kube_statefulset_status_replicas_ready{namespace="keycloak", statefulset="keycloak"} == 0 or absent(kube_statefulset_status_replicas_ready{namespace="keycloak", statefulset="keycloak"}) for: 5m labels: severity: critical service: keycloak annotations: summary: keycloak is down - description: 'keycloak has been unreachable for 5m (SLI: up).' + description: 'keycloak: its StatefulSet has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_statefulset_status_replicas_ready).' diff --git a/apps/wave-1/nextcloud.yaml b/apps/wave-1/nextcloud.yaml index f54636a..8932d10 100644 --- a/apps/wave-1/nextcloud.yaml +++ b/apps/wave-1/nextcloud.yaml @@ -290,26 +290,6 @@ spec: - name: nextcloud port: 80 --- -# ServiceMonitor (R-OBS-01) -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: nextcloud - namespace: nextcloud - labels: - release: kube-prometheus-stack - app.kubernetes.io/name: nextcloud -spec: - selector: - matchLabels: - app.kubernetes.io/name: nextcloud - namespaceSelector: - matchNames: - - nextcloud - endpoints: - - port: metrics - interval: 30s ---- # PrometheusRule SLI alerts (R-OBS-01) apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule @@ -324,11 +304,11 @@ spec: - name: nextcloud.slis rules: - alert: nextcloud-down - expr: up{job="nextcloud"} == 0 + expr: kube_deployment_status_replicas_available{namespace="nextcloud", deployment="nextcloud"} == 0 or absent(kube_deployment_status_replicas_available{namespace="nextcloud", deployment="nextcloud"}) for: 5m labels: severity: critical service: nextcloud annotations: summary: nextcloud is down - description: 'nextcloud has been unreachable for 5m (SLI: up).' + description: 'nextcloud: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available).' diff --git a/apps/wave-1/radarr.yaml b/apps/wave-1/radarr.yaml index dd83afd..6cd03b3 100644 --- a/apps/wave-1/radarr.yaml +++ b/apps/wave-1/radarr.yaml @@ -364,26 +364,6 @@ spec: - name: radarr port: 4180 --- -# ServiceMonitor (R-OBS-01) -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: radarr - namespace: radarr - labels: - release: kube-prometheus-stack - app.kubernetes.io/name: radarr -spec: - selector: - matchLabels: - app.kubernetes.io/name: radarr - namespaceSelector: - matchNames: - - radarr - endpoints: - - port: metrics - interval: 30s ---- # PrometheusRule SLI alerts (R-OBS-01) apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule @@ -398,11 +378,11 @@ spec: - name: radarr.slis rules: - alert: radarr-down - expr: up{job="radarr"} == 0 + expr: kube_deployment_status_replicas_available{namespace="radarr", deployment="radarr"} == 0 or absent(kube_deployment_status_replicas_available{namespace="radarr", deployment="radarr"}) for: 5m labels: severity: critical service: radarr annotations: summary: radarr is down - description: 'radarr has been unreachable for 5m (SLI: up).' + description: 'radarr: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available).' diff --git a/apps/wave-1/semaphore.yaml b/apps/wave-1/semaphore.yaml index 010e334..9b6a820 100644 --- a/apps/wave-1/semaphore.yaml +++ b/apps/wave-1/semaphore.yaml @@ -243,26 +243,6 @@ spec: - name: semaphore port: 3000 --- -# ServiceMonitor (R-OBS-01) -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: semaphore - namespace: semaphore - labels: - release: kube-prometheus-stack - app.kubernetes.io/name: semaphore -spec: - selector: - matchLabels: - app.kubernetes.io/name: semaphore - namespaceSelector: - matchNames: - - semaphore - endpoints: - - port: metrics - interval: 30s ---- # PrometheusRule SLI alerts (R-OBS-01) apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule @@ -277,11 +257,11 @@ spec: - name: semaphore.slis rules: - alert: semaphore-down - expr: up{job="semaphore"} == 0 + expr: kube_deployment_status_replicas_available{namespace="semaphore", deployment="semaphore"} == 0 or absent(kube_deployment_status_replicas_available{namespace="semaphore", deployment="semaphore"}) for: 5m labels: severity: critical service: semaphore annotations: summary: semaphore is down - description: 'semaphore has been unreachable for 5m (SLI: up).' + description: 'semaphore: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available).' diff --git a/apps/wave-1/sonarr.yaml b/apps/wave-1/sonarr.yaml index e5fa7ab..84028d7 100644 --- a/apps/wave-1/sonarr.yaml +++ b/apps/wave-1/sonarr.yaml @@ -364,26 +364,6 @@ spec: - name: sonarr port: 4180 --- -# ServiceMonitor (R-OBS-01) -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: sonarr - namespace: sonarr - labels: - release: kube-prometheus-stack - app.kubernetes.io/name: sonarr -spec: - selector: - matchLabels: - app.kubernetes.io/name: sonarr - namespaceSelector: - matchNames: - - sonarr - endpoints: - - port: metrics - interval: 30s ---- # PrometheusRule SLI alerts (R-OBS-01) apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule @@ -398,11 +378,11 @@ spec: - name: sonarr.slis rules: - alert: sonarr-down - expr: up{job="sonarr"} == 0 + expr: kube_deployment_status_replicas_available{namespace="sonarr", deployment="sonarr"} == 0 or absent(kube_deployment_status_replicas_available{namespace="sonarr", deployment="sonarr"}) for: 5m labels: severity: critical service: sonarr annotations: summary: sonarr is down - description: 'sonarr has been unreachable for 5m (SLI: up).' + description: 'sonarr: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available).' diff --git a/apps/wave-1/tests/app-slis.test.yaml b/apps/wave-1/tests/app-slis.test.yaml new file mode 100644 index 0000000..ad197c5 --- /dev/null +++ b/apps/wave-1/tests/app-slis.test.yaml @@ -0,0 +1,245 @@ +# promtool rule-unit tests for the wave-1 per-app `-down` SLI rules. +# Manifests under test: every PrometheusRule in apps/wave-1/ — nextcloud, radarr, semaphore, sonarr +# (container/Deployment) and keycloak (StatefulSet, see below). Discovered by the SUITE convention; +# the flattened manifests are the RENDERED ones Flux applies. +# +# Run with: platform/monitoring/tests/run.sh apps/wave-1/tests/app-slis.test.yaml +# +# See apps/wave-0/tests/app-slis.test.yaml for the full history: these rules all read +# `up{job=""} == 0` until 2026-09-13 against a job no scrape target produced, and were +# structurally inert on prod. Each is now pinned QUIET on the healthy state, FIRING on no-ready- +# replica, and FIRING when the signal is ABSENT. +# +# KEYCLOAK IS THE INTERESTING ONE. Its server is a StatefulSet run by the Keycloak operator, NOT the +# Deployment this ServiceDefinition would otherwise imply — so its rule reads +# kube_statefulset_status_replicas_ready (catalog/keycloak.yaml declares +# `observability.livenessWorkload: {kind: StatefulSet, name: keycloak}`). Had it been left to the +# default, the `absent()` half would have matched forever and the alert would have been PERMANENTLY +# FIRING, which is the other failure mode this repo has shipped to prod. Verified on prod +# 2026-09-13: kube_statefulset_status_replicas_ready{namespace="keycloak", statefulset="keycloak"} +# = 2, and the only Deployment in that namespace is keycloak-operator. +rule_files: + - generated.rules.yaml + +evaluation_interval: 1m + +tests: + # =========================================================================================== + # 1. HEALTHY WAVE-1 -> every rule SILENT ("not permanently firing"). + # =========================================================================================== + - interval: 1m + name: wave-1 healthy - every app ready, keycloak StatefulSet at 2 replicas - all quiet + input_series: + - series: 'kube_statefulset_status_replicas_ready{namespace="keycloak", statefulset="keycloak"}' + values: '2+0x20' + - series: 'kube_deployment_status_replicas_available{namespace="nextcloud", deployment="nextcloud"}' + values: '1+0x20' + - series: 'kube_deployment_status_replicas_available{namespace="radarr", deployment="radarr"}' + values: '1+0x20' + - series: 'kube_deployment_status_replicas_available{namespace="semaphore", deployment="semaphore"}' + values: '1+0x20' + - series: 'kube_deployment_status_replicas_available{namespace="sonarr", deployment="sonarr"}' + values: '1+0x20' + # The namespace's OTHER workloads are present on purpose: keycloak-operator is a Deployment in + # the keycloak namespace, and keycloak-db is a StatefulSet in it. A rule that matched the + # namespace only (not the object) would be satisfied — or triggered — by these instead of by + # the IdP itself. That is the exact masking KeycloakDown had to be fixed for. + - series: 'kube_deployment_status_replicas_available{namespace="keycloak", deployment="keycloak-operator"}' + values: '1+0x20' + - series: 'kube_statefulset_status_replicas_ready{namespace="keycloak", statefulset="keycloak-db"}' + values: '3+0x20' + alert_rule_test: + - eval_time: 10m + alertname: keycloak-down + exp_alerts: [] + - eval_time: 10m + alertname: nextcloud-down + exp_alerts: [] + - eval_time: 10m + alertname: radarr-down + exp_alerts: [] + - eval_time: 10m + alertname: semaphore-down + exp_alerts: [] + - eval_time: 10m + alertname: sonarr-down + exp_alerts: [] + + # =========================================================================================== + # 2. THE IdP ITSELF IS DOWN while its operator and DB are fine -> ONLY keycloak-down fires. + # This is the anti-masking assertion: the neighbours in that namespace stay ready. + # =========================================================================================== + - interval: 1m + name: wave-1 keycloak StatefulSet has no ready replica (operator + DB fine) - keycloak-down fires + input_series: + - series: 'kube_statefulset_status_replicas_ready{namespace="keycloak", statefulset="keycloak"}' + values: '0+0x20' + - series: 'kube_deployment_status_replicas_available{namespace="keycloak", deployment="keycloak-operator"}' + values: '1+0x20' + - series: 'kube_statefulset_status_replicas_ready{namespace="keycloak", statefulset="keycloak-db"}' + values: '3+0x20' + - series: 'kube_deployment_status_replicas_available{namespace="nextcloud", deployment="nextcloud"}' + values: '1+0x20' + - series: 'kube_deployment_status_replicas_available{namespace="radarr", deployment="radarr"}' + values: '1+0x20' + - series: 'kube_deployment_status_replicas_available{namespace="semaphore", deployment="semaphore"}' + values: '1+0x20' + - series: 'kube_deployment_status_replicas_available{namespace="sonarr", deployment="sonarr"}' + values: '1+0x20' + alert_rule_test: + - eval_time: 10m + alertname: keycloak-down + exp_alerts: + - exp_labels: + severity: critical + service: keycloak + namespace: keycloak + statefulset: keycloak + exp_annotations: + summary: "keycloak is down" + description: "keycloak: its StatefulSet has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_statefulset_status_replicas_ready)." + - eval_time: 10m + alertname: nextcloud-down + exp_alerts: [] + - eval_time: 10m + alertname: radarr-down + exp_alerts: [] + - eval_time: 10m + alertname: semaphore-down + exp_alerts: [] + - eval_time: 10m + alertname: sonarr-down + exp_alerts: [] + + # =========================================================================================== + # 3. EACH DEPLOYMENT-BACKED APP DOWN -> its own rule fires, keycloak stays quiet. + # =========================================================================================== + - interval: 1m + name: wave-1 nextcloud/radarr/semaphore/sonarr have no available replica - each own rule fires + input_series: + - series: 'kube_statefulset_status_replicas_ready{namespace="keycloak", statefulset="keycloak"}' + values: '2+0x20' + - series: 'kube_deployment_status_replicas_available{namespace="nextcloud", deployment="nextcloud"}' + values: '0+0x20' + - series: 'kube_deployment_status_replicas_available{namespace="radarr", deployment="radarr"}' + values: '0+0x20' + - series: 'kube_deployment_status_replicas_available{namespace="semaphore", deployment="semaphore"}' + values: '0+0x20' + - series: 'kube_deployment_status_replicas_available{namespace="sonarr", deployment="sonarr"}' + values: '0+0x20' + alert_rule_test: + - eval_time: 10m + alertname: nextcloud-down + exp_alerts: + - exp_labels: + severity: critical + service: nextcloud + namespace: nextcloud + deployment: nextcloud + exp_annotations: + summary: "nextcloud is down" + description: "nextcloud: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available)." + - eval_time: 10m + alertname: radarr-down + exp_alerts: + - exp_labels: + severity: critical + service: radarr + namespace: radarr + deployment: radarr + exp_annotations: + summary: "radarr is down" + description: "radarr: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available)." + - eval_time: 10m + alertname: semaphore-down + exp_alerts: + - exp_labels: + severity: critical + service: semaphore + namespace: semaphore + deployment: semaphore + exp_annotations: + summary: "semaphore is down" + description: "semaphore: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available)." + - eval_time: 10m + alertname: sonarr-down + exp_alerts: + - exp_labels: + severity: critical + service: sonarr + namespace: sonarr + deployment: sonarr + exp_annotations: + summary: "sonarr is down" + description: "sonarr: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available)." + - eval_time: 10m + alertname: keycloak-down + exp_alerts: [] + + # =========================================================================================== + # 4. THE SIGNAL VANISHES -> the rules FIRE. The defect class itself, asserted on: a rule that can + # only fire on `== 0` goes SILENT when kube-state-metrics or the workload object disappears, + # which is indistinguishable from healthy. Here NOTHING in wave-1 is scraped. + # =========================================================================================== + - interval: 1m + name: wave-1 all liveness series ABSENT (kube-state-metrics gone) - every rule fires + input_series: + # One unrelated series so the test has an input at all; no wave-1 liveness series exists. + - series: 'up{job="kube-state-metrics"}' + values: '0+0x20' + alert_rule_test: + - eval_time: 10m + alertname: keycloak-down + exp_alerts: + - exp_labels: + severity: critical + service: keycloak + namespace: keycloak + statefulset: keycloak + exp_annotations: + summary: "keycloak is down" + description: "keycloak: its StatefulSet has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_statefulset_status_replicas_ready)." + - eval_time: 10m + alertname: nextcloud-down + exp_alerts: + - exp_labels: + severity: critical + service: nextcloud + namespace: nextcloud + deployment: nextcloud + exp_annotations: + summary: "nextcloud is down" + description: "nextcloud: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available)." + - eval_time: 10m + alertname: radarr-down + exp_alerts: + - exp_labels: + severity: critical + service: radarr + namespace: radarr + deployment: radarr + exp_annotations: + summary: "radarr is down" + description: "radarr: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available)." + - eval_time: 10m + alertname: semaphore-down + exp_alerts: + - exp_labels: + severity: critical + service: semaphore + namespace: semaphore + deployment: semaphore + exp_annotations: + summary: "semaphore is down" + description: "semaphore: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available)." + - eval_time: 10m + alertname: sonarr-down + exp_alerts: + - exp_labels: + severity: critical + service: sonarr + namespace: sonarr + deployment: sonarr + exp_annotations: + summary: "sonarr is down" + description: "sonarr: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available)." diff --git a/apps/wave-2/seerr.yaml b/apps/wave-2/seerr.yaml index 9d6e171..a524a8b 100644 --- a/apps/wave-2/seerr.yaml +++ b/apps/wave-2/seerr.yaml @@ -233,26 +233,6 @@ spec: - name: seerr port: 5055 --- -# ServiceMonitor (R-OBS-01) -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: seerr - namespace: seerr - labels: - release: kube-prometheus-stack - app.kubernetes.io/name: seerr -spec: - selector: - matchLabels: - app.kubernetes.io/name: seerr - namespaceSelector: - matchNames: - - seerr - endpoints: - - port: metrics - interval: 30s ---- # PrometheusRule SLI alerts (R-OBS-01) apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule @@ -267,11 +247,11 @@ spec: - name: seerr.slis rules: - alert: seerr-down - expr: up{job="seerr"} == 0 + expr: kube_deployment_status_replicas_available{namespace="seerr", deployment="seerr"} == 0 or absent(kube_deployment_status_replicas_available{namespace="seerr", deployment="seerr"}) for: 5m labels: severity: critical service: seerr annotations: summary: seerr is down - description: 'seerr has been unreachable for 5m (SLI: up).' + description: 'seerr: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available).' diff --git a/apps/wave-2/tests/app-slis.test.yaml b/apps/wave-2/tests/app-slis.test.yaml new file mode 100644 index 0000000..89c18a2 --- /dev/null +++ b/apps/wave-2/tests/app-slis.test.yaml @@ -0,0 +1,69 @@ +# promtool rule-unit tests for the wave-2 per-app `-down` SLI rule (seerr). +# Manifest under test: apps/wave-2/seerr.yaml. Discovered by the SUITE convention; the flattened +# manifest is the RENDERED one Flux applies. +# +# Run with: platform/monitoring/tests/run.sh apps/wave-2/tests/app-slis.test.yaml +# +# seerr (Jellyseerr/Overseerr) exports NO Prometheus metrics of its own, so its `up` SLI could never +# have been met by scraping the app — see apps/wave-0/tests/app-slis.test.yaml for the full history +# of the inert `up{job="seerr"} == 0` rule this replaces. Pinned in both directions plus the +# signal-vanishes case. +rule_files: + - generated.rules.yaml + +evaluation_interval: 1m + +tests: + - interval: 1m + name: wave-2 healthy - seerr has an available replica - quiet + input_series: + - series: 'kube_deployment_status_replicas_available{namespace="seerr", deployment="seerr"}' + values: '1+0x20' + alert_rule_test: + - eval_time: 10m + alertname: seerr-down + exp_alerts: [] + + - interval: 1m + name: wave-2 seerr has no available replica - seerr-down fires + input_series: + - series: 'kube_deployment_status_replicas_available{namespace="seerr", deployment="seerr"}' + values: '0+0x20' + alert_rule_test: + - eval_time: 10m + alertname: seerr-down + exp_alerts: + - exp_labels: + severity: critical + service: seerr + namespace: seerr + deployment: seerr + exp_annotations: + summary: "seerr is down" + description: "seerr: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available)." + # Under `for: 5m` a 2m-old condition must NOT yet be a firing alert, but the expression must + # already be selecting — the distinction an inert rule cannot make. + promql_expr_test: + - expr: 'kube_deployment_status_replicas_available{namespace="seerr", deployment="seerr"} == 0 or absent(kube_deployment_status_replicas_available{namespace="seerr", deployment="seerr"})' + eval_time: 2m + exp_samples: + - labels: 'kube_deployment_status_replicas_available{deployment="seerr", namespace="seerr"}' + value: 0 + + - interval: 1m + name: wave-2 seerr liveness series ABSENT - seerr-down still fires + input_series: + - series: 'up{job="kube-state-metrics"}' + values: '1+0x20' + alert_rule_test: + - eval_time: 10m + alertname: seerr-down + exp_alerts: + - exp_labels: + severity: critical + service: seerr + namespace: seerr + deployment: seerr + exp_annotations: + summary: "seerr is down" + description: "seerr: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available)." diff --git a/catalog/keycloak.yaml b/catalog/keycloak.yaml index 21f482d..30b5dd5 100644 --- a/catalog/keycloak.yaml +++ b/catalog/keycloak.yaml @@ -31,5 +31,15 @@ observability: metrics: true slis: - up # IdP down/unhealthy is a critical alert (docs/10; R-OBS-01) + # WHAT "up" IS for Keycloak. The Keycloak OPERATOR runs the server as a StatefulSet named + # `keycloak` (pods keycloak-0/1), not as the Deployment this ServiceDefinition would otherwise + # imply — the operator owns the workload (platform/keycloak/keycloak-instance.yaml) and this SD + # is the catalog's declaration of intent for it. Without this override the rendered SLI would + # read a Deployment series that does not exist and its `absent()` half would fire FOREVER. + # Verified on prod 2026-09-13: kube_statefulset_status_replicas_ready{namespace="keycloak", + # statefulset="keycloak"} = 2, and the only Deployment in that namespace is keycloak-operator. + livenessWorkload: + kind: StatefulSet + name: keycloak backup: policy: daily-offcluster # DR-critical: rebuilds from DB + git realm config (docs/15, docs/04; R-STATE-03) diff --git a/catalog/render.py b/catalog/render.py index 8205b14..4c12223 100644 --- a/catalog/render.py +++ b/catalog/render.py @@ -81,6 +81,27 @@ BULK_NFS_MOUNT_OPTIONS = ["nfsvers=4.1", "hard", "noatime", "timeo=600", "retran GATEWAY_NAME = "shared" GATEWAY_NS = "gateway" KP_RELEASE = "kube-prometheus-stack" # ServiceMonitor/PrometheusRule discovery label +MONITORING_NS = "monitoring" # where Prometheus runs; the only admitted scrape source + +# The kube-state-metrics series a workload's readiness IS (r_prometheusrule). kube-state-metrics +# is scraped by the bundled kube-prometheus-stack ServiceMonitor, i.e. these exist for EVERY +# workload in the cluster with no per-service exporter — which is the whole point of using them +# for `slis: [up]`. Each entry is (metric, label naming the object). VERIFIED on prod 2026-09-13: +# kube_deployment_status_replicas_available returned a series for all ten deployed app +# Deployments, and kube_statefulset_status_replicas_ready{namespace="keycloak", +# statefulset="keycloak"} returned 2. +LIVENESS_METRICS = { + "Deployment": ("kube_deployment_status_replicas_available", "deployment"), + "StatefulSet": ("kube_statefulset_status_replicas_ready", "statefulset"), + "DaemonSet": ("kube_daemonset_status_number_available", "daemonset"), +} +# A KubeVirt VMI is not a replica count: kubevirt_vmi_info carries the phase as a LABEL, so +# liveness is the EXISTENCE of a running-phase series. The value is lowercase `running` — a +# capital-R matcher selects nothing and is just as inert (the same trap that made +# kubevirt_vmi_phase_count{phase="Failed"} a no-op; verified on prod, where the one live VMI +# reports phase="running"). +VMI_METRIC = "kubevirt_vmi_info" +VMI_RUNNING_PHASE = "running" # egress classes and auth modes fixed by docs/03 / docs/13 + the Kyverno enums. EGRESS_CLASSES = ("internet", "vpn") @@ -135,6 +156,26 @@ def apply_defaults(sd: dict) -> dict: obs = dict(s.get("observability") or {}) obs.setdefault("metrics", True) obs.setdefault("slis", []) + # metricsEndpoint is NEVER defaulted (schema 1.6.0). `metrics: true` says the platform + # collects signals FOR this service; it cannot say the workload SERVES a Prometheus + # endpoint, because for most of this catalog it does not (see r_servicemonitor). When the + # endpoint IS declared, its name/path/interval carry defaults so the common case is + # `metricsEndpoint: {port: 9100}`. + ep = obs.get("metricsEndpoint") + if ep: + obs["metricsEndpoint"] = { + **{"name": "metrics", "path": "/metrics", "interval": "30s"}, + **ep, + } + # livenessWorkload: WHICH Kubernetes object's readiness IS this service being up. Defaulted + # from `kind` — a container renders a Deployment named after the service, a vm renders a + # VirtualMachine whose VMI carries the same name — and overridable for the case where the + # object that actually runs on the cluster is not the one this SD renders (keycloak: the + # Keycloak operator runs a StatefulSet, so a Deployment-shaped rule would `absent()` forever). + lw = dict(obs.get("livenessWorkload") or {}) + lw.setdefault("kind", "VirtualMachineInstance" if s["kind"] == "vm" else "Deployment") + lw.setdefault("name", s["name"]) + obs["livenessWorkload"] = lw s["observability"] = obs # storage entry class default is longhorn (replicated tier); readOnly default is False @@ -488,7 +529,12 @@ def r_deployment(s: dict) -> dict: def r_service(s: dict): ports = _service_ports(s) - if not ports: + # A declared metrics endpoint becomes a NAMED Service port, because that name is the only + # thing a ServiceMonitor endpoint can select on. Appended BELOW rather than inside + # _service_ports so it never leaks into r_gateway_ingress_policy's north-south allow list — + # the metrics port is for the monitoring namespace, not for the Gateway. + ep = s["observability"].get("metricsEndpoint") + if not ports and not ep: return None # forward-auth (Option B): the shared Gateway must reach the oauth2-proxy sidecar, so front it # with a `authproxy` :4180 Service port (the HTTPRoute backendRef targets this). The app UI/API @@ -498,6 +544,10 @@ def r_service(s: dict): ports = ports + [ {"name": "authproxy", "port": OAUTH2_PROXY_PORT, "targetPort": OAUTH2_PROXY_PORT} ] + if ep: + ports = ports + [ + {"name": ep["name"], "port": ep["port"], "targetPort": ep["port"]} + ] return { "apiVersion": "v1", "kind": "Service", @@ -990,7 +1040,33 @@ def _auth_annotations(s: dict) -> dict: def r_servicemonitor(s: dict): - if not s["observability"]["metrics"]: + """ServiceMonitor — rendered ONLY when the SD says WHERE metrics are served. + + THIS FUNCTION USED TO RENDER A MONITOR UNCONDITIONALLY FROM `metrics: true`, hardcoding + `endpoints: [{port: metrics}]`. No Service this renderer emits had ever defined a port of + that name, so all twelve monitors discovered ZERO targets: no `up{job=""}` series + existed anywhere, `up{job=""} == 0` could not fire, and `PrometheusTargetsMissing` + could not see the hole either, because a target that was never discovered has no `up` series + to be 0. Verified on prod 2026-09-13: `count by (job) (up)` listed 24 jobs and not one of + the twelve appeared; `up{job="plex"}` and `up{job="gitea"}` both returned nothing. + + THE ROOT CAUSE WAS NOT A TYPO — it was that `metrics: true` cannot state a fact the workload + does not have. Plex, Pi-hole, Jackett, qBittorrent, Sonarr, Radarr, Seerr, Semaphore and + Nextcloud serve no Prometheus endpoint at all without a separate exporter; Gitea can serve + one but only once its app.ini enables it. So the port name was never the missing piece. A + monitor is now rendered only from `observability.metricsEndpoint`, which names the port the + workload actually serves — and that same declaration renders the Service port (r_service) and + the monitoring-namespace ingress allow (r_monitoring_ingress_policy), so the three halves of + a working scrape cannot be authored apart. + + `metrics: true` still means something without it: logs ship to Loki via the node agent, and + liveness comes from kube-state-metrics (r_prometheusrule) — neither needs a per-service CR. + """ + obs = s["observability"] + if not obs["metrics"]: + return None + ep = obs.get("metricsEndpoint") + if not ep: return None return { "apiVersion": "monitoring.coreos.com/v1", @@ -1003,7 +1079,108 @@ def r_servicemonitor(s: dict): "spec": { "selector": {"matchLabels": {"app.kubernetes.io/name": s["name"]}}, "namespaceSelector": {"matchNames": [s["name"]]}, - "endpoints": [{"port": "metrics", "interval": "30s"}], + "endpoints": [ + {"port": ep["name"], "path": ep["path"], "interval": ep["interval"]} + ], + }, + } + + +def r_monitoring_ingress_policy(s: dict): + """Let the monitoring namespace reach a declared metrics port (R-NET-02, R-OBS-01). + + renderer-contract.md has always said the default-deny baseline carries "a standing allow ... + for observability scraping from the monitoring namespace" — and no renderer output ever did. + Every app namespace ships `ingress: []`, so a scrape of an app metrics port would be DROPPED + by policy even with the Service port and the monitor endpoint in agreement. That is the same + trap Longhorn's own NetworkPolicies set (platform/longhorn/networkpolicy-metrics.yaml exists + for exactly this reason), and it is a THIRD independent way for a metrics SLI to be inert. + + Rendered only alongside a declared metricsEndpoint, and it opens ONLY that port to ONLY the + monitoring namespace: an allow for a port nothing serves would be a standing hole for no + benefit. + """ + ep = s["observability"].get("metricsEndpoint") + if not ep or not s["observability"]["metrics"]: + return None + return { + "apiVersion": "cilium.io/v2", + "kind": "CiliumNetworkPolicy", + "metadata": { + "name": f"{s['name']}-allow-monitoring-scrape", + "namespace": s["name"], + "labels": {"app.kubernetes.io/name": s["name"]}, + }, + "spec": { + "endpointSelector": {"matchLabels": {"app.kubernetes.io/name": s["name"]}}, + "ingress": [ + { + "fromEndpoints": [ + {"matchLabels": {"k8s:io.kubernetes.pod.namespace": MONITORING_NS}} + ], + "toPorts": [ + {"ports": [{"port": str(ep["port"]), "protocol": "TCP"}]} + ], + } + ], + }, + } + + +def _up_sli_rule(s: dict) -> dict: + """`slis: [up]` -> a `-down` alert against a signal THAT EXISTS on this cluster. + + IT USED TO READ `up{job=""} == 0`. That is the textbook expression and it was inert + for all twelve services in the catalog, because `up` only exists for a target Prometheus + actually DISCOVERED and the rendered ServiceMonitor discovered none (see r_servicemonitor). + Worse than a wrong threshold: an inert rule and a correct rule are indistinguishable on a + healthy cluster, and the generic safety net cannot see the difference either — a target that + was never discovered has no `up` series for PrometheusTargetsMissing to find at 0. + + Rewriting the SLI against the app's own /metrics was NOT an option for most of this catalog: + Plex, Pi-hole, Jackett, qBittorrent, Sonarr, Radarr, Seerr, Semaphore and Nextcloud export no + Prometheus metrics without a separate exporter sidecar per app. So the `up` SLI is expressed + against kube-state-metrics, which IS scraped and DOES carry a series for every workload — the + same idiom KeycloakDown in platform/monitoring/alerts.yaml has used since it was fixed. + + The `or absent(...)` half is deliberate and load-bearing: it makes the DISAPPEARANCE of the + signal fire the alert instead of silencing it. A rule that can only fire on `== 0` is a rule + that goes quiet when the workload object (or kube-state-metrics) vanishes, which is precisely + how the old expression failed. It is also why this rule cannot be permanently firing where the + workload exists: the series is present and non-zero on a healthy cluster (verified on prod). + """ + lw = s["observability"]["livenessWorkload"] + kind, obj = lw["kind"], lw["name"] + ns = s["name"] + if kind == "VirtualMachineInstance": + sel = f'{VMI_METRIC}{{namespace="{ns}", name="{obj}", phase="{VMI_RUNNING_PHASE}"}}' + expr = f"absent({sel})" + desc = ( + f"{s['name']}: no KubeVirt VMI in phase {VMI_RUNNING_PHASE} for 5m " + f"(SLI: up, via {VMI_METRIC})." + ) + else: + try: + metric, label = LIVENESS_METRICS[kind] + except KeyError: + raise ValueError( + f"{s['name']}: observability.livenessWorkload.kind {kind!r} has no liveness " + f"metric; known kinds: {sorted(LIVENESS_METRICS)} + VirtualMachineInstance" + ) from None + sel = f'{metric}{{namespace="{ns}", {label}="{obj}"}}' + expr = f"{sel} == 0 or absent({sel})" + desc = ( + f"{s['name']}: its {kind} has no available/ready replica for 5m, or the signal is " + f"gone entirely (SLI: up, via {metric})." + ) + return { + "alert": f"{s['name']}-down".replace("_", "-"), + "expr": expr, + "for": "5m", + "labels": {"severity": "critical", "service": s["name"]}, + "annotations": { + "summary": f"{s['name']} is down", + "description": desc, }, } @@ -1016,18 +1193,7 @@ def r_prometheusrule(s: dict): rules = [] for sli in slis: if sli == "up": - rules.append( - { - "alert": f"{s['name']}-down".replace("_", "-"), - "expr": f'up{{job="{s["name"]}"}} == 0', - "for": "5m", - "labels": {"severity": "critical", "service": s["name"]}, - "annotations": { - "summary": f"{s['name']} is down", - "description": f"{s['name']} has been unreachable for 5m (SLI: up).", - }, - } - ) + rules.append(_up_sli_rule(s)) else: rules.append( { @@ -1386,6 +1552,16 @@ def render(sd: dict, image_automation: bool = True, inbound: list | None = None) ) ) + mon_allow = r_monitoring_ingress_policy(s) + if mon_allow: + docs.append( + ( + "CiliumNetworkPolicy monitoring scrape ingress allow (R-NET-02/R-OBS-01): " + f"{mon_allow['metadata']['name']}", + mon_allow, + ) + ) + ie = r_internet_egress(s) docs.append( ( @@ -2315,6 +2491,154 @@ def _selftest() -> int: "the escaped `$${...}` form must render (and reach the shell as `${...}`)", ) + # 9) observability (R-OBS-01): `metrics: true` alone renders NO ServiceMonitor, and the `up` + # SLI reads a signal that exists. This is the regression test for the defect that made all + # twelve `-down` rules structurally inert on prod (see r_servicemonitor). + obs_sd = { + "name": "obsdemo", + "source": {"image": "a:1"}, + "ingressRoutes": [{"host": "obsdemo.${BASE_DOMAIN}", "port": 8080}], + "auth": {"mode": "none"}, + "observability": {"metrics": True, "slis": ["up"]}, + } + obs_docs = render(obs_sd) + check( + not [d for d in obs_docs if d[1]["kind"] == "ServiceMonitor"], + "metrics: true with NO metricsEndpoint must render no ServiceMonitor — a monitor naming a " + "port the Service does not define discovers zero targets and makes the SLI inert", + ) + pr = next(d[1] for d in obs_docs if d[1]["kind"] == "PrometheusRule") + up_rule = pr["spec"]["groups"][0]["rules"][0] + check(up_rule["alert"] == "obsdemo-down", "slis: [up] must render -down") + check( + "up{job=" not in up_rule["expr"], + "the up SLI must NOT read up{job=...}: no repo-rendered scrape target produces that job, " + "so the rule can never fire (verified inert on prod for all twelve apps)", + ) + check( + up_rule["expr"] + == 'kube_deployment_status_replicas_available{namespace="obsdemo", deployment="obsdemo"} ' + '== 0 or absent(kube_deployment_status_replicas_available{namespace="obsdemo", ' + 'deployment="obsdemo"})', + "container up SLI must read kube_deployment_status_replicas_available with an absent() " + f"half; got: {up_rule['expr']}", + ) + + # 9a) livenessWorkload override: the object that RUNS is not always the one this SD renders. + sts_docs = render( + dict( + obs_sd, + name="obssts", + observability={ + "metrics": True, + "slis": ["up"], + "livenessWorkload": {"kind": "StatefulSet", "name": "obssts"}, + }, + ) + ) + sts_expr = next( + d[1] for d in sts_docs if d[1]["kind"] == "PrometheusRule" + )["spec"]["groups"][0]["rules"][0]["expr"] + check( + "kube_statefulset_status_replicas_ready" in sts_expr, + f"livenessWorkload kind StatefulSet must read the statefulset series; got: {sts_expr}", + ) + + # 9b) vm kind: a VMI has no replica count, so liveness is a running-phase series EXISTING. + vm_pr = next( + d[1] + for d in render( + { + "name": "obsvm", + "kind": "vm", + "source": {"vmTemplate": "quay.io/x/y:1"}, + "storage": [{"name": "root", "size": "6Gi"}], + "observability": {"metrics": True, "slis": ["up"]}, + } + ) + if d[1]["kind"] == "PrometheusRule" + ) + check( + vm_pr["spec"]["groups"][0]["rules"][0]["expr"] + == 'absent(kubevirt_vmi_info{namespace="obsvm", name="obsvm", phase="running"})', + "vm up SLI must read kubevirt_vmi_info with the LOWERCASE running phase: " + f"{vm_pr['spec']['groups'][0]['rules'][0]['expr']}", + ) + + # 9c) metricsEndpoint renders ALL THREE halves of a working scrape, and they AGREE. Any one of + # them missing is an independent way for the scrape to silently discover/deliver nothing. + ep_docs = render( + dict( + obs_sd, + name="obsep", + observability={ + "metrics": True, + "slis": ["up"], + "metricsEndpoint": {"port": 9100}, + }, + ) + ) + ep_svc = next(d[1] for d in ep_docs if d[1]["kind"] == "Service") + ep_sm = next(d[1] for d in ep_docs if d[1]["kind"] == "ServiceMonitor") + svc_port_names = {p["name"] for p in ep_svc["spec"]["ports"]} + ep_ports = {e["port"] for e in ep_sm["spec"]["endpoints"]} + check( + ep_ports <= svc_port_names, + f"ServiceMonitor endpoint port(s) {sorted(ep_ports)} must all exist on the Service " + f"{sorted(svc_port_names)} — this exact disagreement is what made twelve monitors inert", + ) + check( + {"name": "metrics", "port": 9100, "targetPort": 9100} in ep_svc["spec"]["ports"], + "metricsEndpoint must render the named Service port it declares", + ) + mon_cnp = [ + d[1] + for d in ep_docs + if d[1]["kind"] == "CiliumNetworkPolicy" + and d[1]["metadata"]["name"] == "obsep-allow-monitoring-scrape" + ] + check( + len(mon_cnp) == 1, + "a declared metricsEndpoint must render the monitoring-namespace ingress allow — the " + "namespace default-deny has `ingress: []` and would DROP the scrape without it", + ) + if mon_cnp: + ing = mon_cnp[0]["spec"]["ingress"][0] + check( + ing["fromEndpoints"][0]["matchLabels"]["k8s:io.kubernetes.pod.namespace"] + == MONITORING_NS + and ing["toPorts"][0]["ports"] == [{"port": "9100", "protocol": "TCP"}], + "the scrape allow must admit ONLY the monitoring namespace to ONLY the metrics port", + ) + gw_cnp = next( + d[1] + for d in ep_docs + if d[1]["kind"] == "CiliumNetworkPolicy" + and d[1]["metadata"]["name"] == "obsep-allow-gateway-ingress" + ) + gw_ports = {p["port"] for p in gw_cnp["spec"]["ingress"][0]["toPorts"][0]["ports"]} + check( + "9100" not in gw_ports, + "the metrics port must NOT be opened to the Gateway (north-south); it is for the " + f"monitoring namespace only. Gateway allows: {sorted(gw_ports)}", + ) + # A metricsEndpoint the renderer cannot express must be REFUSED, never rendered inert. + try: + render( + dict( + obs_sd, + name="obsbad", + observability={ + "metrics": True, + "slis": ["up"], + "livenessWorkload": {"kind": "CronJob", "name": "obsbad"}, + }, + ) + ) + check(False, "an unknown livenessWorkload.kind must raise, not render a bogus expression") + except ValueError as exc: + check("CronJob" in str(exc), "the refusal must name the unsupported kind") + if failures: for f in failures: sys.stderr.write(f"SELFTEST FAIL: {f}\n") diff --git a/conformance/tests/l0-static/alert-metrics-have-scrape-targets/chainsaw-test.yaml b/conformance/tests/l0-static/alert-metrics-have-scrape-targets/chainsaw-test.yaml index f0fa6dc..fddf42d 100644 --- a/conformance/tests/l0-static/alert-metrics-have-scrape-targets/chainsaw-test.yaml +++ b/conformance/tests/l0-static/alert-metrics-have-scrape-targets/chainsaw-test.yaml @@ -36,18 +36,27 @@ # (4) `up{job="X"}` is audited on its own axis, because `up` always exists and so the metric # check above can never see this failure: where the repo authors a Service and a # ServiceMonitor in the SAME file, the monitor's endpoint port must be a port that Service -# actually defines. All 12 apps/* `-down` SLI rules fail this today (the monitors name -# port `metrics`; no app Service defines one), so they are registered in -# KNOWN_UNWIRED_JOBS with the live proof that their `up{job=...}` series do not exist; +# actually defines. All 12 apps/* `-down` SLI rules used to fail this (the monitors +# named port `metrics`; no app Service defined one) and were registered in +# KNOWN_UNWIRED_JOBS with the live proof that their `up{job=...}` series did not exist. +# THAT REGISTER IS NOW EMPTY BECAUSE THE TWELVE RULES WERE FIXED, not waived: they read +# kube-state-metrics readiness of their own workload object instead (2026-09-13), the dead +# ServiceMonitors are gone, and so is this axis' only subject. The monitor side therefore +# moved into its own check — tests/l0-static/servicemonitor-ports-resolve — which needs no +# alert rule to reach a monitor, resolves by selector across the whole tree instead of by +# same-file adjacency, and proves its resolver on fixtures so it cannot pass merely for +# lack of subjects; # AND (5) the check is LOAD-BEARING: this Test DECLARES `conformance.home.k3s/scope: whole-repo`, # so ci/l0.sh Stage 3b's `--include-scope whole-repo` unions it into EVERY gate # invocation, and ci/gate.sh blocks the merge when the l0 stage fails. # # WHY BOTH REGISTERS ARE RATCHETS AND NOT BLANKET FAILURES: the same reasoning as -# alert-rules-unit-tested. Failing outright on the 12 app SLI rules would make the gate red until -# exporters exist for Plex, Pi-hole, Jackett, qBittorrent, Sonarr, Radarr, Seerr and friends — -# none of which speak Prometheus natively — so the gate would be disabled, and a disabled gate -# protects nothing. Merely printing the gap is how these 13 rules stayed inert for three phases. +# alert-rules-unit-tested. Failing outright on the 12 app SLI rules would have made the gate red +# until exporters existed for Plex, Pi-hole, Jackett, qBittorrent, Sonarr, Radarr, Seerr and friends +# — none of which speak Prometheus natively — so the gate would have been disabled, and a disabled +# gate protects nothing. Merely printing the gap is how these 13 rules stayed inert for three phases. +# (The ratchet did its job: the twelve were closed a day later without an exporter, by reading a +# signal that already existed. The register is empty now, and that is the outcome it was for.) # So each gap is DECLARED, counted on every run, and CANNOT GROW: a new alert on an unclaimed # metric FAILS, a new Service+ServiceMonitor pair with a mismatched port FAILS, and a STALE # register entry (the gap was closed, or the alert deleted) FAILS too, so neither register can rot diff --git a/conformance/tests/l0-static/alert-metrics-have-scrape-targets/check.sh b/conformance/tests/l0-static/alert-metrics-have-scrape-targets/check.sh index e4fe89b..3926515 100755 --- a/conformance/tests/l0-static/alert-metrics-have-scrape-targets/check.sh +++ b/conformance/tests/l0-static/alert-metrics-have-scrape-targets/check.sh @@ -333,38 +333,36 @@ KNOWN_UNWIRED_METRICS: dict[str, str] = { } # ============================================================================================= -# REGISTER 2 — KNOWN_UNWIRED_JOBS. Shrink-only. Each is a `up{job="X"}` selector whose monitor -# names an endpoint port that the Service it selects does not define, so the monitor discovers -# ZERO targets, no `up{job="X"}` series exists, and `up{job="X"} == 0` can never fire. This is the -# nastiest variant of the class: `up` exists in abundance, so nothing about the metric name looks -# wrong, and `PrometheusTargetsMissing` cannot see it either — a target that was never discovered -# has no up series to be 0. +# REGISTER 2 — KNOWN_UNWIRED_JOBS. Shrink-only, and NOW EMPTY. # -# All twelve are the apps/* ServiceDefinition SLI pattern: the rendered ServiceMonitor names -# `port: metrics`, and no app Service defines a port of that name (verified: zero occurrences of -# `name: metrics` across all twelve manifests). VERIFIED on prod 2026-09-13: `count by (job) (up)` -# listed 20 jobs and none of these twelve appeared; all twelve `-down` rules returned no -# series. Closing this needs a Prometheus exporter per app — Plex, Pi-hole, Jackett, qBittorrent, -# Sonarr, Radarr and Seerr do not speak Prometheus natively — so it is its own piece of work, and -# for keycloak specifically the job labels that DO exist are `keycloak-service`/`keycloak-discovery` -# (platform/keycloak/servicemonitor.yaml), so that one is a rename away rather than an exporter. +# Each entry was a `up{job="X"}` selector whose monitor named an endpoint port the Service it +# selects did not define, so the monitor discovered ZERO targets, no `up{job="X"}` series existed, +# and `up{job="X"} == 0` could never fire. The nastiest variant of the class: `up` exists in +# abundance, so nothing about the metric name looks wrong, and `PrometheusTargetsMissing` cannot see +# it either — a target that was never discovered has no up series to be 0. +# +# IT HELD ALL TWELVE apps/* SLI RULES, AND IT IS EMPTY BECAUSE THEY WERE FIXED, not because the gap +# was waived (2026-09-13, branch fix/app-sli-rules-discover-no-targets). The register's own note +# said closing it needed "a Prometheus exporter per app". That was the wrong conclusion from a true +# premise: Plex, Pi-hole, Jackett, qBittorrent, Sonarr, Radarr, Seerr, Semaphore and Nextcloud +# genuinely export no Prometheus metrics, so instead the `up` SLI was rewritten against a signal that +# DOES exist for every workload — kube-state-metrics readiness of the service's own workload object +# (`kube_deployment_status_replicas_available` / `kube_statefulset_status_replicas_ready`), and +# `kubevirt_vmi_info{phase="running"}` for the VM. No exporter, no new scrape target; each rule is +# unit-tested firing AND quiet (apps/wave-*/tests/app-slis.test.yaml) and each expression was +# evaluated live on prod, where eleven of the twelve return no series on a healthy cluster and the +# twelfth (vm-demo) is not deployed at all. +# +# The rendered ServiceMonitors went with them: a monitor for a workload that serves no metrics is +# dead configuration that reads as coverage. So assertion (4) below now has NOTHING to audit under +# apps/ — which is why the monitor-side axis moved into its own check, +# tests/l0-static/servicemonitor-ports-resolve. THAT is where a ServiceMonitor naming a port no +# Service defines fails, with no alert rule required, resolving by selector across the whole tree +# rather than by same-file adjacency, and with its resolver proven on fixtures so it cannot pass for +# lack of subjects. Keeping this register non-empty as "documentation" of a closed gap is exactly the +# fiction the ratchet forbids: the stale-entry assertion below would fail, and rightly. # ============================================================================================= -KNOWN_UNWIRED_JOBS: dict[str, str] = { - "gitea": "apps/wave-0/gitea.yaml — Service defines only port-3000", - "jackett": "apps/wave-0/jackett.yaml — no exporter", - "pihole": "apps/wave-0/pihole.yaml — needs pihole-exporter", - "plex": "apps/wave-0/plex.yaml — no native Prometheus endpoint", - "qbittorrent": "apps/wave-0/qbittorrent.yaml — needs qbittorrent-exporter", - "vm-demo": "apps/wave-0/vm-demo.yaml — a KubeVirt demo VM, no exporter", - "keycloak": "apps/wave-1/keycloak.yaml — the live jobs are keycloak-service / " - "keycloak-discovery (platform/keycloak/servicemonitor.yaml); this SLI rule names " - "neither, so it is a rename away from being live rather than an exporter away", - "nextcloud": "apps/wave-1/nextcloud.yaml — needs nextcloud-exporter (serverinfo token)", - "radarr": "apps/wave-1/radarr.yaml — needs exportarr", - "semaphore": "apps/wave-1/semaphore.yaml — no exporter", - "sonarr": "apps/wave-1/sonarr.yaml — needs exportarr", - "seerr": "apps/wave-2/seerr.yaml — no exporter", -} +KNOWN_UNWIRED_JOBS: dict[str, str] = {} # ============================================================================================= # (0) PARSE EVERY ALERT RULE diff --git a/conformance/tests/l0-static/alert-rules-unit-tested/check.sh b/conformance/tests/l0-static/alert-rules-unit-tested/check.sh index f0a3fe8..18339af 100755 --- a/conformance/tests/l0-static/alert-rules-unit-tested/check.sh +++ b/conformance/tests/l0-static/alert-rules-unit-tested/check.sh @@ -147,25 +147,20 @@ def check(cond, msg): # ============================================================================= # THE DECLARED REGISTER (assertion 7). Shrink-only; see the header for why this is a ratchet. # -# UNTESTED_FILES — PrometheusRule files no test file covers AT ALL. The twelve apps/* entries -# are single `-down` liveness rules generated from one ServiceDefinition template; -# platform/kubevirt/alerts.yaml (4 rules) is the substantial one and is where the next tests -# belong. A file listed here that later gains coverage, or stops carrying a PrometheusRule, -# FAILS as stale. +# UNTESTED_FILES — PrometheusRule files no test file covers AT ALL. A file listed here that later +# gains coverage, or stops carrying a PrometheusRule, FAILS as stale. +# +# THE TWELVE apps/* ENTRIES ARE GONE (2026-09-13). They were the single `-down` liveness rules +# generated from one ServiceDefinition template, and the register's description of them as one +# template was exactly why leaving them untested was so expensive: ONE wrong expression was wrong +# twelve times, and it was — `up{job=""} == 0` against a job no scrape target produced, inert on +# prod for all twelve. They are now covered by apps/wave-0/tests/app-slis.test.yaml, +# apps/wave-1/tests/app-slis.test.yaml and apps/wave-2/tests/app-slis.test.yaml, each rule pinned +# BOTH ways (quiet on the healthy state, firing on no-ready-replica, and firing when the signal +# VANISHES — the failure mode that produced the class). platform/kubevirt/alerts.yaml (4 rules) is +# now the only entry left and is where the next tests belong. # ============================================================================= UNTESTED_FILES = { - "apps/wave-0/gitea.yaml", - "apps/wave-0/jackett.yaml", - "apps/wave-0/pihole.yaml", - "apps/wave-0/plex.yaml", - "apps/wave-0/qbittorrent.yaml", - "apps/wave-0/vm-demo.yaml", - "apps/wave-1/keycloak.yaml", - "apps/wave-1/nextcloud.yaml", - "apps/wave-1/radarr.yaml", - "apps/wave-1/semaphore.yaml", - "apps/wave-1/sonarr.yaml", - "apps/wave-2/seerr.yaml", "platform/kubevirt/alerts.yaml", } diff --git a/conformance/tests/l0-static/servicemonitor-ports-resolve/chainsaw-test.yaml b/conformance/tests/l0-static/servicemonitor-ports-resolve/chainsaw-test.yaml new file mode 100644 index 0000000..0c5ad07 --- /dev/null +++ b/conformance/tests/l0-static/servicemonitor-ports-resolve/chainsaw-test.yaml @@ -0,0 +1,105 @@ +# ============================================================================= +# L0 STATIC — every ServiceMonitor endpoint names a scrape target that RESOLVES +# Catalog: docs/14 L0 row ("Pre-apply, in CI") — runs WITHOUT a cluster. +# +# THE DEFECT THIS CLOSES, AND WHY ITS SIBLING COULD NOT. +# alert-metrics-have-scrape-targets audits scrape wiring FROM THE ALERT SIDE: it walks alert rules +# and asks whether something scrapes the metric each one reads. Its assertion (4) is the `up{job=X}` +# axis, and it is reachable ONLY through an alert that selects that job, only where the Service and +# the ServiceMonitor sit in the SAME FILE. Its own header names the hole. This test is the other +# direction: it walks the MONITORS, and asks of each one whether the target it names can exist — +# with no alert rule required anywhere. +# +# That difference is not academic. On 2026-09-13 all twelve apps/* ServiceMonitors named +# `endpoints[].port: metrics` and NO app Service defined a port of that name, so each discovered +# ZERO targets (verified on prod: `count by (job) (up)` listed 24 jobs, none of the twelve; both +# `up{job="plex"}` and `up{job="gitea"}` returned nothing). That was caught only because each app +# also shipped an `up{job=...}` alert. THE FIX TO THOSE ALERTS DELETED THE ONLY THING THAT MADE THE +# BROKEN MONITORS VISIBLE — they no longer read `up{job=...}` at all — so a monitor that discovers +# nothing would from that moment have been invisible again, and the twelve register entries that +# recorded the gap correctly became stale and were removed. A monitor scraping nothing is a defect +# on its own terms: it is dead configuration that reads as live coverage, and every dashboard, +# recording rule or future alert built on its series inherits the emptiness. +# +# GIVEN every ServiceMonitor and PodMonitor this repo authors, +# WHEN each endpoint's scrape target is resolved against every Service this repo authors, +# THEN (1) every endpoint names a target AT ALL (`port` or `targetPort`) — an endpoint naming +# neither scrapes nothing and is accepted silently by the operator; +# (2) every endpoint that names a `port` names a port that at least one Service its selector +# matches ACTUALLY DEFINES — the exact disagreement that made twelve monitors inert. +# Resolution is by SELECTOR ACROSS THE WHOLE TREE, not by same-file adjacency, so moving +# a Service into its own file cannot defeat it; +# (3) a monitor whose selector matches no Service in this repo is UNAUDITABLE (its Service is +# created by a Helm chart or an operator, so the port name lives outside the tree) and +# must be DECLARED in KNOWN_EXTERNAL_MONITORS, naming who creates the Service. That +# register is shrink-only: a monitor that later gains an in-repo Service, or that is +# deleted, FAILS as stale, so the unauditable set cannot quietly grow; +# (4) THE RESOLVER ITSELF IS PROVEN, on synthetic fixtures inside the check — one agreeing +# pair it must accept, one disagreeing pair it must reject. This is not decoration: after +# the fix there are zero in-repo monitors left for (2) to judge, and a guard whose only +# evidence is "nothing failed" is indistinguishable from a guard that stopped working. +# The fixtures are the exact shape catalog/render.py emits from +# `observability.metricsEndpoint`; +# (5) the GENERATOR cannot reintroduce the class: catalog/render.py must not hardcode an +# endpoint port name — it must take it from the ServiceDefinition, the same declaration +# that renders the Service port. A hardcoded literal is how twelve services acquired an +# identical wrong port name in one edit; +# AND (6) the check is LOAD-BEARING: this Test DECLARES `conformance.home.k3s/scope: whole-repo`, +# so ci/l0.sh Stage 3b's `--include-scope whole-repo` unions it into EVERY gate run, and +# ci/gate.sh blocks the merge when the l0 stage fails. It deliberately adds NO stage to +# ci/l0.sh — tests/l0-static/whole-repo-invariants-wired fails the gate if anyone does. +# +# WHY SCOPE, NOT REQUIREMENTS: ci/touched-requirements.sh maps apps/* to NOTHING, so a change that +# adds a ServiceMonitor under apps/ attributes no requirement and would filter this check out of +# exactly the change that breaks it. Same reasoning as its sibling. +# +# STATUS: EXECUTABLE. `check.sh` is an offline assertion script (PyYAML, as catalog/render.py and +# the sibling check use); conformance/run.sh --layer l0 and ci/l0.sh run it directly, and this +# Chainsaw Test wraps the SAME script. +# +# NOT proven (honest gaps; this is static): +# - That a resolved target is UP, answers, or emits the series an alert wants. The sibling check +# owns the metric-name axis and admits it cannot prove the SERIES NAME either; both are live +# reads and belong to a future L1/L3 check against /api/v1/label/__name__/values. +# - That a NetworkPolicy permits the scrape. The default-deny baseline drops it unless an allow is +# rendered, which is now part of what `observability.metricsEndpoint` emits +# (r_monitoring_ingress_policy) — but this check does not verify the allow, only the ports. +# - For a monitor in KNOWN_EXTERNAL_MONITORS, nothing about the chart's port names. The register +# records WHO owns them so the claim is reviewable, which is all a static check can offer. +# - It does not read ../home-network-k3s. The spec is read-only (R-TEST-06). +# ============================================================================= +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: servicemonitor-ports-resolve + labels: + conformance.home.k3s/R-OBS-01: "true" + annotations: + conformance.home.k3s/requirements: "R-OBS-01" + conformance.home.k3s/layer: "L0" + # WHOLE-REPO SCOPE — this invariant quantifies over every ServiceMonitor/PodMonitor and every + # Service in the tree (apps/wave-*, apps/vms, platform/keycloak, platform/kubevirt, + # platform/oauth2-proxy, platform/monitoring, platform/crafty, platform/homepage, + # runbooks/serving). No single requirement path-maps to that set, and ci/touched-requirements.sh + # maps apps/* to no requirement at all — so a requirement-scoped selection would skip this + # check on precisely the changes that can break it. + conformance.home.k3s/scope: "whole-repo" + conformance.home.k3s/safety-critical: "false" + conformance.home.k3s/executable: "true" + conformance.home.k3s/intent: >- + Every ServiceMonitor/PodMonitor endpoint this repo authors names a scrape target that + resolves — a port some matching Service actually defines — the monitors whose Services come + from outside the tree are enumerated in a shrink-only register, and the resolver that decides + this is itself proven on fixtures, so a monitor can never again be authored, reviewed and + merged against a port nothing serves. +spec: + steps: + - name: resolve-every-monitor-endpoint-against-every-service-this-repo-authors + description: >- + Inventory every ServiceMonitor/PodMonitor and Service in the tree, resolve each endpoint's + port against the Services its selector matches, ratchet the register of monitors whose + Service is chart-owned, prove the resolver on synthetic agreeing/disagreeing fixtures, and + assert the renderer does not hardcode an endpoint port name. + try: + - script: + content: bash ./check.sh diff --git a/conformance/tests/l0-static/servicemonitor-ports-resolve/check.sh b/conformance/tests/l0-static/servicemonitor-ports-resolve/check.sh new file mode 100755 index 0000000..a03038b --- /dev/null +++ b/conformance/tests/l0-static/servicemonitor-ports-resolve/check.sh @@ -0,0 +1,416 @@ +#!/usr/bin/env bash +# ============================================================================= +# L0 STATIC CHECK — every ServiceMonitor/PodMonitor endpoint names a scrape target that RESOLVES. +# See chainsaw-test.yaml's header for the full WHY. In one paragraph: +# +# alert-metrics-have-scrape-targets audits scrape wiring from the ALERT side, and its `up{job=X}` +# axis is reachable only through an alert that selects that job, only for a Service and a +# ServiceMonitor in the SAME FILE. That is how it caught the twelve inert apps/* monitors on +# 2026-09-13 (`endpoints[].port: metrics`, and no app Service defined such a port; on prod +# `count by (job) (up)` listed 24 jobs and none of the twelve appeared). But FIXING those alerts +# removed their `up{job=...}` selectors, which removed the only path by which the broken monitors +# were visible at all. This check walks the MONITORS instead, needs no alert rule, and resolves by +# SELECTOR across the whole tree — so a monitor that discovers nothing fails on its own terms. +# +# ASSERTIONS +# (0) the inventory is non-empty and the parse contract holds (anti-vacuous-pass); +# (1) every endpoint names a scrape target at all — `port` or `targetPort`; +# (2) every endpoint naming a `port` names a port at least one Service its selector matches +# actually DEFINES; +# (3) a monitor whose selector matches no in-repo Service is unauditable (chart/operator-owned +# Service) and must be declared in KNOWN_EXTERNAL_MONITORS — shrink-only, stale entries FAIL; +# (4) the RESOLVER is proven on synthetic fixtures (one agreeing pair, one disagreeing pair), +# because after the fix there are zero in-repo monitors for (2) to judge and a guard whose +# only evidence is "nothing failed" cannot be told apart from one that stopped working; +# (5) catalog/render.py does not HARDCODE an endpoint port name — it derives it from the +# ServiceDefinition, the same declaration that renders the Service port. A hardcoded literal +# is how twelve services acquired one identical wrong port name in a single edit; +# (6) the check is LOAD-BEARING: whole-repo scope declared here, honoured by ci/l0.sh's +# --include-scope union, and ci/gate.sh blocks on an l0 failure. NO hand-written ci/l0.sh +# stage (whole-repo-invariants-wired fails the gate if one appears). +# +# WHAT THIS DOES *NOT* PROVE (honest gaps — it is STATIC): +# - Nothing about liveness: that a resolved target is UP, answers, or emits the series a rule +# wants. The sibling check owns the metric-name axis and admits it cannot prove the series NAME +# either. Both need a live read of /api/v1/label/__name__/values (a future L1/L3 check). +# - Nothing about NetworkPolicy. An app namespace's default-deny baseline is `ingress: []`, so a +# scrape is dropped unless an allow is rendered; that allow is now part of what +# `observability.metricsEndpoint` emits (catalog/render.py r_monitoring_ingress_policy), but this +# check verifies ports, not policy. +# - Nothing about a chart's port names, for a monitor in KNOWN_EXTERNAL_MONITORS. The register +# records WHO owns them so the claim is reviewable; that is all a static check can offer. +# - It does not read ../home-network-k3s. The spec is read-only (R-TEST-06). +# +# Exit 0 = all assertions pass. Exit 1 = a failure. Exit 3 (honest SKIP, which the runner turns into +# rc 3 = UNWIRED, never green, because this test declares the whole-repo scope) only if the repo +# contains no ServiceMonitor or PodMonitor at all — genuinely nothing to audit. +# ============================================================================= +set -euo pipefail +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "${DIR}/../../../.." && pwd)" # repo root +PYTHON="${PYTHON:-python3}" + +# Honest SKIP (never green for a scope-declaring test) if there is nothing to audit at all. +if ! grep -rlE '^[[:space:]]*kind:[[:space:]]*(Service|Pod)Monitor[[:space:]]*$' \ + "${ROOT}/platform" "${ROOT}/apps" >/dev/null 2>&1; then + echo "check SKIPPED: no ServiceMonitor/PodMonitor found under platform/ or apps/ — nothing to audit" >&2 + exit 3 +fi + +cd "${ROOT}" +"${PYTHON}" - "${ROOT}" <<'PY' +"""Static audit: every ServiceMonitor/PodMonitor endpoint's scrape target resolves. + +PyYAML (the same dependency as catalog/render.py and the sibling check) rather than line-based +parsing: a monitor's selector, namespaceSelector and endpoint list are nested structures, and a +line-based reader of them is precisely the approximate parser that silently stops matching — the +failure class this check exists to prevent. +""" +from __future__ import annotations + +import re +import sys +from pathlib import Path + +try: + import yaml +except ImportError: # pragma: no cover + print("check FAIL: PyYAML is required (same dependency as catalog/render.py)", + file=sys.stderr) + raise SystemExit(1) + +ROOT = Path(sys.argv[1]) +sys.dont_write_bytecode = True # keep __pycache__ out of the tracked tree + +FAILURES: list[str] = [] +CHECKS = 0 + + +def check(cond, msg): + global CHECKS + CHECKS += 1 + if not cond: + FAILURES.append(msg) + + +def read(p: Path) -> str: + try: + return p.read_text() + except (OSError, UnicodeDecodeError): + return "" + + +# ============================================================================================= +# REGISTER — KNOWN_EXTERNAL_MONITORS. Shrink-only. +# +# A monitor whose selector matches NO Service in this repo cannot be audited here: the Service is +# created by a Helm chart or an operator, so the port name lives outside the tree. That is a real +# limit, not a pass — so each one is DECLARED, with WHO creates the Service it selects, and the +# register can only shrink. If such a monitor later gains an in-repo Service its entry goes stale +# and FAILS; if it is deleted, likewise. The same ratchet reasoning as the two registers in +# alert-metrics-have-scrape-targets: failing outright would make the gate red for chart-owned +# monitors that are perfectly correct, and a gate that is red for correct code gets disabled. +# +# Keyed by ":". +# ============================================================================================= +KNOWN_EXTERNAL_MONITORS: dict[str, str] = { + "platform/keycloak/servicemonitor.yaml:keycloak": + "selects the operator-created `keycloak-service` (labels app=keycloak) — the Keycloak " + "operator owns that Service. Scrapes by targetPort 9000 (the Quarkus management " + "interface), so it names no Service port at all and assertion (2) has nothing to resolve.", + "platform/kubevirt/servicemonitor.yaml:kubevirt": + "selects the KubeVirt operator's own `kubevirt-prometheus-metrics` Service (label " + "prometheus.kubevirt.io=true), created by the KubeVirt CR, not by this repo. Its `metrics` " + "port is defined there. Live on prod: job kubevirt-prometheus-metrics, 11 targets up.", + "platform/oauth2-proxy/servicemonitor.yaml:oauth2-proxy": + "selects the oauth2-proxy Helm chart's Service (label app=oauth2-proxy); the chart defines " + "its `metrics` port. Live on prod: job oauth2-proxy, 2 targets up.", + "platform/monitoring/flux-podmonitor.yaml:flux-system": + "a PodMonitor, so it resolves against POD ports, not Service ports — the Flux controller " + "Deployments (and their `http-prom` container port) are installed by the Flux bootstrap " + "manifests, not authored here. Live on prod: job flux-system/flux-system, 6 targets up.", +} + +MONITOR_KINDS = ("ServiceMonitor", "PodMonitor") + + +def is_test_fixture(p: Path) -> bool: + """conformance/tests/**/{golden,fixtures}/** is assertion DATA, not deployed manifests. + + The l1-rendering goldens are expected-output copies of rendered apps; the l2/l3 fixtures are + throwaway objects a behavioural test applies itself. Auditing either would be auditing a copy — + the same "reference, not declaration" distinction ci/touched-requirements.sh draws for fixture + paths, and the same exclusion the sibling check makes for goldens. The templates they come from + ARE audited, through apps/ and catalog/. + """ + parts = p.parts + return "conformance" in parts and ("golden" in parts or "fixtures" in parts) + + +# ============================================================ (0) INVENTORY +# services: list of (relpath, namespace, name, labels, {named ports}) +# monitors: list of (relpath, kind, namespace, name, selector, namespaceSelector, [endpoints]) +services: list[tuple] = [] +monitors: list[tuple] = [] +unparseable: list[str] = [] + +for p in sorted(ROOT.rglob("*.yaml")): + if ".git" in p.parts or is_test_fixture(p) or ".sops." in p.name: + continue + txt = read(p) + if not txt: + continue + if not re.search(r"^\s*kind:\s*(Service|ServiceMonitor|PodMonitor)\s*$", txt, re.M): + continue + rel = str(p.relative_to(ROOT)) + try: + docs = [d for d in yaml.safe_load_all(txt) if isinstance(d, dict)] + except yaml.YAMLError as exc: + unparseable.append(f"{rel} ({exc})") + continue + for d in docs: + kind = d.get("kind") + md = d.get("metadata") or {} + spec = d.get("spec") or {} + if kind == "Service": + names = {str(port["name"]) for port in (spec.get("ports") or []) + if isinstance(port, dict) and port.get("name")} + services.append((rel, md.get("namespace"), md.get("name"), + dict(md.get("labels") or {}), names)) + elif kind in MONITOR_KINDS: + eps = spec.get("endpoints") or spec.get("podMetricsEndpoints") or [] + monitors.append((rel, kind, md.get("namespace"), md.get("name"), + dict((spec.get("selector") or {}).get("matchLabels") or {}), + dict(spec.get("namespaceSelector") or {}), + [e for e in eps if isinstance(e, dict)])) + +for u in unparseable: + check(False, f"{u} is not parseable YAML — every monitor and Service in it went unaudited") + +check(len(monitors) > 0, + "no ServiceMonitor/PodMonitor parsed out of the tree, yet the grep gate above found one. The " + "parse contract broke and this check would pass vacuously while every scrape target went " + "unaudited") +check(len(services) >= 10, + f"only {len(services)} Service(s) parsed from the tree. This repo authors more than that " + f"(every apps/wave-* ServiceDefinition renders one), so the inventory has stopped being " + f"populated — assertion (2) would then resolve nothing and pass for the wrong reason") + + +# ============================================================ THE RESOLVER +def monitor_namespaces(mon_ns, ns_selector) -> set | None: + """Namespaces a monitor selects. None means ANY (`any: true`). + + An empty namespaceSelector defaults to the monitor's own namespace — the operator's documented + behaviour, and the case every repo-authored monitor uses. + """ + if ns_selector.get("any") is True: + return None + names = ns_selector.get("matchNames") + if names: + return {str(n) for n in names} + return {mon_ns} if mon_ns else set() + + +def matching_services(selector: dict, namespaces: set | None) -> list[tuple]: + """Services whose labels are a SUPERSET of the selector and whose namespace is in range. + + An EMPTY selector matches every Service in range (Kubernetes label-selector semantics). A + PodMonitor selector is compared against Service labels too — deliberately, so the resolver has + one code path; when that yields no match the monitor lands in the unauditable register with the + reason recorded, which is the honest outcome for a pod-side target this repo does not author. + """ + out = [] + for svc in services: + _, svc_ns, _, labels, _ = svc + if namespaces is not None and svc_ns not in namespaces: + continue + if all(labels.get(k) == v for k, v in selector.items()): + out.append(svc) + return out + + +def resolve(mon: tuple) -> tuple[list[str], list[tuple]]: + """(failures, matched services) for one monitor. The unit assertions (1) and (2) live here so + the SELF-TEST at the bottom exercises the same code path the repo audit does.""" + rel, kind, ns, name, selector, ns_sel, eps = mon + site = f"{rel}:{name}" + problems: list[str] = [] + matched = matching_services(selector, monitor_namespaces(ns, ns_sel)) + + for i, ep in enumerate(eps): + if not ep.get("port") and not ep.get("targetPort"): + problems.append( + f"{site} endpoint[{i}] names NEITHER `port` NOR `targetPort`. The operator accepts " + f"that silently and the monitor scrapes nothing") + if not eps: + problems.append( + f"{site} has NO endpoints at all — it can never produce a target, and nothing about " + f"the object looks wrong") + + named = [(i, str(ep["port"])) for i, ep in enumerate(eps) if ep.get("port")] + if named and matched: + defined: set = set() + for svc in matched: + defined |= svc[4] + for i, port in named: + if port not in defined: + where = ", ".join(f"{s[0]}:{s[2]}" for s in matched) + problems.append( + f"{site} endpoint[{i}] scrapes port {port!r}, which NO Service its selector " + f"matches defines. Matched Service(s): {where}; the port names they define are " + f"{sorted(defined) or ['(none)']}. The monitor therefore discovers ZERO " + f"targets: no `up{{job=\"{name}\"}}` series exists, anything built on its " + f"series is empty, and PrometheusTargetsMissing cannot see it either because a " + f"target that was never discovered has no up series to be 0. Give the Service a " + f"named port the workload actually serves metrics on (for a ServiceDefinition: " + f"`observability.metricsEndpoint`, which renders the Service port, this " + f"endpoint and the monitoring ingress allow together), or scrape by targetPort") + return problems, matched + + +# ============================================================ (1)(2)(3) THE REPO AUDIT +auditable = 0 +for mon in monitors: + rel, kind, ns, name, selector, ns_sel, eps = mon + site = f"{rel}:{name}" + problems, matched = resolve(mon) + for msg in problems: + check(False, msg) + if not problems: + # An endpoint-shape assertion passed for this monitor; count it so the report is honest + # about how much of the run was real work. + check(True, f"{site}: every endpoint names a scrape target") + if matched: + auditable += 1 + check(site not in KNOWN_EXTERNAL_MONITORS, + f"{site} is listed in KNOWN_EXTERNAL_MONITORS but its selector NOW matches " + f"{len(matched)} Service(s) this repo authors — the register is shrink-only: delete " + f"the entry so the monitor is audited for real") + else: + check(site in KNOWN_EXTERNAL_MONITORS, + f"{site} selects {selector or '(everything)'} in namespace(s) " + f"{sorted(monitor_namespaces(ns, ns_sel) or [''])} and NO Service in this repo " + f"matches, so its endpoint ports cannot be audited here. Either the selector is wrong " + f"(a monitor matching nothing discovers nothing), or the Service is chart/operator " + f"owned — in which case DECLARE it in KNOWN_EXTERNAL_MONITORS with who creates it") + +for site in sorted(KNOWN_EXTERNAL_MONITORS): + check(any(f"{m[0]}:{m[3]}" == site for m in monitors), + f"KNOWN_EXTERNAL_MONITORS declares {site}, which is not a monitor in this repo any more " + f"— a stale exemption hides the next real gap; delete the entry") + + +# ============================================================ (4) THE RESOLVER IS PROVEN +# After the twelve apps/* monitors were removed there is NO in-repo monitor left for assertion (2) +# to judge (every survivor is chart-owned, and keycloak's scrapes by targetPort). A guard in that +# state passes whether it works or not. So the resolver is run against two synthetic monitors over +# two synthetic Services, in the exact shape catalog/render.py emits from +# `observability.metricsEndpoint`, and BOTH outcomes are asserted. +_real_services = services +services = [ + ("", "fixture", "agreeing", {"app.kubernetes.io/name": "agreeing"}, + {"port-8080", "metrics"}), + ("", "fixture", "disagreeing", {"app.kubernetes.io/name": "disagreeing"}, + {"port-8080"}), +] +good = ("", "ServiceMonitor", "fixture", "agreeing", + {"app.kubernetes.io/name": "agreeing"}, {"matchNames": ["fixture"]}, + [{"port": "metrics", "path": "/metrics", "interval": "30s"}]) +bad = ("", "ServiceMonitor", "fixture", "disagreeing", + {"app.kubernetes.io/name": "disagreeing"}, {"matchNames": ["fixture"]}, + [{"port": "metrics", "path": "/metrics", "interval": "30s"}]) +empty = ("", "ServiceMonitor", "fixture", "agreeing", + {"app.kubernetes.io/name": "agreeing"}, {"matchNames": ["fixture"]}, + [{"path": "/metrics"}]) + +good_problems, good_matched = resolve(good) +bad_problems, bad_matched = resolve(bad) +empty_problems, _ = resolve(empty) +services = _real_services + +check(good_matched and not good_problems, + f"SELF-TEST: the resolver rejected a CORRECT monitor (Service defines `metrics`, endpoint " + f"scrapes `metrics`): {good_problems or 'selector matched no fixture Service'}. The resolver " + f"is broken in the direction that makes the gate red for correct code") +check(bad_matched and len(bad_problems) == 1 and "discovers ZERO targets" in bad_problems[0], + "SELF-TEST: the resolver ACCEPTED a monitor whose endpoint names a port its Service does not " + "define — the exact defect that left twelve apps/* monitors discovering nothing on prod. " + "Assertion (2) is inert; every pass above proves nothing") +check(len(empty_problems) == 1 and "NEITHER" in empty_problems[0], + "SELF-TEST: the resolver ACCEPTED an endpoint naming neither `port` nor `targetPort`, so " + "assertion (1) is inert") + + +# ============================================================ (5) THE GENERATOR CANNOT REGRESS +RENDER = read(ROOT / "catalog" / "render.py") +check(RENDER != "", "catalog/render.py is unreadable — the generator half cannot be audited") +sm_fn = RENDER.split("def r_servicemonitor")[-1].split("\ndef ")[0] if RENDER else "" +check("def r_servicemonitor" in RENDER, + "catalog/render.py no longer defines r_servicemonitor — this assertion is auditing nothing") +check(not re.search(r'"port":\s*"[a-z][a-z0-9-]*"', sm_fn), + "catalog/render.py's r_servicemonitor HARDCODES a literal endpoint port name. That is the " + "original defect's mechanism: one edit gave twelve services the same wrong port name " + "(`metrics`), which no rendered Service defined, and every `up`-based SLI built on them was " + "structurally inert on prod. The port name must come from the ServiceDefinition's " + "`observability.metricsEndpoint`, the same declaration that renders the Service port") +check("metricsEndpoint" in sm_fn, + "catalog/render.py's r_servicemonitor no longer reads `metricsEndpoint`, so it is back to " + "rendering a monitor from an intent (`metrics: true`) that cannot say where metrics are " + "served — for most of this catalog, nowhere") +SCHEMA = read(ROOT / "schemas" / "servicedefinition.schema.json") +check("metricsEndpoint" in SCHEMA, + "schemas/servicedefinition.schema.json does not define observability.metricsEndpoint, so the " + "field the renderer depends on is not part of the contract and `additionalProperties: false` " + "would reject any ServiceDefinition that declares one") + + +# ============================================================ (6) LOAD-BEARING +SELF = read(ROOT / "conformance" / "tests" / "l0-static" / "servicemonitor-ports-resolve" + / "chainsaw-test.yaml") +check(re.search(r"^\s*conformance\.home\.k3s/scope:\s*[\"']?whole-repo", SELF, re.M) is not None, + "servicemonitor-ports-resolve/chainsaw-test.yaml does not declare " + "conformance.home.k3s/scope: whole-repo, so the merge gate selects this check only when a " + "change happens to touch R-OBS-01. ci/touched-requirements.sh maps apps/* to NOTHING, so " + "adding a ServiceMonitor under apps/ would be filtered out of exactly the change that " + "reintroduces the defect") +L0 = read(ROOT / "ci" / "l0.sh") +GATE = read(ROOT / "ci" / "gate.sh") +check("--include-scope" in L0, + "ci/l0.sh no longer passes --include-scope to the conformance runner, so the L0 stage is a " + "purely requirement-scoped run again and the scope annotation above is inert decoration") +check("servicemonitor-ports-resolve" not in L0, + "ci/l0.sh names this check in a hand-written stage. Whole-repo invariants are selected BY " + "DECLARING the scope; a per-test stage is the pattern tests/l0-static/" + "whole-repo-invariants-wired fails the gate for, because the next invariant's author has to " + "REMEMBER to add one and the omission is silent") +check(re.search(r"^for s in .*\bl0\b.*; do", GATE, re.M) is not None + and re.search(r"fail\).*BLOCK=1", GATE) is not None, + "ci/gate.sh does not set the merge block when the l0 stage fails") + + +# ======================================================================== report +print(f"Monitor inventory: {len(monitors)} ServiceMonitor/PodMonitor(s), {len(services)} " + f"Service(s) authored in this repo") +for rel, kind, ns, name, selector, ns_sel, eps in monitors: + tgt = ", ".join( + str(e.get("port") or f"targetPort {e.get('targetPort')}" or "") for e in eps + ) or "" + mark = "EXTERNAL" if f"{rel}:{name}" in KNOWN_EXTERNAL_MONITORS else "resolved" + print(f" {mark:<8} {kind:<14} {rel}:{name} -> {tgt}") +print(f"\n resolvable against an in-repo Service: {auditable}/{len(monitors)}") +print(f" DECLARED GAP (shrink-only): KNOWN_EXTERNAL_MONITORS: {len(KNOWN_EXTERNAL_MONITORS)} " + f"monitor(s) whose Service is created by a chart/operator outside this tree") +print(" RESOLVER SELF-TEST: agreeing pair accepted, disagreeing pair rejected, " + "target-less endpoint rejected") + +print() +if not FAILURES: + print(f"check OK: servicemonitor-ports-resolve — {CHECKS} assertions (R-OBS-01)") + raise SystemExit(0) +print(f"CHECK FAIL (servicemonitor-ports-resolve): {len(FAILURES)} failed, " + f"{CHECKS - len(FAILURES)} passed", file=sys.stderr) +for f in FAILURES: + print(f" FAIL {f}", file=sys.stderr) +raise SystemExit(1) +PY diff --git a/conformance/tests/l1-rendering/container-workload-render/golden/expected.yaml b/conformance/tests/l1-rendering/container-workload-render/golden/expected.yaml index 857479e..61726d3 100644 --- a/conformance/tests/l1-rendering/container-workload-render/golden/expected.yaml +++ b/conformance/tests/l1-rendering/container-workload-render/golden/expected.yaml @@ -212,26 +212,6 @@ spec: - name: paas-demo port: 4180 --- -# ServiceMonitor (R-OBS-01) -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: paas-demo - namespace: paas-demo - labels: - release: kube-prometheus-stack - app.kubernetes.io/name: paas-demo -spec: - selector: - matchLabels: - app.kubernetes.io/name: paas-demo - namespaceSelector: - matchNames: - - paas-demo - endpoints: - - port: metrics - interval: 30s ---- # Flux ImageRepository (R-UPD) apiVersion: image.toolkit.fluxcd.io/v1 kind: ImageRepository diff --git a/conformance/tests/l1-rendering/default-deny-baseline-render/golden/expected.yaml b/conformance/tests/l1-rendering/default-deny-baseline-render/golden/expected.yaml index 37b31c5..1906e87 100644 --- a/conformance/tests/l1-rendering/default-deny-baseline-render/golden/expected.yaml +++ b/conformance/tests/l1-rendering/default-deny-baseline-render/golden/expected.yaml @@ -72,26 +72,6 @@ spec: - toEntities: - world --- -# ServiceMonitor (R-OBS-01) -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: baseline-demo - namespace: baseline-demo - labels: - release: kube-prometheus-stack - app.kubernetes.io/name: baseline-demo -spec: - selector: - matchLabels: - app.kubernetes.io/name: baseline-demo - namespaceSelector: - matchNames: - - baseline-demo - endpoints: - - port: metrics - interval: 30s ---- # Flux ImageRepository (R-UPD) apiVersion: image.toolkit.fluxcd.io/v1 kind: ImageRepository diff --git a/conformance/tests/l1-rendering/egress-class-render/golden/egress-internet.expected.yaml b/conformance/tests/l1-rendering/egress-class-render/golden/egress-internet.expected.yaml index 4aa6b62..9591994 100644 --- a/conformance/tests/l1-rendering/egress-class-render/golden/egress-internet.expected.yaml +++ b/conformance/tests/l1-rendering/egress-class-render/golden/egress-internet.expected.yaml @@ -239,26 +239,6 @@ spec: - name: sonarr port: 4180 --- -# ServiceMonitor (R-OBS-01) -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: sonarr - namespace: sonarr - labels: - release: kube-prometheus-stack - app.kubernetes.io/name: sonarr -spec: - selector: - matchLabels: - app.kubernetes.io/name: sonarr - namespaceSelector: - matchNames: - - sonarr - endpoints: - - port: metrics - interval: 30s ---- # PrometheusRule SLI alerts (R-OBS-01) apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule @@ -273,14 +253,14 @@ spec: - name: sonarr.slis rules: - alert: sonarr-down - expr: up{job="sonarr"} == 0 + expr: kube_deployment_status_replicas_available{namespace="sonarr", deployment="sonarr"} == 0 or absent(kube_deployment_status_replicas_available{namespace="sonarr", deployment="sonarr"}) for: 5m labels: severity: critical service: sonarr annotations: summary: sonarr is down - description: 'sonarr has been unreachable for 5m (SLI: up).' + description: 'sonarr: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available).' --- # Flux ImageRepository (R-UPD) apiVersion: image.toolkit.fluxcd.io/v1 diff --git a/conformance/tests/l1-rendering/egress-class-render/golden/egress-vpn.expected.yaml b/conformance/tests/l1-rendering/egress-class-render/golden/egress-vpn.expected.yaml index 0980eca..17c0524 100644 --- a/conformance/tests/l1-rendering/egress-class-render/golden/egress-vpn.expected.yaml +++ b/conformance/tests/l1-rendering/egress-class-render/golden/egress-vpn.expected.yaml @@ -239,26 +239,6 @@ spec: - name: sonarr port: 4180 --- -# ServiceMonitor (R-OBS-01) -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: sonarr - namespace: sonarr - labels: - release: kube-prometheus-stack - app.kubernetes.io/name: sonarr -spec: - selector: - matchLabels: - app.kubernetes.io/name: sonarr - namespaceSelector: - matchNames: - - sonarr - endpoints: - - port: metrics - interval: 30s ---- # PrometheusRule SLI alerts (R-OBS-01) apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule @@ -273,14 +253,14 @@ spec: - name: sonarr.slis rules: - alert: sonarr-down - expr: up{job="sonarr"} == 0 + expr: kube_deployment_status_replicas_available{namespace="sonarr", deployment="sonarr"} == 0 or absent(kube_deployment_status_replicas_available{namespace="sonarr", deployment="sonarr"}) for: 5m labels: severity: critical service: sonarr annotations: summary: sonarr is down - description: 'sonarr has been unreachable for 5m (SLI: up).' + description: 'sonarr: its Deployment has no available/ready replica for 5m, or the signal is gone entirely (SLI: up, via kube_deployment_status_replicas_available).' --- # Flux ImageRepository (R-UPD) apiVersion: image.toolkit.fluxcd.io/v1 diff --git a/platform/monitoring/tests/run.sh b/platform/monitoring/tests/run.sh index 56cf6b3..fab6956 100755 --- a/platform/monitoring/tests/run.sh +++ b/platform/monitoring/tests/run.sh @@ -120,8 +120,16 @@ fi discover() { # Companion convention, anywhere in the tree. find "${ROOT}" -path "${ROOT}/.git" -prune -o -name '*.promtool-test.yaml' -print 2>/dev/null - # Suite convention: this directory's own *.test.yaml files. - find "${HERE}" -maxdepth 1 -name '*.test.yaml' -print 2>/dev/null + # Suite convention, ANYWHERE in the tree — not just this directory. + # + # This used to be `find "${HERE}" -maxdepth 1`, which made the suite convention a + # platform/monitoring-only privilege while the header above documented it as a general one. The + # consequence was silent: a suite added under any other component (apps/wave-0/tests/, say) was + # never discovered, never run, and still LOOKED like coverage in review — the same shape of hole + # as the rules this harness exists to test. Found 2026-09-13 adding apps/wave-*/tests/. Scoped by + # NAME only; manifests_for() then refuses (loudly) any *.test.yaml that is not inside a tests/ + # directory, so a misplaced file fails rather than disappearing. + find "${ROOT}" -path "${ROOT}/.git" -prune -o -name '*.test.yaml' -print 2>/dev/null } TESTS=() diff --git a/schemas/renderer-contract.md b/schemas/renderer-contract.md index 991ffca..713c1f7 100644 --- a/schemas/renderer-contract.md +++ b/schemas/renderer-contract.md @@ -26,6 +26,8 @@ platform must render, and cites the requirement each mapping satisfies. Changing | `update.channel` | `stable` | R-UPD-01 | | `update.autoRollbackOnFailedHealth` | `true` | R-UPD-03 | | `observability.metrics` | `true` | R-OBS-01 | +| `observability.metricsEndpoint` | **absent** → the workload serves no Prometheus endpoint; no ServiceMonitor | R-OBS-01 | +| `observability.livenessWorkload` | the object this SD renders (`Deployment` named after the service; `VirtualMachineInstance` for `kind: vm`) | R-OBS-01 | | backup of irreplaceable state | off-cluster (never `reacquirable`) | R-STATE-03 | --- @@ -65,8 +67,11 @@ platform must render, and cites the requirement each mapping satisfies. Changing ### `dependencies[]` (`to` / `port` / `protocol`) — east-west policy - Each namespace first gets the **default-deny baseline** (deny all ingress + egress) applied by - GitOps at namespace creation, with a standing allow only for **DNS to the in-cluster resolver** and - for **observability scraping** from the monitoring namespace. + GitOps at namespace creation, with a standing allow only for **DNS to the in-cluster resolver**. + **Observability scraping is NOT in the baseline** — it is an additive allow rendered per declared + `observability.metricsEndpoint` (see below), scoped to that one port. This paragraph used to claim + the baseline carried a standing monitoring allow; it never did, and an app metrics port would have + been dropped by policy even with the Service port and monitor endpoint in agreement. - Each `dependencies[]` entry renders one **CiliumNetworkPolicy** egress allow: this service → `to` service on `port`/`protocol`. Everything undeclared stays denied. - Empty `dependencies` ⇒ the service can reach **only DNS**. @@ -183,11 +188,35 @@ platform must render, and cites the requirement each mapping satisfies. Changing - **Requirements:** **R-UPD-01** (stable only), **R-UPD-02** (no human/AI authoring), **R-UPD-03** (health-gated auto-rollback). (docs/11, ADR-0009) -### `observability` (`metrics` / `slis[]`) — ServiceMonitor + alerts -- `metrics: true` → a **ServiceMonitor/PodMonitor** so Prometheus scrapes the service; logs are - shipped to Loki by the node/pod agent by default. -- each entry in `slis[]` → at least one **alert rule** (e.g. `up` → service-down alert) routed by - Alertmanager to email; a per-service Grafana dashboard row. +### `observability` (`metrics` / `slis[]` / `metricsEndpoint` / `livenessWorkload`) — ServiceMonitor + alerts +- `metrics: true` → the platform **collects signals** for this service: logs to Loki via the node + agent, and kube-state-metrics readiness for the `up` SLI below. Both are platform-wide scrapes; + neither needs a per-service CR. **It does not assert that the workload serves a Prometheus + endpoint.** *(Corrected 2026-09-13. It used to render a ServiceMonitor unconditionally, hardcoded + to a Service port named `metrics`. No rendered Service ever defined such a port, so all twelve app + monitors discovered ZERO targets — verified on prod: 24 jobs in `up`, not one of the twelve — and + every `up{job=...}` SLI built on them was structurally inert. `metrics: true` could not have been + right: Plex, Pi-hole, Jackett, qBittorrent, Sonarr, Radarr, Seerr, Semaphore and Nextcloud serve no + Prometheus endpoint at all without a separate exporter.)* +- `metricsEndpoint: {port, name?, path?, interval?}` → **the workload does serve metrics, here.** + Renders all three halves of a working scrape from the one declaration, because any one of them + missing makes the scrape silently yield nothing: + 1. a **named Service port** (`name`, default `metrics`) on that `port`; + 2. a **ServiceMonitor** endpoint selecting that port name, on `path` at `interval`; + 3. a **CiliumNetworkPolicy** admitting the `monitoring` namespace to that port — the namespace + default-deny baseline is `ingress: []`, so without it the scrape is dropped by policy. + The metrics port is deliberately **not** added to the Gateway ingress allow: it is east-west for + Prometheus, never north-south. +- each entry in `slis[]` → at least one **alert rule** routed by Alertmanager to email; a per-service + Grafana dashboard row. `up` → a `-down` **critical** alert reading the readiness of + `livenessWorkload` from **kube-state-metrics** (`kube_deployment_status_replicas_available`, + `kube_statefulset_status_replicas_ready`, `kube_daemonset_status_number_available`) or, for a + `VirtualMachineInstance`, the existence of `kubevirt_vmi_info{phase="running"}`. Each rule carries + an `or absent(...)` half, so the signal DISAPPEARING fires the alert rather than silencing it. +- `livenessWorkload: {kind, name}` → which object's readiness **is** this service. Defaults to the + object this SD renders; override it when the running workload is a different one (keycloak's server + is a StatefulSet owned by the Keycloak operator — a Deployment-shaped rule would `absent()` and + fire forever). - **Requirements:** **R-OBS-01** (per-service SLIs/alerts). Platform-health alerts (R-OBS-04) are rendered platform-wide, not per ServiceDefinition. (docs/10) diff --git a/schemas/servicedefinition.schema.json b/schemas/servicedefinition.schema.json index a2caebd..c4211cc 100644 --- a/schemas/servicedefinition.schema.json +++ b/schemas/servicedefinition.schema.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://home.keithjackson.dev/schemas/servicedefinition.schema.json", "title": "ServiceDefinition", - "description": "Operator-facing onboarding contract for the home k3s + Longhorn platform. One ServiceDefinition per file under catalog/. Captures INTENT; the platform compiles it into concrete resources (see schemas/renderer-contract.md). Canonical source of truth: spec docs/13-service-catalog.md. Serves R-GITOPS-02; ties into R-NET-*, R-STATE-*, R-TLS-*, R-DNS-*, R-OBS-*, R-UPD-*, R-IDP-*. Schema version: 1.5.0 (adds optional `inClusterClients` — additional in-cluster clients, named by NAMESPACE, admitted to a declared port of this service DIRECTLY, bypassing the shared Gateway. `ingressRoutes` admits only the Gateway proxy identity and `dependencies` only links catalog services, so a PLATFORM controller dialling a Service ClusterIP had no authorable form at all; that gap is what deadlocked prod's GitOps when its GitRepository moved to the in-cluster Gitea Service. Renders one least-privilege ingress CiliumNetworkPolicy per entry — the named namespace and the declared port, nothing wider). 1.4.0 (makes the `bulk` tier REAL: a bulk storage entry now carries `export` (the absolute NFS export path on the ${BULK_NFS_SERVER} host) and renders a STATIC, pre-bound NFS PersistentVolume + PVC over data that already exists outside the cluster, plus optional `subPath` and `readOnly` on any entry. Resolves the OQ-2 bulk-backing stub for STATIC pre-existing datasets; dynamic bulk provisioning remains unimplemented and unneeded). 1.3.0 added optional pod-level podSecurityContext for the container kind — k8s PodSecurityContext emitted verbatim onto the pod spec; its fsGroup unblocks non-root images e.g. semaphore/UID-1001 writing a mounted Longhorn PVC). 1.2.0 added optional per-container env/envFrom/lifecycle for the container kind — k8s EnvVar/EnvFromSource/Lifecycle shapes emitted verbatim onto the app container; unblocks headless-install/DB-backed apps e.g. gitea/nextcloud/semaphore/unifi/minecraft. 1.1.0 added the shape invariant exposure:public => auth.mode!=none (docs/13 v1.1 / docs/15 / R-IDP-02).", + "description": "Operator-facing onboarding contract for the home k3s + Longhorn platform. One ServiceDefinition per file under catalog/. Captures INTENT; the platform compiles it into concrete resources (see schemas/renderer-contract.md). Canonical source of truth: spec docs/13-service-catalog.md. Serves R-GITOPS-02; ties into R-NET-*, R-STATE-*, R-TLS-*, R-DNS-*, R-OBS-*, R-UPD-*, R-IDP-*. Schema version: 1.6.0 (splits `observability.metrics` from WHERE metrics live: adds optional `metricsEndpoint` (port/name/path/interval) and optional `livenessWorkload` (kind/name). `metrics: true` previously rendered a ServiceMonitor hardcoded to a port named `metrics` that no rendered Service ever defined, so all twelve app monitors discovered ZERO targets and every `up{job=...}` SLI was structurally inert on prod; a monitor is now rendered only from a declared `metricsEndpoint`, which also renders the Service port and the monitoring-namespace ingress allow, and the `up` SLI is expressed against kube-state-metrics -- a signal that exists for every workload without a per-app exporter. `livenessWorkload` names the object whose readiness IS the service, for the case where the running workload is not the one this SD renders). 1.5.0 (adds optional `inClusterClients` — additional in-cluster clients, named by NAMESPACE, admitted to a declared port of this service DIRECTLY, bypassing the shared Gateway. `ingressRoutes` admits only the Gateway proxy identity and `dependencies` only links catalog services, so a PLATFORM controller dialling a Service ClusterIP had no authorable form at all; that gap is what deadlocked prod's GitOps when its GitRepository moved to the in-cluster Gitea Service. Renders one least-privilege ingress CiliumNetworkPolicy per entry — the named namespace and the declared port, nothing wider). 1.4.0 (makes the `bulk` tier REAL: a bulk storage entry now carries `export` (the absolute NFS export path on the ${BULK_NFS_SERVER} host) and renders a STATIC, pre-bound NFS PersistentVolume + PVC over data that already exists outside the cluster, plus optional `subPath` and `readOnly` on any entry. Resolves the OQ-2 bulk-backing stub for STATIC pre-existing datasets; dynamic bulk provisioning remains unimplemented and unneeded). 1.3.0 added optional pod-level podSecurityContext for the container kind — k8s PodSecurityContext emitted verbatim onto the pod spec; its fsGroup unblocks non-root images e.g. semaphore/UID-1001 writing a mounted Longhorn PVC). 1.2.0 added optional per-container env/envFrom/lifecycle for the container kind — k8s EnvVar/EnvFromSource/Lifecycle shapes emitted verbatim onto the app container; unblocks headless-install/DB-backed apps e.g. gitea/nextcloud/semaphore/unifi/minecraft. 1.1.0 added the shape invariant exposure:public => auth.mode!=none (docs/13 v1.1 / docs/15 / R-IDP-02).", "type": "object", "additionalProperties": false, "required": ["name", "source"], @@ -350,7 +350,7 @@ "properties": { "metrics": { "type": "boolean", - "description": "Whether the platform scrapes metrics for this service.", + "description": "Whether the platform collects signals for this service (logs to Loki + kube-state-metrics-derived liveness). It does NOT assert that the workload serves a Prometheus endpoint — say that with metricsEndpoint.", "default": true }, "slis": { @@ -358,6 +358,55 @@ "description": "Per-service alert intents (e.g. up). Each becomes at least one alert rule (R-OBS-01).", "items": { "type": "string", "minLength": 1 }, "default": [] + }, + "metricsEndpoint": { + "type": "object", + "description": "WHERE this workload serves Prometheus metrics. Absent (the default for every catalog entry today) means it serves none, and NO ServiceMonitor is rendered — the renderer used to emit one naming port `metrics` regardless, which no rendered Service ever defined, so all twelve monitors discovered zero targets and every `up`-based SLI was structurally inert. When present it renders all three halves of a working scrape together: the named Service port, the ServiceMonitor endpoint, and the monitoring-namespace ingress allow that the default-deny baseline would otherwise drop.", + "additionalProperties": false, + "required": ["port"], + "properties": { + "port": { + "type": "integer", + "description": "Container/Service port the metrics endpoint listens on.", + "minimum": 1, + "maximum": 65535 + }, + "name": { + "type": "string", + "description": "Service port name the ServiceMonitor endpoint selects on.", + "minLength": 1, + "default": "metrics" + }, + "path": { + "type": "string", + "description": "HTTP path serving the Prometheus exposition format.", + "minLength": 1, + "default": "/metrics" + }, + "interval": { + "type": "string", + "description": "Scrape interval (Prometheus duration).", + "pattern": "^[0-9]+[smh]$", + "default": "30s" + } + } + }, + "livenessWorkload": { + "type": "object", + "description": "WHICH Kubernetes object's readiness constitutes this service being up, for the `up` SLI. Defaults to the object this ServiceDefinition renders (Deployment named after the service for kind: container, VirtualMachineInstance for kind: vm). Override only when the object that actually runs on the cluster is a different one — keycloak's server is a StatefulSet owned by the Keycloak operator, and a Deployment-shaped rule would read an absent series and fire forever.", + "additionalProperties": false, + "properties": { + "kind": { + "type": "string", + "description": "Workload kind whose kube-state-metrics readiness series the SLI reads.", + "enum": ["Deployment", "StatefulSet", "DaemonSet", "VirtualMachineInstance"] + }, + "name": { + "type": "string", + "description": "Object name (defaults to the service name).", + "minLength": 1 + } + } } } },