From 6a7e9d15e7a93c588dcd20aee0444418c96b89ba Mon Sep 17 00:00:00 2001
From: Keith Jackson <keithallenjackson@gmail.com>
Date: Wed, 26 Aug 2026 10:20:48 -0500
Subject: [PATCH] fix(kubevirt): drop broken http CDI ServiceMonitor causing
 false CDIOperatorDown

The cdi-operator manages its own `service-monitor-cdi` (scheme https +
cdi-metrics-reader-token) which backs CDI's shipped recording rules
(kubevirt_cdi_operator_up / cluster:kubevirt_cdi_operator_up:sum) and the
CDIOperatorDown alert. Our extra `cdi` ServiceMonitor scraped the same
cdi-prometheus-metrics service (HTTPS :8443) over http, so every scrape
failed with 400 Bad Request, adding down (up=0) targets for the CDI pods.
That poisoned the up-aggregation the recording rule sums, driving it to 0
and firing a false CDIOperatorDown even though the operator was Running,
Ready, and Available=True. Remove the duplicate; keep the kubevirt SM
(virt-operator ships none of its own).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 platform/kubevirt/servicemonitor.yaml | 30 +++++++--------------------
 1 file changed, 8 insertions(+), 22 deletions(-)

diff --git a/platform/kubevirt/servicemonitor.yaml b/platform/kubevirt/servicemonitor.yaml
index e9e27ef..a85b774 100644
--- a/platform/kubevirt/servicemonitor.yaml
+++ b/platform/kubevirt/servicemonitor.yaml
@@ -8,8 +8,14 @@
 # Prometheus discovers ServiceMonitors cluster-wide (serviceMonitorSelectorNilUsesHelmValues:false in
 # platform/monitoring/kube-prometheus-stack.yaml), so these can live beside their target Services.
 # NOTE (verify at bring-up, same convention as alerts.yaml): KubeVirt serves metrics over HTTPS with a
-# self-signed internal cert (insecureSkipVerify) on the port named `metrics`; CDI serves plain HTTP on
-# its `metrics` port. Port names/scheme are confirmed against the deployed Services on first reconcile.
+# self-signed internal cert (insecureSkipVerify) on the port named `metrics`. virt-operator does NOT
+# ship its own ServiceMonitor, so this one is required.
+#
+# CDI is intentionally NOT scraped here: the cdi-operator manages its own `service-monitor-cdi`
+# (scheme https + a cdi-metrics-reader-token, owned by the cdi-operator Deployment) which backs CDI's
+# shipped recording rules (kubevirt_cdi_operator_up / cluster:kubevirt_cdi_operator_up:sum) and the
+# CDIOperatorDown alert. A second http ServiceMonitor here added down (400) targets for the same pods,
+# poisoning that up-aggregation to 0 and firing a false CDIOperatorDown — so it was removed.
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
@@ -31,23 +37,3 @@ spec:
         insecureSkipVerify: true
       honorLabels: true
       interval: 30s
----
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
-  name: cdi
-  namespace: cdi
-  labels:
-    release: kube-prometheus-stack
-spec:
-  selector:
-    matchLabels:
-      prometheus.cdi.kubevirt.io: "true"
-  namespaceSelector:
-    matchNames:
-      - cdi
-  endpoints:
-    - port: metrics
-      scheme: http
-      honorLabels: true
-      interval: 30s