--- # kube-prometheus-stack — Prometheus + Alertmanager + Grafana + kube-state-metrics + # node-exporter + the standard Kubernetes exporters/rules (docs/10). Serves R-OBS-01/02/03. # # Retention/PVC sizes are overlay knobs (R-ENV-02) resolved from cluster-vars via Flux # postBuild.substituteFrom. Everything else is DEV-SCOPED for the ~22 GiB workstation ceiling: # single replicas, modest requests/limits, ephemeral (emptyDir) Alertmanager/Grafana state, and # the k3s-embedded control-plane scrape jobs disabled (they are not separately reachable on k3s and # would otherwise show as permanently-down targets and burn RAM/alert noise). apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: name: kube-prometheus-stack namespace: monitoring spec: interval: 30m # The 65->88 catch-up is a large multi-major jump (operator + KSM + node-exporter roll, plus the # Prometheus/Alertmanager StatefulSets). The default 5m Helm timeout is too short for the full rollout # to reach Ready, so Flux logged "context deadline exceeded" and rolled back to 65.8.1 four times. # A generous timeout lets one attempt finish; remediateLastFailure:false keeps partial progress instead # of yo-yoing back to the old chart (a re-reconcile then drives it forward). CRDs are pre-applied # server-side (Helm never updates crds/ on upgrade), so schema is already at operator v0.93.1. timeout: 20m0s chart: spec: chart: kube-prometheus-stack version: ">=88.0.0 <89.0.0" # k8s-1.36 catch-up (po v0.93; CRDs pre-applied server-side, Helm won't update them). Renovate maintains sourceRef: kind: HelmRepository name: prometheus-community namespace: monitoring install: remediation: retries: 3 upgrade: remediation: retries: 2 remediateLastFailure: false cleanupOnFail: true values: # --- k3s trim: control-plane components are embedded in the k3s server process and are not # scrapable as discrete endpoints. Disabling avoids permanently-firing "target down" noise. kubeControllerManager: enabled: false kubeScheduler: enabled: false kubeEtcd: enabled: false kubeProxy: enabled: false # --- Prometheus --- prometheus: prometheusSpec: replicas: 1 retention: ${MONITORING_PROMETHEUS_RETENTION} # overlay knob (dev 3d / prod long) # SIZE-based retention cap (docs/10, R-OBS). `retention` alone prunes only by AGE — on a young # cluster (< retention window) NOTHING is pruned yet, so a busy TSDB fills the PVC before the # time window elapses. A full /prometheus wedges ALL WAL commits ("no space left on device"): # scrapes still 200 but samples are never stored, the head freezes, and every absent()/==0 # platform alert (Keycloak/KubeAPI/Kubelet/VPN/KubeVirt/CDI…) fires falsely because instant # queries at "now" return empty. retentionSize makes Prometheus drop the OLDEST blocks once the # block set approaches this cap, so the disk never fills regardless of cluster age or churn. # Kept BELOW the PVC size (leaves headroom for the WAL + head + compaction scratch, which are # NOT counted by retentionSize). Prometheus size units are *iB (GiB), distinct from the K8s PVC # `Gi`, so it is a separate overlay knob rather than a reuse of the PVC-size knob. retentionSize: ${MONITORING_PROMETHEUS_RETENTION_SIZE} # overlay knob (dev 5GiB / base 8GiB / prod 45GiB) # Discover ServiceMonitors/PodMonitors/Rules from ALL namespaces (flux, cert-manager, # longhorn, velero, keycloak, cilium/hubble ship their own) — not just this release's. ruleSelectorNilUsesHelmValues: false serviceMonitorSelectorNilUsesHelmValues: false podMonitorSelectorNilUsesHelmValues: false probeSelectorNilUsesHelmValues: false # Modest footprint for a home dev cluster (few hundred series). Limit gives headroom for # WAL replay / compaction spikes without letting Prometheus eat the node. resources: requests: cpu: 100m memory: 512Mi limits: memory: 3Gi # 1Gi OOMKilled (exit 137) during TSDB compaction/checkpoint; # 2Gi ran pinned at ~98% (2002Mi) with no room for compaction/WAL-replay # spikes (the classic Prometheus OOM). Nodes are 24Gi at 38-44% used # (~14Gi free each), so 3Gi gives real headroom. storageSpec: volumeClaimTemplate: spec: storageClassName: longhorn # replicated tier (ADR-0003) accessModes: ["ReadWriteOnce"] resources: requests: storage: ${MONITORING_PROMETHEUS_PVC_SIZE} # overlay knob (dev 5Gi / prod large) # --- Alertmanager: routes by severity to email (R-OBS-02) --- alertmanager: alertmanagerSpec: replicas: 1 # DEV TRIM: no PVC — silences/notification-log live in emptyDir. Acceptable in dev (a # restart re-fires active alerts). Prod overlay can add a small storage block. resources: requests: cpu: 10m memory: 32Mi limits: memory: 96Mi # SMTP credentials are an OPERATOR HAND-OFF: a SOPS-encrypted Secret, referenced BY NAME # only (docs/07). Mounted at /etc/alertmanager/secrets// so the config below can read # the password from a file (never inline plaintext). secrets: - alertmanager-smtp # Routing/receivers. Non-secret SMTP endpoint fields (smarthost/from/to/username) are # operator-supplied placeholders (REPLACE-ME, same convention as velero); the PASSWORD is # read from the mounted SOPS secret via *_file. Both severities email; subject/route differ. config: global: resolve_timeout: 5m smtp_smarthost: "smtp.gmail.com:587" # operator hand-off (e.g. smtp.gmail.com:587) smtp_from: "alertmanager@keithjackson.dev" # operator hand-off (From: address) smtp_auth_username: "keithallenjackson@gmail.com" # operator hand-off (SMTP login) smtp_auth_password_file: /etc/alertmanager/secrets/alertmanager-smtp/smtp-password smtp_require_tls: true route: group_by: ["alertname", "namespace"] group_wait: 30s group_interval: 5m repeat_interval: 12h receiver: email-warning routes: - matchers: ['severity = "critical"'] receiver: email-critical # page-worthy — notify fast, remind often group_wait: 10s repeat_interval: 1h receivers: - name: email-warning email_configs: - to: "keithallenjackson@gmail.com" # alert inbox (operator: change if a different inbox is wanted) send_resolved: true headers: subject: '[WARNING] {{ .CommonLabels.alertname }} ({{ .CommonLabels.namespace }})' - name: email-critical email_configs: - to: "keithallenjackson@gmail.com" # alert inbox (operator: change if a different inbox is wanted) send_resolved: true headers: subject: '[CRITICAL] {{ .CommonLabels.alertname }} ({{ .CommonLabels.namespace }})' # --- Grafana: dashboards (R-OBS-03), surfaced in the C2 UI (docs/12) behind Keycloak SSO --- grafana: enabled: true # Force a pod roll when the OIDC wiring changes so Grafana loads the grafana-oidc-credentials # client-secret (env valueFrom is resolved at pod start; the secret was added after first boot, so # generic_oauth started disabled). Bump this value to re-roll Grafana on future OIDC secret changes. podAnnotations: homelab.keithjackson.dev/oidc-rollout: "1" # Admin bootstrap + (later) OIDC client are SOPS secrets referenced BY NAME (WS-SEC/ADR-0014). admin: existingSecret: grafana-admin-credentials userKey: admin-user passwordKey: admin-password # DEV TRIM: emptyDir (default). Dashboards are provisioned from ConfigMaps (sidecar), so no # durable Grafana DB is needed to iterate. Prod can enable persistence. persistence: enabled: false # Auto-discover dashboard ConfigMaps labeled grafana_dashboard across namespaces (per-service # dashboards rendered from ServiceDefinitions land here — docs/10 onboarding output). sidecar: dashboards: enabled: true searchNamespace: ALL label: grafana_dashboard datasources: enabled: true resources: requests: cpu: 50m memory: 384Mi # Grafana 13.x (image 13.2.0, kps 88) baseline is well above Grafana 11.x (kps 65) limits: memory: 768Mi # 256Mi OOMKilled (exit 137) Grafana 13.x during apiserver/unified-storage # startup — the silent SIGKILL that made the whole 65->88 Helm upgrade # time out (the new Grafana pod never went Ready). 512Mi then ran pinned at # ~98% (499Mi) with no headroom — one render/GC spike from OOM. 768Mi holds # with real headroom (nodes 24Gi, ~14Gi free each). # --- Keycloak native-OIDC SSO (ADR-0014, deliverable #4 SSO proof) --- # Grafana is a confidential OIDC client of the Keycloak `homelab` realm. The client SECRET comes # from the SOPS-encrypted grafana-oidc-credentials Secret via env (referenced by name only, # R-IDP-04). optional:true so the RUNNING Grafana keeps starting before the operator fills the # secret — local admin login (below) stays available as the fallback until SSO is proven. envValueFrom: GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: secretKeyRef: name: grafana-oidc-credentials key: client-secret optional: true grafana.ini: server: root_url: https://grafana.${BASE_DOMAIN} auth.generic_oauth: enabled: true name: Keycloak client_id: grafana scopes: "openid email profile" auth_url: https://keycloak.${BASE_DOMAIN}/realms/homelab/protocol/openid-connect/auth token_url: https://keycloak.${BASE_DOMAIN}/realms/homelab/protocol/openid-connect/token api_url: https://keycloak.${BASE_DOMAIN}/realms/homelab/protocol/openid-connect/userinfo login_attribute_path: preferred_username email_attribute_path: email # Map Keycloak groups (the `groups` claim) to Grafana roles (R-IDP-06): platform-admins -> # Admin, media-users -> Editor, everyone else -> Viewer. role_attribute_path: "contains(groups[*], 'platform-admins') && 'Admin' || contains(groups[*], 'media-users') && 'Editor' || 'Viewer'" # Keep local admin login enabled as break-glass until SSO is proven end-to-end; flip to # disable_login_form:true afterwards (docs/15 break-glass, R-IDP-05). # TODO(WS-SEC, post-proof): set auth.disable_login_form=true once SSO login is verified. # --- Bundled exporters / operator: keep small --- prometheusOperator: resources: requests: cpu: 50m memory: 96Mi limits: memory: 256Mi kube-state-metrics: # Expose the egress-gateway NODE LABEL as a metric label so alerts can ask "does this cluster # have a VPN egress gateway at all?". KSM exposes NO custom labels by default, so without this # `kube_node_labels` does not exist and any join against it silently matches nothing. # Consumed by `VPNTunnelDown` (platform/monitoring/alerts.yaml): the alert is armed only on a # cluster that actually has a labelled gateway node, and ARMS ITSELF the moment one is labelled # — which is how prod (no gateway today) stops emailing a permanently-firing critical alert # without anyone having to remember to re-enable it later. Scoped to one label on one resource: # 3 extra series, no cardinality concern. Label name is sanitized by KSM to # `label_homelab_keithjackson_dev_egress_gateway` (`label_` + non-alphanumerics -> `_`). metricLabelsAllowlist: - "nodes=[homelab.keithjackson.dev/egress-gateway]" # --- VELERO SCHEDULE OBJECTS AS METRICS (added 2026-09-13) --------------------------------- # Velero exports NOTHING that enumerates Schedule objects and nothing that exposes # `spec.paused`. Verified on prod against velero v1.18.1: all 35 velero_* metric names that # carry a `schedule` label are created by a backup ATTEMPT, so a Schedule that has never run # has NO series at all — a throwaway `paused: true` Schedule produced zero lines in # `curl velero:8085/metrics`. Consequence: `VeleroNoRecentBackup` # (`time() - velero_backup_last_successful_timestamp{schedule!=""}`) was structurally INERT for # a never-successful schedule, because `time() - ` yields no series. prod sat # with `velero-daily` (the VOLUME-DATA schedule) paused and unbacked-up, and nothing alerted. # # No PromQL can fix that — the fact is not in the TSDB — so the Schedule CRs themselves are # exported here. KSM's custom-resource-state collector is the right mechanism precisely # because it is ALREADY the platform's CRD-to-metrics path (this same block exports # kube_node_labels for VPNTunnelDown); it needs no new exporter, image, ServiceMonitor or # scrape job, and the series land on the existing kube-state-metrics target. # # Consumed by VeleroScheduleNeverBackedUp / VeleroSchedulePaused / VeleroNoVolumeDataBackup # (platform/monitoring/alerts.yaml). Cost: 4 series per Schedule — 8 on prod today. # # Chart mechanics, verified by rendering kube-prometheus-stack 88.6.5 (KSM subchart 8.4.1, # KSM v2.20.0) and by running that exact KSM image against prod's API: # * `customResourceState.enabled` mounts this `config` as a ConfigMap at # /etc/customresourcestate/config.yaml and adds --custom-resource-state-config-file. # `--custom-resource-state-only` is NOT set, so the standard kube_* metrics keep flowing. # * the chart auto-adds the apiextensions.k8s.io/customresourcedefinitions list+watch rule # (rbac.customResourceState.createClusterRoleRules, default true). That rule is MANDATORY, # not cosmetic: without it KSM logs "customresourcedefinitions ... is forbidden" and emits # the CRS metrics as an EMPTY SET — silent, exactly the inert-alert failure mode again. # Measured: the first probe run, with velero.io access but no CRD access, produced 0 series. # * `rbac.extraRules` below is the only RBAC this repo must supply (read-only, one CRD). # * `metricNamePrefix` deliberately yields `kube_velero_schedule_*`, NOT `velero_*`: these are # kube-state-metrics series about Kubernetes objects, and must not be mistaken for # something Velero itself exports. # * `nilIsZero` matters — a Schedule that simply omits `spec.paused` (the chart renders # `velero-daily-resources-only` that way) must still publish paused=0 rather than nothing, # or `== 1` would be evaluated against a missing series. # * the KSM Deployment carries NO checksum annotation over this ConfigMap, so a change to # `config` alone does not restart the pod: after editing it, `kubectl -n monitoring rollout # restart deploy/kube-prometheus-stack-kube-state-metrics`. Enabling it the first time # changes the pod spec (new arg + volume), so the initial rollout is automatic. rbac: extraRules: - apiGroups: ["velero.io"] resources: ["schedules"] verbs: ["get", "list", "watch"] customResourceState: enabled: true config: spec: resources: - groupVersionKind: group: velero.io version: v1 kind: Schedule metricNamePrefix: kube_velero_schedule labelsFromPath: # `schedule` (not `name`) so these join 1:1 with Velero's own `schedule` label. schedule: [metadata, name] namespace: [metadata, namespace] metrics: - name: created help: "Unix creation time of the Velero Schedule object." each: type: Gauge gauge: # KSM parses the RFC3339 timestamp to a unix float (verified live). # This is what gives VeleroScheduleNeverBackedUp a grace period, so a # newly added schedule is not alerted before its first run is even due. path: [metadata, creationTimestamp] - name: paused help: "1 if the Velero Schedule is paused and creating no backups." each: type: Gauge gauge: path: [spec, paused] nilIsZero: true - name: protects_volume_data help: "1 if the Schedule captures pod volume DATA, not just Kubernetes objects." each: type: Gauge gauge: # THE anti-masking signal: read from the field that defines it, so a # resources-only schedule can never stand in for a volume-data one. path: [spec, template, defaultVolumesToFsBackup] nilIsZero: true - name: info help: "Descriptive metadata for the Velero Schedule (cron, phase)." each: type: Info info: # NOTE: KSM OMITS a label whose path is absent (a Schedule with no # status.phase yet yields no `phase` label at all), so label sets differ # between series here. Context for dashboards — never join on this metric. labelsFromPath: cron: [spec, schedule] phase: [status, phase] resources: requests: cpu: 20m memory: 64Mi limits: memory: 128Mi prometheus-node-exporter: resources: requests: cpu: 20m memory: 32Mi limits: memory: 64Mi