--- # kube-prometheus-stack — Prometheus + Alertmanager + Grafana + kube-state-metrics + # node-exporter + the standard Kubernetes exporters/rules (docs/10). Serves R-OBS-01/02/03. # # Retention/PVC sizes are overlay knobs (R-ENV-02) resolved from cluster-vars via Flux # postBuild.substituteFrom. Everything else is DEV-SCOPED for the ~22 GiB workstation ceiling: # single replicas, modest requests/limits, ephemeral (emptyDir) Alertmanager/Grafana state, and # the k3s-embedded control-plane scrape jobs disabled (they are not separately reachable on k3s and # would otherwise show as permanently-down targets and burn RAM/alert noise). apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: name: kube-prometheus-stack namespace: monitoring spec: interval: 30m # The 65->88 catch-up is a large multi-major jump (operator + KSM + node-exporter roll, plus the # Prometheus/Alertmanager StatefulSets). The default 5m Helm timeout is too short for the full rollout # to reach Ready, so Flux logged "context deadline exceeded" and rolled back to 65.8.1 four times. # A generous timeout lets one attempt finish; remediateLastFailure:false keeps partial progress instead # of yo-yoing back to the old chart (a re-reconcile then drives it forward). CRDs are pre-applied # server-side (Helm never updates crds/ on upgrade), so schema is already at operator v0.93.1. timeout: 20m0s chart: spec: chart: kube-prometheus-stack version: ">=88.0.0 <89.0.0" # k8s-1.36 catch-up (po v0.93; CRDs pre-applied server-side, Helm won't update them). Renovate maintains sourceRef: kind: HelmRepository name: prometheus-community namespace: monitoring install: remediation: retries: 3 upgrade: remediation: retries: 2 remediateLastFailure: false cleanupOnFail: true values: # --- k3s trim: control-plane components are embedded in the k3s server process and are not # scrapable as discrete endpoints. Disabling avoids permanently-firing "target down" noise. kubeControllerManager: enabled: false kubeScheduler: enabled: false kubeEtcd: enabled: false kubeProxy: enabled: false # --- Prometheus --- prometheus: prometheusSpec: replicas: 1 retention: ${MONITORING_PROMETHEUS_RETENTION} # overlay knob (dev 3d / prod long) # SIZE-based retention cap (docs/10, R-OBS). `retention` alone prunes only by AGE — on a young # cluster (< retention window) NOTHING is pruned yet, so a busy TSDB fills the PVC before the # time window elapses. A full /prometheus wedges ALL WAL commits ("no space left on device"): # scrapes still 200 but samples are never stored, the head freezes, and every absent()/==0 # platform alert (Keycloak/KubeAPI/Kubelet/VPN/KubeVirt/CDI…) fires falsely because instant # queries at "now" return empty. retentionSize makes Prometheus drop the OLDEST blocks once the # block set approaches this cap, so the disk never fills regardless of cluster age or churn. # Kept BELOW the PVC size (leaves headroom for the WAL + head + compaction scratch, which are # NOT counted by retentionSize). Prometheus size units are *iB (GiB), distinct from the K8s PVC # `Gi`, so it is a separate overlay knob rather than a reuse of the PVC-size knob. retentionSize: ${MONITORING_PROMETHEUS_RETENTION_SIZE} # overlay knob (dev 5GiB / base 8GiB / prod 45GiB) # Discover ServiceMonitors/PodMonitors/Rules from ALL namespaces (flux, cert-manager, # longhorn, velero, keycloak, cilium/hubble ship their own) — not just this release's. ruleSelectorNilUsesHelmValues: false serviceMonitorSelectorNilUsesHelmValues: false podMonitorSelectorNilUsesHelmValues: false probeSelectorNilUsesHelmValues: false # Modest footprint for a home dev cluster (few hundred series). Limit gives headroom for # WAL replay / compaction spikes without letting Prometheus eat the node. resources: requests: cpu: 100m memory: 512Mi limits: memory: 3Gi # 1Gi OOMKilled (exit 137) during TSDB compaction/checkpoint; # 2Gi ran pinned at ~98% (2002Mi) with no room for compaction/WAL-replay # spikes (the classic Prometheus OOM). Nodes are 24Gi at 38-44% used # (~14Gi free each), so 3Gi gives real headroom. storageSpec: volumeClaimTemplate: spec: storageClassName: longhorn # replicated tier (ADR-0003) accessModes: ["ReadWriteOnce"] resources: requests: storage: ${MONITORING_PROMETHEUS_PVC_SIZE} # overlay knob (dev 5Gi / prod large) # --- Alertmanager: routes by severity to email (R-OBS-02) --- alertmanager: alertmanagerSpec: replicas: 1 # DEV TRIM: no PVC — silences/notification-log live in emptyDir. Acceptable in dev (a # restart re-fires active alerts). Prod overlay can add a small storage block. resources: requests: cpu: 10m memory: 32Mi limits: memory: 96Mi # SMTP credentials are an OPERATOR HAND-OFF: a SOPS-encrypted Secret, referenced BY NAME # only (docs/07). Mounted at /etc/alertmanager/secrets// so the config below can read # the password from a file (never inline plaintext). secrets: - alertmanager-smtp # Routing/receivers. Non-secret SMTP endpoint fields (smarthost/from/to/username) are # operator-supplied placeholders (REPLACE-ME, same convention as velero); the PASSWORD is # read from the mounted SOPS secret via *_file. Both severities email; subject/route differ. config: global: resolve_timeout: 5m smtp_smarthost: "smtp.gmail.com:587" # operator hand-off (e.g. smtp.gmail.com:587) smtp_from: "alertmanager@keithjackson.dev" # operator hand-off (From: address) smtp_auth_username: "keithallenjackson@gmail.com" # operator hand-off (SMTP login) smtp_auth_password_file: /etc/alertmanager/secrets/alertmanager-smtp/smtp-password smtp_require_tls: true route: group_by: ["alertname", "namespace"] group_wait: 30s group_interval: 5m repeat_interval: 12h receiver: email-warning routes: - matchers: ['severity = "critical"'] receiver: email-critical # page-worthy — notify fast, remind often group_wait: 10s repeat_interval: 1h receivers: - name: email-warning email_configs: - to: "keithallenjackson@gmail.com" # alert inbox (operator: change if a different inbox is wanted) send_resolved: true headers: subject: '[WARNING] {{ .CommonLabels.alertname }} ({{ .CommonLabels.namespace }})' - name: email-critical email_configs: - to: "keithallenjackson@gmail.com" # alert inbox (operator: change if a different inbox is wanted) send_resolved: true headers: subject: '[CRITICAL] {{ .CommonLabels.alertname }} ({{ .CommonLabels.namespace }})' # --- Grafana: dashboards (R-OBS-03), surfaced in the C2 UI (docs/12) behind Keycloak SSO --- grafana: enabled: true # Force a pod roll when the OIDC wiring changes so Grafana loads the grafana-oidc-credentials # client-secret (env valueFrom is resolved at pod start; the secret was added after first boot, so # generic_oauth started disabled). Bump this value to re-roll Grafana on future OIDC secret changes. podAnnotations: homelab.keithjackson.dev/oidc-rollout: "1" # Admin bootstrap + (later) OIDC client are SOPS secrets referenced BY NAME (WS-SEC/ADR-0014). admin: existingSecret: grafana-admin-credentials userKey: admin-user passwordKey: admin-password # DEV TRIM: emptyDir (default). Dashboards are provisioned from ConfigMaps (sidecar), so no # durable Grafana DB is needed to iterate. Prod can enable persistence. persistence: enabled: false # Auto-discover dashboard ConfigMaps labeled grafana_dashboard across namespaces (per-service # dashboards rendered from ServiceDefinitions land here — docs/10 onboarding output). sidecar: dashboards: enabled: true searchNamespace: ALL label: grafana_dashboard datasources: enabled: true resources: requests: cpu: 50m memory: 384Mi # Grafana 13.x (image 13.2.0, kps 88) baseline is well above Grafana 11.x (kps 65) limits: memory: 768Mi # 256Mi OOMKilled (exit 137) Grafana 13.x during apiserver/unified-storage # startup — the silent SIGKILL that made the whole 65->88 Helm upgrade # time out (the new Grafana pod never went Ready). 512Mi then ran pinned at # ~98% (499Mi) with no headroom — one render/GC spike from OOM. 768Mi holds # with real headroom (nodes 24Gi, ~14Gi free each). # --- Keycloak native-OIDC SSO (ADR-0014, deliverable #4 SSO proof) --- # Grafana is a confidential OIDC client of the Keycloak `homelab` realm. The client SECRET comes # from the SOPS-encrypted grafana-oidc-credentials Secret via env (referenced by name only, # R-IDP-04). optional:true so the RUNNING Grafana keeps starting before the operator fills the # secret — local admin login (below) stays available as the fallback until SSO is proven. envValueFrom: GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: secretKeyRef: name: grafana-oidc-credentials key: client-secret optional: true grafana.ini: server: root_url: https://grafana.${BASE_DOMAIN} auth.generic_oauth: enabled: true name: Keycloak client_id: grafana scopes: "openid email profile" auth_url: https://keycloak.${BASE_DOMAIN}/realms/homelab/protocol/openid-connect/auth token_url: https://keycloak.${BASE_DOMAIN}/realms/homelab/protocol/openid-connect/token api_url: https://keycloak.${BASE_DOMAIN}/realms/homelab/protocol/openid-connect/userinfo login_attribute_path: preferred_username email_attribute_path: email # Map Keycloak groups (the `groups` claim) to Grafana roles (R-IDP-06): platform-admins -> # Admin, media-users -> Editor, everyone else -> Viewer. role_attribute_path: "contains(groups[*], 'platform-admins') && 'Admin' || contains(groups[*], 'media-users') && 'Editor' || 'Viewer'" # Keep local admin login enabled as break-glass until SSO is proven end-to-end; flip to # disable_login_form:true afterwards (docs/15 break-glass, R-IDP-05). # TODO(WS-SEC, post-proof): set auth.disable_login_form=true once SSO login is verified. # --- Bundled exporters / operator: keep small --- prometheusOperator: resources: requests: cpu: 50m memory: 96Mi limits: memory: 256Mi kube-state-metrics: # Expose the egress-gateway NODE LABEL as a metric label so alerts can ask "does this cluster # have a VPN egress gateway at all?". KSM exposes NO custom labels by default, so without this # `kube_node_labels` does not exist and any join against it silently matches nothing. # Consumed by `VPNTunnelDown` (platform/monitoring/alerts.yaml): the alert is armed only on a # cluster that actually has a labelled gateway node, and ARMS ITSELF the moment one is labelled # — which is how prod (no gateway today) stops emailing a permanently-firing critical alert # without anyone having to remember to re-enable it later. Scoped to one label on one resource: # 3 extra series, no cardinality concern. Label name is sanitized by KSM to # `label_homelab_keithjackson_dev_egress_gateway` (`label_` + non-alphanumerics -> `_`). metricLabelsAllowlist: - "nodes=[homelab.keithjackson.dev/egress-gateway]" resources: requests: cpu: 20m memory: 64Mi limits: memory: 128Mi prometheus-node-exporter: resources: requests: cpu: 20m memory: 32Mi limits: memory: 64Mi