home-network-k3s-impl/platform/monitoring/kube-prometheus-stack.yaml

---
# kube-prometheus-stack — Prometheus + Alertmanager + Grafana + kube-state-metrics +
# node-exporter + the standard Kubernetes exporters/rules (docs/10). Serves R-OBS-01/02/03.
#
# Retention/PVC sizes are overlay knobs (R-ENV-02) resolved from cluster-vars via Flux
# postBuild.substituteFrom. Everything else is DEV-SCOPED for the ~22 GiB workstation ceiling:
# single replicas, modest requests/limits, ephemeral (emptyDir) Alertmanager/Grafana state, and
# the k3s-embedded control-plane scrape jobs disabled (they are not separately reachable on k3s and
# would otherwise show as permanently-down targets and burn RAM/alert noise).
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
  name: kube-prometheus-stack
  namespace: monitoring
spec:
  interval: 30m
  # The 65->88 catch-up is a large multi-major jump (operator + KSM + node-exporter roll, plus the
  # Prometheus/Alertmanager StatefulSets). The default 5m Helm timeout is too short for the full rollout
  # to reach Ready, so Flux logged "context deadline exceeded" and rolled back to 65.8.1 four times.
  # A generous timeout lets one attempt finish; remediateLastFailure:false keeps partial progress instead
  # of yo-yoing back to the old chart (a re-reconcile then drives it forward). CRDs are pre-applied
  # server-side (Helm never updates crds/ on upgrade), so schema is already at operator v0.93.1.
  timeout: 20m0s
  chart:
    spec:
      chart: kube-prometheus-stack
      version: ">=88.0.0 <89.0.0"   # k8s-1.36 catch-up (po v0.93; CRDs pre-applied server-side, Helm won't update them). Renovate maintains
      sourceRef:
        kind: HelmRepository
        name: prometheus-community
        namespace: monitoring
  install:
    remediation:
      retries: 3
  upgrade:
    remediation:
      retries: 2
      remediateLastFailure: false
    cleanupOnFail: true
  values:
    # --- k3s trim: control-plane components are embedded in the k3s server process and are not
    #     scrapable as discrete endpoints. Disabling avoids permanently-firing "target down" noise.
    kubeControllerManager:
      enabled: false
    kubeScheduler:
      enabled: false
    kubeEtcd:
      enabled: false
    kubeProxy:
      enabled: false

    # --- Prometheus ---
    prometheus:
      prometheusSpec:
        replicas: 1
        retention: ${MONITORING_PROMETHEUS_RETENTION}     # overlay knob (dev 3d / prod long)
        # SIZE-based retention cap (docs/10, R-OBS). `retention` alone prunes only by AGE — on a young
        # cluster (< retention window) NOTHING is pruned yet, so a busy TSDB fills the PVC before the
        # time window elapses. A full /prometheus wedges ALL WAL commits ("no space left on device"):
        # scrapes still 200 but samples are never stored, the head freezes, and every absent()/==0
        # platform alert (Keycloak/KubeAPI/Kubelet/VPN/KubeVirt/CDI…) fires falsely because instant
        # queries at "now" return empty. retentionSize makes Prometheus drop the OLDEST blocks once the
        # block set approaches this cap, so the disk never fills regardless of cluster age or churn.
        # Kept BELOW the PVC size (leaves headroom for the WAL + head + compaction scratch, which are
        # NOT counted by retentionSize). Prometheus size units are *iB (GiB), distinct from the K8s PVC
        # `Gi`, so it is a separate overlay knob rather than a reuse of the PVC-size knob.
        retentionSize: ${MONITORING_PROMETHEUS_RETENTION_SIZE}   # overlay knob (dev 5GiB / base 8GiB / prod 45GiB)
        # Discover ServiceMonitors/PodMonitors/Rules from ALL namespaces (flux, cert-manager,
        # longhorn, velero, keycloak, cilium/hubble ship their own) — not just this release's.
        ruleSelectorNilUsesHelmValues: false
        serviceMonitorSelectorNilUsesHelmValues: false
        podMonitorSelectorNilUsesHelmValues: false
        probeSelectorNilUsesHelmValues: false
        # Modest footprint for a home dev cluster (few hundred series). Limit gives headroom for
        # WAL replay / compaction spikes without letting Prometheus eat the node.
        resources:
          requests:
            cpu: 100m
            memory: 512Mi
          limits:
            memory: 3Gi                        # 1Gi OOMKilled (exit 137) during TSDB compaction/checkpoint;
                                               # 2Gi ran pinned at ~98% (2002Mi) with no room for compaction/WAL-replay
                                               # spikes (the classic Prometheus OOM). Nodes are 24Gi at 38-44% used
                                               # (~14Gi free each), so 3Gi gives real headroom.
        storageSpec:
          volumeClaimTemplate:
            spec:
              storageClassName: longhorn                  # replicated tier (ADR-0003)
              accessModes: ["ReadWriteOnce"]
              resources:
                requests:
                  storage: ${MONITORING_PROMETHEUS_PVC_SIZE}   # overlay knob (dev 5Gi / prod large)

    # --- Alertmanager: routes by severity to email (R-OBS-02) ---
    alertmanager:
      alertmanagerSpec:
        replicas: 1
        # DEV TRIM: no PVC — silences/notification-log live in emptyDir. Acceptable in dev (a
        # restart re-fires active alerts). Prod overlay can add a small storage block.
        resources:
          requests:
            cpu: 10m
            memory: 32Mi
          limits:
            memory: 96Mi
        # SMTP credentials are an OPERATOR HAND-OFF: a SOPS-encrypted Secret, referenced BY NAME
        # only (docs/07). Mounted at /etc/alertmanager/secrets/<name>/ so the config below can read
        # the password from a file (never inline plaintext).
        secrets:
          - alertmanager-smtp
      # Routing/receivers. Non-secret SMTP endpoint fields (smarthost/from/to/username) are
      # operator-supplied placeholders (REPLACE-ME, same convention as velero); the PASSWORD is
      # read from the mounted SOPS secret via *_file. Both severities email; subject/route differ.
      config:
        global:
          resolve_timeout: 5m
          smtp_smarthost: "smtp.gmail.com:587"          # operator hand-off (e.g. smtp.gmail.com:587)
          smtp_from: "[email protected]"                    # operator hand-off (From: address)
          smtp_auth_username: "[email protected]"           # operator hand-off (SMTP login)
          smtp_auth_password_file: /etc/alertmanager/secrets/alertmanager-smtp/smtp-password
          smtp_require_tls: true
        route:
          group_by: ["alertname", "namespace"]
          group_wait: 30s
          group_interval: 5m
          repeat_interval: 12h
          receiver: email-warning
          routes:
            - matchers: ['severity = "critical"']
              receiver: email-critical
              # page-worthy — notify fast, remind often
              group_wait: 10s
              repeat_interval: 1h
        receivers:
          - name: email-warning
            email_configs:
              - to: "[email protected]"    # alert inbox (operator: change if a different inbox is wanted)
                send_resolved: true
                headers:
                  subject: '[WARNING] {{ .CommonLabels.alertname }} ({{ .CommonLabels.namespace }})'
          - name: email-critical
            email_configs:
              - to: "[email protected]"    # alert inbox (operator: change if a different inbox is wanted)
                send_resolved: true
                headers:
                  subject: '[CRITICAL] {{ .CommonLabels.alertname }} ({{ .CommonLabels.namespace }})'

    # --- Grafana: dashboards (R-OBS-03), surfaced in the C2 UI (docs/12) behind Keycloak SSO ---
    grafana:
      enabled: true
      # Force a pod roll when the OIDC wiring changes so Grafana loads the grafana-oidc-credentials
      # client-secret (env valueFrom is resolved at pod start; the secret was added after first boot, so
      # generic_oauth started disabled). Bump this value to re-roll Grafana on future OIDC secret changes.
      podAnnotations:
        homelab.keithjackson.dev/oidc-rollout: "1"
      # Admin bootstrap + (later) OIDC client are SOPS secrets referenced BY NAME (WS-SEC/ADR-0014).
      admin:
        existingSecret: grafana-admin-credentials
        userKey: admin-user
        passwordKey: admin-password
      # DEV TRIM: emptyDir (default). Dashboards are provisioned from ConfigMaps (sidecar), so no
      # durable Grafana DB is needed to iterate. Prod can enable persistence.
      persistence:
        enabled: false
      # Auto-discover dashboard ConfigMaps labeled grafana_dashboard across namespaces (per-service
      # dashboards rendered from ServiceDefinitions land here — docs/10 onboarding output).
      sidecar:
        dashboards:
          enabled: true
          searchNamespace: ALL
          label: grafana_dashboard
        datasources:
          enabled: true
      resources:
        requests:
          cpu: 50m
          memory: 384Mi                     # Grafana 13.x (image 13.2.0, kps 88) baseline is well above Grafana 11.x (kps 65)
        limits:
          memory: 768Mi                      # 256Mi OOMKilled (exit 137) Grafana 13.x during apiserver/unified-storage
                                             # startup — the silent SIGKILL that made the whole 65->88 Helm upgrade
                                             # time out (the new Grafana pod never went Ready). 512Mi then ran pinned at
                                             # ~98% (499Mi) with no headroom — one render/GC spike from OOM. 768Mi holds
                                             # with real headroom (nodes 24Gi, ~14Gi free each).
      # --- Keycloak native-OIDC SSO (ADR-0014, deliverable #4 SSO proof) ---
      # Grafana is a confidential OIDC client of the Keycloak `homelab` realm. The client SECRET comes
      # from the SOPS-encrypted grafana-oidc-credentials Secret via env (referenced by name only,
      # R-IDP-04). optional:true so the RUNNING Grafana keeps starting before the operator fills the
      # secret — local admin login (below) stays available as the fallback until SSO is proven.
      envValueFrom:
        GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET:
          secretKeyRef:
            name: grafana-oidc-credentials
            key: client-secret
            optional: true
      grafana.ini:
        server:
          root_url: https://grafana.${BASE_DOMAIN}
        auth.generic_oauth:
          enabled: true
          name: Keycloak
          client_id: grafana
          scopes: "openid email profile"
          auth_url: https://keycloak.${BASE_DOMAIN}/realms/homelab/protocol/openid-connect/auth
          token_url: https://keycloak.${BASE_DOMAIN}/realms/homelab/protocol/openid-connect/token
          api_url: https://keycloak.${BASE_DOMAIN}/realms/homelab/protocol/openid-connect/userinfo
          login_attribute_path: preferred_username
          email_attribute_path: email
          # Map Keycloak groups (the `groups` claim) to Grafana roles (R-IDP-06): platform-admins ->
          # Admin, media-users -> Editor, everyone else -> Viewer.
          role_attribute_path: "contains(groups[*], 'platform-admins') && 'Admin' || contains(groups[*], 'media-users') && 'Editor' || 'Viewer'"
          # Keep local admin login enabled as break-glass until SSO is proven end-to-end; flip to
          # disable_login_form:true afterwards (docs/15 break-glass, R-IDP-05).
        # TODO(WS-SEC, post-proof): set auth.disable_login_form=true once SSO login is verified.

    # --- Bundled exporters / operator: keep small ---
    prometheusOperator:
      resources:
        requests:
          cpu: 50m
          memory: 96Mi
        limits:
          memory: 256Mi
    kube-state-metrics:
      resources:
        requests:
          cpu: 20m
          memory: 64Mi
        limits:
          memory: 128Mi
    prometheus-node-exporter:
      resources:
        requests:
          cpu: 20m
          memory: 32Mi
        limits:
          memory: 64Mi