home-network-k3s-impl/platform/monitoring/kube-prometheus-stack.yaml

---
# kube-prometheus-stack — Prometheus + Alertmanager + Grafana + kube-state-metrics +
# node-exporter + the standard Kubernetes exporters/rules (docs/10). Serves R-OBS-01/02/03.
#
# Retention/PVC sizes are overlay knobs (R-ENV-02) resolved from cluster-vars via Flux
# postBuild.substituteFrom. Everything else is DEV-SCOPED for the ~22 GiB workstation ceiling:
# single replicas, modest requests/limits, ephemeral (emptyDir) Alertmanager/Grafana state, and
# the k3s-embedded control-plane scrape jobs disabled (they are not separately reachable on k3s and
# would otherwise show as permanently-down targets and burn RAM/alert noise).
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
  name: kube-prometheus-stack
  namespace: monitoring
spec:
  interval: 30m
  chart:
    spec:
      chart: kube-prometheus-stack
      version: ">=65.0.0 <66.0.0"   # TODO(WS-OBS): pin at bring-up; Renovate maintains
      sourceRef:
        kind: HelmRepository
        name: prometheus-community
        namespace: monitoring
  install:
    remediation:
      retries: 3
  upgrade:
    remediation:
      retries: 3
  values:
    # --- k3s trim: control-plane components are embedded in the k3s server process and are not
    #     scrapable as discrete endpoints. Disabling avoids permanently-firing "target down" noise.
    kubeControllerManager:
      enabled: false
    kubeScheduler:
      enabled: false
    kubeEtcd:
      enabled: false
    kubeProxy:
      enabled: false

    # --- Prometheus ---
    prometheus:
      prometheusSpec:
        replicas: 1
        retention: ${MONITORING_PROMETHEUS_RETENTION}     # overlay knob (dev 3d / prod long)
        # Discover ServiceMonitors/PodMonitors/Rules from ALL namespaces (flux, cert-manager,
        # longhorn, velero, keycloak, cilium/hubble ship their own) — not just this release's.
        ruleSelectorNilUsesHelmValues: false
        serviceMonitorSelectorNilUsesHelmValues: false
        podMonitorSelectorNilUsesHelmValues: false
        probeSelectorNilUsesHelmValues: false
        # Modest footprint for a home dev cluster (few hundred series). Limit gives headroom for
        # WAL replay / compaction spikes without letting Prometheus eat the node.
        resources:
          requests:
            cpu: 100m
            memory: 512Mi
          limits:
            memory: 2Gi                        # 1Gi OOMKilled (exit 137) during TSDB compaction/checkpoint
                                               # spikes; 2Gi holds. Nodes have headroom (~2Gi free each).
        storageSpec:
          volumeClaimTemplate:
            spec:
              storageClassName: longhorn                  # replicated tier (ADR-0003)
              accessModes: ["ReadWriteOnce"]
              resources:
                requests:
                  storage: ${MONITORING_PROMETHEUS_PVC_SIZE}   # overlay knob (dev 5Gi / prod large)

    # --- Alertmanager: routes by severity to email (R-OBS-02) ---
    alertmanager:
      alertmanagerSpec:
        replicas: 1
        # DEV TRIM: no PVC — silences/notification-log live in emptyDir. Acceptable in dev (a
        # restart re-fires active alerts). Prod overlay can add a small storage block.
        resources:
          requests:
            cpu: 10m
            memory: 32Mi
          limits:
            memory: 96Mi
        # SMTP credentials are an OPERATOR HAND-OFF: a SOPS-encrypted Secret, referenced BY NAME
        # only (docs/07). Mounted at /etc/alertmanager/secrets/<name>/ so the config below can read
        # the password from a file (never inline plaintext).
        secrets:
          - alertmanager-smtp
      # Routing/receivers. Non-secret SMTP endpoint fields (smarthost/from/to/username) are
      # operator-supplied placeholders (REPLACE-ME, same convention as velero); the PASSWORD is
      # read from the mounted SOPS secret via *_file. Both severities email; subject/route differ.
      config:
        global:
          resolve_timeout: 5m
          smtp_smarthost: "smtp.gmail.com:587"          # operator hand-off (e.g. smtp.gmail.com:587)
          smtp_from: "[email protected]"                    # operator hand-off (From: address)
          smtp_auth_username: "[email protected]"           # operator hand-off (SMTP login)
          smtp_auth_password_file: /etc/alertmanager/secrets/alertmanager-smtp/smtp-password
          smtp_require_tls: true
        route:
          group_by: ["alertname", "namespace"]
          group_wait: 30s
          group_interval: 5m
          repeat_interval: 12h
          receiver: email-warning
          routes:
            - matchers: ['severity = "critical"']
              receiver: email-critical
              # page-worthy — notify fast, remind often
              group_wait: 10s
              repeat_interval: 1h
        receivers:
          - name: email-warning
            email_configs:
              - to: "[email protected]"    # alert inbox (operator: change if a different inbox is wanted)
                send_resolved: true
                headers:
                  subject: '[WARNING] {{ .CommonLabels.alertname }} ({{ .CommonLabels.namespace }})'
          - name: email-critical
            email_configs:
              - to: "[email protected]"    # alert inbox (operator: change if a different inbox is wanted)
                send_resolved: true
                headers:
                  subject: '[CRITICAL] {{ .CommonLabels.alertname }} ({{ .CommonLabels.namespace }})'

    # --- Grafana: dashboards (R-OBS-03), surfaced in the C2 UI (docs/12) behind Keycloak SSO ---
    grafana:
      enabled: true
      # Admin bootstrap + (later) OIDC client are SOPS secrets referenced BY NAME (WS-SEC/ADR-0014).
      admin:
        existingSecret: grafana-admin-credentials
        userKey: admin-user
        passwordKey: admin-password
      # DEV TRIM: emptyDir (default). Dashboards are provisioned from ConfigMaps (sidecar), so no
      # durable Grafana DB is needed to iterate. Prod can enable persistence.
      persistence:
        enabled: false
      # Auto-discover dashboard ConfigMaps labeled grafana_dashboard across namespaces (per-service
      # dashboards rendered from ServiceDefinitions land here — docs/10 onboarding output).
      sidecar:
        dashboards:
          enabled: true
          searchNamespace: ALL
          label: grafana_dashboard
        datasources:
          enabled: true
      resources:
        requests:
          cpu: 50m
          memory: 128Mi
        limits:
          memory: 256Mi
      # TODO(WS-SEC/WS-C2, later): Keycloak OIDC client (auth.generic_oauth) + root_url behind the
      #   Gateway; disable local login once SSO is proven. Secret referenced by name only.

    # --- Bundled exporters / operator: keep small ---
    prometheusOperator:
      resources:
        requests:
          cpu: 50m
          memory: 96Mi
        limits:
          memory: 256Mi
    kube-state-metrics:
      resources:
        requests:
          cpu: 20m
          memory: 64Mi
        limits:
          memory: 128Mi
    prometheus-node-exporter:
      resources:
        requests:
          cpu: 20m
          memory: 32Mi
        limits:
          memory: 64Mi