home-network-k3s-impl/platform/monitoring/kube-prometheus-stack.yaml

---
# kube-prometheus-stack — Prometheus + Alertmanager + Grafana + kube-state-metrics +
# node-exporter + the standard Kubernetes exporters/rules (docs/10). Serves R-OBS-01/02/03.
#
# Retention/PVC sizes are overlay knobs (R-ENV-02) resolved from cluster-vars via Flux
# postBuild.substituteFrom. Everything else is DEV-SCOPED for the ~22 GiB workstation ceiling:
# single replicas, modest requests/limits, ephemeral (emptyDir) Alertmanager/Grafana state, and
# the k3s-embedded control-plane scrape jobs disabled (they are not separately reachable on k3s and
# would otherwise show as permanently-down targets and burn RAM/alert noise).
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
  name: kube-prometheus-stack
  namespace: monitoring
spec:
  interval: 30m
  chart:
    spec:
      chart: kube-prometheus-stack
      version: ">=65.0.0 <66.0.0"   # TODO(WS-OBS): pin at bring-up; Renovate maintains
      sourceRef:
        kind: HelmRepository
        name: prometheus-community
        namespace: monitoring
  install:
    remediation:
      retries: 3
  upgrade:
    remediation:
      retries: 3
  values:
    # --- k3s trim: control-plane components are embedded in the k3s server process and are not
    #     scrapable as discrete endpoints. Disabling avoids permanently-firing "target down" noise.
    kubeControllerManager:
      enabled: false
    kubeScheduler:
      enabled: false
    kubeEtcd:
      enabled: false
    kubeProxy:
      enabled: false

    # --- Prometheus ---
    prometheus:
      prometheusSpec:
        replicas: 1
        retention: ${MONITORING_PROMETHEUS_RETENTION}     # overlay knob (dev 3d / prod long)
        # Discover ServiceMonitors/PodMonitors/Rules from ALL namespaces (flux, cert-manager,
        # longhorn, velero, keycloak, cilium/hubble ship their own) — not just this release's.
        ruleSelectorNilUsesHelmValues: false
        serviceMonitorSelectorNilUsesHelmValues: false
        podMonitorSelectorNilUsesHelmValues: false
        probeSelectorNilUsesHelmValues: false
        # Modest footprint for a home dev cluster (few hundred series). Limit gives headroom for
        # WAL replay / compaction spikes without letting Prometheus eat the node.
        resources:
          requests:
            cpu: 100m
            memory: 384Mi
          limits:
            memory: 1Gi
        storageSpec:
          volumeClaimTemplate:
            spec:
              storageClassName: ${MONITORING_STORAGE_CLASS}   # replicated tier (ADR-0003); cluster-vars knob
              accessModes: ["ReadWriteOnce"]
              resources:
                requests:
                  storage: ${MONITORING_PROMETHEUS_PVC_SIZE}   # overlay knob (dev 5Gi / prod large)

    # --- Alertmanager: routes by severity to email (R-OBS-02) ---
    alertmanager:
      alertmanagerSpec:
        replicas: 1
        # DEV TRIM: no PVC — silences/notification-log live in emptyDir. Acceptable in dev (a
        # restart re-fires active alerts). Prod overlay can add a small storage block.
        resources:
          requests:
            cpu: 10m
            memory: 32Mi
          limits:
            memory: 96Mi
        # SMTP is an OPERATOR HAND-OFF DEFERRED to Phase 4 — no creds yet, and the by-name
        # `alertmanager-smtp` Secret does NOT exist. Mounting a missing Secret wedges Alertmanager
        # in ContainerCreating, so the mount is DISABLED until the hand-off. Re-enable verbatim
        # (uncomment) when the SOPS `alertmanager-smtp` Secret is delivered. TODO(Phase-4 SMTP).
        # secrets:
        #   - alertmanager-smtp
      # Routing stays live; RECEIVERS ARE NULL until the SMTP hand-off (alerts still fire, route,
      # and show in the Alertmanager UI + Grafana — they just aren't emailed). The smtp_* globals
      # and email_configs are retained below, commented, so re-enabling is a pure uncomment.
      config:
        global:
          resolve_timeout: 5m
          # TODO(Phase-4 SMTP hand-off): uncomment together with the alertmanager-smtp Secret mount.
          # smtp_smarthost: "smtp.example.com:587"
          # smtp_from: "[email protected]"
          # smtp_auth_username: "<smtp-login>"
          # smtp_auth_password_file: /etc/alertmanager/secrets/alertmanager-smtp/smtp-password
          # smtp_require_tls: true
        route:
          group_by: ["alertname", "namespace"]
          group_wait: 30s
          group_interval: 5m
          repeat_interval: 12h
          receiver: email-warning
          routes:
            - matchers: ['severity = "critical"']
              receiver: email-critical
              # page-worthy — notify fast, remind often
              group_wait: 10s
              repeat_interval: 1h
        receivers:
          # NULL receivers until Phase-4 SMTP. email_configs retained (commented) for verbatim re-enable.
          - name: email-warning
            # email_configs:
            #   - to: "<alert-inbox>"
            #     send_resolved: true
            #     headers:
            #       subject: '[WARNING] {{ .CommonLabels.alertname }} ({{ .CommonLabels.namespace }})'
          - name: email-critical
            # email_configs:
            #   - to: "<alert-inbox>"
            #     send_resolved: true
            #     headers:
            #       subject: '[CRITICAL] {{ .CommonLabels.alertname }} ({{ .CommonLabels.namespace }})'

    # --- Grafana: dashboards (R-OBS-03), surfaced in the C2 UI (docs/12) behind Keycloak SSO ---
    grafana:
      enabled: true
      # Admin bootstrap + (later) OIDC client are SOPS secrets referenced BY NAME (WS-SEC/ADR-0014).
      # cluster-vars knob: prod = the SOPS `grafana-admin-credentials` Secret; dev = "" (empty),
      # which makes the chart fall back to its default admin password (dev-only, NAT-isolated) so
      # Grafana starts without an operator secret. TODO: supply a dev Secret if wider dev access.
      admin:
        existingSecret: ${GRAFANA_ADMIN_SECRET}
        userKey: admin-user
        passwordKey: admin-password
      # DEV TRIM: emptyDir (default). Dashboards are provisioned from ConfigMaps (sidecar), so no
      # durable Grafana DB is needed to iterate. Prod can enable persistence.
      persistence:
        enabled: false
      # Auto-discover dashboard ConfigMaps labeled grafana_dashboard across namespaces (per-service
      # dashboards rendered from ServiceDefinitions land here — docs/10 onboarding output).
      sidecar:
        dashboards:
          enabled: true
          searchNamespace: ALL
          label: grafana_dashboard
        datasources:
          enabled: true
      resources:
        requests:
          cpu: 50m
          memory: 128Mi
        limits:
          memory: 256Mi
      # TODO(WS-SEC/WS-C2, later): Keycloak OIDC client (auth.generic_oauth) + root_url behind the
      #   Gateway; disable local login once SSO is proven. Secret referenced by name only.

    # --- Bundled exporters / operator: keep small ---
    prometheusOperator:
      resources:
        requests:
          cpu: 50m
          memory: 96Mi
        limits:
          memory: 256Mi
    kube-state-metrics:
      resources:
        requests:
          cpu: 20m
          memory: 64Mi
        limits:
          memory: 128Mi
    prometheus-node-exporter:
      resources:
        requests:
          cpu: 20m
          memory: 32Mi
        limits:
          memory: 64Mi