---
# kube-prometheus-stack — Prometheus + Alertmanager + Grafana + kube-state-metrics +
# node-exporter + the standard Kubernetes exporters/rules (docs/10). Serves R-OBS-01/02/03.
#
# Retention/PVC sizes are overlay knobs (R-ENV-02) resolved from cluster-vars via Flux
# postBuild.substituteFrom. Everything else is DEV-SCOPED for the ~22 GiB workstation ceiling:
# single replicas, modest requests/limits, ephemeral (emptyDir) Alertmanager/Grafana state, and
# the k3s-embedded control-plane scrape jobs disabled (they are not separately reachable on k3s and
# would otherwise show as permanently-down targets and burn RAM/alert noise).
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
  name: kube-prometheus-stack
  namespace: monitoring
spec:
  interval: 30m
  chart:
    spec:
      chart: kube-prometheus-stack
      version: ">=65.0.0 <66.0.0"   # TODO(WS-OBS): pin at bring-up; Renovate maintains
      sourceRef:
        kind: HelmRepository
        name: prometheus-community
        namespace: monitoring
  install:
    remediation:
      retries: 3
  upgrade:
    remediation:
      retries: 3
  values:
    # --- k3s trim: control-plane components are embedded in the k3s server process and are not
    #     scrapable as discrete endpoints. Disabling avoids permanently-firing "target down" noise.
    kubeControllerManager:
      enabled: false
    kubeScheduler:
      enabled: false
    kubeEtcd:
      enabled: false
    kubeProxy:
      enabled: false

    # --- Prometheus ---
    prometheus:
      prometheusSpec:
        replicas: 1
        retention: ${MONITORING_PROMETHEUS_RETENTION}     # overlay knob (dev 3d / prod long)
        # Discover ServiceMonitors/PodMonitors/Rules from ALL namespaces (flux, cert-manager,
        # longhorn, velero, keycloak, cilium/hubble ship their own) — not just this release's.
        ruleSelectorNilUsesHelmValues: false
        serviceMonitorSelectorNilUsesHelmValues: false
        podMonitorSelectorNilUsesHelmValues: false
        probeSelectorNilUsesHelmValues: false
        # Modest footprint for a home dev cluster (few hundred series). Limit gives headroom for
        # WAL replay / compaction spikes without letting Prometheus eat the node.
        resources:
          requests:
            cpu: 100m
            memory: 512Mi
          limits:
            memory: 2Gi                        # 1Gi OOMKilled (exit 137) during TSDB compaction/checkpoint
                                               # spikes; 2Gi holds. Nodes have headroom (~2Gi free each).
        storageSpec:
          volumeClaimTemplate:
            spec:
              storageClassName: longhorn                  # replicated tier (ADR-0003)
              accessModes: ["ReadWriteOnce"]
              resources:
                requests:
                  storage: ${MONITORING_PROMETHEUS_PVC_SIZE}   # overlay knob (dev 5Gi / prod large)

    # --- Alertmanager: routes by severity to email (R-OBS-02) ---
    alertmanager:
      alertmanagerSpec:
        replicas: 1
        # DEV TRIM: no PVC — silences/notification-log live in emptyDir. Acceptable in dev (a
        # restart re-fires active alerts). Prod overlay can add a small storage block.
        resources:
          requests:
            cpu: 10m
            memory: 32Mi
          limits:
            memory: 96Mi
        # SMTP credentials are an OPERATOR HAND-OFF: a SOPS-encrypted Secret, referenced BY NAME
        # only (docs/07). Mounted at /etc/alertmanager/secrets/<name>/ so the config below can read
        # the password from a file (never inline plaintext).
        secrets:
          - alertmanager-smtp
      # Routing/receivers. Non-secret SMTP endpoint fields (smarthost/from/to/username) are
      # operator-supplied placeholders (REPLACE-ME, same convention as velero); the PASSWORD is
      # read from the mounted SOPS secret via *_file. Both severities email; subject/route differ.
      config:
        global:
          resolve_timeout: 5m
          smtp_smarthost: "smtp.gmail.com:587"          # operator hand-off (e.g. smtp.gmail.com:587)
          smtp_from: "alertmanager@keithjackson.dev"                    # operator hand-off (From: address)
          smtp_auth_username: "keithallenjackson@gmail.com"           # operator hand-off (SMTP login)
          smtp_auth_password_file: /etc/alertmanager/secrets/alertmanager-smtp/smtp-password
          smtp_require_tls: true
        route:
          group_by: ["alertname", "namespace"]
          group_wait: 30s
          group_interval: 5m
          repeat_interval: 12h
          receiver: email-warning
          routes:
            - matchers: ['severity = "critical"']
              receiver: email-critical
              # page-worthy — notify fast, remind often
              group_wait: 10s
              repeat_interval: 1h
        receivers:
          - name: email-warning
            email_configs:
              - to: "keithallenjackson@gmail.com"    # alert inbox (operator: change if a different inbox is wanted)
                send_resolved: true
                headers:
                  subject: '[WARNING] {{ .CommonLabels.alertname }} ({{ .CommonLabels.namespace }})'
          - name: email-critical
            email_configs:
              - to: "keithallenjackson@gmail.com"    # alert inbox (operator: change if a different inbox is wanted)
                send_resolved: true
                headers:
                  subject: '[CRITICAL] {{ .CommonLabels.alertname }} ({{ .CommonLabels.namespace }})'

    # --- Grafana: dashboards (R-OBS-03), surfaced in the C2 UI (docs/12) behind Keycloak SSO ---
    grafana:
      enabled: true
      # Admin bootstrap + (later) OIDC client are SOPS secrets referenced BY NAME (WS-SEC/ADR-0014).
      admin:
        existingSecret: grafana-admin-credentials
        userKey: admin-user
        passwordKey: admin-password
      # DEV TRIM: emptyDir (default). Dashboards are provisioned from ConfigMaps (sidecar), so no
      # durable Grafana DB is needed to iterate. Prod can enable persistence.
      persistence:
        enabled: false
      # Auto-discover dashboard ConfigMaps labeled grafana_dashboard across namespaces (per-service
      # dashboards rendered from ServiceDefinitions land here — docs/10 onboarding output).
      sidecar:
        dashboards:
          enabled: true
          searchNamespace: ALL
          label: grafana_dashboard
        datasources:
          enabled: true
      resources:
        requests:
          cpu: 50m
          memory: 128Mi
        limits:
          memory: 256Mi
      # --- Keycloak native-OIDC SSO (ADR-0014, deliverable #4 SSO proof) ---
      # Grafana is a confidential OIDC client of the Keycloak `homelab` realm. The client SECRET comes
      # from the SOPS-encrypted grafana-oidc-credentials Secret via env (referenced by name only,
      # R-IDP-04). optional:true so the RUNNING Grafana keeps starting before the operator fills the
      # secret — local admin login (below) stays available as the fallback until SSO is proven.
      envValueFrom:
        GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET:
          secretKeyRef:
            name: grafana-oidc-credentials
            key: client-secret
            optional: true
      grafana.ini:
        server:
          root_url: https://grafana.home.keithjackson.dev
        auth.generic_oauth:
          enabled: true
          name: Keycloak
          client_id: grafana
          scopes: "openid email profile"
          auth_url: https://keycloak.home.keithjackson.dev/realms/homelab/protocol/openid-connect/auth
          token_url: https://keycloak.home.keithjackson.dev/realms/homelab/protocol/openid-connect/token
          api_url: https://keycloak.home.keithjackson.dev/realms/homelab/protocol/openid-connect/userinfo
          login_attribute_path: preferred_username
          email_attribute_path: email
          # Map Keycloak groups (the `groups` claim) to Grafana roles (R-IDP-06): platform-admins ->
          # Admin, media-users -> Editor, everyone else -> Viewer.
          role_attribute_path: "contains(groups[*], 'platform-admins') && 'Admin' || contains(groups[*], 'media-users') && 'Editor' || 'Viewer'"
          # Keep local admin login enabled as break-glass until SSO is proven end-to-end; flip to
          # disable_login_form:true afterwards (docs/15 break-glass, R-IDP-05).
        # TODO(WS-SEC, post-proof): set auth.disable_login_form=true once SSO login is verified.

    # --- Bundled exporters / operator: keep small ---
    prometheusOperator:
      resources:
        requests:
          cpu: 50m
          memory: 96Mi
        limits:
          memory: 256Mi
    kube-state-metrics:
      resources:
        requests:
          cpu: 20m
          memory: 64Mi
        limits:
          memory: 128Mi
    prometheus-node-exporter:
      resources:
        requests:
          cpu: 20m
          memory: 32Mi
        limits:
          memory: 64Mi