jakman85
bc0982c213
Grafana's pod predated the grafana-oidc-credentials secret, so generic_oauth started with an empty client_secret (disabled) → /login/generic_oauth didn't redirect to Keycloak. Add a podAnnotation to force one roll; bump on future OIDC secret changes.
211 lines
9.2 KiB
YAML
211 lines
9.2 KiB
YAML
---
|
|
# kube-prometheus-stack — Prometheus + Alertmanager + Grafana + kube-state-metrics +
|
|
# node-exporter + the standard Kubernetes exporters/rules (docs/10). Serves R-OBS-01/02/03.
|
|
#
|
|
# Retention/PVC sizes are overlay knobs (R-ENV-02) resolved from cluster-vars via Flux
|
|
# postBuild.substituteFrom. Everything else is DEV-SCOPED for the ~22 GiB workstation ceiling:
|
|
# single replicas, modest requests/limits, ephemeral (emptyDir) Alertmanager/Grafana state, and
|
|
# the k3s-embedded control-plane scrape jobs disabled (they are not separately reachable on k3s and
|
|
# would otherwise show as permanently-down targets and burn RAM/alert noise).
|
|
apiVersion: helm.toolkit.fluxcd.io/v2
|
|
kind: HelmRelease
|
|
metadata:
|
|
name: kube-prometheus-stack
|
|
namespace: monitoring
|
|
spec:
|
|
interval: 30m
|
|
chart:
|
|
spec:
|
|
chart: kube-prometheus-stack
|
|
version: ">=65.0.0 <66.0.0" # TODO(WS-OBS): pin at bring-up; Renovate maintains
|
|
sourceRef:
|
|
kind: HelmRepository
|
|
name: prometheus-community
|
|
namespace: monitoring
|
|
install:
|
|
remediation:
|
|
retries: 3
|
|
upgrade:
|
|
remediation:
|
|
retries: 3
|
|
values:
|
|
# --- k3s trim: control-plane components are embedded in the k3s server process and are not
|
|
# scrapable as discrete endpoints. Disabling avoids permanently-firing "target down" noise.
|
|
kubeControllerManager:
|
|
enabled: false
|
|
kubeScheduler:
|
|
enabled: false
|
|
kubeEtcd:
|
|
enabled: false
|
|
kubeProxy:
|
|
enabled: false
|
|
|
|
# --- Prometheus ---
|
|
prometheus:
|
|
prometheusSpec:
|
|
replicas: 1
|
|
retention: ${MONITORING_PROMETHEUS_RETENTION} # overlay knob (dev 3d / prod long)
|
|
# Discover ServiceMonitors/PodMonitors/Rules from ALL namespaces (flux, cert-manager,
|
|
# longhorn, velero, keycloak, cilium/hubble ship their own) — not just this release's.
|
|
ruleSelectorNilUsesHelmValues: false
|
|
serviceMonitorSelectorNilUsesHelmValues: false
|
|
podMonitorSelectorNilUsesHelmValues: false
|
|
probeSelectorNilUsesHelmValues: false
|
|
# Modest footprint for a home dev cluster (few hundred series). Limit gives headroom for
|
|
# WAL replay / compaction spikes without letting Prometheus eat the node.
|
|
resources:
|
|
requests:
|
|
cpu: 100m
|
|
memory: 512Mi
|
|
limits:
|
|
memory: 2Gi # 1Gi OOMKilled (exit 137) during TSDB compaction/checkpoint
|
|
# spikes; 2Gi holds. Nodes have headroom (~2Gi free each).
|
|
storageSpec:
|
|
volumeClaimTemplate:
|
|
spec:
|
|
storageClassName: longhorn # replicated tier (ADR-0003)
|
|
accessModes: ["ReadWriteOnce"]
|
|
resources:
|
|
requests:
|
|
storage: ${MONITORING_PROMETHEUS_PVC_SIZE} # overlay knob (dev 5Gi / prod large)
|
|
|
|
# --- Alertmanager: routes by severity to email (R-OBS-02) ---
|
|
alertmanager:
|
|
alertmanagerSpec:
|
|
replicas: 1
|
|
# DEV TRIM: no PVC — silences/notification-log live in emptyDir. Acceptable in dev (a
|
|
# restart re-fires active alerts). Prod overlay can add a small storage block.
|
|
resources:
|
|
requests:
|
|
cpu: 10m
|
|
memory: 32Mi
|
|
limits:
|
|
memory: 96Mi
|
|
# SMTP credentials are an OPERATOR HAND-OFF: a SOPS-encrypted Secret, referenced BY NAME
|
|
# only (docs/07). Mounted at /etc/alertmanager/secrets/<name>/ so the config below can read
|
|
# the password from a file (never inline plaintext).
|
|
secrets:
|
|
- alertmanager-smtp
|
|
# Routing/receivers. Non-secret SMTP endpoint fields (smarthost/from/to/username) are
|
|
# operator-supplied placeholders (REPLACE-ME, same convention as velero); the PASSWORD is
|
|
# read from the mounted SOPS secret via *_file. Both severities email; subject/route differ.
|
|
config:
|
|
global:
|
|
resolve_timeout: 5m
|
|
smtp_smarthost: "smtp.gmail.com:587" # operator hand-off (e.g. smtp.gmail.com:587)
|
|
smtp_from: "[email protected]" # operator hand-off (From: address)
|
|
smtp_auth_username: "[email protected]" # operator hand-off (SMTP login)
|
|
smtp_auth_password_file: /etc/alertmanager/secrets/alertmanager-smtp/smtp-password
|
|
smtp_require_tls: true
|
|
route:
|
|
group_by: ["alertname", "namespace"]
|
|
group_wait: 30s
|
|
group_interval: 5m
|
|
repeat_interval: 12h
|
|
receiver: email-warning
|
|
routes:
|
|
- matchers: ['severity = "critical"']
|
|
receiver: email-critical
|
|
# page-worthy — notify fast, remind often
|
|
group_wait: 10s
|
|
repeat_interval: 1h
|
|
receivers:
|
|
- name: email-warning
|
|
email_configs:
|
|
- to: "[email protected]" # alert inbox (operator: change if a different inbox is wanted)
|
|
send_resolved: true
|
|
headers:
|
|
subject: '[WARNING] {{ .CommonLabels.alertname }} ({{ .CommonLabels.namespace }})'
|
|
- name: email-critical
|
|
email_configs:
|
|
- to: "[email protected]" # alert inbox (operator: change if a different inbox is wanted)
|
|
send_resolved: true
|
|
headers:
|
|
subject: '[CRITICAL] {{ .CommonLabels.alertname }} ({{ .CommonLabels.namespace }})'
|
|
|
|
# --- Grafana: dashboards (R-OBS-03), surfaced in the C2 UI (docs/12) behind Keycloak SSO ---
|
|
grafana:
|
|
enabled: true
|
|
# Force a pod roll when the OIDC wiring changes so Grafana loads the grafana-oidc-credentials
|
|
# client-secret (env valueFrom is resolved at pod start; the secret was added after first boot, so
|
|
# generic_oauth started disabled). Bump this value to re-roll Grafana on future OIDC secret changes.
|
|
podAnnotations:
|
|
homelab.keithjackson.dev/oidc-rollout: "1"
|
|
# Admin bootstrap + (later) OIDC client are SOPS secrets referenced BY NAME (WS-SEC/ADR-0014).
|
|
admin:
|
|
existingSecret: grafana-admin-credentials
|
|
userKey: admin-user
|
|
passwordKey: admin-password
|
|
# DEV TRIM: emptyDir (default). Dashboards are provisioned from ConfigMaps (sidecar), so no
|
|
# durable Grafana DB is needed to iterate. Prod can enable persistence.
|
|
persistence:
|
|
enabled: false
|
|
# Auto-discover dashboard ConfigMaps labeled grafana_dashboard across namespaces (per-service
|
|
# dashboards rendered from ServiceDefinitions land here — docs/10 onboarding output).
|
|
sidecar:
|
|
dashboards:
|
|
enabled: true
|
|
searchNamespace: ALL
|
|
label: grafana_dashboard
|
|
datasources:
|
|
enabled: true
|
|
resources:
|
|
requests:
|
|
cpu: 50m
|
|
memory: 128Mi
|
|
limits:
|
|
memory: 256Mi
|
|
# --- Keycloak native-OIDC SSO (ADR-0014, deliverable #4 SSO proof) ---
|
|
# Grafana is a confidential OIDC client of the Keycloak `homelab` realm. The client SECRET comes
|
|
# from the SOPS-encrypted grafana-oidc-credentials Secret via env (referenced by name only,
|
|
# R-IDP-04). optional:true so the RUNNING Grafana keeps starting before the operator fills the
|
|
# secret — local admin login (below) stays available as the fallback until SSO is proven.
|
|
envValueFrom:
|
|
GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET:
|
|
secretKeyRef:
|
|
name: grafana-oidc-credentials
|
|
key: client-secret
|
|
optional: true
|
|
grafana.ini:
|
|
server:
|
|
root_url: https://grafana.home.keithjackson.dev
|
|
auth.generic_oauth:
|
|
enabled: true
|
|
name: Keycloak
|
|
client_id: grafana
|
|
scopes: "openid email profile"
|
|
auth_url: https://keycloak.home.keithjackson.dev/realms/homelab/protocol/openid-connect/auth
|
|
token_url: https://keycloak.home.keithjackson.dev/realms/homelab/protocol/openid-connect/token
|
|
api_url: https://keycloak.home.keithjackson.dev/realms/homelab/protocol/openid-connect/userinfo
|
|
login_attribute_path: preferred_username
|
|
email_attribute_path: email
|
|
# Map Keycloak groups (the `groups` claim) to Grafana roles (R-IDP-06): platform-admins ->
|
|
# Admin, media-users -> Editor, everyone else -> Viewer.
|
|
role_attribute_path: "contains(groups[*], 'platform-admins') && 'Admin' || contains(groups[*], 'media-users') && 'Editor' || 'Viewer'"
|
|
# Keep local admin login enabled as break-glass until SSO is proven end-to-end; flip to
|
|
# disable_login_form:true afterwards (docs/15 break-glass, R-IDP-05).
|
|
# TODO(WS-SEC, post-proof): set auth.disable_login_form=true once SSO login is verified.
|
|
|
|
# --- Bundled exporters / operator: keep small ---
|
|
prometheusOperator:
|
|
resources:
|
|
requests:
|
|
cpu: 50m
|
|
memory: 96Mi
|
|
limits:
|
|
memory: 256Mi
|
|
kube-state-metrics:
|
|
resources:
|
|
requests:
|
|
cpu: 20m
|
|
memory: 64Mi
|
|
limits:
|
|
memory: 128Mi
|
|
prometheus-node-exporter:
|
|
resources:
|
|
requests:
|
|
cpu: 20m
|
|
memory: 32Mi
|
|
limits:
|
|
memory: 64Mi
|