jakman85andClaude Opus 4.8
3e61f817b6
Both were pinned at ~98% of their limits (Prometheus 2002/2048Mi, Grafana 499/512Mi) with no headroom for TSDB compaction / dashboard-render spikes — the classic OOM setups. Nodes are 24Gi at 38-44% used (~14Gi free each; the '~22 GiB ceiling' comment predates the 16->24Gi node bump), so the extra headroom is comfortable. Grafana request 256->384Mi, Prometheus request unchanged. Co-Authored-By: Claude Opus 4.8 <[email protected]>
237 lines
12 KiB
YAML
237 lines
12 KiB
YAML
---
|
|
# kube-prometheus-stack — Prometheus + Alertmanager + Grafana + kube-state-metrics +
|
|
# node-exporter + the standard Kubernetes exporters/rules (docs/10). Serves R-OBS-01/02/03.
|
|
#
|
|
# Retention/PVC sizes are overlay knobs (R-ENV-02) resolved from cluster-vars via Flux
|
|
# postBuild.substituteFrom. Everything else is DEV-SCOPED for the ~22 GiB workstation ceiling:
|
|
# single replicas, modest requests/limits, ephemeral (emptyDir) Alertmanager/Grafana state, and
|
|
# the k3s-embedded control-plane scrape jobs disabled (they are not separately reachable on k3s and
|
|
# would otherwise show as permanently-down targets and burn RAM/alert noise).
|
|
apiVersion: helm.toolkit.fluxcd.io/v2
|
|
kind: HelmRelease
|
|
metadata:
|
|
name: kube-prometheus-stack
|
|
namespace: monitoring
|
|
spec:
|
|
interval: 30m
|
|
# The 65->88 catch-up is a large multi-major jump (operator + KSM + node-exporter roll, plus the
|
|
# Prometheus/Alertmanager StatefulSets). The default 5m Helm timeout is too short for the full rollout
|
|
# to reach Ready, so Flux logged "context deadline exceeded" and rolled back to 65.8.1 four times.
|
|
# A generous timeout lets one attempt finish; remediateLastFailure:false keeps partial progress instead
|
|
# of yo-yoing back to the old chart (a re-reconcile then drives it forward). CRDs are pre-applied
|
|
# server-side (Helm never updates crds/ on upgrade), so schema is already at operator v0.93.1.
|
|
timeout: 20m0s
|
|
chart:
|
|
spec:
|
|
chart: kube-prometheus-stack
|
|
version: ">=88.0.0 <89.0.0" # k8s-1.36 catch-up (po v0.93; CRDs pre-applied server-side, Helm won't update them). Renovate maintains
|
|
sourceRef:
|
|
kind: HelmRepository
|
|
name: prometheus-community
|
|
namespace: monitoring
|
|
install:
|
|
remediation:
|
|
retries: 3
|
|
upgrade:
|
|
remediation:
|
|
retries: 2
|
|
remediateLastFailure: false
|
|
cleanupOnFail: true
|
|
values:
|
|
# --- k3s trim: control-plane components are embedded in the k3s server process and are not
|
|
# scrapable as discrete endpoints. Disabling avoids permanently-firing "target down" noise.
|
|
kubeControllerManager:
|
|
enabled: false
|
|
kubeScheduler:
|
|
enabled: false
|
|
kubeEtcd:
|
|
enabled: false
|
|
kubeProxy:
|
|
enabled: false
|
|
|
|
# --- Prometheus ---
|
|
prometheus:
|
|
prometheusSpec:
|
|
replicas: 1
|
|
retention: ${MONITORING_PROMETHEUS_RETENTION} # overlay knob (dev 3d / prod long)
|
|
# SIZE-based retention cap (docs/10, R-OBS). `retention` alone prunes only by AGE — on a young
|
|
# cluster (< retention window) NOTHING is pruned yet, so a busy TSDB fills the PVC before the
|
|
# time window elapses. A full /prometheus wedges ALL WAL commits ("no space left on device"):
|
|
# scrapes still 200 but samples are never stored, the head freezes, and every absent()/==0
|
|
# platform alert (Keycloak/KubeAPI/Kubelet/VPN/KubeVirt/CDI…) fires falsely because instant
|
|
# queries at "now" return empty. retentionSize makes Prometheus drop the OLDEST blocks once the
|
|
# block set approaches this cap, so the disk never fills regardless of cluster age or churn.
|
|
# Kept BELOW the PVC size (leaves headroom for the WAL + head + compaction scratch, which are
|
|
# NOT counted by retentionSize). Prometheus size units are *iB (GiB), distinct from the K8s PVC
|
|
# `Gi`, so it is a separate overlay knob rather than a reuse of the PVC-size knob.
|
|
retentionSize: ${MONITORING_PROMETHEUS_RETENTION_SIZE} # overlay knob (dev 5GiB / base 8GiB / prod 45GiB)
|
|
# Discover ServiceMonitors/PodMonitors/Rules from ALL namespaces (flux, cert-manager,
|
|
# longhorn, velero, keycloak, cilium/hubble ship their own) — not just this release's.
|
|
ruleSelectorNilUsesHelmValues: false
|
|
serviceMonitorSelectorNilUsesHelmValues: false
|
|
podMonitorSelectorNilUsesHelmValues: false
|
|
probeSelectorNilUsesHelmValues: false
|
|
# Modest footprint for a home dev cluster (few hundred series). Limit gives headroom for
|
|
# WAL replay / compaction spikes without letting Prometheus eat the node.
|
|
resources:
|
|
requests:
|
|
cpu: 100m
|
|
memory: 512Mi
|
|
limits:
|
|
memory: 3Gi # 1Gi OOMKilled (exit 137) during TSDB compaction/checkpoint;
|
|
# 2Gi ran pinned at ~98% (2002Mi) with no room for compaction/WAL-replay
|
|
# spikes (the classic Prometheus OOM). Nodes are 24Gi at 38-44% used
|
|
# (~14Gi free each), so 3Gi gives real headroom.
|
|
storageSpec:
|
|
volumeClaimTemplate:
|
|
spec:
|
|
storageClassName: longhorn # replicated tier (ADR-0003)
|
|
accessModes: ["ReadWriteOnce"]
|
|
resources:
|
|
requests:
|
|
storage: ${MONITORING_PROMETHEUS_PVC_SIZE} # overlay knob (dev 5Gi / prod large)
|
|
|
|
# --- Alertmanager: routes by severity to email (R-OBS-02) ---
|
|
alertmanager:
|
|
alertmanagerSpec:
|
|
replicas: 1
|
|
# DEV TRIM: no PVC — silences/notification-log live in emptyDir. Acceptable in dev (a
|
|
# restart re-fires active alerts). Prod overlay can add a small storage block.
|
|
resources:
|
|
requests:
|
|
cpu: 10m
|
|
memory: 32Mi
|
|
limits:
|
|
memory: 96Mi
|
|
# SMTP credentials are an OPERATOR HAND-OFF: a SOPS-encrypted Secret, referenced BY NAME
|
|
# only (docs/07). Mounted at /etc/alertmanager/secrets/<name>/ so the config below can read
|
|
# the password from a file (never inline plaintext).
|
|
secrets:
|
|
- alertmanager-smtp
|
|
# Routing/receivers. Non-secret SMTP endpoint fields (smarthost/from/to/username) are
|
|
# operator-supplied placeholders (REPLACE-ME, same convention as velero); the PASSWORD is
|
|
# read from the mounted SOPS secret via *_file. Both severities email; subject/route differ.
|
|
config:
|
|
global:
|
|
resolve_timeout: 5m
|
|
smtp_smarthost: "smtp.gmail.com:587" # operator hand-off (e.g. smtp.gmail.com:587)
|
|
smtp_from: "[email protected]" # operator hand-off (From: address)
|
|
smtp_auth_username: "[email protected]" # operator hand-off (SMTP login)
|
|
smtp_auth_password_file: /etc/alertmanager/secrets/alertmanager-smtp/smtp-password
|
|
smtp_require_tls: true
|
|
route:
|
|
group_by: ["alertname", "namespace"]
|
|
group_wait: 30s
|
|
group_interval: 5m
|
|
repeat_interval: 12h
|
|
receiver: email-warning
|
|
routes:
|
|
- matchers: ['severity = "critical"']
|
|
receiver: email-critical
|
|
# page-worthy — notify fast, remind often
|
|
group_wait: 10s
|
|
repeat_interval: 1h
|
|
receivers:
|
|
- name: email-warning
|
|
email_configs:
|
|
- to: "[email protected]" # alert inbox (operator: change if a different inbox is wanted)
|
|
send_resolved: true
|
|
headers:
|
|
subject: '[WARNING] {{ .CommonLabels.alertname }} ({{ .CommonLabels.namespace }})'
|
|
- name: email-critical
|
|
email_configs:
|
|
- to: "[email protected]" # alert inbox (operator: change if a different inbox is wanted)
|
|
send_resolved: true
|
|
headers:
|
|
subject: '[CRITICAL] {{ .CommonLabels.alertname }} ({{ .CommonLabels.namespace }})'
|
|
|
|
# --- Grafana: dashboards (R-OBS-03), surfaced in the C2 UI (docs/12) behind Keycloak SSO ---
|
|
grafana:
|
|
enabled: true
|
|
# Force a pod roll when the OIDC wiring changes so Grafana loads the grafana-oidc-credentials
|
|
# client-secret (env valueFrom is resolved at pod start; the secret was added after first boot, so
|
|
# generic_oauth started disabled). Bump this value to re-roll Grafana on future OIDC secret changes.
|
|
podAnnotations:
|
|
homelab.keithjackson.dev/oidc-rollout: "1"
|
|
# Admin bootstrap + (later) OIDC client are SOPS secrets referenced BY NAME (WS-SEC/ADR-0014).
|
|
admin:
|
|
existingSecret: grafana-admin-credentials
|
|
userKey: admin-user
|
|
passwordKey: admin-password
|
|
# DEV TRIM: emptyDir (default). Dashboards are provisioned from ConfigMaps (sidecar), so no
|
|
# durable Grafana DB is needed to iterate. Prod can enable persistence.
|
|
persistence:
|
|
enabled: false
|
|
# Auto-discover dashboard ConfigMaps labeled grafana_dashboard across namespaces (per-service
|
|
# dashboards rendered from ServiceDefinitions land here — docs/10 onboarding output).
|
|
sidecar:
|
|
dashboards:
|
|
enabled: true
|
|
searchNamespace: ALL
|
|
label: grafana_dashboard
|
|
datasources:
|
|
enabled: true
|
|
resources:
|
|
requests:
|
|
cpu: 50m
|
|
memory: 384Mi # Grafana 13.x (image 13.2.0, kps 88) baseline is well above Grafana 11.x (kps 65)
|
|
limits:
|
|
memory: 768Mi # 256Mi OOMKilled (exit 137) Grafana 13.x during apiserver/unified-storage
|
|
# startup — the silent SIGKILL that made the whole 65->88 Helm upgrade
|
|
# time out (the new Grafana pod never went Ready). 512Mi then ran pinned at
|
|
# ~98% (499Mi) with no headroom — one render/GC spike from OOM. 768Mi holds
|
|
# with real headroom (nodes 24Gi, ~14Gi free each).
|
|
# --- Keycloak native-OIDC SSO (ADR-0014, deliverable #4 SSO proof) ---
|
|
# Grafana is a confidential OIDC client of the Keycloak `homelab` realm. The client SECRET comes
|
|
# from the SOPS-encrypted grafana-oidc-credentials Secret via env (referenced by name only,
|
|
# R-IDP-04). optional:true so the RUNNING Grafana keeps starting before the operator fills the
|
|
# secret — local admin login (below) stays available as the fallback until SSO is proven.
|
|
envValueFrom:
|
|
GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET:
|
|
secretKeyRef:
|
|
name: grafana-oidc-credentials
|
|
key: client-secret
|
|
optional: true
|
|
grafana.ini:
|
|
server:
|
|
root_url: https://grafana.home.keithjackson.dev
|
|
auth.generic_oauth:
|
|
enabled: true
|
|
name: Keycloak
|
|
client_id: grafana
|
|
scopes: "openid email profile"
|
|
auth_url: https://keycloak.home.keithjackson.dev/realms/homelab/protocol/openid-connect/auth
|
|
token_url: https://keycloak.home.keithjackson.dev/realms/homelab/protocol/openid-connect/token
|
|
api_url: https://keycloak.home.keithjackson.dev/realms/homelab/protocol/openid-connect/userinfo
|
|
login_attribute_path: preferred_username
|
|
email_attribute_path: email
|
|
# Map Keycloak groups (the `groups` claim) to Grafana roles (R-IDP-06): platform-admins ->
|
|
# Admin, media-users -> Editor, everyone else -> Viewer.
|
|
role_attribute_path: "contains(groups[*], 'platform-admins') && 'Admin' || contains(groups[*], 'media-users') && 'Editor' || 'Viewer'"
|
|
# Keep local admin login enabled as break-glass until SSO is proven end-to-end; flip to
|
|
# disable_login_form:true afterwards (docs/15 break-glass, R-IDP-05).
|
|
# TODO(WS-SEC, post-proof): set auth.disable_login_form=true once SSO login is verified.
|
|
|
|
# --- Bundled exporters / operator: keep small ---
|
|
prometheusOperator:
|
|
resources:
|
|
requests:
|
|
cpu: 50m
|
|
memory: 96Mi
|
|
limits:
|
|
memory: 256Mi
|
|
kube-state-metrics:
|
|
resources:
|
|
requests:
|
|
cpu: 20m
|
|
memory: 64Mi
|
|
limits:
|
|
memory: 128Mi
|
|
prometheus-node-exporter:
|
|
resources:
|
|
requests:
|
|
cpu: 20m
|
|
memory: 32Mi
|
|
limits:
|
|
memory: 64Mi
|