jakman85andClaude Opus 5
8b36242643
Working set measured at 111 MiB with a 6h peak of 123.3 MiB against a 128Mi limit, so it died on every spike — exit 137, roughly every 11 minutes, 91 restarts in 16h. The consequence is worse than the restarts. Thirteen alert rules read kube_* series, so each kill made them intermittently unable to fire, which is the quiet version of the inert-alert class found elsewhere today; KubePodNotReady and KubeDeploymentReplicasMismatch were both sitting pending on it. Self-inflicted: enabling customResourceState for the Velero Schedule metrics earlier today added a CRD informer plus the mandatory CRD list/watch, and that marginal cost crossed a limit already sitting about 4 MiB above the working set. Raised to 384Mi, roughly 3x the observed peak, which costs nothing on nodes reporting 18.7-22.3 GiB available. Co-Authored-By: Claude Opus 5 <[email protected]>
357 lines
20 KiB
YAML
357 lines
20 KiB
YAML
---
|
|
# kube-prometheus-stack — Prometheus + Alertmanager + Grafana + kube-state-metrics +
|
|
# node-exporter + the standard Kubernetes exporters/rules (docs/10). Serves R-OBS-01/02/03.
|
|
#
|
|
# Retention/PVC sizes are overlay knobs (R-ENV-02) resolved from cluster-vars via Flux
|
|
# postBuild.substituteFrom. Everything else is DEV-SCOPED for the ~22 GiB workstation ceiling:
|
|
# single replicas, modest requests/limits, ephemeral (emptyDir) Alertmanager/Grafana state, and
|
|
# the k3s-embedded control-plane scrape jobs disabled (they are not separately reachable on k3s and
|
|
# would otherwise show as permanently-down targets and burn RAM/alert noise).
|
|
apiVersion: helm.toolkit.fluxcd.io/v2
|
|
kind: HelmRelease
|
|
metadata:
|
|
name: kube-prometheus-stack
|
|
namespace: monitoring
|
|
spec:
|
|
interval: 30m
|
|
# The 65->88 catch-up is a large multi-major jump (operator + KSM + node-exporter roll, plus the
|
|
# Prometheus/Alertmanager StatefulSets). The default 5m Helm timeout is too short for the full rollout
|
|
# to reach Ready, so Flux logged "context deadline exceeded" and rolled back to 65.8.1 four times.
|
|
# A generous timeout lets one attempt finish; remediateLastFailure:false keeps partial progress instead
|
|
# of yo-yoing back to the old chart (a re-reconcile then drives it forward). CRDs are pre-applied
|
|
# server-side (Helm never updates crds/ on upgrade), so schema is already at operator v0.93.1.
|
|
timeout: 20m0s
|
|
chart:
|
|
spec:
|
|
chart: kube-prometheus-stack
|
|
version: ">=88.0.0 <89.0.0" # k8s-1.36 catch-up (po v0.93; CRDs pre-applied server-side, Helm won't update them). Renovate maintains
|
|
sourceRef:
|
|
kind: HelmRepository
|
|
name: prometheus-community
|
|
namespace: monitoring
|
|
install:
|
|
remediation:
|
|
retries: 3
|
|
upgrade:
|
|
remediation:
|
|
retries: 2
|
|
remediateLastFailure: false
|
|
cleanupOnFail: true
|
|
values:
|
|
# --- k3s trim: control-plane components are embedded in the k3s server process and are not
|
|
# scrapable as discrete endpoints. Disabling avoids permanently-firing "target down" noise.
|
|
kubeControllerManager:
|
|
enabled: false
|
|
kubeScheduler:
|
|
enabled: false
|
|
kubeEtcd:
|
|
enabled: false
|
|
kubeProxy:
|
|
enabled: false
|
|
|
|
# --- Prometheus ---
|
|
prometheus:
|
|
prometheusSpec:
|
|
replicas: 1
|
|
retention: ${MONITORING_PROMETHEUS_RETENTION} # overlay knob (dev 3d / prod long)
|
|
# SIZE-based retention cap (docs/10, R-OBS). `retention` alone prunes only by AGE — on a young
|
|
# cluster (< retention window) NOTHING is pruned yet, so a busy TSDB fills the PVC before the
|
|
# time window elapses. A full /prometheus wedges ALL WAL commits ("no space left on device"):
|
|
# scrapes still 200 but samples are never stored, the head freezes, and every absent()/==0
|
|
# platform alert (Keycloak/KubeAPI/Kubelet/VPN/KubeVirt/CDI…) fires falsely because instant
|
|
# queries at "now" return empty. retentionSize makes Prometheus drop the OLDEST blocks once the
|
|
# block set approaches this cap, so the disk never fills regardless of cluster age or churn.
|
|
# Kept BELOW the PVC size (leaves headroom for the WAL + head + compaction scratch, which are
|
|
# NOT counted by retentionSize). Prometheus size units are *iB (GiB), distinct from the K8s PVC
|
|
# `Gi`, so it is a separate overlay knob rather than a reuse of the PVC-size knob.
|
|
retentionSize: ${MONITORING_PROMETHEUS_RETENTION_SIZE} # overlay knob (dev 5GiB / base 8GiB / prod 45GiB)
|
|
# Discover ServiceMonitors/PodMonitors/Rules from ALL namespaces (flux, cert-manager,
|
|
# longhorn, velero, keycloak, cilium/hubble ship their own) — not just this release's.
|
|
ruleSelectorNilUsesHelmValues: false
|
|
serviceMonitorSelectorNilUsesHelmValues: false
|
|
podMonitorSelectorNilUsesHelmValues: false
|
|
probeSelectorNilUsesHelmValues: false
|
|
# Modest footprint for a home dev cluster (few hundred series). Limit gives headroom for
|
|
# WAL replay / compaction spikes without letting Prometheus eat the node.
|
|
resources:
|
|
requests:
|
|
cpu: 100m
|
|
memory: 512Mi
|
|
limits:
|
|
memory: 3Gi # 1Gi OOMKilled (exit 137) during TSDB compaction/checkpoint;
|
|
# 2Gi ran pinned at ~98% (2002Mi) with no room for compaction/WAL-replay
|
|
# spikes (the classic Prometheus OOM). Nodes are 24Gi at 38-44% used
|
|
# (~14Gi free each), so 3Gi gives real headroom.
|
|
storageSpec:
|
|
volumeClaimTemplate:
|
|
spec:
|
|
storageClassName: longhorn # replicated tier (ADR-0003)
|
|
accessModes: ["ReadWriteOnce"]
|
|
resources:
|
|
requests:
|
|
storage: ${MONITORING_PROMETHEUS_PVC_SIZE} # overlay knob (dev 5Gi / prod large)
|
|
|
|
# --- Alertmanager: routes by severity to email (R-OBS-02) ---
|
|
alertmanager:
|
|
alertmanagerSpec:
|
|
replicas: 1
|
|
# DEV TRIM: no PVC — silences/notification-log live in emptyDir. Acceptable in dev (a
|
|
# restart re-fires active alerts). Prod overlay can add a small storage block.
|
|
resources:
|
|
requests:
|
|
cpu: 10m
|
|
memory: 32Mi
|
|
limits:
|
|
memory: 96Mi
|
|
# SMTP credentials are an OPERATOR HAND-OFF: a SOPS-encrypted Secret, referenced BY NAME
|
|
# only (docs/07). Mounted at /etc/alertmanager/secrets/<name>/ so the config below can read
|
|
# the password from a file (never inline plaintext).
|
|
secrets:
|
|
- alertmanager-smtp
|
|
# Routing/receivers. Non-secret SMTP endpoint fields (smarthost/from/to/username) are
|
|
# operator-supplied placeholders (REPLACE-ME, same convention as velero); the PASSWORD is
|
|
# read from the mounted SOPS secret via *_file. Both severities email; subject/route differ.
|
|
config:
|
|
global:
|
|
resolve_timeout: 5m
|
|
smtp_smarthost: "smtp.gmail.com:587" # operator hand-off (e.g. smtp.gmail.com:587)
|
|
smtp_from: "[email protected]" # operator hand-off (From: address)
|
|
smtp_auth_username: "[email protected]" # operator hand-off (SMTP login)
|
|
smtp_auth_password_file: /etc/alertmanager/secrets/alertmanager-smtp/smtp-password
|
|
smtp_require_tls: true
|
|
route:
|
|
group_by: ["alertname", "namespace"]
|
|
group_wait: 30s
|
|
group_interval: 5m
|
|
repeat_interval: 12h
|
|
receiver: email-warning
|
|
routes:
|
|
- matchers: ['severity = "critical"']
|
|
receiver: email-critical
|
|
# page-worthy — notify fast, remind often
|
|
group_wait: 10s
|
|
repeat_interval: 1h
|
|
receivers:
|
|
- name: email-warning
|
|
email_configs:
|
|
- to: "[email protected]" # alert inbox (operator: change if a different inbox is wanted)
|
|
send_resolved: true
|
|
headers:
|
|
subject: '[WARNING] {{ .CommonLabels.alertname }} ({{ .CommonLabels.namespace }})'
|
|
- name: email-critical
|
|
email_configs:
|
|
- to: "[email protected]" # alert inbox (operator: change if a different inbox is wanted)
|
|
send_resolved: true
|
|
headers:
|
|
subject: '[CRITICAL] {{ .CommonLabels.alertname }} ({{ .CommonLabels.namespace }})'
|
|
|
|
# --- Grafana: dashboards (R-OBS-03), surfaced in the C2 UI (docs/12) behind Keycloak SSO ---
|
|
grafana:
|
|
enabled: true
|
|
# Force a pod roll when the OIDC wiring changes so Grafana loads the grafana-oidc-credentials
|
|
# client-secret (env valueFrom is resolved at pod start; the secret was added after first boot, so
|
|
# generic_oauth started disabled). Bump this value to re-roll Grafana on future OIDC secret changes.
|
|
podAnnotations:
|
|
homelab.keithjackson.dev/oidc-rollout: "1"
|
|
# Admin bootstrap + (later) OIDC client are SOPS secrets referenced BY NAME (WS-SEC/ADR-0014).
|
|
admin:
|
|
existingSecret: grafana-admin-credentials
|
|
userKey: admin-user
|
|
passwordKey: admin-password
|
|
# DEV TRIM: emptyDir (default). Dashboards are provisioned from ConfigMaps (sidecar), so no
|
|
# durable Grafana DB is needed to iterate. Prod can enable persistence.
|
|
persistence:
|
|
enabled: false
|
|
# Auto-discover dashboard ConfigMaps labeled grafana_dashboard across namespaces (per-service
|
|
# dashboards rendered from ServiceDefinitions land here — docs/10 onboarding output).
|
|
sidecar:
|
|
dashboards:
|
|
enabled: true
|
|
searchNamespace: ALL
|
|
label: grafana_dashboard
|
|
datasources:
|
|
enabled: true
|
|
resources:
|
|
requests:
|
|
cpu: 50m
|
|
memory: 384Mi # Grafana 13.x (image 13.2.0, kps 88) baseline is well above Grafana 11.x (kps 65)
|
|
limits:
|
|
memory: 768Mi # 256Mi OOMKilled (exit 137) Grafana 13.x during apiserver/unified-storage
|
|
# startup — the silent SIGKILL that made the whole 65->88 Helm upgrade
|
|
# time out (the new Grafana pod never went Ready). 512Mi then ran pinned at
|
|
# ~98% (499Mi) with no headroom — one render/GC spike from OOM. 768Mi holds
|
|
# with real headroom (nodes 24Gi, ~14Gi free each).
|
|
# --- Keycloak native-OIDC SSO (ADR-0014, deliverable #4 SSO proof) ---
|
|
# Grafana is a confidential OIDC client of the Keycloak `homelab` realm. The client SECRET comes
|
|
# from the SOPS-encrypted grafana-oidc-credentials Secret via env (referenced by name only,
|
|
# R-IDP-04). optional:true so the RUNNING Grafana keeps starting before the operator fills the
|
|
# secret — local admin login (below) stays available as the fallback until SSO is proven.
|
|
envValueFrom:
|
|
GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET:
|
|
secretKeyRef:
|
|
name: grafana-oidc-credentials
|
|
key: client-secret
|
|
optional: true
|
|
grafana.ini:
|
|
server:
|
|
root_url: https://grafana.${BASE_DOMAIN}
|
|
auth.generic_oauth:
|
|
enabled: true
|
|
name: Keycloak
|
|
client_id: grafana
|
|
scopes: "openid email profile"
|
|
auth_url: https://keycloak.${BASE_DOMAIN}/realms/homelab/protocol/openid-connect/auth
|
|
token_url: https://keycloak.${BASE_DOMAIN}/realms/homelab/protocol/openid-connect/token
|
|
api_url: https://keycloak.${BASE_DOMAIN}/realms/homelab/protocol/openid-connect/userinfo
|
|
login_attribute_path: preferred_username
|
|
email_attribute_path: email
|
|
# Map Keycloak groups (the `groups` claim) to Grafana roles (R-IDP-06): platform-admins ->
|
|
# Admin, media-users -> Editor, everyone else -> Viewer.
|
|
role_attribute_path: "contains(groups[*], 'platform-admins') && 'Admin' || contains(groups[*], 'media-users') && 'Editor' || 'Viewer'"
|
|
# Keep local admin login enabled as break-glass until SSO is proven end-to-end; flip to
|
|
# disable_login_form:true afterwards (docs/15 break-glass, R-IDP-05).
|
|
# TODO(WS-SEC, post-proof): set auth.disable_login_form=true once SSO login is verified.
|
|
|
|
# --- Bundled exporters / operator: keep small ---
|
|
prometheusOperator:
|
|
resources:
|
|
requests:
|
|
cpu: 50m
|
|
memory: 96Mi
|
|
limits:
|
|
memory: 256Mi
|
|
kube-state-metrics:
|
|
# Expose the egress-gateway NODE LABEL as a metric label so alerts can ask "does this cluster
|
|
# have a VPN egress gateway at all?". KSM exposes NO custom labels by default, so without this
|
|
# `kube_node_labels` does not exist and any join against it silently matches nothing.
|
|
# Consumed by `VPNTunnelDown` (platform/monitoring/alerts.yaml): the alert is armed only on a
|
|
# cluster that actually has a labelled gateway node, and ARMS ITSELF the moment one is labelled
|
|
# — which is how prod (no gateway today) stops emailing a permanently-firing critical alert
|
|
# without anyone having to remember to re-enable it later. Scoped to one label on one resource:
|
|
# 3 extra series, no cardinality concern. Label name is sanitized by KSM to
|
|
# `label_homelab_keithjackson_dev_egress_gateway` (`label_` + non-alphanumerics -> `_`).
|
|
metricLabelsAllowlist:
|
|
- "nodes=[homelab.keithjackson.dev/egress-gateway]"
|
|
# --- VELERO SCHEDULE OBJECTS AS METRICS (added 2026-09-13) ---------------------------------
|
|
# Velero exports NOTHING that enumerates Schedule objects and nothing that exposes
|
|
# `spec.paused`. Verified on prod against velero v1.18.1: all 35 velero_* metric names that
|
|
# carry a `schedule` label are created by a backup ATTEMPT, so a Schedule that has never run
|
|
# has NO series at all — a throwaway `paused: true` Schedule produced zero lines in
|
|
# `curl velero:8085/metrics`. Consequence: `VeleroNoRecentBackup`
|
|
# (`time() - velero_backup_last_successful_timestamp{schedule!=""}`) was structurally INERT for
|
|
# a never-successful schedule, because `time() - <absent series>` yields no series. prod sat
|
|
# with `velero-daily` (the VOLUME-DATA schedule) paused and unbacked-up, and nothing alerted.
|
|
#
|
|
# No PromQL can fix that — the fact is not in the TSDB — so the Schedule CRs themselves are
|
|
# exported here. KSM's custom-resource-state collector is the right mechanism precisely
|
|
# because it is ALREADY the platform's CRD-to-metrics path (this same block exports
|
|
# kube_node_labels for VPNTunnelDown); it needs no new exporter, image, ServiceMonitor or
|
|
# scrape job, and the series land on the existing kube-state-metrics target.
|
|
#
|
|
# Consumed by VeleroScheduleNeverBackedUp / VeleroSchedulePaused / VeleroNoVolumeDataBackup
|
|
# (platform/monitoring/alerts.yaml). Cost: 4 series per Schedule — 8 on prod today.
|
|
#
|
|
# Chart mechanics, verified by rendering kube-prometheus-stack 88.6.5 (KSM subchart 8.4.1,
|
|
# KSM v2.20.0) and by running that exact KSM image against prod's API:
|
|
# * `customResourceState.enabled` mounts this `config` as a ConfigMap at
|
|
# /etc/customresourcestate/config.yaml and adds --custom-resource-state-config-file.
|
|
# `--custom-resource-state-only` is NOT set, so the standard kube_* metrics keep flowing.
|
|
# * the chart auto-adds the apiextensions.k8s.io/customresourcedefinitions list+watch rule
|
|
# (rbac.customResourceState.createClusterRoleRules, default true). That rule is MANDATORY,
|
|
# not cosmetic: without it KSM logs "customresourcedefinitions ... is forbidden" and emits
|
|
# the CRS metrics as an EMPTY SET — silent, exactly the inert-alert failure mode again.
|
|
# Measured: the first probe run, with velero.io access but no CRD access, produced 0 series.
|
|
# * `rbac.extraRules` below is the only RBAC this repo must supply (read-only, one CRD).
|
|
# * `metricNamePrefix` deliberately yields `kube_velero_schedule_*`, NOT `velero_*`: these are
|
|
# kube-state-metrics series about Kubernetes objects, and must not be mistaken for
|
|
# something Velero itself exports.
|
|
# * `nilIsZero` matters — a Schedule that simply omits `spec.paused` (the chart renders
|
|
# `velero-daily-resources-only` that way) must still publish paused=0 rather than nothing,
|
|
# or `== 1` would be evaluated against a missing series.
|
|
# * the KSM Deployment carries NO checksum annotation over this ConfigMap, so a change to
|
|
# `config` alone does not restart the pod: after editing it, `kubectl -n monitoring rollout
|
|
# restart deploy/kube-prometheus-stack-kube-state-metrics`. Enabling it the first time
|
|
# changes the pod spec (new arg + volume), so the initial rollout is automatic.
|
|
rbac:
|
|
extraRules:
|
|
- apiGroups: ["velero.io"]
|
|
resources: ["schedules"]
|
|
verbs: ["get", "list", "watch"]
|
|
customResourceState:
|
|
enabled: true
|
|
config:
|
|
spec:
|
|
resources:
|
|
- groupVersionKind:
|
|
group: velero.io
|
|
version: v1
|
|
kind: Schedule
|
|
metricNamePrefix: kube_velero_schedule
|
|
labelsFromPath:
|
|
# `schedule` (not `name`) so these join 1:1 with Velero's own `schedule` label.
|
|
schedule: [metadata, name]
|
|
namespace: [metadata, namespace]
|
|
metrics:
|
|
- name: created
|
|
help: "Unix creation time of the Velero Schedule object."
|
|
each:
|
|
type: Gauge
|
|
gauge:
|
|
# KSM parses the RFC3339 timestamp to a unix float (verified live).
|
|
# This is what gives VeleroScheduleNeverBackedUp a grace period, so a
|
|
# newly added schedule is not alerted before its first run is even due.
|
|
path: [metadata, creationTimestamp]
|
|
- name: paused
|
|
help: "1 if the Velero Schedule is paused and creating no backups."
|
|
each:
|
|
type: Gauge
|
|
gauge:
|
|
path: [spec, paused]
|
|
nilIsZero: true
|
|
- name: protects_volume_data
|
|
help: "1 if the Schedule captures pod volume DATA, not just Kubernetes objects."
|
|
each:
|
|
type: Gauge
|
|
gauge:
|
|
# THE anti-masking signal: read from the field that defines it, so a
|
|
# resources-only schedule can never stand in for a volume-data one.
|
|
path: [spec, template, defaultVolumesToFsBackup]
|
|
nilIsZero: true
|
|
- name: info
|
|
help: "Descriptive metadata for the Velero Schedule (cron, phase)."
|
|
each:
|
|
type: Info
|
|
info:
|
|
# NOTE: KSM OMITS a label whose path is absent (a Schedule with no
|
|
# status.phase yet yields no `phase` label at all), so label sets differ
|
|
# between series here. Context for dashboards — never join on this metric.
|
|
labelsFromPath:
|
|
cron: [spec, schedule]
|
|
phase: [status, phase]
|
|
# MEMORY RAISED 2026-09-13 after 91 OOMKills. Measured, not guessed: working set 111 MiB
|
|
# with a 6h peak of 123.3 MiB against the old 128Mi limit — KSM was dying on every spike,
|
|
# roughly every 11 minutes (exit 137). That matters beyond the restarts: THIRTEEN alert
|
|
# rules read kube_* series, so each kill made them intermittently inert, and an alert that
|
|
# is only sometimes able to fire is the quiet version of the inert-alert class this repo
|
|
# keeps getting bitten by (KubePodNotReady and KubeDeploymentReplicasMismatch were both
|
|
# stuck pending on it). The trigger was enabling customResourceState below for the Velero
|
|
# Schedule metrics — it adds a CRD informer plus the mandatory CRD list/watch, and that
|
|
# marginal cost crossed a limit already sitting ~4 MiB above the working set.
|
|
# 384Mi is ~3x the observed peak and costs nothing here: the nodes report 18.7-22.3 GiB
|
|
# available. Re-measure with
|
|
# max_over_time(container_memory_working_set_bytes{container="kube-state-metrics"}[6h])
|
|
# before trimming it back.
|
|
resources:
|
|
requests:
|
|
cpu: 20m
|
|
memory: 192Mi
|
|
limits:
|
|
memory: 384Mi
|
|
prometheus-node-exporter:
|
|
resources:
|
|
requests:
|
|
cpu: 20m
|
|
memory: 32Mi
|
|
limits:
|
|
memory: 64Mi
|