--- # kube-prometheus-stack — Prometheus + Alertmanager + Grafana + kube-state-metrics + # node-exporter + the standard Kubernetes exporters/rules (docs/10). Serves R-OBS-01/02/03. # # Retention/PVC sizes are overlay knobs (R-ENV-02) resolved from cluster-vars via Flux # postBuild.substituteFrom. Everything else is DEV-SCOPED for the ~22 GiB workstation ceiling: # single replicas, modest requests/limits, ephemeral (emptyDir) Alertmanager/Grafana state, and # the k3s-embedded control-plane scrape jobs disabled (they are not separately reachable on k3s and # would otherwise show as permanently-down targets and burn RAM/alert noise). apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: name: kube-prometheus-stack namespace: monitoring spec: interval: 30m # The 65->88 catch-up is a large multi-major jump (operator + KSM + node-exporter roll, plus the # Prometheus/Alertmanager StatefulSets). The default 5m Helm timeout is too short for the full rollout # to reach Ready, so Flux logged "context deadline exceeded" and rolled back to 65.8.1 four times. # A generous timeout lets one attempt finish; remediateLastFailure:false keeps partial progress instead # of yo-yoing back to the old chart (a re-reconcile then drives it forward). CRDs are pre-applied # server-side (Helm never updates crds/ on upgrade), so schema is already at operator v0.93.1. timeout: 20m0s chart: spec: chart: kube-prometheus-stack version: ">=88.0.0 <89.0.0" # k8s-1.36 catch-up (po v0.93; CRDs pre-applied server-side, Helm won't update them). Renovate maintains sourceRef: kind: HelmRepository name: prometheus-community namespace: monitoring install: remediation: retries: 3 upgrade: remediation: retries: 2 remediateLastFailure: false cleanupOnFail: true values: # --- k3s trim: control-plane components are embedded in the k3s server process and are not # scrapable as discrete endpoints. Disabling avoids permanently-firing "target down" noise. kubeControllerManager: enabled: false kubeScheduler: enabled: false kubeEtcd: enabled: false kubeProxy: enabled: false # --- Prometheus --- prometheus: prometheusSpec: replicas: 1 retention: ${MONITORING_PROMETHEUS_RETENTION} # overlay knob (dev 3d / prod long) # SIZE-based retention cap (docs/10, R-OBS). `retention` alone prunes only by AGE — on a young # cluster (< retention window) NOTHING is pruned yet, so a busy TSDB fills the PVC before the # time window elapses. A full /prometheus wedges ALL WAL commits ("no space left on device"): # scrapes still 200 but samples are never stored, the head freezes, and every absent()/==0 # platform alert (Keycloak/KubeAPI/Kubelet/VPN/KubeVirt/CDI…) fires falsely because instant # queries at "now" return empty. retentionSize makes Prometheus drop the OLDEST blocks once the # block set approaches this cap, so the disk never fills regardless of cluster age or churn. # Kept BELOW the PVC size (leaves headroom for the WAL + head + compaction scratch, which are # NOT counted by retentionSize). Prometheus size units are *iB (GiB), distinct from the K8s PVC # `Gi`, so it is a separate overlay knob rather than a reuse of the PVC-size knob. retentionSize: ${MONITORING_PROMETHEUS_RETENTION_SIZE} # overlay knob (dev 5GiB / base 8GiB / prod 45GiB) # Discover ServiceMonitors/PodMonitors/Rules from ALL namespaces (flux, cert-manager, # longhorn, velero, keycloak, cilium/hubble ship their own) — not just this release's. ruleSelectorNilUsesHelmValues: false serviceMonitorSelectorNilUsesHelmValues: false podMonitorSelectorNilUsesHelmValues: false probeSelectorNilUsesHelmValues: false # Modest footprint for a home dev cluster (few hundred series). Limit gives headroom for # WAL replay / compaction spikes without letting Prometheus eat the node. resources: requests: cpu: 100m memory: 512Mi limits: memory: 3Gi # 1Gi OOMKilled (exit 137) during TSDB compaction/checkpoint; # 2Gi ran pinned at ~98% (2002Mi) with no room for compaction/WAL-replay # spikes (the classic Prometheus OOM). Nodes are 24Gi at 38-44% used # (~14Gi free each), so 3Gi gives real headroom. storageSpec: volumeClaimTemplate: spec: storageClassName: longhorn # replicated tier (ADR-0003) accessModes: ["ReadWriteOnce"] resources: requests: storage: ${MONITORING_PROMETHEUS_PVC_SIZE} # overlay knob (dev 5Gi / prod large) # --- Alertmanager: routes by severity to email (R-OBS-02) --- alertmanager: alertmanagerSpec: replicas: 1 # DEV TRIM: no PVC — silences/notification-log live in emptyDir. Acceptable in dev (a # restart re-fires active alerts). Prod overlay can add a small storage block. resources: requests: cpu: 10m memory: 32Mi limits: memory: 96Mi # SMTP credentials are an OPERATOR HAND-OFF: a SOPS-encrypted Secret, referenced BY NAME # only (docs/07). Mounted at /etc/alertmanager/secrets// so the config below can read # the password from a file (never inline plaintext). secrets: - alertmanager-smtp # Routing/receivers. Non-secret SMTP endpoint fields (smarthost/from/to/username) are # operator-supplied placeholders (REPLACE-ME, same convention as velero); the PASSWORD is # read from the mounted SOPS secret via *_file. Both severities email; subject/route differ. config: global: resolve_timeout: 5m smtp_smarthost: "smtp.gmail.com:587" # operator hand-off (e.g. smtp.gmail.com:587) smtp_from: "alertmanager@keithjackson.dev" # operator hand-off (From: address) smtp_auth_username: "keithallenjackson@gmail.com" # operator hand-off (SMTP login) smtp_auth_password_file: /etc/alertmanager/secrets/alertmanager-smtp/smtp-password smtp_require_tls: true route: group_by: ["alertname", "namespace"] group_wait: 30s group_interval: 5m repeat_interval: 12h receiver: email-warning routes: - matchers: ['severity = "critical"'] receiver: email-critical # page-worthy — notify fast, remind often group_wait: 10s repeat_interval: 1h # DEPLOY NOTICES, not health alerts (added 2026-09-29). FluxRolloutInProgress fires # the moment the applied revision diverges from main and RESOLVES when every # Kustomization has caught up — so the FIRED mail is "a deploy started" and the # RESOLVED mail is "it finished". Both halves are the product, which is why # send_resolved must stay true on the receiver below. # # A DEDICATED ROUTE, because the default receiver is `email-warning`: without it a # routine deploy arrives subject-lined "[WARNING]" and becomes indistinguishable from a # real platform fault, which is how alert mail stops being read at all. `severity: info` # is used by NO other rule in this repo (grepped 2026-09-29: zero matches), so this # route claims exactly the deploy notices and nothing else. Keep it that way — giving a # genuine fault severity: info would silently route it here and re-label it a deploy. - matchers: ['severity = "info"'] receiver: email-deploy # One mail per ROLLOUT, not per Kustomization: the alert fires once per lagging # Kustomization (16 possible), and grouping on alertname alone collapses them into a # single mail that lists all of them. The default group_by would have split them by # namespace as well — harmless today (all 16 are in flux-system) and wrong the moment # a Kustomization lives elsewhere. group_by: ["alertname"] group_wait: 30s # 2m, not the 5m default: group_interval also bounds how long a group's RESOLVED # notification waits, and holding "the deploy finished" four minutes behind the # deploy finishing makes the completion half of this feature feel broken. group_interval: 2m # A deploy notice is a statement about one revision, so re-sending it on the 12h # default would just repeat old news. 24h is long enough never to fire twice for one # rollout, while still re-nagging if a rollout is genuinely stuck for a whole day # (FluxRolloutStuck, severity warning, is the real escalation for that case). repeat_interval: 24h receivers: - name: email-warning email_configs: - to: "keithallenjackson@gmail.com" # alert inbox (operator: change if a different inbox is wanted) send_resolved: true headers: subject: '[WARNING] {{ .CommonLabels.alertname }} ({{ .CommonLabels.namespace }})' - name: email-critical email_configs: - to: "keithallenjackson@gmail.com" # alert inbox (operator: change if a different inbox is wanted) send_resolved: true headers: subject: '[CRITICAL] {{ .CommonLabels.alertname }} ({{ .CommonLabels.namespace }})' # Flux deploy notices (the `severity = "info"` route above). - name: email-deploy email_configs: - to: "keithallenjackson@gmail.com" # alert inbox (operator: change if a different inbox is wanted) # BOTH transitions are the deliverable here: firing = "Flux started applying a new # revision", resolved = "every Kustomization reached it". Turning this off would # delete the completion notification, not merely reduce noise. send_resolved: true headers: # `[FLUX] applying` / `[FLUX] applied` — readable in a notification preview # without opening the mail, and never the word WARNING for a healthy deploy. # .Status is "firing" or "resolved" in Alertmanager's notification template data. subject: '[FLUX] {{ if eq .Status "firing" }}applying{{ else }}applied{{ end }} — {{ .CommonLabels.alertname }}' # --- Grafana: dashboards (R-OBS-03), surfaced in the C2 UI (docs/12) behind Keycloak SSO --- grafana: enabled: true # Force a pod roll when the OIDC wiring changes so Grafana loads the grafana-oidc-credentials # client-secret (env valueFrom is resolved at pod start; the secret was added after first boot, so # generic_oauth started disabled). Bump this value to re-roll Grafana on future OIDC secret changes. podAnnotations: homelab.keithjackson.dev/oidc-rollout: "1" # Admin bootstrap + (later) OIDC client are SOPS secrets referenced BY NAME (WS-SEC/ADR-0014). admin: existingSecret: grafana-admin-credentials userKey: admin-user passwordKey: admin-password # DEV TRIM: emptyDir (default). Dashboards are provisioned from ConfigMaps (sidecar), so no # durable Grafana DB is needed to iterate. Prod can enable persistence. persistence: enabled: false # Auto-discover dashboard ConfigMaps labeled grafana_dashboard across namespaces (per-service # dashboards rendered from ServiceDefinitions land here — docs/10 onboarding output). sidecar: dashboards: enabled: true searchNamespace: ALL label: grafana_dashboard datasources: enabled: true resources: requests: cpu: 50m memory: 384Mi # Grafana 13.x (image 13.2.0, kps 88) baseline is well above Grafana 11.x (kps 65) limits: memory: 768Mi # 256Mi OOMKilled (exit 137) Grafana 13.x during apiserver/unified-storage # startup — the silent SIGKILL that made the whole 65->88 Helm upgrade # time out (the new Grafana pod never went Ready). 512Mi then ran pinned at # ~98% (499Mi) with no headroom — one render/GC spike from OOM. 768Mi holds # with real headroom (nodes 24Gi, ~14Gi free each). # --- Keycloak native-OIDC SSO (ADR-0014, deliverable #4 SSO proof) --- # Grafana is a confidential OIDC client of the Keycloak `homelab` realm. The client SECRET comes # from the SOPS-encrypted grafana-oidc-credentials Secret via env (referenced by name only, # R-IDP-04). optional:true so the RUNNING Grafana keeps starting before the operator fills the # secret — local admin login (below) stays available as the fallback until SSO is proven. envValueFrom: GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: secretKeyRef: name: grafana-oidc-credentials key: client-secret optional: true grafana.ini: server: root_url: https://grafana.${BASE_DOMAIN} auth.generic_oauth: enabled: true name: Keycloak client_id: grafana scopes: "openid email profile" auth_url: https://keycloak.${BASE_DOMAIN}/realms/homelab/protocol/openid-connect/auth token_url: https://keycloak.${BASE_DOMAIN}/realms/homelab/protocol/openid-connect/token api_url: https://keycloak.${BASE_DOMAIN}/realms/homelab/protocol/openid-connect/userinfo login_attribute_path: preferred_username email_attribute_path: email # Map Keycloak groups (the `groups` claim) to Grafana roles (R-IDP-06): platform-admins -> # Admin, media-users -> Editor, everyone else -> Viewer. role_attribute_path: "contains(groups[*], 'platform-admins') && 'Admin' || contains(groups[*], 'media-users') && 'Editor' || 'Viewer'" # Keep local admin login enabled as break-glass until SSO is proven end-to-end; flip to # disable_login_form:true afterwards (docs/15 break-glass, R-IDP-05). # TODO(WS-SEC, post-proof): set auth.disable_login_form=true once SSO login is verified. # --- Bundled exporters / operator: keep small --- prometheusOperator: resources: requests: cpu: 50m memory: 96Mi limits: memory: 256Mi kube-state-metrics: # Expose the egress-gateway NODE LABEL as a metric label so alerts can ask "does this cluster # have a VPN egress gateway at all?". KSM exposes NO custom labels by default, so without this # `kube_node_labels` does not exist and any join against it silently matches nothing. # Consumed by `VPNTunnelDown` (platform/monitoring/alerts.yaml): the alert is armed only on a # cluster that actually has a labelled gateway node, and ARMS ITSELF the moment one is labelled # — which is how prod (no gateway today) stops emailing a permanently-firing critical alert # without anyone having to remember to re-enable it later. Scoped to one label on one resource: # 3 extra series, no cardinality concern. Label name is sanitized by KSM to # `label_homelab_keithjackson_dev_egress_gateway` (`label_` + non-alphanumerics -> `_`). metricLabelsAllowlist: - "nodes=[homelab.keithjackson.dev/egress-gateway,homelab.keithjackson.dev/egress-uplink]" # --- VELERO SCHEDULE OBJECTS AS METRICS (added 2026-09-13) --------------------------------- # Velero exports NOTHING that enumerates Schedule objects and nothing that exposes # `spec.paused`. Verified on prod against velero v1.18.1: all 35 velero_* metric names that # carry a `schedule` label are created by a backup ATTEMPT, so a Schedule that has never run # has NO series at all — a throwaway `paused: true` Schedule produced zero lines in # `curl velero:8085/metrics`. Consequence: `VeleroNoRecentBackup` # (`time() - velero_backup_last_successful_timestamp{schedule!=""}`) was structurally INERT for # a never-successful schedule, because `time() - ` yields no series. prod sat # with `velero-daily` (the VOLUME-DATA schedule) paused and unbacked-up, and nothing alerted. # # No PromQL can fix that — the fact is not in the TSDB — so the Schedule CRs themselves are # exported here. KSM's custom-resource-state collector is the right mechanism precisely # because it is ALREADY the platform's CRD-to-metrics path (this same block exports # kube_node_labels for VPNTunnelDown); it needs no new exporter, image, ServiceMonitor or # scrape job, and the series land on the existing kube-state-metrics target. # # Consumed by VeleroScheduleNeverBackedUp / VeleroSchedulePaused / VeleroNoVolumeDataBackup # (platform/monitoring/alerts.yaml). Cost: 4 series per Schedule — 8 on prod today. # # Chart mechanics, verified by rendering kube-prometheus-stack 88.6.5 (KSM subchart 8.4.1, # KSM v2.20.0) and by running that exact KSM image against prod's API: # * `customResourceState.enabled` mounts this `config` as a ConfigMap at # /etc/customresourcestate/config.yaml and adds --custom-resource-state-config-file. # `--custom-resource-state-only` is NOT set, so the standard kube_* metrics keep flowing. # * the chart auto-adds the apiextensions.k8s.io/customresourcedefinitions list+watch rule # (rbac.customResourceState.createClusterRoleRules, default true). That rule is MANDATORY, # not cosmetic: without it KSM logs "customresourcedefinitions ... is forbidden" and emits # the CRS metrics as an EMPTY SET — silent, exactly the inert-alert failure mode again. # Measured: the first probe run, with velero.io access but no CRD access, produced 0 series. # * `rbac.extraRules` below is the only RBAC this repo must supply (read-only, one CRD). # * `metricNamePrefix` deliberately yields `kube_velero_schedule_*`, NOT `velero_*`: these are # kube-state-metrics series about Kubernetes objects, and must not be mistaken for # something Velero itself exports. # * `nilIsZero` matters — a Schedule that simply omits `spec.paused` (the chart renders # `velero-daily-resources-only` that way) must still publish paused=0 rather than nothing, # or `== 1` would be evaluated against a missing series. # * the KSM Deployment carries NO checksum annotation over this ConfigMap, so a change to # `config` alone does not restart the pod: after editing it, `kubectl -n monitoring rollout # restart deploy/kube-prometheus-stack-kube-state-metrics`. Enabling it the first time # changes the pod spec (new arg + volume), so the initial rollout is automatic. # FORCE A POD ROLL WHEN `customResourceState.config` BELOW CHANGES. The chart renders that # config into a ConfigMap and mounts it, but the KSM Deployment carries NO checksum annotation # over it — documented in the block above, and the reason the Velero metrics needed a manual # `rollout restart` when they were first tuned. So a PR that edits ONLY `config` changes the # ConfigMap, leaves the pod spec byte-identical, and KSM keeps running the OLD collector set: # the new metrics never appear and every alert reading them is inert, while the PR is merged # and green. That is the exact shape of defect this repo keeps getting bitten by, so it is # closed declaratively rather than by remembering an operator step. BUMP THIS VALUE IN THE # SAME COMMIT as any `customResourceState.config` edit and Flux rolls the pod for you. # (Same mechanism as grafana.podAnnotations/oidc-rollout above.) podAnnotations: homelab.keithjackson.dev/crs-config-rollout: "1" rbac: extraRules: - apiGroups: ["velero.io"] resources: ["schedules"] verbs: ["get", "list", "watch"] # Flux applied-revision metrics (the kube_flux_* block at the end of `config` below). # MANDATORY for the same measured reason as velero.io above: without read access KSM # publishes the custom-resource-state metrics as an EMPTY SET and logs a forbidden # error, which would leave FluxRolloutInProgress/FluxRolloutStuck inert — silently. - apiGroups: ["kustomize.toolkit.fluxcd.io"] resources: ["kustomizations"] verbs: ["get", "list", "watch"] - apiGroups: ["source.toolkit.fluxcd.io"] resources: ["gitrepositories"] verbs: ["get", "list", "watch"] customResourceState: enabled: true config: spec: resources: - groupVersionKind: group: velero.io version: v1 kind: Schedule metricNamePrefix: kube_velero_schedule labelsFromPath: # `schedule` (not `name`) so these join 1:1 with Velero's own `schedule` label. schedule: [metadata, name] namespace: [metadata, namespace] metrics: - name: created help: "Unix creation time of the Velero Schedule object." each: type: Gauge gauge: # KSM parses the RFC3339 timestamp to a unix float (verified live). # This is what gives VeleroScheduleNeverBackedUp a grace period, so a # newly added schedule is not alerted before its first run is even due. path: [metadata, creationTimestamp] - name: paused help: "1 if the Velero Schedule is paused and creating no backups." each: type: Gauge gauge: path: [spec, paused] nilIsZero: true - name: protects_volume_data help: "1 if the Schedule captures pod volume DATA, not just Kubernetes objects." each: type: Gauge gauge: # THE anti-masking signal: read from the field that defines it, so a # resources-only schedule can never stand in for a volume-data one. path: [spec, template, defaultVolumesToFsBackup] nilIsZero: true - name: info help: "Descriptive metadata for the Velero Schedule (cron, phase)." each: type: Info info: # NOTE: KSM OMITS a label whose path is absent (a Schedule with no # status.phase yet yields no `phase` label at all), so label sets differ # between series here. Context for dashboards — never join on this metric. labelsFromPath: cron: [spec, schedule] phase: [status, phase] # --- FLUX APPLIED REVISION AS METRICS (added 2026-09-29) -------------------------- # WHY. Nothing in the TSDB could answer "is the cluster running the commit that is on # main?", so "Flux started applying a git change" and "Flux is idle" were # indistinguishable and a deploy could start, stall halfway and never finish with # nothing to say so. Measured on prod 2026-09-29, all three candidate signals fail: # * gotk_* (scraped by flux-podmonitor.yaml) carries NO revision label anywhere — # gotk_reconcile_condition 120 series, gotk_suspend_status 40, not one with a # revision. It can say a Kustomization is unhealthy, never which commit it holds. # * the EVENT stream cannot be filtered into a deploy signal: a Kustomization emits # ReconciliationSucceeded on EVERY interval tick — 101 events in one hour across # 16 Kustomizations — with the message "Reconciliation finished in 324ms, next run # in 10m0s", byte-identical whether the apply changed anything or not. # * `Progressing` is worse, not better: five Kustomizations re-apply the same object # on every single tick (2505/2504/2501/2495/2490 on one event object each), so # anything keyed on it fires five times every ten minutes, forever. # The applied REVISION is the one signal that moves only when git moves, and it lives # in these two objects' status and nowhere else: # GitRepository/flux-system .status.artifact.revision = what main is (poll 1m) # Kustomization/* .status.lastAppliedRevision = what is applied # Exporting both makes "applied != desired" a PromQL question, which is what # FluxRolloutInProgress / FluxRolloutStuck ask (platform/monitoring/alerts.yaml). # # NAMING: kube_flux_*, NOT gotk_* — the same rule as kube_velero_schedule_* above. # These are kube-state-metrics series ABOUT Kubernetes objects and must never be # mistaken for the gotk_* family the Flux controllers themselves export. # # VERIFIED ON PROD before writing this (2026-09-29): both CRDs serve `v1` as the # STORAGE version (kustomizations.kustomize.toolkit.fluxcd.io, # gitrepositories.source.toolkit.fluxcd.io) and both status paths below are populated # — artifact.revision=main@sha1:b93594b7…, and all 16 Kustomizations reporting that # same revision as lastAppliedRevision. # # Cost: 2 series per Kustomization (16 today) + 1 for the GitRepository = 33. - groupVersionKind: group: kustomize.toolkit.fluxcd.io version: v1 kind: Kustomization metricNamePrefix: kube_flux_kustomization labelsFromPath: name: [metadata, name] namespace: [metadata, namespace] metrics: - name: info help: "Revision a Flux Kustomization has applied (status.lastAppliedRevision)." each: type: Info info: # KSM OMITS a label whose path is absent (the same behaviour noted for the # Velero `phase` label above), so a Kustomization that has never applied # anything publishes this series with NO `revision` label at all rather than # an empty one. Every rule that reads it therefore matches revision!="" — # otherwise those would collapse together under the empty string and a # never-applied Kustomization would compare equal to a converged one. labelsFromPath: revision: [status, lastAppliedRevision] - name: suspended help: "1 if the Flux Kustomization is suspended (spec.suspend)." each: type: Gauge gauge: # nilIsZero is MANDATORY, exactly as for the Velero `paused` gauge above: # measured 2026-09-29, NONE of the 16 Kustomizations sets spec.suspend at # all. Without it this metric would not exist, and the `unless` that stops a # suspended Kustomization holding a rollout notification open forever would # silently match nothing. path: [spec, suspend] nilIsZero: true - groupVersionKind: group: source.toolkit.fluxcd.io version: v1 kind: GitRepository metricNamePrefix: kube_flux_gitrepository labelsFromPath: name: [metadata, name] namespace: [metadata, namespace] metrics: - name: info help: "Revision of the artifact a Flux GitRepository has fetched (what main is)." each: type: Info info: labelsFromPath: revision: [status, artifact, revision] # MEMORY RAISED 2026-09-13 after 91 OOMKills. Measured, not guessed: working set 111 MiB # with a 6h peak of 123.3 MiB against the old 128Mi limit — KSM was dying on every spike, # roughly every 11 minutes (exit 137). That matters beyond the restarts: THIRTEEN alert # rules read kube_* series, so each kill made them intermittently inert, and an alert that # is only sometimes able to fire is the quiet version of the inert-alert class this repo # keeps getting bitten by (KubePodNotReady and KubeDeploymentReplicasMismatch were both # stuck pending on it). The trigger was enabling customResourceState below for the Velero # Schedule metrics — it adds a CRD informer plus the mandatory CRD list/watch, and that # marginal cost crossed a limit already sitting ~4 MiB above the working set. # 384Mi is ~3x the observed peak and costs nothing here: the nodes report 18.7-22.3 GiB # available. Re-measure with # max_over_time(container_memory_working_set_bytes{container="kube-state-metrics"}[6h]) # before trimming it back. resources: requests: cpu: 20m memory: 192Mi limits: memory: 384Mi prometheus-node-exporter: # --- NFS OBSERVABILITY (added 2026-09-14) ------------------------------------------------- # WHY. Five in-tree NFS PVs on 192.168.2.102 carry the whole media library and the legacy # Nextcloud dataset, and I/O PSI counts an NFS stall as an I/O stall. That is why prod-1's # 20.98% PSI spike at 02:40Z on 2026-09-14 looked inexplicable from the block layer: devices # at 0.09-0.21 utilisation, 1.4-4.8 ms per write, zero NFS retransmissions. The one signal # that moved was 114 NFS reads/s on prod-1 against 0 on the other two nodes. # # WHAT IS ALREADY THERE, SO NOBODY "ENABLES" IT TWICE. The `nfs` collector is ON BY DEFAULT # and its series are confirmed present on prod (node_nfs_rpcs_total, node_nfs_requests_total # by method/proto, node_nfs_rpc_retransmissions_total — all three nodes, 2026-09-14). It is # host-wide, read from /proc/net/rpc/nfs, which works because this DaemonSet is hostNetwork. # It gives op COUNTS and retransmissions. It gives NO LATENCY and no per-mount breakdown, so # it can say "the server dropped requests" but not "the export is slow". # # WHAT THIS ARGUMENT ADDS. `mountstats` reads /proc/.../mountstats and emits per-mount, # per-operation counters including cumulative RPC queue time and round-trip time, which # divided by the request count is the per-operation NFS latency this platform has never had. # That is the missing half of the I/O-cause question: with it, "PSI is high and the export is # answering slowly" and "PSI is high because WE are asking for a lot" become distinguishable, # and so does the case where neither holds and the load is coming from the Proxmox media # stack that still has operational control of the same files. # # COST, MEASURED-ADJACENT NOT ASSUMED. mountstats is per-mount and per-operation, so it is # the highest-cardinality collector in node-exporter. NFSv4.1 exposes ~70 operations and the # collector emits ~8 counters per operation plus ~15 mount-level series, so roughly 575 # series per mounted export, and the mounts are per POD (kubelet mounts an in-tree NFS PV # once per pod), so they churn on every restart. With five exports that is ~2.9k series # against a head of 451,650 measured on prod 2026-09-14 — about 0.6%, inside the 3Gi # Prometheus limit and the 45GiB retentionSize. Re-measure after rollout with # count({__name__=~"node_mountstats_.*"}) # and if it lands materially above ~5k series, drop the collector rather than growing the PVC. # # VERIFY BEFORE TRUSTING — THIS MAY LAND INERT, AND THAT MUST BE CHECKED NOT ASSUMED. # node-exporter runs with --path.procfs=/host/proc and hostPID, so it reads ITS OWN mount # namespace, not the host's. The kubelet-created NFS mounts should still be visible because # the `root` volume is a hostPath of / with mountPropagation: HostToContainer, which # propagates them in — they will appear with mount points prefixed /host/root/var/lib/kubelet/ # (the `export` label, e.g. 192.168.2.102:/mnt/Storage/plex, is the stable thing to aggregate # by). The node-exporter image is distroless with no shell, so this could not be tested from # outside the collector; note that node_filesystem_* currently reports only ext4 and tmpfs, # which is explained by the kubelet mount-point exclusion above but is NOT proof either way. # AFTER THIS ROLLS OUT, RUN: # count({__name__=~"node_mountstats_.*"}) -> must be non-zero # count(node_mountstats_nfs_operations_requests_total) by (export) # If it is empty, REVERT this argument — a collector that is on and produces nothing is the # same lie as an alert rule on a metric nobody scrapes. No alert rule references # node_mountstats_* yet, deliberately, for exactly that reason; the per-export latency rule # is the follow-up once the series are confirmed, and runbooks/node-io.md records the gap. # # extraArgs REPLACES, IT DOES NOT APPEND — and the chart's DEFAULT for this key is not empty. # prometheus-node-exporter's own values.yaml ships two filesystem excludes in `extraArgs` # (confirmed against the chart, and against the live DaemonSet's argv on prod). Listing only # `--collector.mountstats` here would silently DROP both, so the filesystem collector would # start reporting every container overlay and every kubelet bind mount: a large cardinality # jump and a behaviour change in NodeDiskFillingUp, which alerts per mountpoint. Both # defaults are therefore restated verbatim below. If the chart's defaults change on an # upgrade, this list must be re-reconciled against # helm show values prometheus-community/prometheus-node-exporter | grep -A3 '^extraArgs:' extraArgs: - --collector.filesystem.mount-points-exclude=^/(dev|proc|sys|run/containerd/.+|var/lib/docker/.+|var/lib/kubelet/.+)($|/) - --collector.filesystem.fs-types-exclude=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs|erofs)$ - --collector.mountstats resources: requests: cpu: 20m memory: 32Mi limits: # mountstats parses a per-mount, per-operation table on every scrape. 64Mi was sized for # the default collector set; raised with headroom for that parse rather than discovering # the ceiling as an OOMKill on the exporter that every node alert depends on. memory: 96Mi