jakman85andClaude Opus 4.8
d519ede4e3
gluetun retirement removes the gluetun_vpn_up gauge and the in-cluster gateway pod. Point
VPNTunnelDown at absent(node_network_receive_bytes_total{device=tun0}) — tun0 exists iff the
host OpenVPN tunnel is up (node_network_up unreliable: tun0 operstate=unknown). This also
fires if the gateway node/node-exporter is down (series goes stale). Remove the pod-readiness
EgressGatewayUnhealthy alert — there is no gateway pod under ADR-0015.
Co-Authored-By: Claude Opus 4.8 <[email protected]>
368 lines
17 KiB
YAML
368 lines
17 KiB
YAML
---
|
|
# Platform-health alert rules (docs/10, R-OBS-04). These complement the default node/kubernetes
|
|
# rules bundled by kube-prometheus-stack and add FIRST-CLASS alerts for the platform's own moving
|
|
# parts: nodes, Longhorn, Flux, cert-manager, networking/VPN, backups, identity, and the monitoring
|
|
# stack's own capacity ("don't let the monitor blind itself").
|
|
#
|
|
# Each rule maps to a docs/10 platform-health bullet and carries a severity (critical = page-worthy
|
|
# vs warning = degraded/capacity) plus enough annotation context (summary + runbook) to act.
|
|
#
|
|
# The `release: kube-prometheus-stack` label makes the operator's Prometheus select these rules
|
|
# (ruleSelector). Some expressions depend on exporters/ServiceMonitors owned by other workstreams
|
|
# (Flux, cert-manager, Longhorn, Velero, Keycloak, Cilium/Hubble ship their own metrics); the rules
|
|
# are authored to fire as soon as those series arrive. Metric names flagged TODO need confirming at
|
|
# bring-up against the deployed exporter.
|
|
#
|
|
# runbook_url points at a docs base; swap for the real runbook location when it exists.
|
|
|
|
# =====================================================================================
|
|
# Nodes
|
|
# =====================================================================================
|
|
apiVersion: monitoring.coreos.com/v1
|
|
kind: PrometheusRule
|
|
metadata:
|
|
name: platform-health-nodes
|
|
namespace: monitoring
|
|
labels:
|
|
release: kube-prometheus-stack
|
|
platform-health: "true"
|
|
spec:
|
|
groups:
|
|
- name: platform-health.nodes
|
|
rules:
|
|
- alert: NodeExporterDown
|
|
expr: up{job="node-exporter"} == 0
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Node exporter down on {{ $labels.instance }}"
|
|
description: "Prometheus cannot scrape node-exporter on {{ $labels.instance }} for 5m — the node may be down or unreachable."
|
|
runbook_url: "https://home.keithjackson.dev/runbooks/node-down"
|
|
- alert: NodeMemoryPressureHigh
|
|
expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) > 0.90
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Memory pressure high on {{ $labels.instance }}"
|
|
description: "Available memory below 10% for 10m on {{ $labels.instance }} — the ~22 GiB box is under pressure."
|
|
runbook_url: "https://home.keithjackson.dev/runbooks/node-memory"
|
|
- alert: NodeDiskFillingUp
|
|
expr: |
|
|
(node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs"}
|
|
/ node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs"}) < 0.10
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Disk filling up on {{ $labels.instance }} ({{ $labels.mountpoint }})"
|
|
description: "Less than 10% free on {{ $labels.mountpoint }} of {{ $labels.instance }} for 15m."
|
|
runbook_url: "https://home.keithjackson.dev/runbooks/node-disk"
|
|
|
|
# =====================================================================================
|
|
# Storage — Longhorn
|
|
# =====================================================================================
|
|
---
|
|
apiVersion: monitoring.coreos.com/v1
|
|
kind: PrometheusRule
|
|
metadata:
|
|
name: platform-health-longhorn
|
|
namespace: monitoring
|
|
labels:
|
|
release: kube-prometheus-stack
|
|
platform-health: "true"
|
|
spec:
|
|
groups:
|
|
- name: platform-health.longhorn
|
|
rules:
|
|
# longhorn_volume_robustness: 0=unknown 1=healthy 2=degraded 3=faulted
|
|
- alert: LonghornVolumeDegraded
|
|
expr: longhorn_volume_robustness == 2
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Longhorn volume degraded: {{ $labels.volume }}"
|
|
description: "Volume {{ $labels.volume }} has been degraded (replica rebuild / missing replica) for 10m."
|
|
runbook_url: "https://home.keithjackson.dev/runbooks/longhorn-degraded"
|
|
- alert: LonghornVolumeFaulted
|
|
expr: longhorn_volume_robustness == 3
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Longhorn volume FAULTED: {{ $labels.volume }}"
|
|
description: "Volume {{ $labels.volume }} is faulted — data at risk / offline. Immediate attention required."
|
|
runbook_url: "https://home.keithjackson.dev/runbooks/longhorn-faulted"
|
|
- alert: LonghornVolumeCapacityHigh
|
|
expr: (longhorn_volume_actual_size_bytes / longhorn_volume_capacity_bytes) > 0.85
|
|
for: 30m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Longhorn volume near capacity: {{ $labels.volume }}"
|
|
description: "Volume {{ $labels.volume }} is over 85% used for 30m."
|
|
runbook_url: "https://home.keithjackson.dev/runbooks/longhorn-capacity"
|
|
- alert: LonghornNodeStorageHigh
|
|
expr: (longhorn_node_storage_usage_bytes / longhorn_node_storage_capacity_bytes) > 0.85
|
|
for: 30m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Longhorn node storage near capacity: {{ $labels.node }}"
|
|
description: "Longhorn node {{ $labels.node }} is over 85% of its storage capacity for 30m."
|
|
runbook_url: "https://home.keithjackson.dev/runbooks/longhorn-capacity"
|
|
|
|
# =====================================================================================
|
|
# GitOps — Flux
|
|
# =====================================================================================
|
|
---
|
|
apiVersion: monitoring.coreos.com/v1
|
|
kind: PrometheusRule
|
|
metadata:
|
|
name: platform-health-flux
|
|
namespace: monitoring
|
|
labels:
|
|
release: kube-prometheus-stack
|
|
platform-health: "true"
|
|
spec:
|
|
groups:
|
|
- name: platform-health.flux
|
|
rules:
|
|
- alert: FluxReconciliationFailure
|
|
expr: |
|
|
max by (namespace, name, kind) (
|
|
gotk_reconcile_condition{type="Ready", status="False"}
|
|
) == 1
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Flux resource not ready: {{ $labels.kind }}/{{ $labels.name }}"
|
|
description: "{{ $labels.kind }} {{ $labels.namespace }}/{{ $labels.name }} has failed to reconcile (Ready=False) for 15m — GitOps drift/failure (R-GITOPS-03)."
|
|
runbook_url: "https://home.keithjackson.dev/runbooks/flux-reconcile"
|
|
- alert: FluxResourceSuspended
|
|
expr: |
|
|
max by (namespace, name, kind) (gotk_suspend_status) == 1
|
|
for: 1h
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Flux resource suspended: {{ $labels.kind }}/{{ $labels.name }}"
|
|
description: "{{ $labels.kind }} {{ $labels.namespace }}/{{ $labels.name }} has been suspended for 1h — reconciliation/drift-correction is off."
|
|
runbook_url: "https://home.keithjackson.dev/runbooks/flux-suspended"
|
|
|
|
# =====================================================================================
|
|
# Certificates — cert-manager
|
|
# =====================================================================================
|
|
---
|
|
apiVersion: monitoring.coreos.com/v1
|
|
kind: PrometheusRule
|
|
metadata:
|
|
name: platform-health-certificates
|
|
namespace: monitoring
|
|
labels:
|
|
release: kube-prometheus-stack
|
|
platform-health: "true"
|
|
spec:
|
|
groups:
|
|
- name: platform-health.certificates
|
|
rules:
|
|
- alert: CertificateExpiringSoon
|
|
expr: (certmanager_certificate_expiration_timestamp_seconds - time()) < (14 * 24 * 3600)
|
|
for: 1h
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Certificate expiring soon: {{ $labels.name }}"
|
|
description: "Certificate {{ $labels.namespace }}/{{ $labels.name }} expires in under 14 days."
|
|
runbook_url: "https://home.keithjackson.dev/runbooks/cert-expiry"
|
|
- alert: CertificateExpiryImminent
|
|
expr: (certmanager_certificate_expiration_timestamp_seconds - time()) < (3 * 24 * 3600)
|
|
for: 1h
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Certificate expiry imminent: {{ $labels.name }}"
|
|
description: "Certificate {{ $labels.namespace }}/{{ $labels.name }} expires in under 3 days and has not renewed."
|
|
runbook_url: "https://home.keithjackson.dev/runbooks/cert-expiry"
|
|
- alert: CertificateNotReady
|
|
expr: certmanager_certificate_ready_status{condition="False"} == 1
|
|
for: 1h
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Certificate not ready / issuance failing: {{ $labels.name }}"
|
|
description: "Certificate {{ $labels.namespace }}/{{ $labels.name }} has been in a not-ready state for 1h (ACME issuance failure?)."
|
|
runbook_url: "https://home.keithjackson.dev/runbooks/cert-issuance"
|
|
|
|
# =====================================================================================
|
|
# Networking — VPN / egress-gateway / denied flows (Hubble)
|
|
# =====================================================================================
|
|
---
|
|
apiVersion: monitoring.coreos.com/v1
|
|
kind: PrometheusRule
|
|
metadata:
|
|
name: platform-health-networking
|
|
namespace: monitoring
|
|
labels:
|
|
release: kube-prometheus-stack
|
|
platform-health: "true"
|
|
spec:
|
|
groups:
|
|
- name: platform-health.networking
|
|
rules:
|
|
# VPN tunnel health (ADR-0015: node-level host OpenVPN tunnel on the egress-gateway node, steered
|
|
# by Cilium Egress Gateway — there is NO in-cluster gateway pod). OpenVPN creates/destroys the tun0
|
|
# device with the connection, so the tun0 netdev series exists iff the tunnel is up. node_network_up
|
|
# is unreliable here (tun0 operstate=unknown => 0), so key on the byte-counter's PRESENCE. absent()
|
|
# also fires if the gateway node / its node-exporter is down (the tun0 series goes stale), covering
|
|
# gateway-node-down as well. Metric confirmed live on the dev cluster 2026-08-24.
|
|
- alert: VPNTunnelDown
|
|
expr: absent(node_network_receive_bytes_total{device="tun0"})
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Egress VPN tunnel down"
|
|
description: "The egress-gateway VPN tunnel is down (or its metric is absent) for 5m — outbound traffic could leak or stall (R-SEC-03 leak risk)."
|
|
runbook_url: "https://home.keithjackson.dev/runbooks/vpn-down"
|
|
# (ADR-0015) The former `EgressGatewayUnhealthy` pod-readiness alert was REMOVED with the gluetun
|
|
# retirement: there is no in-cluster egress-gateway pod anymore (the tunnel is a host systemd
|
|
# service, steering is Cilium eBPF). Tunnel + gateway-node health is covered by `VPNTunnelDown`
|
|
# above (tun0 presence, which also goes absent if the gateway node is down); Cilium datapath
|
|
# health is covered by the Cilium platform alerts.
|
|
- alert: HubbleDeniedFlowSpike
|
|
# Cilium/Hubble denied (policy-drop) flows. Requires hubble metrics enabled (docs/03).
|
|
expr: sum(rate(hubble_drop_total{reason="POLICY_DENIED"}[5m])) > 5
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Spike in policy-denied network flows"
|
|
description: "Policy-denied flow rate is elevated for 10m — a NetworkPolicy regression or an intrusion probe (R-SEC-03)."
|
|
runbook_url: "https://home.keithjackson.dev/runbooks/denied-flows"
|
|
|
|
# =====================================================================================
|
|
# Backups — Velero
|
|
# =====================================================================================
|
|
---
|
|
apiVersion: monitoring.coreos.com/v1
|
|
kind: PrometheusRule
|
|
metadata:
|
|
name: platform-health-backups
|
|
namespace: monitoring
|
|
labels:
|
|
release: kube-prometheus-stack
|
|
platform-health: "true"
|
|
spec:
|
|
groups:
|
|
- name: platform-health.backups
|
|
rules:
|
|
- alert: VeleroBackupFailure
|
|
expr: increase(velero_backup_failure_total[1h]) > 0
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Velero backup failed: schedule {{ $labels.schedule }}"
|
|
description: "One or more Velero backups failed in the last hour (R-STATE-03 — off-cluster DR at risk)."
|
|
runbook_url: "https://home.keithjackson.dev/runbooks/backup-failure"
|
|
- alert: VeleroBackupPartialFailure
|
|
expr: increase(velero_backup_partial_failure_total[1h]) > 0
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Velero backup partially failed"
|
|
description: "A Velero backup completed with partial failures in the last hour — some resources/volumes were skipped."
|
|
runbook_url: "https://home.keithjackson.dev/runbooks/backup-partial"
|
|
- alert: VeleroNoRecentBackup
|
|
expr: (time() - velero_backup_last_successful_timestamp{schedule!=""}) > (24 * 3600)
|
|
for: 1h
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "No successful Velero backup in 24h: {{ $labels.schedule }}"
|
|
description: "Schedule {{ $labels.schedule }} has not produced a successful backup in over 24h (missed schedule)."
|
|
runbook_url: "https://home.keithjackson.dev/runbooks/backup-missed"
|
|
|
|
# =====================================================================================
|
|
# Identity — Keycloak (SSO is a critical dependency — docs/15)
|
|
# =====================================================================================
|
|
---
|
|
apiVersion: monitoring.coreos.com/v1
|
|
kind: PrometheusRule
|
|
metadata:
|
|
name: platform-health-identity
|
|
namespace: monitoring
|
|
labels:
|
|
release: kube-prometheus-stack
|
|
platform-health: "true"
|
|
spec:
|
|
groups:
|
|
- name: platform-health.identity
|
|
rules:
|
|
- alert: KeycloakDown
|
|
expr: |
|
|
max(kube_pod_status_ready{namespace="keycloak", condition="true"}) == 0
|
|
or absent(kube_pod_status_ready{namespace="keycloak"})
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Keycloak IdP down"
|
|
description: "No ready Keycloak pod for 5m — SSO is unavailable and dependent services cannot authenticate (docs/15)."
|
|
runbook_url: "https://home.keithjackson.dev/runbooks/keycloak-down"
|
|
- alert: KeycloakDatabaseDown
|
|
# Keycloak's PostgreSQL (Longhorn-backed, DR-critical). Metric from postgres-exporter or
|
|
# the bundled chart; confirm label set at bring-up.
|
|
expr: |
|
|
max(kube_pod_status_ready{namespace="keycloak", pod=~".*postgres.*", condition="true"}) == 0
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Keycloak database unavailable"
|
|
description: "Keycloak's PostgreSQL is not ready for 5m — the IdP will fail (docs/15)."
|
|
runbook_url: "https://home.keithjackson.dev/runbooks/keycloak-db"
|
|
|
|
# =====================================================================================
|
|
# The monitor's own capacity — don't let the monitor blind itself (docs/10)
|
|
# =====================================================================================
|
|
---
|
|
apiVersion: monitoring.coreos.com/v1
|
|
kind: PrometheusRule
|
|
metadata:
|
|
name: platform-health-monitoring-self
|
|
namespace: monitoring
|
|
labels:
|
|
release: kube-prometheus-stack
|
|
platform-health: "true"
|
|
spec:
|
|
groups:
|
|
- name: platform-health.monitoring-self
|
|
rules:
|
|
- alert: MonitoringVolumeFillingUp
|
|
expr: |
|
|
(kubelet_volume_stats_available_bytes{namespace="monitoring"}
|
|
/ kubelet_volume_stats_capacity_bytes{namespace="monitoring"}) < 0.15
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Monitoring PVC filling up: {{ $labels.persistentvolumeclaim }}"
|
|
description: "PVC {{ $labels.persistentvolumeclaim }} (Prometheus/Loki) is under 15% free — retention/PVC size may need raising before the monitor goes blind."
|
|
runbook_url: "https://home.keithjackson.dev/runbooks/monitoring-capacity"
|
|
- alert: PrometheusTargetsMissing
|
|
expr: |
|
|
(count(up == 0) / count(up)) > 0.10
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "More than 10% of Prometheus targets are down"
|
|
description: "Over 10% of scrape targets have been down for 15m — a broad monitoring blind spot."
|
|
runbook_url: "https://home.keithjackson.dev/runbooks/targets-down"
|