Autonomous session: Phase-2 platform authoring + Flux bootstrap progress
Author-only Phase-2 components (platform suspended; cluster untouched): - platform/longhorn: 3-replica SC + bulk external-mount indirection (R-STATE) - platform/monitoring: kube-prometheus-stack + Loki + Alloy (dev-trimmed) + 8 alert rules (R-OBS) - platform/cilium: parity fix (k8sServiceHost=10.73.0.11 + masquerade) safe to adopt Phase 2 - conformance: partial L2 tests (worker hit session limit; incomplete — see DECISIONS-LOG K2) - bootstrap/gitea: fix rootless-podman perms (host bind-mount -> named volume) Flux installed on the cluster; host Gitea up + repo pushed; GitRepository BLOCKED by host firewalld (VM->10.73.0.1:3000) — needs one operator sudo command (DECISIONS-LOG). Known issue K1: cluster-vars substitution-var case mismatch (fix before un-suspend). Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
This commit is contained in:
jakman85andClaude Opus 4.8 committed
1 parent
397f29e99a
commit
cc89b4b830
19 files changed
+1567
-221
No files matched your search
@@ -1,11 +1,20 @@
|
||||
---
|
||||
# Cilium — CNI, kube-proxy replacement, Gateway API, Egress Gateway, Hubble (ADR-0002).
|
||||
# Phase 0 STUB. Values are PLACEHOLDERS; fill in during Phase 1/2 (WS-NET).
|
||||
# Serves: R-NET-01..07, R-DNS-01, R-SEC-02. Owner: WS-NET (bootstrap shared with WS-INFRA).
|
||||
#
|
||||
# IMPORTANT (cross-workstream contract): Cilium's INITIAL install is performed by the Ansible
|
||||
# bootstrap (spec docs/02, Phase 1), replacing k3s flannel/kube-proxy. This HelmRelease manages
|
||||
# Cilium CONFIG going forward. Its values MUST match the Ansible bootstrap values (kubeProxyReplacement,
|
||||
# k8sServiceHost/Port) or Flux and the bootstrap will fight. Owner: WS-NET + WS-INFRA.
|
||||
# ============================ PARITY CONTRACT ============================
|
||||
# Cilium's INITIAL install is performed by the Ansible k3s bootstrap (spec docs/02, Phase 1) —
|
||||
# a cluster needs a CNI before Flux can run. This HelmRelease ADOPTS and manages Cilium's config
|
||||
# thereafter (docs/05). The values below MUST EQUAL the Ansible bootstrap values, or Flux and the
|
||||
# bootstrap will fight over the running CNI and break cluster networking.
|
||||
#
|
||||
# SOURCE OF TRUTH (do not diverge without updating both):
|
||||
# infra/ansible/roles/cilium/templates/cilium-values.yaml.j2
|
||||
# infra/ansible/roles/cilium/defaults/main.yml (cilium_version, k3s_api_host/port)
|
||||
#
|
||||
# Every value tagged [parity] below is asserted identical to the Ansible render. If you change one
|
||||
# here, change it in the Ansible role in the same PR. (docs/02 Cilium-ownership contract.)
|
||||
# ========================================================================
|
||||
apiVersion: helm.toolkit.fluxcd.io/v2
|
||||
kind: HelmRelease
|
||||
metadata:
|
||||
@@ -16,7 +25,9 @@ spec:
|
||||
chart:
|
||||
spec:
|
||||
chart: cilium
|
||||
version: ">=1.16.0 <1.17.0" # TODO(WS-NET): pin exact version at bring-up; Renovate maintains
|
||||
# [parity] Pinned to the exact version the Ansible bootstrap installs
|
||||
# (cilium_version in roles/cilium/defaults/main.yml). Renovate bumps both together.
|
||||
version: "1.16.5"
|
||||
sourceRef:
|
||||
kind: HelmRepository
|
||||
name: cilium
|
||||
@@ -27,21 +38,35 @@ spec:
|
||||
upgrade:
|
||||
remediation:
|
||||
retries: 3
|
||||
# Placeholder values — see values.yaml for the annotated skeleton. Inlined here so the intent
|
||||
# (Cilium REPLACES flannel + kube-proxy + Traefik) is visible at the release.
|
||||
# Values are inlined so the intent (Cilium REPLACES flannel + kube-proxy + Traefik) is visible at
|
||||
# the release. MUST equal the Ansible bootstrap render (see PARITY CONTRACT header).
|
||||
values:
|
||||
# --- kube-proxy replacement (ADR-0001/0002: k3s kube-proxy disabled) ---
|
||||
# [parity] MUST equal Ansible: kubeProxyReplacement: true
|
||||
kubeProxyReplacement: true
|
||||
# TODO(WS-INFRA): k8sServiceHost/k8sServicePort must equal the Ansible bootstrap's API endpoint.
|
||||
k8sServiceHost: "REPLACE-ME" # e.g. the k3s API VIP/host
|
||||
# [parity] MUST equal Ansible: k8sServiceHost/k8sServicePort = the k3s API endpoint
|
||||
# (k3s_api_host/k3s_api_port). This is the primary control-plane node IP. Setting these
|
||||
# wrong (e.g. the old "REPLACE-ME") breaks Cilium's connection to the API server on adopt.
|
||||
k8sServiceHost: "10.73.0.11"
|
||||
k8sServicePort: 6443
|
||||
# --- Gateway API (ingress; replaces Traefik) — docs/09, ADR-0002 ---
|
||||
# [parity] MUST equal Ansible: gatewayAPI.enabled: true
|
||||
gatewayAPI:
|
||||
enabled: true
|
||||
# --- Egress Gateway (egress-class steering; part of VPN model with ADR-0013) ---
|
||||
# --- Egress Gateway (egress-class steering; part of the VPN model, ADR-0013) ---
|
||||
# [parity] MUST equal Ansible: egressGateway.enabled: true
|
||||
egressGateway:
|
||||
enabled: true
|
||||
# --- Masquerade (REQUIRED by Egress Gateway) — Phase-1 crash fix ---
|
||||
# [parity] MUST equal Ansible: enableIPv4Masquerade: true + bpf.masquerade: true.
|
||||
# This exact pairing (ipv4 masquerade via eBPF, the correct companion to kubeProxyReplacement)
|
||||
# was the fix for the Phase-1 agent crash. Enabling one without the other regresses it —
|
||||
# keep both, and keep them equal to the Ansible values.
|
||||
enableIPv4Masquerade: true
|
||||
bpf:
|
||||
masquerade: true
|
||||
# --- Hubble (flow observability / isolation audit) ---
|
||||
# [parity] MUST equal Ansible: hubble.{enabled,relay.enabled,ui.enabled}: true
|
||||
hubble:
|
||||
enabled: true
|
||||
relay:
|
||||
@@ -49,7 +74,21 @@ spec:
|
||||
ui:
|
||||
enabled: true # exposure gated behind oauth2-proxy forward-auth later (ADR-0014)
|
||||
# --- L2 announcement for the ingress VIP on the flat LAN (docs/09, OQ-1 resolved) ---
|
||||
# [parity] MUST equal Ansible: l2announcements.enabled: true. IP pool +
|
||||
# CiliumL2AnnouncementPolicy (reserved 10.73.0.240-.250) authored by WS-NET in Phase 2.
|
||||
l2announcements:
|
||||
enabled: true
|
||||
# --- cgroup: k3s ships its own cgroup mount; point Cilium at it, don't auto-mount ---
|
||||
# [parity] MUST equal Ansible cgroup block. Not in the deliverable's headline list, but it IS
|
||||
# part of the working CNI config on k3s: reconciling without it lets Cilium try to auto-mount
|
||||
# cgroupv2 and break on adopt. Kept in parity for a safe Phase-2 adoption.
|
||||
cgroup:
|
||||
autoMount:
|
||||
enabled: false
|
||||
hostRoot: /run/cilium/cgroupv2
|
||||
# NOTE (NON-parity): operator.replicas is deliberately omitted here. The Ansible dev render sets
|
||||
# replicas: 1 to save RAM, but defaults/main.yml declares it explicitly NOT a parity field
|
||||
# (Flux/prod may differ). Only the [parity] values above must match the bootstrap.
|
||||
#
|
||||
# TODO(WS-NET, Phase 2): default-deny baseline, CiliumL2AnnouncementPolicy + IP pool,
|
||||
# Egress Gateway policies, Hubble metrics export to kube-prometheus-stack.
|
||||
+45
-11
@@ -1,13 +1,47 @@
|
||||
# Cilium Helm values — PLACEHOLDER skeleton (Phase 0). Authoritative values live inline in
|
||||
# helmrelease.yaml for now; this file documents the knobs WS-NET must set in Phase 1/2.
|
||||
# Cilium Helm values — REFERENCE COPY of the parity set (docs/02 Cilium-ownership contract).
|
||||
#
|
||||
# MUST match the Ansible bootstrap (docs/02) for the fields the bootstrap also sets.
|
||||
# The AUTHORITATIVE values that Flux reconciles are inlined in helmrelease.yaml. This file is a
|
||||
# readable, un-nested mirror of that value set so WS-NET can diff it against the live cluster and
|
||||
# the Ansible bootstrap render at a glance. Keep the two in sync.
|
||||
#
|
||||
# kubeProxyReplacement: true # k3s kube-proxy disabled (ADR-0001/0002)
|
||||
# k8sServiceHost / k8sServicePort # = Ansible bootstrap API endpoint (WS-INFRA contract)
|
||||
# gatewayAPI.enabled: true # ingress via Gateway API, no Traefik (docs/09)
|
||||
# egressGateway.enabled: true # egress-class steering (ADR-0002, VPN model ADR-0013)
|
||||
# hubble.{enabled,relay,ui} # flow observability + isolation audit
|
||||
# l2announcements.enabled: true # ingress VIP on flat LAN (docs/09, OQ-1)
|
||||
# ipam / ipv4NativeRoutingCIDR # TODO: pin to the cluster pod/LAN CIDRs at bring-up
|
||||
# encryption (wireguard/ipsec) # TODO(WS-SEC/WS-NET): decide east-west encryption later
|
||||
# SOURCE OF TRUTH (all [parity] fields MUST equal these):
|
||||
# infra/ansible/roles/cilium/templates/cilium-values.yaml.j2
|
||||
# infra/ansible/roles/cilium/defaults/main.yml (cilium_version: 1.16.5, k3s_api_host/port)
|
||||
#
|
||||
# chart version (helmrelease.yaml chart.spec.version): 1.16.5 # [parity] = cilium_version
|
||||
|
||||
# [parity] k3s kube-proxy disabled → Cilium replaces it (ADR-0001/0002)
|
||||
kubeProxyReplacement: true
|
||||
# [parity] = the k3s API endpoint (primary control-plane node IP) + port
|
||||
k8sServiceHost: "10.73.0.11"
|
||||
k8sServicePort: 6443
|
||||
# [parity] ingress via Gateway API, no Traefik (docs/09)
|
||||
gatewayAPI:
|
||||
enabled: true
|
||||
# [parity] egress-class steering (ADR-0002; VPN model ADR-0013)
|
||||
egressGateway:
|
||||
enabled: true
|
||||
# [parity] Egress Gateway REQUIRES masquerade; ipv4 + eBPF is the Phase-1 crash fix — keep BOTH
|
||||
enableIPv4Masquerade: true
|
||||
bpf:
|
||||
masquerade: true
|
||||
# [parity] flow observability / isolation audit
|
||||
hubble:
|
||||
enabled: true
|
||||
relay:
|
||||
enabled: true
|
||||
ui:
|
||||
enabled: true
|
||||
# [parity] ingress VIP announcement on the flat LAN (docs/09, OQ-1)
|
||||
l2announcements:
|
||||
enabled: true
|
||||
# [parity] k3s provides the cgroupv2 mount; do not auto-mount
|
||||
cgroup:
|
||||
autoMount:
|
||||
enabled: false
|
||||
hostRoot: /run/cilium/cgroupv2
|
||||
# NON-parity (Flux/prod may differ; dev sets 1 to save RAM) — omitted from the HelmRelease:
|
||||
# operator.replicas
|
||||
#
|
||||
# TODO (Phase 2, WS-NET): ipam / ipv4NativeRoutingCIDR pinned to cluster CIDRs; east-west
|
||||
# encryption (wireguard/ipsec) decision (WS-SEC/WS-NET).
|
||||
+59
-16
@@ -4,28 +4,71 @@
|
||||
|
||||
- **Governing ADR:** [ADR-0003 — Distributed storage: Longhorn (+ bulk tier + Velero)](../../../home-network-k3s/decisions/adr-0003-storage-longhorn.md)
|
||||
- **Spec:** [docs/04 — Storage & State](../../../home-network-k3s/docs/04-storage-and-state.md)
|
||||
- **Serves:** R-STATE-01..04
|
||||
- **Serves:** R-STATE-01, R-STATE-02, R-STATE-04 (R-STATE-03 is Velero's, Phase 4)
|
||||
- **Owner:** WS-STORAGE
|
||||
|
||||
## What this stub is
|
||||
## What this is
|
||||
|
||||
Flux `HelmRelease` + two `StorageClass`es:
|
||||
- **`longhorn`** — the **default**, **3-replica** replicated class (replica count is the
|
||||
`${LONGHORN_DEFAULT_REPLICA_COUNT}` overlay knob). Durable home for app/config state and KubeVirt
|
||||
VM disks (via CDI).
|
||||
- **`bulk`** — non-replicated tier for large media (Plex library, download staging). **STUB**: its
|
||||
backing store is **OQ-2 (unresolved)** — escalate before finalizing. Plex's metadata DB stays on
|
||||
the 3-replica class regardless (ADR-0003).
|
||||
A real, dev-appropriate Flux `HelmRelease` for Longhorn plus two `StorageClass`es:
|
||||
|
||||
- **`longhorn`** — the cluster's **only default** StorageClass, **3 replicas** with **hard node
|
||||
anti-affinity** (one replica per node). Durable home for app/config state, databases, and
|
||||
KubeVirt VM disks (via CDI). Replica count is the `${LONGHORN_DEFAULT_REPLICA_COUNT}` overlay
|
||||
knob (cluster-vars, =3 in dev).
|
||||
- **`bulk`** — thin indirection for the **external** (outside-cluster) media tier: Plex library,
|
||||
download staging, arr handoff. **STUB placeholder** modelled as an external NFS mount; its backing
|
||||
is **OQ-2 (unresolved)** and will not provision until answered. Plex's metadata DB stays on the
|
||||
3-replica `longhorn` class regardless (docs/04, ADR-0003).
|
||||
|
||||
## Requirement mapping
|
||||
|
||||
| Requirement | Summary | How this component serves it |
|
||||
|---|---|---|
|
||||
| **R-STATE-01** | Stateful data survives destroy/rebuild of the workload | Volumes are Longhorn PVCs, independent of pod/VM lifecycle; `reclaimPolicy: Delete` on the class ties data to the PVC, not the workload. Destroy the pod/Deployment/VM → PVC + data persist → rebuild re-attaches. (Per-service rebuild test is the Phase-2 exit gate; T-1.5 proves node-level rebuild.) |
|
||||
| **R-STATE-02** | Persistent storage replicated across the 3 nodes | `numberOfReplicas=3` on the default class + `replicaSoftAntiAffinity: false` (hard) → exactly one replica per node, so any single node loss keeps volumes available. Replicas live on the dedicated `/var/lib/longhorn` data disk. `replicaAutoBalance: best-effort` re-spreads on node return. |
|
||||
| **R-STATE-04** | Bulk media has a capacity strategy distinct from the replicated tier | Separate `bulk` StorageClass (NON-replicated, external NFS mount) keeps multi-TB media off the 3x tier; only Plex's small metadata DB uses `longhorn`. |
|
||||
|
||||
## Dev-scoped choices (change for prod)
|
||||
|
||||
- **`reclaimPolicy: Delete`** on `longhorn` — convenient in dev; prod may prefer `Retain` for
|
||||
accidental-delete protection (backups still cover DR — R-STATE-03).
|
||||
- **`storageOverProvisioningPercentage: 100`** — no thin over-provisioning in dev (small disks);
|
||||
prod may raise it.
|
||||
- **`storageReservedPercentageForDefaultDisk: 20` / `storageMinimalAvailablePercentage: 25`** —
|
||||
headroom guardrails tuned for small dev disks; revisit for prod disk sizes.
|
||||
- **Replica count = 3** comes from `cluster-vars.longhorn_default_replica_count`; both dev and prod
|
||||
are 3 today (3 nodes), so the value is identical — the knob exists so it is never forked per env.
|
||||
|
||||
## Guardrails baked in (docs/04)
|
||||
|
||||
- Reserved disk headroom (never fill a node to 100%).
|
||||
- Hard replica anti-affinity across nodes (no two replicas of a volume on one node).
|
||||
- Single-zone anti-affinity relaxed (3 mini-PCs = one failure zone).
|
||||
- Volume/disk health + degraded-replica alerting is WS-OBS's job (docs/10) — not configured here.
|
||||
|
||||
## Node prerequisites (already handled, informational)
|
||||
|
||||
Each node presents a dedicated Longhorn data disk (`/dev/vdb` → ext4) mounted at
|
||||
`/var/lib/longhorn`, and installs `open-iscsi` + `nfs-common` + kernel modules — all via
|
||||
`infra/cloud-init/user-data.yaml.tftpl`. This component assumes that floor.
|
||||
|
||||
## Contract (consumed by everyone with state)
|
||||
|
||||
Per `roadmap/workstreams.md`, WS-STORAGE exposes the `longhorn` (replicated) and `bulk` (media)
|
||||
StorageClasses; WS-APPS/WS-VM/WS-OBS reference them from ServiceDefinitions. Do not place bulk media
|
||||
on the replicated tier or app DBs on the bulk tier.
|
||||
StorageClasses; WS-APPS/WS-VM/WS-OBS reference them from ServiceDefinitions. **Do not place bulk
|
||||
media on the replicated tier, or app DBs on the bulk tier.**
|
||||
|
||||
## TODO (later phases)
|
||||
## Open decisions / TODO (later phases)
|
||||
|
||||
- **OQ-2 (human):** decide the bulk-tier backing (NAS/NFS vs dedicated disk). Blocks `bulk` finalize.
|
||||
- **Phase 2 (WS-STORAGE):** dedicated Longhorn data disk path (docs/02); prove a PVC survives
|
||||
pod destroy/rebuild (R-STATE-01, Phase-2 exit gate).
|
||||
- **Phase 4 (WS-STORAGE):** wire off-cluster backup target with Velero (R-STATE-03).
|
||||
- **OQ-2 (human):** decide the `bulk`-tier backing (NAS/NFS host + share, vs dedicated large disk).
|
||||
Until answered, the `bulk` class is a non-functional placeholder (`server`/`share` = REPLACE-ME).
|
||||
Blocks `bulk` finalize; does **not** block the core platform (docs/04).
|
||||
- **Version pin (WS-STORAGE):** `>=1.7.0 <1.8.0` is a range; pin exactly at bring-up, Renovate
|
||||
maintains.
|
||||
- **Phase 4 (WS-STORAGE):** off-cluster backup target via `defaultSettings.backupTarget` +
|
||||
SOPS-encrypted credential secret, wired alongside `platform/velero/` (R-STATE-03).
|
||||
|
||||
## Build check
|
||||
|
||||
`kubectl kustomize platform/longhorn` renders the namespace, HelmRepository, HelmRelease, and both
|
||||
StorageClasses. (Structural only — not applied to a cluster here; the PM validates + deploys.)
|
||||
@@ -1,5 +1,9 @@
|
||||
---
|
||||
# Longhorn — distributed replicated block storage (ADR-0003). Phase 0 STUB.
|
||||
# Longhorn — distributed replicated block storage across the 3 nodes (ADR-0003).
|
||||
# Serves R-STATE-01 (state survives destroy/rebuild) and R-STATE-02 (3-way replication).
|
||||
#
|
||||
# Dev-appropriate values. Environment differences enter ONLY via ${...} substitution from
|
||||
# cluster-vars (R-ENV-02); everything else is identical base config for dev and prod.
|
||||
apiVersion: helm.toolkit.fluxcd.io/v2
|
||||
kind: HelmRelease
|
||||
metadata:
|
||||
@@ -16,21 +20,60 @@ spec:
|
||||
name: longhorn
|
||||
namespace: longhorn-system
|
||||
install:
|
||||
createNamespace: false
|
||||
createNamespace: false # namespace.yaml owns it (GitOps-managed, labelled)
|
||||
remediation:
|
||||
retries: 3
|
||||
upgrade:
|
||||
remediation:
|
||||
retries: 3
|
||||
values:
|
||||
# We define StorageClasses ourselves (storageclasses.yaml) so the default 3-replica SC and the
|
||||
# bulk tier are explicit and GitOps-managed. Disable the chart's built-in default SC.
|
||||
# We own the StorageClasses (storageclasses.yaml) so the default 3-replica class and the bulk
|
||||
# tier are explicit and GitOps-managed. Disable the chart's built-in default SC so there is
|
||||
# exactly one default-class annotation in the cluster.
|
||||
persistence:
|
||||
defaultClass: false
|
||||
# ${LONGHORN_DEFAULT_REPLICA_COUNT} resolved from cluster-vars (R-ENV-02).
|
||||
defaultClassReplicaCount: ${LONGHORN_DEFAULT_REPLICA_COUNT}
|
||||
|
||||
defaultSettings:
|
||||
# TODO(WS-STORAGE, Phase 2): set defaultDataPath to the dedicated Longhorn data disk
|
||||
# (docs/02 — a dedicated data disk is presented to Longhorn on each node).
|
||||
# --- data path (R-STATE-02) ---
|
||||
# Dedicated Longhorn data disk (/dev/vdb → ext4) is mounted here by cloud-init on every node
|
||||
# (infra/cloud-init/user-data.yaml.tftpl). Longhorn stores replicas on this disk only.
|
||||
defaultDataPath: "/var/lib/longhorn"
|
||||
# TODO(WS-STORAGE, Phase 4): configure off-cluster backup target here (see velero/ + ADR-0003).
|
||||
|
||||
# --- replication (R-STATE-02) ---
|
||||
# Global fallback replica count. The `longhorn` StorageClass sets numberOfReplicas explicitly
|
||||
# (authoritative); this keeps any volume created without an explicit count consistent.
|
||||
# Substituted from cluster-vars.longhorn_default_replica_count (=3 in dev).
|
||||
defaultReplicaCount: ${LONGHORN_DEFAULT_REPLICA_COUNT}
|
||||
|
||||
# --- replica anti-affinity across nodes (R-STATE-02 durability guarantee) ---
|
||||
# HARD anti-affinity: never place two replicas of the same volume on the same node, so a
|
||||
# single node loss cannot take down more than one replica. On a 3-node / 3-replica cluster
|
||||
# this yields exactly one replica per node.
|
||||
replicaSoftAntiAffinity: false
|
||||
# Single zone (3 mini-PCs, no failure zones): allow replicas within the one zone.
|
||||
replicaZoneSoftAntiAffinity: true
|
||||
# When a downed node returns, rebalance replicas back toward one-per-node (best-effort so it
|
||||
# never blocks scheduling).
|
||||
replicaAutoBalance: "best-effort"
|
||||
|
||||
# --- capacity guardrails (docs/04 "reserve disk headroom") ---
|
||||
# Reserve 20% of the dedicated data disk so a node never fills to 100% (which wedges the
|
||||
# kubelet and Longhorn alike). Dev disks are small, so headroom matters.
|
||||
storageReservedPercentageForDefaultDisk: "20"
|
||||
# Stop scheduling new replicas onto a disk once free space drops below 25%.
|
||||
storageMinimalAvailablePercentage: "25"
|
||||
# Dev: no thin over-provisioning — requested capacity must physically fit (1:1).
|
||||
storageOverProvisioningPercentage: "100"
|
||||
|
||||
# --- misc dev-appropriate defaults ---
|
||||
# Keep replicas spread for durability rather than co-locating with the pod.
|
||||
defaultDataLocality: "disabled"
|
||||
# Reclaim replica space of deleted/failed volumes promptly in dev.
|
||||
orphanAutoDeletion: true
|
||||
|
||||
# Longhorn's own manager/UI stay in-cluster; UI exposure (if any) is gated behind
|
||||
# oauth2-proxy forward-auth (ADR-0014) by WS-SEC — not exposed here.
|
||||
|
||||
# TODO(WS-STORAGE, Phase 4): off-cluster backup target (defaultSettings.backupTarget +
|
||||
# backupTargetCredentialSecret) is configured alongside Velero (ADR-0003, R-STATE-03).
|
||||
# Credentials referenced BY NAME only, SOPS-encrypted (WS-SEC) — never inline here.
|
||||
@@ -1,6 +1,10 @@
|
||||
---
|
||||
# Default replicated StorageClass — 3 replicas (ADR-0003). The durable home for app/config state
|
||||
# and KubeVirt VM disks. Replica count is an overlay knob (R-ENV-02).
|
||||
# Default replicated StorageClass — 3 replicas (ADR-0003, R-STATE-02). The durable home for
|
||||
# app/config state, databases, and KubeVirt VM disks (via CDI). Destroying a pod/VM leaves the PVC
|
||||
# and its data intact; rebuilding re-attaches it (R-STATE-01).
|
||||
#
|
||||
# This is the cluster's ONLY default StorageClass (the chart's built-in default is disabled in
|
||||
# helmrelease.yaml), so PVCs without an explicit storageClassName land here.
|
||||
apiVersion: storage.k8s.io/v1
|
||||
kind: StorageClass
|
||||
metadata:
|
||||
@@ -9,26 +13,40 @@ metadata:
|
||||
storageclass.kubernetes.io/is-default-class: "true"
|
||||
provisioner: driver.longhorn.io
|
||||
allowVolumeExpansion: true
|
||||
reclaimPolicy: Delete
|
||||
reclaimPolicy: Delete # dev: reclaim space when the PVC is deleted
|
||||
volumeBindingMode: Immediate
|
||||
parameters:
|
||||
numberOfReplicas: "${LONGHORN_DEFAULT_REPLICA_COUNT}" # from cluster-vars (default 3)
|
||||
staleReplicaTimeout: "30"
|
||||
# 3 replicas across 3 nodes — substituted from cluster-vars.longhorn_default_replica_count.
|
||||
# HARD node anti-affinity in helmrelease.yaml (replicaSoftAntiAffinity=false) guarantees these
|
||||
# replicas land on distinct nodes, so any single node loss keeps the volume available.
|
||||
numberOfReplicas: "${LONGHORN_DEFAULT_REPLICA_COUNT}"
|
||||
staleReplicaTimeout: "30" # minutes before a down replica is considered stale and rebuilt
|
||||
dataLocality: "disabled" # keep replicas spread for durability, not co-located with the pod
|
||||
fsType: "ext4"
|
||||
---
|
||||
# Bulk media tier — large, NON-replicated data (Plex library, download staging). ADR-0003 / docs/04.
|
||||
# NOTE: backing store (NAS/NFS vs dedicated large disk) is OQ-2, unresolved — this is a STUB.
|
||||
# Plex's metadata DB stays on the 3-replica `longhorn` class regardless (ADR-0003).
|
||||
# Bulk media tier — large, NON-replicated data (Plex library, download staging, arr handoff).
|
||||
# docs/04 / R-STATE-04: 3x replicating multi-TB media across 3 mini-PCs is impractical, so bulk
|
||||
# media lives OUTSIDE the cluster and is consumed as an external NFS/SMB mount via this thin
|
||||
# indirection. Services reference storageClassName: bulk; the concrete backing is filled in later
|
||||
# WITHOUT touching service definitions.
|
||||
#
|
||||
# ⚠️ STUB — OQ-2 UNRESOLVED (human operator decision). The provisioner + parameters below are
|
||||
# PLACEHOLDERS: they name the standard external NFS CSI driver but the server/share are REPLACE-ME.
|
||||
# This class will NOT provision until OQ-2 is answered and the NFS CSI driver is installed. That is
|
||||
# intentional — per docs/04 the bulk backing does not block the core platform.
|
||||
#
|
||||
# Plex's config/metadata DB is small and MUST stay on the 3-replica `longhorn` class regardless
|
||||
# (docs/04, ADR-0003) — only the large media files use `bulk`.
|
||||
apiVersion: storage.k8s.io/v1
|
||||
kind: StorageClass
|
||||
metadata:
|
||||
name: bulk
|
||||
provisioner: driver.longhorn.io # TODO(WS-STORAGE): may become an NFS provisioner per OQ-2
|
||||
provisioner: nfs.csi.k8s.io # TODO(OQ-2): external NFS/SMB backing — driver + endpoint TBD
|
||||
allowVolumeExpansion: true
|
||||
reclaimPolicy: Retain
|
||||
reclaimPolicy: Retain # bulk media is precious/reacquirable — never auto-delete on PVC removal
|
||||
volumeBindingMode: Immediate
|
||||
parameters:
|
||||
numberOfReplicas: "1" # non-replicated bulk tier (capacity, not durability)
|
||||
staleReplicaTimeout: "30"
|
||||
fsType: "ext4"
|
||||
# TODO(WS-STORAGE): finalize once OQ-2 (bulk backing) is answered by the human operator.
|
||||
server: REPLACE-ME # TODO(OQ-2): NAS/NFS host holding the bulk media share
|
||||
share: REPLACE-ME # TODO(OQ-2): exported path (e.g. /export/media)
|
||||
mountOptions:
|
||||
- nfsvers=4.1
|
||||
@@ -1,7 +1,21 @@
|
||||
# Longhorn Helm values — PLACEHOLDER (Phase 0). Authoritative values inline in helmrelease.yaml.
|
||||
# Longhorn Helm values — REFERENCE ONLY (not consumed by kustomization.yaml).
|
||||
#
|
||||
# persistence.defaultClass: false # we own StorageClasses (storageclasses.yaml)
|
||||
# persistence.defaultClassReplicaCount: ${LONGHORN_DEFAULT_REPLICA_COUNT} # overlay knob
|
||||
# defaultSettings.defaultDataPath: <data disk> # TODO(WS-STORAGE): dedicated Longhorn disk (docs/02)
|
||||
# defaultSettings.backupTarget: <off-cluster> # TODO(WS-STORAGE, Phase 4): DR target (ADR-0003)
|
||||
# Longhorn UI exposure gated behind oauth2-proxy forward-auth (ADR-0014).
|
||||
# The authoritative values are inline in helmrelease.yaml (matching the sibling pattern across
|
||||
# platform/ — cilium, cert-manager, velero, etc.). This file documents the knobs that matter so a
|
||||
# reviewer can see intent without reading the HelmRelease. Keep it in sync with helmrelease.yaml.
|
||||
#
|
||||
# persistence:
|
||||
# defaultClass: false # we own StorageClasses (storageclasses.yaml)
|
||||
#
|
||||
# defaultSettings:
|
||||
# defaultDataPath: /var/lib/longhorn # dedicated data disk (infra/cloud-init)
|
||||
# defaultReplicaCount: ${LONGHORN_DEFAULT_REPLICA_COUNT} # overlay knob (=3 in dev)
|
||||
# replicaSoftAntiAffinity: false # HARD anti-affinity: one replica per node
|
||||
# replicaZoneSoftAntiAffinity: true # single zone (3 mini-PCs, no failure zones)
|
||||
# replicaAutoBalance: best-effort # rebalance toward one-per-node when a node returns
|
||||
# storageReservedPercentageForDefaultDisk: "20" # reserve disk headroom (docs/04 guardrail)
|
||||
# storageMinimalAvailablePercentage: "25" # stop scheduling below 25% free
|
||||
# storageOverProvisioningPercentage: "100" # dev: no thin over-provisioning (1:1)
|
||||
# defaultDataLocality: disabled # keep replicas spread for durability
|
||||
# orphanAutoDeletion: true # reclaim orphaned replica space promptly
|
||||
# backupTarget: <off-cluster> # TODO(WS-STORAGE, Phase 4): DR target (ADR-0003)
|
||||
+104
-18
@@ -1,31 +1,117 @@
|
||||
# platform/monitoring
|
||||
|
||||
**Observability stack** — kube-prometheus-stack (Prometheus + Alertmanager + Grafana) + **Loki** +
|
||||
**Grafana Alloy**.
|
||||
**Grafana Alloy** + platform-health **PrometheusRules**.
|
||||
|
||||
- **Spec:** [docs/10 — Observability & Alerting](../../../home-network-k3s/docs/10-observability-and-alerting.md)
|
||||
(named in [docs/01](../../../home-network-k3s/docs/01-architecture-overview.md); no dedicated ADR)
|
||||
- **Serves:** R-OBS-01..04
|
||||
- **Serves:** R-OBS-01/02/03/04 (supports R-C2-02, R-SEC-03, R-STATE-03)
|
||||
- **Owner:** WS-OBS
|
||||
|
||||
## What this stub is
|
||||
## Contents
|
||||
|
||||
Three Flux `HelmRelease`s (kube-prometheus-stack, loki, alloy) with placeholder values. Retention and
|
||||
PVC sizes are overlay knobs (`${MONITORING_*}`, R-ENV-02): short/small in dev, long/large in prod.
|
||||
Persistent data uses the replicated `longhorn` StorageClass.
|
||||
| File | What |
|
||||
|---|---|
|
||||
| `namespace.yaml` | `monitoring` namespace |
|
||||
| `helmrepositories.yaml` | `prometheus-community` + `grafana` Flux `HelmRepository`s |
|
||||
| `kube-prometheus-stack.yaml` | `HelmRelease` — Prometheus, Alertmanager (→ SMTP email), Grafana, kube-state-metrics, node-exporter |
|
||||
| `loki.yaml` | `HelmRelease` — Loki single-binary, filesystem, compactor-enforced retention |
|
||||
| `alloy.yaml` | `HelmRelease` — Grafana Alloy DaemonSet, tails pod logs → Loki |
|
||||
| `alerts.yaml` | Platform-health `PrometheusRule`s (R-OBS-04) |
|
||||
| `kustomization.yaml` | builds all of the above |
|
||||
|
||||
## Contract (observability)
|
||||
## Requirement mapping
|
||||
|
||||
The "observability contract" (`roadmap/workstreams.md`): how a service declares metrics/SLIs/alerts
|
||||
and gets dashboards. ServiceMonitors/dashboards are rendered from ServiceDefinitions by WS-APPS/WS-VM
|
||||
against this stack. Grafana is part of the C2 UI composition (docs/12), behind Keycloak SSO
|
||||
(ADR-0014).
|
||||
- **R-OBS-01 (what is monitored):** Prometheus + node-exporter + kube-state-metrics for metrics;
|
||||
Alloy → Loki for logs. `serviceMonitorSelectorNilUsesHelmValues: false` so ServiceMonitors from
|
||||
every namespace (and those rendered from ServiceDefinitions, docs/13) are scraped by default.
|
||||
- **R-OBS-02 (alert delivery):** Alertmanager routes by `severity` (`critical` vs `warning`) to
|
||||
**email**. SMTP creds are a SOPS secret referenced **by name only** (see below).
|
||||
- **R-OBS-03 (dashboards):** Grafana, with the dashboard/datasource sidecar watching
|
||||
`grafana_dashboard`-labeled ConfigMaps across all namespaces (per-service dashboards land here).
|
||||
Grafana is part of the C2 UI composition (docs/12), later behind Keycloak SSO (ADR-0014).
|
||||
- **R-OBS-04 (platform health):** `alerts.yaml` — first-class alerts for **nodes**, **Longhorn**,
|
||||
**Flux**, **cert-manager**, **networking/VPN/Hubble**, **Velero backups**, **Keycloak/IdP**, and
|
||||
the **monitor's own capacity**. Split `critical`/`warning` per the docs/10 severity model.
|
||||
|
||||
## TODO (later phases)
|
||||
## SMTP secret (operator hand-off)
|
||||
|
||||
- **Phase 4 (WS-OBS):** Alertmanager → Gmail SMTP receiver (creds SOPS-encrypted, name only);
|
||||
Loki retention via compactor (`retention_enabled: true` — a period alone does nothing); Alloy log
|
||||
scrape config → Loki; platform-health alerts. **Reconciliation errors + drift are alerting
|
||||
conditions (R-OBS-04).**
|
||||
- **Exit gate (Phase 4):** an injected fault emails an alert; dashboards render (R-OBS-01/02/03/04).
|
||||
- **Grafana OIDC** client via Keycloak (WS-SEC/WS-C2).
|
||||
Alertmanager reads the SMTP **password** from a mounted SOPS-encrypted Secret — referenced by name,
|
||||
never inlined. The operator must provide (out of band, SOPS-encrypted per docs/07):
|
||||
|
||||
```
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: alertmanager-smtp # <-- referenced in kube-prometheus-stack.yaml (alertmanagerSpec.secrets)
|
||||
namespace: monitoring
|
||||
stringData:
|
||||
smtp-password: "<app password>" # mounted at /etc/alertmanager/secrets/alertmanager-smtp/smtp-password
|
||||
```
|
||||
|
||||
The **non-secret** SMTP fields (`smtp_smarthost`, `smtp_from`, `smtp_auth_username`, and the
|
||||
receiver `to:` address) are `REPLACE-ME` placeholders in `kube-prometheus-stack.yaml` for the
|
||||
operator to fill (same convention as `velero/`). Grafana's admin bootstrap likewise references a
|
||||
SOPS secret **by name** — `grafana-admin-credentials` (keys `admin-user` / `admin-password`).
|
||||
|
||||
## Dev trims (the ~22 GiB RAM ceiling)
|
||||
|
||||
This subtree is authored to stay light on the single dev workstation:
|
||||
|
||||
- **Loki `chunksCache`/`resultsCache` DISABLED** — the grafana/loki chart's default memcached caches
|
||||
request **~8-9 GiB each**; leaving them on would blow the budget by itself. Also disabled: the
|
||||
nginx `gateway`, `lokiCanary`, and `test` hooks. Loki runs **SingleBinary** (one pod).
|
||||
- **Single replicas** everywhere (Prometheus, Alertmanager, Grafana, Loki).
|
||||
- **Modest requests/limits** on every component (see values). Ephemeral (emptyDir) state for
|
||||
Alertmanager and Grafana in dev.
|
||||
- **k3s control-plane scrape jobs disabled** (`kubeControllerManager`/`kubeScheduler`/`kubeEtcd`/
|
||||
`kubeProxy`) — embedded in k3s, not separately scrapable; keeps targets green and RAM down.
|
||||
|
||||
Approximate steady-state **memory requests** (what the scheduler reserves):
|
||||
|
||||
| Component | Req | Notes |
|
||||
|---|---:|---|
|
||||
| Prometheus | 384Mi | limit 1Gi |
|
||||
| Alertmanager | 32Mi | |
|
||||
| Grafana | 128Mi | |
|
||||
| prometheus-operator | 96Mi | |
|
||||
| kube-state-metrics | 64Mi | |
|
||||
| node-exporter ×3 | 96Mi | 32Mi/node DaemonSet |
|
||||
| Loki (single binary) | 192Mi | limit 512Mi |
|
||||
| Alloy ×3 | 288Mi | 96Mi/node DaemonSet |
|
||||
| **Total requests** | **~1.3 GiB** | working set typically ~1.5-2.5 GiB; limit ceiling ~3 GiB |
|
||||
|
||||
Comfortably within the 22 GiB box alongside the rest of the platform.
|
||||
|
||||
## Environment knobs (R-ENV-02)
|
||||
|
||||
Only three substitution vars touch this subtree; the dev overlay (`clusters/dev`) patches
|
||||
`cluster-vars` to dev values:
|
||||
|
||||
| Var | base | dev |
|
||||
|---|---|---|
|
||||
| `${MONITORING_PROMETHEUS_RETENTION}` | `7d` | `3d` |
|
||||
| `${MONITORING_PROMETHEUS_PVC_SIZE}` | `10Gi` | `5Gi` |
|
||||
| `${MONITORING_LOKI_PVC_SIZE}` | `10Gi` | `5Gi` |
|
||||
|
||||
PVCs use the replicated `longhorn` StorageClass (ADR-0003).
|
||||
|
||||
## Notes / known gaps for bring-up
|
||||
|
||||
- **Loki log retention has no dedicated knob.** Only the three vars above exist. Loki's
|
||||
`retention_period` is hardcoded to `72h` (compactor-enforced) to mirror dev metrics retention.
|
||||
Promote to a `cluster-vars` knob (in `clusters/`, out of this subtree) if per-env log retention is
|
||||
wanted.
|
||||
- Several R-OBS-04 rules depend on exporters/ServiceMonitors owned by **other** workstreams (Flux,
|
||||
cert-manager, Longhorn, Velero, Keycloak, Cilium/Hubble). Rules are authored to fire once those
|
||||
series arrive; metric names marked `TODO(WS-*)` in `alerts.yaml` (notably the **egress VPN**
|
||||
metric) need confirming against the deployed exporter.
|
||||
- **Hubble** network-flow visibility itself is delivered by `platform/cilium` (docs/03); this subtree
|
||||
only consumes its metrics for denied-flow alerts.
|
||||
|
||||
## Validate (no deploy)
|
||||
|
||||
```
|
||||
kubectl kustomize platform/monitoring
|
||||
```
|
||||
|
||||
Author-only — nothing here is applied to a cluster; Flux reconciles it at deploy time.
|
||||
@@ -0,0 +1,369 @@
|
||||
---
|
||||
# Platform-health alert rules (docs/10, R-OBS-04). These complement the default node/kubernetes
|
||||
# rules bundled by kube-prometheus-stack and add FIRST-CLASS alerts for the platform's own moving
|
||||
# parts: nodes, Longhorn, Flux, cert-manager, networking/VPN, backups, identity, and the monitoring
|
||||
# stack's own capacity ("don't let the monitor blind itself").
|
||||
#
|
||||
# Each rule maps to a docs/10 platform-health bullet and carries a severity (critical = page-worthy
|
||||
# vs warning = degraded/capacity) plus enough annotation context (summary + runbook) to act.
|
||||
#
|
||||
# The `release: kube-prometheus-stack` label makes the operator's Prometheus select these rules
|
||||
# (ruleSelector). Some expressions depend on exporters/ServiceMonitors owned by other workstreams
|
||||
# (Flux, cert-manager, Longhorn, Velero, Keycloak, Cilium/Hubble ship their own metrics); the rules
|
||||
# are authored to fire as soon as those series arrive. Metric names flagged TODO need confirming at
|
||||
# bring-up against the deployed exporter.
|
||||
#
|
||||
# runbook_url points at a docs base; swap for the real runbook location when it exists.
|
||||
|
||||
# =====================================================================================
|
||||
# Nodes
|
||||
# =====================================================================================
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: platform-health-nodes
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
platform-health: "true"
|
||||
spec:
|
||||
groups:
|
||||
- name: platform-health.nodes
|
||||
rules:
|
||||
- alert: NodeExporterDown
|
||||
expr: up{job="node-exporter"} == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Node exporter down on {{ $labels.instance }}"
|
||||
description: "Prometheus cannot scrape node-exporter on {{ $labels.instance }} for 5m — the node may be down or unreachable."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/node-down"
|
||||
- alert: NodeMemoryPressureHigh
|
||||
expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) > 0.90
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Memory pressure high on {{ $labels.instance }}"
|
||||
description: "Available memory below 10% for 10m on {{ $labels.instance }} — the ~22 GiB box is under pressure."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/node-memory"
|
||||
- alert: NodeDiskFillingUp
|
||||
expr: |
|
||||
(node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs"}
|
||||
/ node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs"}) < 0.10
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Disk filling up on {{ $labels.instance }} ({{ $labels.mountpoint }})"
|
||||
description: "Less than 10% free on {{ $labels.mountpoint }} of {{ $labels.instance }} for 15m."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/node-disk"
|
||||
|
||||
# =====================================================================================
|
||||
# Storage — Longhorn
|
||||
# =====================================================================================
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: platform-health-longhorn
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
platform-health: "true"
|
||||
spec:
|
||||
groups:
|
||||
- name: platform-health.longhorn
|
||||
rules:
|
||||
# longhorn_volume_robustness: 0=unknown 1=healthy 2=degraded 3=faulted
|
||||
- alert: LonghornVolumeDegraded
|
||||
expr: longhorn_volume_robustness == 2
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Longhorn volume degraded: {{ $labels.volume }}"
|
||||
description: "Volume {{ $labels.volume }} has been degraded (replica rebuild / missing replica) for 10m."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/longhorn-degraded"
|
||||
- alert: LonghornVolumeFaulted
|
||||
expr: longhorn_volume_robustness == 3
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Longhorn volume FAULTED: {{ $labels.volume }}"
|
||||
description: "Volume {{ $labels.volume }} is faulted — data at risk / offline. Immediate attention required."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/longhorn-faulted"
|
||||
- alert: LonghornVolumeCapacityHigh
|
||||
expr: (longhorn_volume_actual_size_bytes / longhorn_volume_capacity_bytes) > 0.85
|
||||
for: 30m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Longhorn volume near capacity: {{ $labels.volume }}"
|
||||
description: "Volume {{ $labels.volume }} is over 85% used for 30m."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/longhorn-capacity"
|
||||
- alert: LonghornNodeStorageHigh
|
||||
expr: (longhorn_node_storage_usage_bytes / longhorn_node_storage_capacity_bytes) > 0.85
|
||||
for: 30m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Longhorn node storage near capacity: {{ $labels.node }}"
|
||||
description: "Longhorn node {{ $labels.node }} is over 85% of its storage capacity for 30m."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/longhorn-capacity"
|
||||
|
||||
# =====================================================================================
|
||||
# GitOps — Flux
|
||||
# =====================================================================================
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: platform-health-flux
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
platform-health: "true"
|
||||
spec:
|
||||
groups:
|
||||
- name: platform-health.flux
|
||||
rules:
|
||||
- alert: FluxReconciliationFailure
|
||||
expr: |
|
||||
max by (namespace, name, kind) (
|
||||
gotk_reconcile_condition{type="Ready", status="False"}
|
||||
) == 1
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Flux resource not ready: {{ $labels.kind }}/{{ $labels.name }}"
|
||||
description: "{{ $labels.kind }} {{ $labels.namespace }}/{{ $labels.name }} has failed to reconcile (Ready=False) for 15m — GitOps drift/failure (R-GITOPS-03)."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/flux-reconcile"
|
||||
- alert: FluxResourceSuspended
|
||||
expr: |
|
||||
max by (namespace, name, kind) (gotk_suspend_status) == 1
|
||||
for: 1h
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Flux resource suspended: {{ $labels.kind }}/{{ $labels.name }}"
|
||||
description: "{{ $labels.kind }} {{ $labels.namespace }}/{{ $labels.name }} has been suspended for 1h — reconciliation/drift-correction is off."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/flux-suspended"
|
||||
|
||||
# =====================================================================================
|
||||
# Certificates — cert-manager
|
||||
# =====================================================================================
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: platform-health-certificates
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
platform-health: "true"
|
||||
spec:
|
||||
groups:
|
||||
- name: platform-health.certificates
|
||||
rules:
|
||||
- alert: CertificateExpiringSoon
|
||||
expr: (certmanager_certificate_expiration_timestamp_seconds - time()) < (14 * 24 * 3600)
|
||||
for: 1h
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Certificate expiring soon: {{ $labels.name }}"
|
||||
description: "Certificate {{ $labels.namespace }}/{{ $labels.name }} expires in under 14 days."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/cert-expiry"
|
||||
- alert: CertificateExpiryImminent
|
||||
expr: (certmanager_certificate_expiration_timestamp_seconds - time()) < (3 * 24 * 3600)
|
||||
for: 1h
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Certificate expiry imminent: {{ $labels.name }}"
|
||||
description: "Certificate {{ $labels.namespace }}/{{ $labels.name }} expires in under 3 days and has not renewed."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/cert-expiry"
|
||||
- alert: CertificateNotReady
|
||||
expr: certmanager_certificate_ready_status{condition="False"} == 1
|
||||
for: 1h
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Certificate not ready / issuance failing: {{ $labels.name }}"
|
||||
description: "Certificate {{ $labels.namespace }}/{{ $labels.name }} has been in a not-ready state for 1h (ACME issuance failure?)."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/cert-issuance"
|
||||
|
||||
# =====================================================================================
|
||||
# Networking — VPN / egress-gateway / denied flows (Hubble)
|
||||
# =====================================================================================
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: platform-health-networking
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
platform-health: "true"
|
||||
spec:
|
||||
groups:
|
||||
- name: platform-health.networking
|
||||
rules:
|
||||
# VPN tunnel health. The concrete metric depends on the egress-gateway VPN sidecar chosen by
|
||||
# WS-NET (docs/03). Written against a gluetun-style up gauge; confirm/adjust at bring-up.
|
||||
- alert: VPNTunnelDown
|
||||
# TODO(WS-NET): confirm metric name of the egress VPN exporter (docs/03).
|
||||
expr: max(gluetun_vpn_up) == 0 or absent(gluetun_vpn_up)
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Egress VPN tunnel down"
|
||||
description: "The egress-gateway VPN tunnel is down (or its metric is absent) for 5m — outbound traffic could leak or stall (R-SEC-03 leak risk)."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/vpn-down"
|
||||
- alert: EgressGatewayUnhealthy
|
||||
expr: |
|
||||
max by (pod) (kube_pod_status_ready{namespace="egress", condition="true"}) == 0
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Egress gateway unhealthy"
|
||||
description: "No ready egress-gateway pod for 10m — VPN-scoped egress is unavailable."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/egress-gateway"
|
||||
- alert: HubbleDeniedFlowSpike
|
||||
# Cilium/Hubble denied (policy-drop) flows. Requires hubble metrics enabled (docs/03).
|
||||
expr: sum(rate(hubble_drop_total{reason="POLICY_DENIED"}[5m])) > 5
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Spike in policy-denied network flows"
|
||||
description: "Policy-denied flow rate is elevated for 10m — a NetworkPolicy regression or an intrusion probe (R-SEC-03)."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/denied-flows"
|
||||
|
||||
# =====================================================================================
|
||||
# Backups — Velero
|
||||
# =====================================================================================
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: platform-health-backups
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
platform-health: "true"
|
||||
spec:
|
||||
groups:
|
||||
- name: platform-health.backups
|
||||
rules:
|
||||
- alert: VeleroBackupFailure
|
||||
expr: increase(velero_backup_failure_total[1h]) > 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Velero backup failed: schedule {{ $labels.schedule }}"
|
||||
description: "One or more Velero backups failed in the last hour (R-STATE-03 — off-cluster DR at risk)."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/backup-failure"
|
||||
- alert: VeleroBackupPartialFailure
|
||||
expr: increase(velero_backup_partial_failure_total[1h]) > 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Velero backup partially failed"
|
||||
description: "A Velero backup completed with partial failures in the last hour — some resources/volumes were skipped."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/backup-partial"
|
||||
- alert: VeleroNoRecentBackup
|
||||
expr: (time() - velero_backup_last_successful_timestamp{schedule!=""}) > (24 * 3600)
|
||||
for: 1h
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "No successful Velero backup in 24h: {{ $labels.schedule }}"
|
||||
description: "Schedule {{ $labels.schedule }} has not produced a successful backup in over 24h (missed schedule)."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/backup-missed"
|
||||
|
||||
# =====================================================================================
|
||||
# Identity — Keycloak (SSO is a critical dependency — docs/15)
|
||||
# =====================================================================================
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: platform-health-identity
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
platform-health: "true"
|
||||
spec:
|
||||
groups:
|
||||
- name: platform-health.identity
|
||||
rules:
|
||||
- alert: KeycloakDown
|
||||
expr: |
|
||||
max(kube_pod_status_ready{namespace="keycloak", condition="true"}) == 0
|
||||
or absent(kube_pod_status_ready{namespace="keycloak"})
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Keycloak IdP down"
|
||||
description: "No ready Keycloak pod for 5m — SSO is unavailable and dependent services cannot authenticate (docs/15)."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/keycloak-down"
|
||||
- alert: KeycloakDatabaseDown
|
||||
# Keycloak's PostgreSQL (Longhorn-backed, DR-critical). Metric from postgres-exporter or
|
||||
# the bundled chart; confirm label set at bring-up.
|
||||
expr: |
|
||||
max(kube_pod_status_ready{namespace="keycloak", pod=~".*postgres.*", condition="true"}) == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Keycloak database unavailable"
|
||||
description: "Keycloak's PostgreSQL is not ready for 5m — the IdP will fail (docs/15)."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/keycloak-db"
|
||||
|
||||
# =====================================================================================
|
||||
# The monitor's own capacity — don't let the monitor blind itself (docs/10)
|
||||
# =====================================================================================
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: platform-health-monitoring-self
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
platform-health: "true"
|
||||
spec:
|
||||
groups:
|
||||
- name: platform-health.monitoring-self
|
||||
rules:
|
||||
- alert: MonitoringVolumeFillingUp
|
||||
expr: |
|
||||
(kubelet_volume_stats_available_bytes{namespace="monitoring"}
|
||||
/ kubelet_volume_stats_capacity_bytes{namespace="monitoring"}) < 0.15
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Monitoring PVC filling up: {{ $labels.persistentvolumeclaim }}"
|
||||
description: "PVC {{ $labels.persistentvolumeclaim }} (Prometheus/Loki) is under 15% free — retention/PVC size may need raising before the monitor goes blind."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/monitoring-capacity"
|
||||
- alert: PrometheusTargetsMissing
|
||||
expr: |
|
||||
(count(up == 0) / count(up)) > 0.10
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "More than 10% of Prometheus targets are down"
|
||||
description: "Over 10% of scrape targets have been down for 15m — a broad monitoring blind spot."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/targets-down"
|
||||
@@ -1,5 +1,7 @@
|
||||
---
|
||||
# Grafana Alloy — telemetry collector / log shipping to Loki (docs/10). Phase 0 STUB.
|
||||
# Grafana Alloy — log-shipping agent (docs/10). Serves R-OBS-01. Runs as a DaemonSet, discovers
|
||||
# pods on each node, and forwards container logs to Loki. DEV-SCOPED: modest per-node resources,
|
||||
# in-cluster clustering disabled (not needed for log tailing on a 3-node box).
|
||||
apiVersion: helm.toolkit.fluxcd.io/v2
|
||||
kind: HelmRelease
|
||||
metadata:
|
||||
@@ -21,5 +23,73 @@ spec:
|
||||
upgrade:
|
||||
remediation:
|
||||
retries: 3
|
||||
values: {}
|
||||
# TODO(WS-OBS, Phase 4): Alloy config to scrape pod/node logs -> Loki; K8s events; metrics relay.
|
||||
values:
|
||||
controller:
|
||||
type: daemonset # one collector per node — tails all pod logs locally
|
||||
alloy:
|
||||
clustering:
|
||||
enabled: false # log tailing needs no cluster ring on a 3-node dev box
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 96Mi
|
||||
limits:
|
||||
memory: 256Mi
|
||||
# Alloy pipeline: discover pods -> relabel identity -> ship to the Loki single binary.
|
||||
configMap:
|
||||
content: |-
|
||||
logging {
|
||||
level = "info"
|
||||
format = "logfmt"
|
||||
}
|
||||
|
||||
discovery.kubernetes "pods" {
|
||||
role = "pod"
|
||||
}
|
||||
|
||||
// Carry useful identity labels; keep cardinality bounded (no per-request labels).
|
||||
discovery.relabel "pod_logs" {
|
||||
targets = discovery.kubernetes.pods.targets
|
||||
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_namespace"]
|
||||
target_label = "namespace"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_pod_name"]
|
||||
target_label = "pod"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_pod_container_name"]
|
||||
target_label = "container"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_pod_node_name"]
|
||||
target_label = "node"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"]
|
||||
target_label = "app"
|
||||
}
|
||||
}
|
||||
|
||||
loki.source.kubernetes "pods" {
|
||||
targets = discovery.relabel.pod_logs.output
|
||||
forward_to = [loki.process.default.receiver]
|
||||
}
|
||||
|
||||
loki.process "default" {
|
||||
forward_to = [loki.write.default.receiver]
|
||||
stage.static_labels {
|
||||
values = {
|
||||
job = "kubernetes-pods",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
loki.write "default" {
|
||||
endpoint {
|
||||
// Loki single-binary service (gateway disabled).
|
||||
url = "http://loki.monitoring.svc.cluster.local:3100/loki/api/v1/push"
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,12 @@
|
||||
---
|
||||
# kube-prometheus-stack — Prometheus + Alertmanager + Grafana (docs/10). Phase 0 STUB.
|
||||
# Retention/sizes are overlay knobs (R-ENV-02). Alertmanager emails via SMTP.
|
||||
# kube-prometheus-stack — Prometheus + Alertmanager + Grafana + kube-state-metrics +
|
||||
# node-exporter + the standard Kubernetes exporters/rules (docs/10). Serves R-OBS-01/02/03.
|
||||
#
|
||||
# Retention/PVC sizes are overlay knobs (R-ENV-02) resolved from cluster-vars via Flux
|
||||
# postBuild.substituteFrom. Everything else is DEV-SCOPED for the ~22 GiB workstation ceiling:
|
||||
# single replicas, modest requests/limits, ephemeral (emptyDir) Alertmanager/Grafana state, and
|
||||
# the k3s-embedded control-plane scrape jobs disabled (they are not separately reachable on k3s and
|
||||
# would otherwise show as permanently-down targets and burn RAM/alert noise).
|
||||
apiVersion: helm.toolkit.fluxcd.io/v2
|
||||
kind: HelmRelease
|
||||
metadata:
|
||||
@@ -23,22 +29,148 @@ spec:
|
||||
remediation:
|
||||
retries: 3
|
||||
values:
|
||||
# --- k3s trim: control-plane components are embedded in the k3s server process and are not
|
||||
# scrapable as discrete endpoints. Disabling avoids permanently-firing "target down" noise.
|
||||
kubeControllerManager:
|
||||
enabled: false
|
||||
kubeScheduler:
|
||||
enabled: false
|
||||
kubeEtcd:
|
||||
enabled: false
|
||||
kubeProxy:
|
||||
enabled: false
|
||||
|
||||
# --- Prometheus ---
|
||||
prometheus:
|
||||
prometheusSpec:
|
||||
retention: ${MONITORING_PROMETHEUS_RETENTION} # overlay knob (dev short / prod long)
|
||||
replicas: 1
|
||||
retention: ${MONITORING_PROMETHEUS_RETENTION} # overlay knob (dev 3d / prod long)
|
||||
# Discover ServiceMonitors/PodMonitors/Rules from ALL namespaces (flux, cert-manager,
|
||||
# longhorn, velero, keycloak, cilium/hubble ship their own) — not just this release's.
|
||||
ruleSelectorNilUsesHelmValues: false
|
||||
serviceMonitorSelectorNilUsesHelmValues: false
|
||||
podMonitorSelectorNilUsesHelmValues: false
|
||||
probeSelectorNilUsesHelmValues: false
|
||||
# Modest footprint for a home dev cluster (few hundred series). Limit gives headroom for
|
||||
# WAL replay / compaction spikes without letting Prometheus eat the node.
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 384Mi
|
||||
limits:
|
||||
memory: 1Gi
|
||||
storageSpec:
|
||||
volumeClaimTemplate:
|
||||
spec:
|
||||
storageClassName: longhorn # replicated tier (ADR-0003)
|
||||
accessModes: ["ReadWriteOnce"]
|
||||
resources:
|
||||
requests:
|
||||
storage: ${MONITORING_PROMETHEUS_PVC_SIZE}
|
||||
grafana:
|
||||
# Grafana behind Keycloak SSO (native OIDC) — ADR-0014. C2 UI composition (docs/12).
|
||||
# TODO(WS-SEC/WS-C2): OIDC client + admin password from SOPS secret (name only).
|
||||
enabled: true
|
||||
storage: ${MONITORING_PROMETHEUS_PVC_SIZE} # overlay knob (dev 5Gi / prod large)
|
||||
|
||||
# --- Alertmanager: routes by severity to email (R-OBS-02) ---
|
||||
alertmanager:
|
||||
alertmanagerSpec: {}
|
||||
# TODO(WS-OBS, Phase 4): Alertmanager → Gmail SMTP receiver; SMTP creds SOPS-encrypted (name
|
||||
# only). Reconciliation errors + drift are alerting conditions (R-OBS-04). Prove: injected
|
||||
# fault emails an alert (Phase-4 exit gate).
|
||||
alertmanagerSpec:
|
||||
replicas: 1
|
||||
# DEV TRIM: no PVC — silences/notification-log live in emptyDir. Acceptable in dev (a
|
||||
# restart re-fires active alerts). Prod overlay can add a small storage block.
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 32Mi
|
||||
limits:
|
||||
memory: 96Mi
|
||||
# SMTP credentials are an OPERATOR HAND-OFF: a SOPS-encrypted Secret, referenced BY NAME
|
||||
# only (docs/07). Mounted at /etc/alertmanager/secrets/<name>/ so the config below can read
|
||||
# the password from a file (never inline plaintext).
|
||||
secrets:
|
||||
- alertmanager-smtp
|
||||
# Routing/receivers. Non-secret SMTP endpoint fields (smarthost/from/to/username) are
|
||||
# operator-supplied placeholders (REPLACE-ME, same convention as velero); the PASSWORD is
|
||||
# read from the mounted SOPS secret via *_file. Both severities email; subject/route differ.
|
||||
config:
|
||||
global:
|
||||
resolve_timeout: 5m
|
||||
smtp_smarthost: "REPLACE-ME:587" # operator hand-off (e.g. smtp.gmail.com:587)
|
||||
smtp_from: "REPLACE-ME" # operator hand-off (From: address)
|
||||
smtp_auth_username: "REPLACE-ME" # operator hand-off (SMTP login)
|
||||
smtp_auth_password_file: /etc/alertmanager/secrets/alertmanager-smtp/smtp-password
|
||||
smtp_require_tls: true
|
||||
route:
|
||||
group_by: ["alertname", "namespace"]
|
||||
group_wait: 30s
|
||||
group_interval: 5m
|
||||
repeat_interval: 12h
|
||||
receiver: email-warning
|
||||
routes:
|
||||
- matchers: ['severity = "critical"']
|
||||
receiver: email-critical
|
||||
# page-worthy — notify fast, remind often
|
||||
group_wait: 10s
|
||||
repeat_interval: 1h
|
||||
receivers:
|
||||
- name: email-warning
|
||||
email_configs:
|
||||
- to: "REPLACE-ME" # operator hand-off (alert inbox)
|
||||
send_resolved: true
|
||||
headers:
|
||||
subject: '[WARNING] {{ .CommonLabels.alertname }} ({{ .CommonLabels.namespace }})'
|
||||
- name: email-critical
|
||||
email_configs:
|
||||
- to: "REPLACE-ME" # operator hand-off (alert inbox)
|
||||
send_resolved: true
|
||||
headers:
|
||||
subject: '[CRITICAL] {{ .CommonLabels.alertname }} ({{ .CommonLabels.namespace }})'
|
||||
|
||||
# --- Grafana: dashboards (R-OBS-03), surfaced in the C2 UI (docs/12) behind Keycloak SSO ---
|
||||
grafana:
|
||||
enabled: true
|
||||
# Admin bootstrap + (later) OIDC client are SOPS secrets referenced BY NAME (WS-SEC/ADR-0014).
|
||||
admin:
|
||||
existingSecret: grafana-admin-credentials
|
||||
userKey: admin-user
|
||||
passwordKey: admin-password
|
||||
# DEV TRIM: emptyDir (default). Dashboards are provisioned from ConfigMaps (sidecar), so no
|
||||
# durable Grafana DB is needed to iterate. Prod can enable persistence.
|
||||
persistence:
|
||||
enabled: false
|
||||
# Auto-discover dashboard ConfigMaps labeled grafana_dashboard across namespaces (per-service
|
||||
# dashboards rendered from ServiceDefinitions land here — docs/10 onboarding output).
|
||||
sidecar:
|
||||
dashboards:
|
||||
enabled: true
|
||||
searchNamespace: ALL
|
||||
label: grafana_dashboard
|
||||
datasources:
|
||||
enabled: true
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
memory: 256Mi
|
||||
# TODO(WS-SEC/WS-C2, later): Keycloak OIDC client (auth.generic_oauth) + root_url behind the
|
||||
# Gateway; disable local login once SSO is proven. Secret referenced by name only.
|
||||
|
||||
# --- Bundled exporters / operator: keep small ---
|
||||
prometheusOperator:
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 96Mi
|
||||
limits:
|
||||
memory: 256Mi
|
||||
kube-state-metrics:
|
||||
resources:
|
||||
requests:
|
||||
cpu: 20m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
memory: 128Mi
|
||||
prometheus-node-exporter:
|
||||
resources:
|
||||
requests:
|
||||
cpu: 20m
|
||||
memory: 32Mi
|
||||
limits:
|
||||
memory: 64Mi
|
||||
@@ -7,3 +7,4 @@ resources:
|
||||
- kube-prometheus-stack.yaml
|
||||
- loki.yaml
|
||||
- alloy.yaml
|
||||
- alerts.yaml
|
||||
@@ -1,5 +1,13 @@
|
||||
---
|
||||
# Loki — log aggregation (docs/10). Phase 0 STUB. PVC size is an overlay knob (R-ENV-02).
|
||||
# Loki — log aggregation (docs/10). Serves R-OBS-01. PVC size is an overlay knob (R-ENV-02).
|
||||
#
|
||||
# DEV-SCOPED for the ~22 GiB ceiling. Deployment mode is SingleBinary (one pod, filesystem
|
||||
# storage) — the scalable read/write/backend split is prod-only. CRITICAL RAM TRIM: the grafana
|
||||
# loki chart's default memcached caches (chunksCache/resultsCache) request ~8-9 GiB EACH; both are
|
||||
# disabled here. The nginx `gateway`, self-monitoring, and the canary/test hooks are disabled too.
|
||||
#
|
||||
# Retention is enforced by the COMPACTOR (retention_enabled: true) — a retention_period alone does
|
||||
# nothing (docs/10, and a known-gotcha carried from the legacy stack).
|
||||
apiVersion: helm.toolkit.fluxcd.io/v2
|
||||
kind: HelmRelease
|
||||
metadata:
|
||||
@@ -22,12 +30,63 @@ spec:
|
||||
remediation:
|
||||
retries: 3
|
||||
values:
|
||||
# TODO(WS-OBS, Phase 4): single-binary vs scalable deployment mode; retention via compactor
|
||||
# (retention_enabled: true — a period alone does nothing); storage backing.
|
||||
deploymentMode: SingleBinary
|
||||
|
||||
loki:
|
||||
auth_enabled: false
|
||||
commonConfig:
|
||||
replication_factor: 1 # single binary — no ring replication
|
||||
storage:
|
||||
type: filesystem
|
||||
schemaConfig:
|
||||
configs:
|
||||
- from: "2024-04-01"
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: loki_index_
|
||||
period: 24h
|
||||
limits_config:
|
||||
# NOTE: Loki's LOG retention has no dedicated cluster-vars knob (only the 3 monitoring vars
|
||||
# exist). Held to a dev-friendly 72h to mirror the dev metrics retention; promote to an
|
||||
# overlay knob if per-env log retention is ever needed. See README / report.
|
||||
retention_period: 72h
|
||||
compactor:
|
||||
retention_enabled: true # REQUIRED — a period alone does not delete (docs/10)
|
||||
delete_request_store: filesystem
|
||||
|
||||
singleBinary:
|
||||
replicas: 1
|
||||
persistence:
|
||||
storageClass: longhorn
|
||||
size: ${MONITORING_LOKI_PVC_SIZE}
|
||||
enabled: true
|
||||
storageClass: longhorn # replicated tier (ADR-0003)
|
||||
size: ${MONITORING_LOKI_PVC_SIZE} # overlay knob (dev 5Gi / prod large)
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 192Mi
|
||||
limits:
|
||||
memory: 512Mi
|
||||
|
||||
# Not used in SingleBinary mode — pinned to 0 so a chart default can't spin up extra pods.
|
||||
backend:
|
||||
replicas: 0
|
||||
read:
|
||||
replicas: 0
|
||||
write:
|
||||
replicas: 0
|
||||
|
||||
# --- RAM trims: kill the memory-hungry / non-essential-for-dev pieces ---
|
||||
chunksCache:
|
||||
enabled: false # default memcached requests ~8-9Gi — must stay off in dev
|
||||
resultsCache:
|
||||
enabled: false
|
||||
gateway:
|
||||
enabled: false # no nginx front door needed for a single binary in-cluster
|
||||
lokiCanary:
|
||||
enabled: false
|
||||
test:
|
||||
enabled: false
|
||||
# Alloy ships logs here (in-cluster service DNS, no gateway):
|
||||
# http://loki.monitoring.svc.cluster.local:3100/loki/api/v1/push
|
||||
Reference in new issue
Block a user