Autonomous session: Phase-2 platform authoring + Flux bootstrap progress
Author-only Phase-2 components (platform suspended; cluster untouched): - platform/longhorn: 3-replica SC + bulk external-mount indirection (R-STATE) - platform/monitoring: kube-prometheus-stack + Loki + Alloy (dev-trimmed) + 8 alert rules (R-OBS) - platform/cilium: parity fix (k8sServiceHost=10.73.0.11 + masquerade) safe to adopt Phase 2 - conformance: partial L2 tests (worker hit session limit; incomplete — see DECISIONS-LOG K2) - bootstrap/gitea: fix rootless-podman perms (host bind-mount -> named volume) Flux installed on the cluster; host Gitea up + repo pushed; GitRepository BLOCKED by host firewalld (VM->10.73.0.1:3000) — needs one operator sudo command (DECISIONS-LOG). Known issue K1: cluster-vars substitution-var case mismatch (fix before un-suspend). Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
This commit is contained in:
jakman85andClaude Opus 4.8 committed
1 parent
397f29e99a
commit
cc89b4b830
19 files changed
+1567
-221
No files matched your search
@@ -1,6 +1,12 @@
|
||||
---
|
||||
# kube-prometheus-stack — Prometheus + Alertmanager + Grafana (docs/10). Phase 0 STUB.
|
||||
# Retention/sizes are overlay knobs (R-ENV-02). Alertmanager emails via SMTP.
|
||||
# kube-prometheus-stack — Prometheus + Alertmanager + Grafana + kube-state-metrics +
|
||||
# node-exporter + the standard Kubernetes exporters/rules (docs/10). Serves R-OBS-01/02/03.
|
||||
#
|
||||
# Retention/PVC sizes are overlay knobs (R-ENV-02) resolved from cluster-vars via Flux
|
||||
# postBuild.substituteFrom. Everything else is DEV-SCOPED for the ~22 GiB workstation ceiling:
|
||||
# single replicas, modest requests/limits, ephemeral (emptyDir) Alertmanager/Grafana state, and
|
||||
# the k3s-embedded control-plane scrape jobs disabled (they are not separately reachable on k3s and
|
||||
# would otherwise show as permanently-down targets and burn RAM/alert noise).
|
||||
apiVersion: helm.toolkit.fluxcd.io/v2
|
||||
kind: HelmRelease
|
||||
metadata:
|
||||
@@ -23,22 +29,148 @@ spec:
|
||||
remediation:
|
||||
retries: 3
|
||||
values:
|
||||
# --- k3s trim: control-plane components are embedded in the k3s server process and are not
|
||||
# scrapable as discrete endpoints. Disabling avoids permanently-firing "target down" noise.
|
||||
kubeControllerManager:
|
||||
enabled: false
|
||||
kubeScheduler:
|
||||
enabled: false
|
||||
kubeEtcd:
|
||||
enabled: false
|
||||
kubeProxy:
|
||||
enabled: false
|
||||
|
||||
# --- Prometheus ---
|
||||
prometheus:
|
||||
prometheusSpec:
|
||||
retention: ${MONITORING_PROMETHEUS_RETENTION} # overlay knob (dev short / prod long)
|
||||
replicas: 1
|
||||
retention: ${MONITORING_PROMETHEUS_RETENTION} # overlay knob (dev 3d / prod long)
|
||||
# Discover ServiceMonitors/PodMonitors/Rules from ALL namespaces (flux, cert-manager,
|
||||
# longhorn, velero, keycloak, cilium/hubble ship their own) — not just this release's.
|
||||
ruleSelectorNilUsesHelmValues: false
|
||||
serviceMonitorSelectorNilUsesHelmValues: false
|
||||
podMonitorSelectorNilUsesHelmValues: false
|
||||
probeSelectorNilUsesHelmValues: false
|
||||
# Modest footprint for a home dev cluster (few hundred series). Limit gives headroom for
|
||||
# WAL replay / compaction spikes without letting Prometheus eat the node.
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 384Mi
|
||||
limits:
|
||||
memory: 1Gi
|
||||
storageSpec:
|
||||
volumeClaimTemplate:
|
||||
spec:
|
||||
storageClassName: longhorn # replicated tier (ADR-0003)
|
||||
accessModes: ["ReadWriteOnce"]
|
||||
resources:
|
||||
requests:
|
||||
storage: ${MONITORING_PROMETHEUS_PVC_SIZE}
|
||||
grafana:
|
||||
# Grafana behind Keycloak SSO (native OIDC) — ADR-0014. C2 UI composition (docs/12).
|
||||
# TODO(WS-SEC/WS-C2): OIDC client + admin password from SOPS secret (name only).
|
||||
enabled: true
|
||||
storage: ${MONITORING_PROMETHEUS_PVC_SIZE} # overlay knob (dev 5Gi / prod large)
|
||||
|
||||
# --- Alertmanager: routes by severity to email (R-OBS-02) ---
|
||||
alertmanager:
|
||||
alertmanagerSpec: {}
|
||||
# TODO(WS-OBS, Phase 4): Alertmanager → Gmail SMTP receiver; SMTP creds SOPS-encrypted (name
|
||||
# only). Reconciliation errors + drift are alerting conditions (R-OBS-04). Prove: injected
|
||||
# fault emails an alert (Phase-4 exit gate).
|
||||
alertmanagerSpec:
|
||||
replicas: 1
|
||||
# DEV TRIM: no PVC — silences/notification-log live in emptyDir. Acceptable in dev (a
|
||||
# restart re-fires active alerts). Prod overlay can add a small storage block.
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 32Mi
|
||||
limits:
|
||||
memory: 96Mi
|
||||
# SMTP credentials are an OPERATOR HAND-OFF: a SOPS-encrypted Secret, referenced BY NAME
|
||||
# only (docs/07). Mounted at /etc/alertmanager/secrets/<name>/ so the config below can read
|
||||
# the password from a file (never inline plaintext).
|
||||
secrets:
|
||||
- alertmanager-smtp
|
||||
# Routing/receivers. Non-secret SMTP endpoint fields (smarthost/from/to/username) are
|
||||
# operator-supplied placeholders (REPLACE-ME, same convention as velero); the PASSWORD is
|
||||
# read from the mounted SOPS secret via *_file. Both severities email; subject/route differ.
|
||||
config:
|
||||
global:
|
||||
resolve_timeout: 5m
|
||||
smtp_smarthost: "REPLACE-ME:587" # operator hand-off (e.g. smtp.gmail.com:587)
|
||||
smtp_from: "REPLACE-ME" # operator hand-off (From: address)
|
||||
smtp_auth_username: "REPLACE-ME" # operator hand-off (SMTP login)
|
||||
smtp_auth_password_file: /etc/alertmanager/secrets/alertmanager-smtp/smtp-password
|
||||
smtp_require_tls: true
|
||||
route:
|
||||
group_by: ["alertname", "namespace"]
|
||||
group_wait: 30s
|
||||
group_interval: 5m
|
||||
repeat_interval: 12h
|
||||
receiver: email-warning
|
||||
routes:
|
||||
- matchers: ['severity = "critical"']
|
||||
receiver: email-critical
|
||||
# page-worthy — notify fast, remind often
|
||||
group_wait: 10s
|
||||
repeat_interval: 1h
|
||||
receivers:
|
||||
- name: email-warning
|
||||
email_configs:
|
||||
- to: "REPLACE-ME" # operator hand-off (alert inbox)
|
||||
send_resolved: true
|
||||
headers:
|
||||
subject: '[WARNING] {{ .CommonLabels.alertname }} ({{ .CommonLabels.namespace }})'
|
||||
- name: email-critical
|
||||
email_configs:
|
||||
- to: "REPLACE-ME" # operator hand-off (alert inbox)
|
||||
send_resolved: true
|
||||
headers:
|
||||
subject: '[CRITICAL] {{ .CommonLabels.alertname }} ({{ .CommonLabels.namespace }})'
|
||||
|
||||
# --- Grafana: dashboards (R-OBS-03), surfaced in the C2 UI (docs/12) behind Keycloak SSO ---
|
||||
grafana:
|
||||
enabled: true
|
||||
# Admin bootstrap + (later) OIDC client are SOPS secrets referenced BY NAME (WS-SEC/ADR-0014).
|
||||
admin:
|
||||
existingSecret: grafana-admin-credentials
|
||||
userKey: admin-user
|
||||
passwordKey: admin-password
|
||||
# DEV TRIM: emptyDir (default). Dashboards are provisioned from ConfigMaps (sidecar), so no
|
||||
# durable Grafana DB is needed to iterate. Prod can enable persistence.
|
||||
persistence:
|
||||
enabled: false
|
||||
# Auto-discover dashboard ConfigMaps labeled grafana_dashboard across namespaces (per-service
|
||||
# dashboards rendered from ServiceDefinitions land here — docs/10 onboarding output).
|
||||
sidecar:
|
||||
dashboards:
|
||||
enabled: true
|
||||
searchNamespace: ALL
|
||||
label: grafana_dashboard
|
||||
datasources:
|
||||
enabled: true
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
memory: 256Mi
|
||||
# TODO(WS-SEC/WS-C2, later): Keycloak OIDC client (auth.generic_oauth) + root_url behind the
|
||||
# Gateway; disable local login once SSO is proven. Secret referenced by name only.
|
||||
|
||||
# --- Bundled exporters / operator: keep small ---
|
||||
prometheusOperator:
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 96Mi
|
||||
limits:
|
||||
memory: 256Mi
|
||||
kube-state-metrics:
|
||||
resources:
|
||||
requests:
|
||||
cpu: 20m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
memory: 128Mi
|
||||
prometheus-node-exporter:
|
||||
resources:
|
||||
requests:
|
||||
cpu: 20m
|
||||
memory: 32Mi
|
||||
limits:
|
||||
memory: 64Mi
|
||||
Reference in new issue
Block a user