feat(phase4): deploy monitoring (kube-prometheus-stack + Loki + Alloy + Alertmanager)

Enable monitoring in platform/kustomization.yaml (Phase 4). Grafana admin + Alertmanager SMTP secrets are provisioned (SOPS) and wired; Longhorn SC is up for Prometheus/Loki PVCs. Set the Alertmanager receiver 'to:' to a valid inbox ([email protected]) so the config loads (REPLACE-ME is not a valid address); operator can change it. NOTE: smtp_from ([email protected]) differs from the Gmail auth user — Gmail may reject/rewrite unless a send-as alias is configured; that only affects email delivery at alert time, not stack health. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
2026-08-21 22:06:23 -05:00
1 parent 75764580d8
commit dfde5292d7
@@ -0,0 +1,46 @@
+# =============================================================================
+# TEMPLATE — NOT APPLIED, NOT ENCRYPTED (.yaml.example is ignored by kustomize
+# and never auto-encrypted by SOPS). Reference scaffold only.
+#
+# (a) Copy to:      platform/monitoring/alertmanager-smtp.sops.yaml
+# (b) kustomization: add this line under `resources:` in
+#                    platform/monitoring/kustomization.yaml
+#                       - alertmanager-smtp.sops.yaml
+# (c) Encrypt:       sops -e -i platform/monitoring/alertmanager-smtp.sops.yaml
+# (d) Consumed by:   Alertmanager (kube-prometheus-stack). Only the SMTP
+#                    PASSWORD lives here. It is mounted at
+#                    /etc/alertmanager/secrets/alertmanager-smtp/smtp-password. [Phase 4]
+#
+# NOTE: The non-secret SMTP endpoint fields — smtp_smarthost, smtp_from,
+# smtp_auth_username, and the receiver `to:` addresses — are configured in
+# platform/monitoring/kube-prometheus-stack.yaml (currently REPLACE-ME
+# placeholders there). Only the password is this secret.
+# =============================================================================
+apiVersion: v1
+kind: Secret
+metadata:
+    name: alertmanager-smtp
+    namespace: monitoring
+type: Opaque
+stringData:
+    smtp-password: ENC[AES256_GCM,data:MDBdjOjZXSoieAwO0GJVN7RhTQ==,iv:40Kwqv4fO5QjehYXf0fNGPA+asInXfzUDChuQv7HYrM=,tag:HgMu9y2+gqufSAQ9MqKiOA==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age175l8mdpdfu7hq0aml3exs3gykarwjmuvr23sq56s3wjejnyk44wsp4mfzg
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBiamFxdXVUR0VxMlNkeUFL
+            OXZqc3d5V1RGQVhHMW15dmllakNZeDVUQzJVCnVtRHBCOWJOSmsxVVRFSEZBTkFi
+            V3FkSzVDMkU5aWNxR0lVMGpheEk1OUkKLS0tIE5hQ3A4eDV4eklWVFB6NjZkaUFT
+            UHhPc0FDbDNuT2ZERHJwYllCY1U2NmsKF2Ct1RzFMHYW5JNy8v0kS8oV96eecH9b
+            nkG3ve8Ku1Ttw4O8htQcJ0qs6rE4+rXXNrhszmFOWDHZEKBd5CaWiw==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2026-08-22T02:35:27Z"
+    mac: ENC[AES256_GCM,data:3/KTeIFvRZKDQFH7WvkJLPH2eyTl8g5LPLKBvV7Dg0u+ZfSkNU1zql3LzLbApiqTXcP2393pZyO+11AO5TK4ObqfFVYxqyTFega1CCnkgrHygFGmt70jwlgCXbWqkC+AGG8XOUb96M0tCrRbJjz/C3eoFjEmqTZLNfqn+smNO3U=,iv:KgC0NsxHWCbeidHa8Yt+F9Qh1YslHvKtdYr+4tM/IB8=,tag:GtDE/7fuTbPZMVE6C/oiLQ==,type:str]
+    pgp: []
+    encrypted_regex: ^(data|stringData)$
+    version: 3.9.4
@@ -0,0 +1,41 @@
+# =============================================================================
+# TEMPLATE — NOT APPLIED, NOT ENCRYPTED (.yaml.example is ignored by kustomize
+# and never auto-encrypted by SOPS). Reference scaffold only.
+#
+# (a) Copy to:      platform/monitoring/grafana-admin-credentials.sops.yaml
+# (b) kustomization: add this line under `resources:` in
+#                    platform/monitoring/kustomization.yaml
+#                       - grafana-admin-credentials.sops.yaml
+# (c) Encrypt:       sops -e -i platform/monitoring/grafana-admin-credentials.sops.yaml
+# (d) Consumed by:   kube-prometheus-stack -> grafana.admin.existingSecret
+#                    (userKey: admin-user, passwordKey: admin-password). [Phase 4]
+# =============================================================================
+apiVersion: v1
+kind: Secret
+metadata:
+    name: grafana-admin-credentials
+    namespace: monitoring
+type: Opaque
+stringData:
+    admin-user: ENC[AES256_GCM,data:lBUukEo=,iv:hbLPBZujuBp44qhqGAO+ZrNtBStW5RdblWQmgLv5fbI=,tag:c4AyEyeyYMGYJL80ipMMcQ==,type:str]
+    admin-password: ENC[AES256_GCM,data:n53uTxnmIpbKomR//0dm,iv:cx5v29rYPtO0Z2iNmv0ahj/al1Ye/7Mu5MyK0quXpMI=,tag:xAnKhR88oRUwzgL8tuZOJA==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age175l8mdpdfu7hq0aml3exs3gykarwjmuvr23sq56s3wjejnyk44wsp4mfzg
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB4U3V5TUNqcDhEK2NHK1F0
+            VkV4YmQ2bHVYM1ExQytzNDNmRi9QeFNCMjJzClp4SnlVYmlXTER6N05wQldDRzZv
+            enFZcng4UTExbkF5S2ltWTV5UUkyWTQKLS0tIE12NEhZRk1Ya3JTTjJFQlgwN1Jr
+            cHhySmRuZ2xpTXMyRVJ3ZThGVm5OSk0K32w53bZMjE8E1kKtHxczDbI1DECgm4Nd
+            APFB2qmf7PP5vRwfiZEsypy19a+sg02oq4WZD9OcwHUElPdOs1Am+w==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2026-08-22T02:29:14Z"
+    mac: ENC[AES256_GCM,data:o/Z26bcQRG8rHIoHOWHPAr7Bbdn1+FQDdPizBYsl0u167Gn4DGV4J6HxC9loPnRUNVT/0wyy7PbPDfkyxosjVt9amBGc1t7iC7K/RK2nP2IQffGUbfWiCZqXrt0SVHn+wlvtADQxieAzHLMHD1aZhVei3o5WxpzQYHb9iofDIuU=,iv:0IVvtrI9klknXLrtHIJYYEuziTTBu4GUUpqWIVWJp+g=,tag:6MxmKMY7sQLxG0D0qqyNEA==,type:str]
+    pgp: []
+    encrypted_regex: ^(data|stringData)$
+    version: 3.9.4
@@ -91,9 +91,9 @@ spec:
      config:
        global:
          resolve_timeout: 5m
-          smtp_smarthost: "REPLACE-ME:587"          # operator hand-off (e.g. smtp.gmail.com:587)
-          smtp_from: "REPLACE-ME"                    # operator hand-off (From: address)
-          smtp_auth_username: "REPLACE-ME"           # operator hand-off (SMTP login)
+          smtp_smarthost: "smtp.gmail.com:587"          # operator hand-off (e.g. smtp.gmail.com:587)
+          smtp_from: "[email protected]"                    # operator hand-off (From: address)
+          smtp_auth_username: "[email protected]"           # operator hand-off (SMTP login)
          smtp_auth_password_file: /etc/alertmanager/secrets/alertmanager-smtp/smtp-password
          smtp_require_tls: true
        route:
@@ -111,13 +111,13 @@ spec:
        receivers:
          - name: email-warning
            email_configs:
-              - to: "REPLACE-ME"                     # operator hand-off (alert inbox)
+              - to: "[email protected]"    # alert inbox (operator: change if a different inbox is wanted)
                send_resolved: true
                headers:
                  subject: '[WARNING] {{ .CommonLabels.alertname }} ({{ .CommonLabels.namespace }})'
          - name: email-critical
            email_configs:
-              - to: "REPLACE-ME"                     # operator hand-off (alert inbox)
+              - to: "[email protected]"    # alert inbox (operator: change if a different inbox is wanted)
                send_resolved: true
                headers:
                  subject: '[CRITICAL] {{ .CommonLabels.alertname }} ({{ .CommonLabels.namespace }})'
@@ -8,3 +8,5 @@ resources:
  - loki.yaml
  - alloy.yaml
  - alerts.yaml
+  - grafana-admin-credentials.sops.yaml
+  - alertmanager-smtp.sops.yaml