diff --git a/apps/wave-0/nextcloud-db.yaml b/apps/wave-0/nextcloud-db.yaml index 754ba03..2e5b51f 100644 --- a/apps/wave-0/nextcloud-db.yaml +++ b/apps/wave-0/nextcloud-db.yaml @@ -37,10 +37,17 @@ metadata: name: nextcloud-db namespace: nextcloud-db spec: - # Single Postgres instance on 3-replica Longhorn is acceptable at home scale (ADR-0014, same as - # keycloak-db). The storage is already 3-way replicated by Longhorn; raise to 3 for Postgres-level - # HA as a future hardening. - instances: 1 + # THREE instances — one primary + two hot standbys (2026-09-07), same as keycloak-db; see + # platform/keycloak/postgres-cluster.yaml for the full rationale. Short version: Longhorn's 3-way + # replication saves the DATA on node loss, not the SERVICE, and with `instances: 1` all three CNPG + # primaries had drifted onto k3s-prod-2, so one node took Nextcloud down alongside SSO and + # Semaphore. A streaming standby lets CNPG promote in seconds and repoint the -rw Service. + # Gated on the Barman archive below actually working (ContinuousArchiving=True with a real + # firstRecoverabilityPoint) — standbys replicate corruption, they are not a backup. + # + # Scaling out is non-disruptive: CNPG creates -2 then -3 sequentially, each seeded by + # pg_basebackup from the running primary, which is neither restarted nor disconnected. + instances: 3 imageName: ghcr.io/cloudnative-pg/postgresql:16.4 # Stamp app.kubernetes.io/name=nextcloud-db onto every generated pod/service so the rendered # nextcloud egress CiliumNetworkPolicy's toEndpoints selector matches the primary pod (R-NET-02). diff --git a/apps/wave-0/semaphore-db.yaml b/apps/wave-0/semaphore-db.yaml index 3991548..65e364f 100644 --- a/apps/wave-0/semaphore-db.yaml +++ b/apps/wave-0/semaphore-db.yaml @@ -38,10 +38,17 @@ metadata: name: semaphore-db namespace: semaphore-db spec: - # Single Postgres instance on 3-replica Longhorn is acceptable at home scale (ADR-0014, same as - # keycloak-db / nextcloud-db). The storage is already 3-way replicated by Longhorn; raise to 3 for - # Postgres-level HA as a future hardening. - instances: 1 + # THREE instances — one primary + two hot standbys (2026-09-07), same as keycloak-db / + # nextcloud-db; see platform/keycloak/postgres-cluster.yaml for the full rationale. Short version: + # Longhorn's 3-way replication saves the DATA on node loss, not the SERVICE, and with + # `instances: 1` all three CNPG primaries had drifted onto k3s-prod-2, so one node took Semaphore + # down alongside SSO and Nextcloud. A streaming standby lets CNPG promote in seconds and repoint + # the -rw Service. Gated on the Barman archive below actually working (ContinuousArchiving=True + # with a real firstRecoverabilityPoint) — standbys replicate corruption, they are not a backup. + # + # Scaling out is non-disruptive: CNPG creates -2 then -3 sequentially, each seeded by + # pg_basebackup from the running primary, which is neither restarted nor disconnected. + instances: 3 imageName: ghcr.io/cloudnative-pg/postgresql:16.4 # Stamp app.kubernetes.io/name=semaphore-db onto every generated pod/service so the rendered # semaphore egress CiliumNetworkPolicy's toEndpoints selector matches the primary pod (R-NET-02). diff --git a/platform/keycloak/postgres-cluster.yaml b/platform/keycloak/postgres-cluster.yaml index bcfd330..a50e61e 100644 --- a/platform/keycloak/postgres-cluster.yaml +++ b/platform/keycloak/postgres-cluster.yaml @@ -12,10 +12,35 @@ metadata: name: keycloak-db namespace: keycloak spec: - # Single Postgres instance on 3-replica Longhorn is acceptable for the home scale (ADR-0014: - # "single replica with DB on Longhorn + backups is acceptable"). Raise to 3 for Postgres-level HA - # as a future hardening (prod) — the storage is already 3-way replicated by Longhorn. - instances: 1 + # THREE instances — one primary + two hot standbys, one per node (2026-09-07). + # + # WAS `instances: 1`, on ADR-0014's "single replica with DB on Longhorn + backups is acceptable". + # Longhorn's 3-way replication saves the DATA when a node dies; it does NOT save the SERVICE. With + # one instance, losing the node means rescheduling the pod, re-attaching the volume and replaying + # WAL before Postgres will accept a connection again — minutes, and only if scheduling cooperates. + # + # WHY NOW (the 2026-09-07 incident): all three CNPG primaries — keycloak-db, nextcloud-db and + # semaphore-db — were co-resident on k3s-prod-2, because with `instances: 1` nothing in the spec + # ever said they must not be. One node took out SSO, Nextcloud and Semaphore together, and with + # SSO went kubectl OIDC, Grafana, Headlamp, Gitea and forward-auth. Three instances give this + # Cluster something to fail OVER to: CNPG promotes a streaming standby in seconds and repoints the + # -rw Service, with no volume re-attach and no restore. The `affinity.podAntiAffinityType: + # required` below is what makes the three land on three DIFFERENT nodes. + # + # UNBLOCKED BY WORKING BACKUPS, NOT A SUBSTITUTE FOR THEM. Replicas copy corruption and deletions + # faithfully; only the Barman archive below is a backup. Adding standbys while archiving was + # broken would have multiplied the disk-full clock described further down, so this change waited + # for all three Clusters to report ContinuousArchiving=True with a real firstRecoverabilityPoint. + # + # COST: two more 5Gi PVCs here (Longhorn thin-provisions; the live pgdata is ~600 MiB), ~30 GiB + # PROVISIONED across all three DBs and well under 3 GiB actually consumed. Longhorn allows 200% + # over-provisioning on both clusters and both have headroom. + # + # APPLYING THIS IS NON-DISRUPTIVE: CNPG scales out by creating -2 then -3 sequentially, each + # seeded by pg_basebackup from the RUNNING primary. The primary is not restarted and clients are + # not disconnected. (The anti-affinity change is a separate commit precisely because that one DOES + # cause a switchover — see it for the blast radius.) + instances: 3 imageName: ghcr.io/cloudnative-pg/postgresql:16.4 storage: storageClass: longhorn