diff --git a/apps/wave-0/nextcloud-db.yaml b/apps/wave-0/nextcloud-db.yaml index 8a70f78..41d3b6a 100644 --- a/apps/wave-0/nextcloud-db.yaml +++ b/apps/wave-0/nextcloud-db.yaml @@ -69,6 +69,21 @@ spec: enablePodAntiAffinity: true podAntiAffinityType: required primaryUpdateMethod: switchover + # SYNCHRONOUS REPLICATION — quorum of ONE standby, PREFERRED durability (2026-09-07). Full + # rationale in platform/keycloak/postgres-cluster.yaml. Short version: async replication loses the + # last few transactions when a primary dies before its standbys catch up, and `method: any` + + # `number: 1` makes COMMIT wait for one standby to confirm. dataDurability MUST stay `preferred`, + # never `required` — `required` blocks writes outright when fewer than `number` healthy standbys + # are available, which on a 3-node cluster sharing nodes with etcd and the CNI turns one degraded + # standby into a total write outage. `preferred` relaxes to async instead. + # + # NON-DISRUPTIVE: compiles to `synchronous_standby_names`, a reload-only PostgreSQL parameter — + # no restart, no switchover, no client disconnect. Land it after the anti-affinity roll. + postgresql: + synchronous: + method: any + number: 1 + dataDurability: preferred storage: storageClass: longhorn size: 5Gi diff --git a/apps/wave-0/semaphore-db.yaml b/apps/wave-0/semaphore-db.yaml index 111bb7e..e588da3 100644 --- a/apps/wave-0/semaphore-db.yaml +++ b/apps/wave-0/semaphore-db.yaml @@ -70,6 +70,21 @@ spec: enablePodAntiAffinity: true podAntiAffinityType: required primaryUpdateMethod: switchover + # SYNCHRONOUS REPLICATION — quorum of ONE standby, PREFERRED durability (2026-09-07). Full + # rationale in platform/keycloak/postgres-cluster.yaml. Short version: async replication loses the + # last few transactions when a primary dies before its standbys catch up, and `method: any` + + # `number: 1` makes COMMIT wait for one standby to confirm. dataDurability MUST stay `preferred`, + # never `required` — `required` blocks writes outright when fewer than `number` healthy standbys + # are available, which on a 3-node cluster sharing nodes with etcd and the CNI turns one degraded + # standby into a total write outage. `preferred` relaxes to async instead. + # + # NON-DISRUPTIVE: compiles to `synchronous_standby_names`, a reload-only PostgreSQL parameter — + # no restart, no switchover, no client disconnect. Land it after the anti-affinity roll. + postgresql: + synchronous: + method: any + number: 1 + dataDurability: preferred storage: storageClass: longhorn size: 5Gi diff --git a/platform/keycloak/postgres-cluster.yaml b/platform/keycloak/postgres-cluster.yaml index 5e9fd3a..1750269 100644 --- a/platform/keycloak/postgres-cluster.yaml +++ b/platform/keycloak/postgres-cluster.yaml @@ -75,6 +75,39 @@ spec: # AND a postgresql parameter in the same reconcile. Renovate bumping the image alone is fine; # never combine an image bump with a postgresql.parameters change in one commit. primaryUpdateMethod: switchover + # SYNCHRONOUS REPLICATION — quorum of ONE standby, PREFERRED durability (2026-09-07). + # + # Standbys are asynchronous by default: COMMIT returns as soon as the PRIMARY has flushed locally, + # so a primary that dies before its standbys catch up takes the last few transactions with it and + # the promoted standby comes up quietly missing them. On the IdP database that is a just-created + # user, a rotated credential or a consent silently disappearing at exactly the moment of a + # failover — the hardest class of bug to notice and the worst one to have. `method: any` + + # `number: 1` makes COMMIT wait for ONE of the two standbys to confirm, which is the entire point + # of having them. + # + # dataDurability: preferred — AND NEVER `required`. `required` is a STRICT quorum: if fewer than + # `number` healthy standbys are available, writes BLOCK. Postgres here shares three nodes with + # etcd and the CNI, so one degraded standby — a node reboot, a Longhorn replica rebuild, a Cilium + # restart — would turn a partial degradation into a TOTAL WRITE OUTAGE on SSO, and SSO is the + # dependency of kubectl OIDC, Grafana, Headlamp, Gitea and forward-auth. `preferred` keeps + # synchronous commit whenever a healthy standby is there and relaxes the required count + # dynamically when one is not: durability when we can have it, availability when we cannot. That + # is the right trade for a 3-node home cluster; a 5-node cluster with a spare could revisit it. + # (dataDurability is only honoured while standbyNamesPre/standbyNamesPost are unset — they are.) + # + # NON-DISRUPTIVE TO APPLY: this compiles to `synchronous_standby_names`, which PostgreSQL treats + # as a RELOAD-only parameter. No pod restart, no switchover, no client disconnect — CNPG reloads + # the config in place. Landing it AFTER the anti-affinity roll is deliberate: it wants standbys + # that already exist and are already on their final nodes, and it must not be folded into a roll. + # + # Field shape verified against the live clusters.postgresql.cnpg.io CRD (CNPG 1.30.0, identical + # on prod and dev): spec.postgresql.synchronous requires `method` (enum any|first) and `number` + # (>0); `dataDurability` is enum required|preferred. + postgresql: + synchronous: + method: any + number: 1 + dataDurability: preferred storage: storageClass: longhorn size: 5Gi