diff --git a/apps/wave-0/nextcloud-db.yaml b/apps/wave-0/nextcloud-db.yaml index 2e5b51f..8a70f78 100644 --- a/apps/wave-0/nextcloud-db.yaml +++ b/apps/wave-0/nextcloud-db.yaml @@ -54,6 +54,21 @@ spec: inheritedMetadata: labels: app.kubernetes.io/name: nextcloud-db + # HARD anti-affinity — the three instances MUST land on three DIFFERENT nodes; and roll the + # primary by SWITCHOVER rather than CNPG's default in-place `restart`, which would delete and + # recreate the primary pod. Full rationale in platform/keycloak/postgres-cluster.yaml. Short + # version: CNPG's default `preferred` is a scheduler hint, so when CNPG recreates an instance + # after a node loss it may stack two instances on one node — which is how `instances: 3` decays + # back into the single-node failure domain this change removes. `required` makes it a constraint; + # the cost is that an instance whose node is gone stays Pending, which is correct (2 of 3 still + # serve). Satisfiable on both clusters: prod and dev each have 3 schedulable, untainted nodes. + # + # APPLYING THIS CHANGES THE POD SPEC, so CNPG rolls the instances: standbys first, then one + # switchover. Expect a single brief connection reset for Nextcloud. + affinity: + enablePodAntiAffinity: true + podAntiAffinityType: required + primaryUpdateMethod: switchover storage: storageClass: longhorn size: 5Gi diff --git a/apps/wave-0/semaphore-db.yaml b/apps/wave-0/semaphore-db.yaml index 65e364f..111bb7e 100644 --- a/apps/wave-0/semaphore-db.yaml +++ b/apps/wave-0/semaphore-db.yaml @@ -55,6 +55,21 @@ spec: inheritedMetadata: labels: app.kubernetes.io/name: semaphore-db + # HARD anti-affinity — the three instances MUST land on three DIFFERENT nodes; and roll the + # primary by SWITCHOVER rather than CNPG's default in-place `restart`, which would delete and + # recreate the primary pod. Full rationale in platform/keycloak/postgres-cluster.yaml. Short + # version: CNPG's default `preferred` is a scheduler hint, so when CNPG recreates an instance + # after a node loss it may stack two instances on one node — which is how `instances: 3` decays + # back into the single-node failure domain this change removes. `required` makes it a constraint; + # the cost is that an instance whose node is gone stays Pending, which is correct (2 of 3 still + # serve). Satisfiable on both clusters: prod and dev each have 3 schedulable, untainted nodes. + # + # APPLYING THIS CHANGES THE POD SPEC, so CNPG rolls the instances: standbys first, then one + # switchover. Expect a single brief connection reset for Semaphore. + affinity: + enablePodAntiAffinity: true + podAntiAffinityType: required + primaryUpdateMethod: switchover storage: storageClass: longhorn size: 5Gi diff --git a/platform/keycloak/postgres-cluster.yaml b/platform/keycloak/postgres-cluster.yaml index a50e61e..5e9fd3a 100644 --- a/platform/keycloak/postgres-cluster.yaml +++ b/platform/keycloak/postgres-cluster.yaml @@ -42,6 +42,39 @@ spec: # cause a switchover — see it for the blast radius.) instances: 3 imageName: ghcr.io/cloudnative-pg/postgresql:16.4 + # HARD anti-affinity — the three instances MUST land on three DIFFERENT nodes (2026-09-07). + # + # CNPG's default is `preferred`, which is only a scheduler HINT: the scheduler may put two + # instances on one node whenever that scores better, and it WILL do so on the path that matters — + # when CNPG recreates an instance after losing a node, the surviving nodes are the only candidates + # and one of them already runs an instance. So `preferred` is precisely how `instances: 3` decays + # back into the single-node failure domain this whole change exists to remove; nothing pins the + # good layout in place. `required` makes it a scheduling CONSTRAINT instead of a preference. + # + # THE TRADE, STATED PLAINLY: on a 3-node cluster with no spare node, an instance whose node is + # gone stays Pending until a node is available. That is the CORRECT behaviour here. A replacement + # squeezed onto a node that already hosts an instance buys nothing — both die with that node — + # while making the cluster LOOK three-way replicated. Availability is unaffected either way: + # 2 of 3 instances still serve and CNPG promotes one of them. + # + # Satisfiable on both clusters: prod and dev each have 3 schedulable, untainted nodes (checked + # read-only before merge). topologyKey defaults to kubernetes.io/hostname — the failure domain + # that matters here. enablePodAntiAffinity defaults to true; stated explicitly so this block can + # be read without knowing the default. + affinity: + enablePodAntiAffinity: true + podAntiAffinityType: required + # Roll the PRIMARY by SWITCHOVER, not by restarting it in place. CNPG's default for + # primaryUpdateMethod is `restart`, which deletes and recreates the primary pod — tens of seconds + # with no Postgres serving, and on THIS database that is an SSO outage. With standbys present + # (instances: 3 above) a switchover promotes an already-updated, already-streaming replica and + # demotes the old primary, so the interruption is a connection reset rather than a cold start. + # This is the field that makes the anti-affinity roll below a switchover rather than an outage. + # + # KNOWN CONSTRAINT: under `switchover` the operator REJECTS an update that changes `imageName` + # AND a postgresql parameter in the same reconcile. Renovate bumping the image alone is fine; + # never combine an image bump with a postgresql.parameters change in one commit. + primaryUpdateMethod: switchover storage: storageClass: longhorn size: 5Gi