From 37a6c94168c276e44595a97b613ce17b914aba83 Mon Sep 17 00:00:00 2001 From: Keith Jackson Date: Sat, 12 Sep 2026 18:05:49 -0500 Subject: [PATCH] fix(vpn): two gateways on one pinned PIA server evict each other every 2 minutes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Standing up prod broke dev, and the mechanism is worth spelling out because "same PIA account, well inside its connection limit" is true and still not enough. The vaulted .ovpn pins ONE server. With both gateways using it, dev and prod landed on the SAME server AND were handed the SAME tunnel address — both journals read `[Server-10779-2a] Peer Connection Initiated with [AF_INET]158.173.241.112:1198` and `ifconfig 10.72.192.7 255.255.192.0`. Each reconnect steals that address on that server; the loser then receives nothing, trips `--ping-restart` after 60s, reconnects and steals it back. Measured cycle: dev restarting at 22:57:05, 22:59:03, 23:01:05 UTC with prod interleaved about a minute off, each ~2 minutes apart, indefinitely. Neither environment had reliable vpn-class egress — prod's probes looked green only because they happened to land inside an up-window. Fix: `vpn_egress_gateway_pia_remote` (" ", empty = honour the .ovpn) rewrites just the `remote` line of the deployed .ovpn, and prod's group_vars sets it to 158.173.241.135. That is another member of the same PIA Stockholm LB pool — `sweden.privacy.network` resolved from k3s-prod-3 to 158.173.241.{34,55,112,135,239} — and every server in the pool takes the same credentials and the same inline CA, so only the address differs. Dev keeps the .ovpn's .112 untouched, which is why the override is per-group rather than in the role or a second vaulted .ovpn. Two details that matter for whoever touches this next: * The regex is anchored to a line STARTING with `remote` plus whitespace, so `remote-cert-tls` and `remote-random` are not collateral damage. The task keeps `no_log: true` — the transformation never widens what is logged. * A PIA server address is NOT a credential, so it lives in group_vars/vpn_egress_gateway_prod/vars.yml in the clear, next to the vault rather than inside it. Keeping it out of the vault is what makes it reviewable. If prod ever gets its own PIA account, this override can be deleted again. Co-Authored-By: Claude Opus 5 --- .../vpn_egress_gateway_prod/vars.yml | 22 +++++++++++++++++++ .../vpn_egress_gateway/defaults/main.yml | 17 ++++++++++++++ .../roles/vpn_egress_gateway/tasks/main.yml | 11 +++++++++- 3 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 infra/ansible/inventory/group_vars/vpn_egress_gateway_prod/vars.yml diff --git a/infra/ansible/inventory/group_vars/vpn_egress_gateway_prod/vars.yml b/infra/ansible/inventory/group_vars/vpn_egress_gateway_prod/vars.yml new file mode 100644 index 0000000..19e2ed4 --- /dev/null +++ b/infra/ansible/inventory/group_vars/vpn_egress_gateway_prod/vars.yml @@ -0,0 +1,22 @@ +--- +# Non-secret group_vars for the PROD egress gateway (k3s-prod-3). Credentials live in the +# vaulted vault.yml beside this file. + +# PROD DIALS ITS OWN PIA SERVER — and it has to. +# +# The vaulted .ovpn pins ONE server, 158.173.241.112. Measured 2026-09-12: with dev +# (k3s-dev-3) and prod (k3s-prod-3) both dialling it, PIA handed BOTH sessions the SAME +# tunnel address — `Server-10779-2a ... ifconfig 10.72.192.7` in both journals. Each +# reconnect steals that address from the other; the loser then sees no data for 60s, hits +# `--ping-restart`, reconnects and steals it back. Both tunnels flapped on a ~2-minute cycle +# (dev 22:57 / 22:59 / 23:01 UTC, prod interleaved a minute off) and neither environment had +# reliable vpn-class egress. This is NOT the account's connection limit — that has room. +# +# 158.173.241.135 is another member of the same PIA Stockholm LB pool: `sweden.privacy.network` +# resolved from k3s-prod-3 to 158.173.241.{34,55,112,135,239}. Any server in the pool accepts +# the same credentials and the same inline CA, so only the address differs. Dev keeps the +# .ovpn's pinned .112 untouched, which is why the override lives HERE and not in the role. +# +# A public server address is not a credential, so this is plain group_vars, not vault. +# If prod is ever moved to its own PIA account, this can go away again. +vpn_egress_gateway_pia_remote: "158.173.241.135 1198" diff --git a/infra/ansible/roles/vpn_egress_gateway/defaults/main.yml b/infra/ansible/roles/vpn_egress_gateway/defaults/main.yml index 359d1ea..817ad04 100644 --- a/infra/ansible/roles/vpn_egress_gateway/defaults/main.yml +++ b/infra/ansible/roles/vpn_egress_gateway/defaults/main.yml @@ -20,6 +20,23 @@ vpn_egress_gateway_pia_password: "{{ vault_vpn_egress_gateway_pia_password }}" # The FULL text of the PIA .ovpn (inline /, cipher/auth, remote line). vpn_egress_gateway_pia_ovpn: "{{ vault_vpn_egress_gateway_pia_ovpn }}" +# OVERRIDE THE .ovpn's PINNED `remote` — " ", empty to honour the .ovpn. +# +# WHY THIS EXISTS (measured 2026-09-12): the operator's .ovpn pins ONE PIA server IP. Point +# two gateways at the same vaulted .ovpn and both dial the SAME server, and PIA hands BOTH +# THE SAME tunnel address — dev and prod were both `Server-10779-2a` +# (158.173.241.112:1198) with `ifconfig 10.72.192.7`. Each reconnect then steals that +# address from the other, the loser sees no data for 60s, hits `--ping-restart`, reconnects +# and steals it back: both tunnels flapped on a ~2-minute cycle and neither was reliably +# up. It is NOT a connection-LIMIT problem (the account has room) — it is one pinned server. +# +# So each environment's gateway needs its own server. The PIA Stockholm LB pool +# (`sweden.privacy.network`) resolved to 158.173.241.{34,55,112,135,239}; any PIA server in +# the pool accepts the same credentials and CA, so this is a per-environment knob rather +# than a second vaulted .ovpn. NOT a secret — a public server address — so it belongs in +# group_vars//vars.yml, not the vault. +vpn_egress_gateway_pia_remote: "" + # --- on-disk layout ------------------------------------------------------------- vpn_egress_gateway_config_dir: /etc/openvpn/client vpn_egress_gateway_scripts_dir: /etc/openvpn/scripts diff --git a/infra/ansible/roles/vpn_egress_gateway/tasks/main.yml b/infra/ansible/roles/vpn_egress_gateway/tasks/main.yml index 401a13f..f9a7761 100644 --- a/infra/ansible/roles/vpn_egress_gateway/tasks/main.yml +++ b/infra/ansible/roles/vpn_egress_gateway/tasks/main.yml @@ -28,9 +28,18 @@ - "{{ vpn_egress_gateway_config_dir }}" - "{{ vpn_egress_gateway_scripts_dir }}" +# `vpn_egress_gateway_pia_remote`, when set, rewrites ONLY the `remote ` line so +# this environment's gateway dials its own PIA server. Two gateways sharing the .ovpn's single +# pinned server are handed the SAME tunnel address and evict each other every ~2 minutes — see +# the variable's comment in defaults/main.yml. The regex is anchored to a line that STARTS with +# `remote` followed by whitespace, so `remote-cert-tls` / `remote-random` are untouched. - name: Deploy the operator's proven PIA .ovpn (from Ansible vault) ansible.builtin.copy: - content: "{{ vpn_egress_gateway_pia_ovpn }}" + content: >- + {{ vpn_egress_gateway_pia_ovpn if (vpn_egress_gateway_pia_remote | length == 0) + else (vpn_egress_gateway_pia_ovpn + | regex_replace('(?m)^[ \t]*remote[ \t]+\S+.*$', + 'remote ' ~ vpn_egress_gateway_pia_remote)) }} dest: "{{ vpn_egress_gateway_ovpn_path }}" owner: root group: root