From 95a55952ccaf917df0abec9537590d0b888f56cd Mon Sep 17 00:00:00 2001 From: Keith Jackson Date: Sat, 22 Aug 2026 18:15:11 -0500 Subject: [PATCH] =?UTF-8?q?fix(vpn):=20disable=20vpn-egress=20=E2=80=94=20?= =?UTF-8?q?network=20blackholes=20PIA=20IPs=20(not=20ports/MTU/cluster)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Decisive diagnosis: OpenVPN's outbound init packets are 14 bytes with zero replies (so MTU was never the blocker), pod UDP egress is healthy (NTP replies), but TCP 501/443 to PIA IPs time out while Google/Cloudflare are OPEN — across many PIA servers. This 192.168.2.x WiFi segment blackholes PIA's IP ranges (and intercepts DNS). Not code-fixable. Reverted the port/MTU red-herring env to PIA defaults; disabled vpn-egress pending an operator network fix (move to the unfiltered 192.168.1.x LAN, or unblock PIA on the router). Full trace in DECISIONS-LOG. Co-Authored-By: Claude Opus 4.8 --- DECISIONS-LOG.md | 21 +++++++++++++++++++++ clusters/base/cluster-vars.yaml | 6 ++---- platform/kustomization.yaml | 12 +++++++----- platform/vpn-egress/gluetun-gateway.yaml | 16 ++++------------ 4 files changed, 34 insertions(+), 21 deletions(-) diff --git a/DECISIONS-LOG.md b/DECISIONS-LOG.md index 1479dd8..a94931c 100644 --- a/DECISIONS-LOG.md +++ b/DECISIONS-LOG.md @@ -560,3 +560,24 @@ return traffic) and had me check MTU at each layer. Findings via `ping -M do` sw Cilium VXLAN unchanged. Port kept at 8080 (confirmed PIA responds). Re-enabled `vpn-egress`. Values hardcoded as quoted strings (int-coercion gotcha). Alternatives not taken: Cilium native routing (pod MTU→1500) or WireGuard/thrnz (MTU-robust) — both still available if the MTU shrink proves insufficient. + +### Phase 3 VPN — ACTUAL ACTUAL root cause: the network blackholes PIA's IP ranges (2026-08-22) +The MTU shrink did NOT fix it. A node-side tcpdump of the handshake was decisive: OpenVPN's outbound init +packets are **14 bytes** (nowhere near any MTU) and get **zero replies** — so MTU was never the handshake +blocker either. Confirmed pod UDP egress is healthy (pod NTP/123 reply OK). Then the real test — raw +TCP/ICMP reachability to PIA IPs vs controls (from host AND pod): +- **PIA** (46.246.8.55, 212.112.19.22): TCP **501 and 443 time out** (silent drop, not refused), UDP no + reply, ICMP no reply — across many PIA servers gluetun cycled. +- **Google / Cloudflare**: TCP/443 OPEN, UDP replies, ICMP replies. + +→ **This network (192.168.2.x, WiFi) blackholes traffic to PIA's IP ranges.** It also transparently +intercepts DNS/:53. Everything else (HTTPS, DNS-via-proxy, NTP, general internet) works. So it is neither +ports, nor MTU (both red herrings — the Cilium VXLAN 1450 pod MTU is real but irrelevant to a 14-byte +handshake), nor the cluster/config — it is upstream network filtering of PIA specifically. + +**Action → operator (network, not code):** move the k3s host to the unfiltered main LAN (192.168.1.x / +wired), or unblock PIA's ranges on the 192.168.2.x router, then re-enable `vpn-egress` (no code change). +WireGuard/thrnz would NOT help — same PIA IPs are blocked; if switching providers is ever considered, pick +one whose ranges this network permits. Reverted the port/MTU red-herring env from gluetun-gateway.yaml +(back to PIA defaults); `vpn-egress` disabled so it doesn't crashloop/alert. **Config is correct and +complete — the only blocker is the network.** diff --git a/clusters/base/cluster-vars.yaml b/clusters/base/cluster-vars.yaml index 8d84364..f0c916e 100644 --- a/clusters/base/cluster-vars.yaml +++ b/clusters/base/cluster-vars.yaml @@ -43,10 +43,8 @@ data: # which lacks PF; NOT a US region, which also lack PF) CLUSTER_POD_CIDR: "10.0.0.0/8" # Cilium cluster-pool pod CIDR (pods get 10.x); Gluetun # FIREWALL_OUTBOUND_SUBNETS = allow these to egress via the tunnel - # NOTE: PIA OpenVPN port is HARDCODED "8080" in gluetun-gateway.yaml, not a cluster-var — Flux/kustomize - # strips quotes around a bare "${VAR}" so a numeric value coerces to int, which k8s rejects for an env - # value (same gotcha as the Longhorn numberOfReplicas fix). 8080 chosen: 1197 conflicts with the home - # OpenVPN server, 53 is DNS-intercepted here; both confirmed by tcpdump. See DECISIONS-LOG. + # NOTE: Phase-3 VPN (vpn-egress) is DISABLED — the network blackholes traffic to PIA's IP ranges + # (operator fix needed, not a config issue). Uses PIA's default OpenVPN UDP port. See DECISIONS-LOG. # --- identity / auth (ADR-0014) --- KEYCLOAK_REPLICAS: "1" # dev 1 | prod 2 (HA) diff --git a/platform/kustomization.yaml b/platform/kustomization.yaml index 0ea233f..e73e609 100644 --- a/platform/kustomization.yaml +++ b/platform/kustomization.yaml @@ -17,11 +17,13 @@ resources: - cert-manager # Let's Encrypt DNS-01 wildcard — needs SOPS `cloudflare-api-token` - externaldns # auto DNS records — needs SOPS `cloudflare-api-token` - gateway # Gateway API listeners (ingress; replaces Traefik) - # ---- Phase 3 — Egress / VPN (PIA OpenVPN-UDP, ADR-0013 amendment). Root cause of earlier failures: - # Cilium VXLAN gives pods MTU 1450 (node/host/WAN all 1500), so OpenVPN's 1500-byte handshake - # packets were dropped. Fix: OPENVPN_MTU/MSSFIX=1350 in gluetun-gateway.yaml so packets fit 1450. - # See DECISIONS-LOG "Phase 3 VPN". ---- - - vpn-egress # shared Gluetun PIA/OpenVPN-UDP egress gateway + kill-switch (ADR-0013) + # ---- Phase 3 — Egress / VPN — DISABLED: BLOCKED at the network (operator action needed). Definitive + # finding: this 192.168.2.x WiFi segment BLACKHOLES traffic to PIA's IP ranges — TCP 501/443 to PIA + # time out, UDP + ICMP get no reply across many PIA servers, while Google/Cloudflare work on all. + # It also intercepts DNS/:53. Not MTU, not ports, not the cluster — the config is correct. Fix at + # the network (move to the unfiltered 192.168.1.x LAN / wired, or unblock PIA on the router), then + # re-enable. See DECISIONS-LOG "Phase 3 VPN — network blocks PIA". ---- + # - vpn-egress # re-enable once the network permits PIA (no code change needed) # ---- Phase 4 — Observability (creds provisioned via SOPS: grafana-admin, alertmanager-smtp) ---- - monitoring # kube-prometheus-stack + Loki + Alloy + Alertmanager # ---- Later phases: enabled when their phase opens ---- diff --git a/platform/vpn-egress/gluetun-gateway.yaml b/platform/vpn-egress/gluetun-gateway.yaml index 1875392..6164f33 100644 --- a/platform/vpn-egress/gluetun-gateway.yaml +++ b/platform/vpn-egress/gluetun-gateway.yaml @@ -40,18 +40,10 @@ spec: value: "private internet access" - name: VPN_TYPE value: "openvpn" # OpenVPN-UDP — satisfies "protocol != TCP" (R-NET); ADR-0013 amended - - name: OPENVPN_ENDPOINT_PORT - value: "8080" # 8080: PIA confirmed responding here; avoid 53 (this network - # intercepts DNS/:53). Port is NOT the root cause — MTU is (below). - # Hardcoded string (not ${var}): Flux/kustomize strips quotes on - # "${VAR}" → int coercion k8s rejects (Longhorn-fix gotcha). - # --- MTU: Cilium runs VXLAN (routing-mode tunnel), so pods get MTU 1450 (1500 − 50 encap) while - # the node/host/WAN are all 1500. OpenVPN's default 1500-byte packets exceed 1450 and are - # dropped (PMTUD broken). Shrink tun-mtu so packets fit: 1350 + ~52 overhead ≈ 1402 < 1450. --- - - name: OPENVPN_MTU - value: "1350" # tun-mtu; keeps encrypted UDP packets under the 1450 pod MTU - - name: OPENVPN_MSSFIX - value: "1350" # clamp TCP-in-tunnel to match, avoid in-tunnel fragmentation + # NOTE (2026-08-22): PIA's default OpenVPN UDP (1197) is used. Earlier port/MTU overrides were + # removed — they were chasing symptoms. The real blocker is that this network BLACKHOLES traffic + # to PIA's IP ranges (TCP 501/443 time out, UDP/ICMP no reply — while Google/Cloudflare work). + # Fixable only at the network, not here. See DECISIONS-LOG "Phase 3 VPN — network blocks PIA". # PIA credentials from the SOPS secret (Gluetun's PIA OpenVPN creds env): - name: OPENVPN_USER valueFrom: