Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f21e89d9d2 | ||
|
|
2d7d302c7e |
No files matched your search
@@ -101,37 +101,10 @@ data:
|
||||
# ship the literal string into the cluster.
|
||||
|
||||
# k3s API endpoint Cilium's kube-proxy replacement dials (platform/cilium/helmrelease.yaml).
|
||||
#
|
||||
# *** THIS IS THE CONTROL-PLANE VIP (kube-vip), NOT A NODE ADDRESS. *** Operator decision
|
||||
# 2026-09-06. It was each environment's PRIMARY NODE IP until then, which made the CNI's
|
||||
# control-plane path depend on one machine in a cluster whose entire purpose is HA: verified on
|
||||
# live prod, all three cilium-agents carried KUBERNETES_SERVICE_HOST=192.168.2.10 (k3s-prod-1's
|
||||
# own address) while kube-vip held 192.168.2.9. Losing that node leaves the datapath running off
|
||||
# Cilium's cache while NO service or policy update lands — a silent, unbounded staleness, not an
|
||||
# outage you would notice. A virtual address that follows the control plane is the fix, and it is
|
||||
# what kube-vip was stood up to provide.
|
||||
#
|
||||
# dev -> 192.168.1.49 (= kube_vip_address, group_vars/all.yml; base default, below)
|
||||
# prod -> 192.168.2.9 (= kube_vip_address, group_vars/k3s_prod/vars.yml)
|
||||
#
|
||||
# THE NAME IS NOW SLIGHTLY WRONG AND IS KEPT ANYWAY: renaming the knob would be a second,
|
||||
# unrelated change to every substituting Kustomization at the same time as a live CNI endpoint
|
||||
# move. Read it as "the API host Cilium dials", which is what it always meant.
|
||||
#
|
||||
# [parity] MUST equal the Ansible bootstrap's **k3s_apiserver_endpoint** for the SAME environment
|
||||
# (infra/ansible/inventory/group_vars/all.yml -> roles/cilium/templates/cilium-values.yaml.j2,
|
||||
# already inventory-parameterized). NOT k3s_api_host — that variable still exists and still holds
|
||||
# the FOUNDER's address, because the cold-bootstrap `--server` join needs a machine that is
|
||||
# already up before any VIP exists. The two are deliberately separate; see the TWO API ADDRESSES
|
||||
# block at the top of group_vars/all.yml.
|
||||
#
|
||||
# Wrong value = Cilium cannot reach the API server on adopt and the CNI breaks. Port stays 6443
|
||||
# in both envs, so it is NOT a knob.
|
||||
#
|
||||
# TLS needs nothing: k3s_tls_sans has always carried kube_vip_address alongside every node IP, so
|
||||
# both addresses are already in the serving cert (verified live 2026-09-06 — prod answers through
|
||||
# https://192.168.2.9:6443 under full CA verification). No cert reissue, no k3s restart.
|
||||
K3S_API_HOST: "192.168.1.49"
|
||||
# [parity] MUST equal the Ansible bootstrap's k3s_api_host (infra/ansible/roles/cilium/defaults/
|
||||
# main.yml -> cilium-values.yaml.j2, already inventory-parameterized). Wrong value = Cilium cannot
|
||||
# reach the API server on adopt and the CNI breaks. Port stays 6443 in both envs, so it is NOT a knob.
|
||||
K3S_API_HOST: "192.168.1.50"
|
||||
|
||||
# CiliumLoadBalancerIPPool `default-pool` (platform-config/cilium/l2-ip-pool.yaml): the VIP range
|
||||
# L2-announced on the flat LAN. Must sit OUTSIDE the node IPs and any DHCP scope, and must match the
|
||||
|
||||
@@ -187,25 +187,11 @@ patches:
|
||||
- op: replace
|
||||
path: /data/GATEWAY_VIP
|
||||
value: "192.168.1.53"
|
||||
# k3s API endpoint Cilium's kube-proxy replacement dials — DEV'S CONTROL-PLANE VIP,
|
||||
# the address kube-vip floats, NOT the primary node .50.
|
||||
# [parity] equals the Ansible inventory's k3s_apiserver_endpoint for dev
|
||||
# (= kube_vip_address, group_vars/all.yml). See clusters/base/cluster-vars.yaml for why.
|
||||
#
|
||||
# *** THE ONE EXCEPTION TO "NEVER LET THIS BLOCK DRIFT FROM THE LIVE CLUSTER" ***
|
||||
# Every other value in this block is what dev is running right now. This one is not:
|
||||
# the live dev cilium-agents dial 192.168.1.50 (k3s-1). Changing it here is a
|
||||
# DELIBERATE, OPERATOR-APPROVED move (2026-09-06) and reconciling it ROLLS THE CILIUM
|
||||
# AGENT DAEMONSET on dev — one node at a time (updateStrategy.maxUnavailable: 1, kept
|
||||
# in parity for exactly this reason). Do not push this to dev-live casually, and not
|
||||
# while dev's DNS migration is mid-flight; the L2 datapath survives an agent restart
|
||||
# but a wrong value here is the 2026-09-01 incident. .49 is in dev's serving-cert SANs
|
||||
# already (k3s_tls_sans) and dev's own kubeconfig validates against it via
|
||||
# tls-server-name, so TLS is not the risk — the risk is reachability, and .49 is on the
|
||||
# same LAN as the nodes and ARP-announced by kube-vip on every server.
|
||||
# k3s API endpoint Cilium's kube-proxy replacement dials — the primary control-plane node
|
||||
# (k3s-1). [parity] equals the Ansible inventory's k3s_api_host for dev.
|
||||
- op: replace
|
||||
path: /data/K3S_API_HOST
|
||||
value: "192.168.1.49"
|
||||
value: "192.168.1.50"
|
||||
# Cilium LB IP pool = the tofu reservation cilium_l2_vip_range .53-.60, outside the node IPs
|
||||
# (.50/.51/.52) and outside the DHCP scope. .53 Gateway, .54 unifi, .55 crafty; .56-.60 free.
|
||||
- op: replace
|
||||
|
||||
@@ -185,9 +185,25 @@ patches:
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
spec:
|
||||
# TEMPORARY bootstrap source — workstation container, NOT in any cluster. See the block
|
||||
# above for the cutover to https://gitea.home.keithjackson.dev/... . Do not treat as final.
|
||||
url: http://192.168.2.132:3000/Home-Network/home-network-k3s-impl.git
|
||||
# CUTOVER DONE 2026-09-06 (operator direction: "move the repo to prod"). Prod now reconciles
|
||||
# from PROD'S OWN in-cluster Gitea. The workstation bootstrap container
|
||||
# (http://192.168.2.132:3000) is retired as prod's source — it was a podman container on the
|
||||
# operator's workstation holding a DHCP lease on WIFI, i.e. production's source of truth was a
|
||||
# laptop. It is kept only as a recovery path (bootstrap/gitea/bootstrap-gitea.sh).
|
||||
#
|
||||
# THIS LINE IS SELF-REFERENTIAL AND THAT IS THE WHOLE DIFFICULTY. The root Kustomization
|
||||
# renders ./clusters/prod, which contains this patch — so a live `kubectl patch` of
|
||||
# GitRepository.spec.url is REVERTED on the next reconcile unless the authored value moves
|
||||
# too. The cutover therefore had to be: (1) push the repo to prod's Gitea, (2) fast-forward
|
||||
# its `main` to the exact commit prod was already running so the source swap changed nothing
|
||||
# else, (3) live-patch the url to open a window, (4) land THIS commit through that window so
|
||||
# the authored value agrees. Dev hit the identical trap on 2026-09-05
|
||||
# (infra/PROD-PROVISIONING.md §10a "D0/D1 as executed").
|
||||
#
|
||||
# Verified before the switch: a throwaway pod INSIDE prod resolved this host through
|
||||
# in-cluster CoreDNS to the Gateway VIP and cloned over the public LE cert with no CA
|
||||
# supplied. The workstation being able to reach it proves nothing about pod egress.
|
||||
url: https://gitea.prod.home.keithjackson.dev/Home-Network/home-network-k3s-impl.git
|
||||
ref:
|
||||
branch: main
|
||||
- target:
|
||||
@@ -340,32 +356,14 @@ patches:
|
||||
- op: replace
|
||||
path: /data/CILIUM_LB_POOL_STOP
|
||||
value: "192.168.2.30"
|
||||
# --- still UNDECIDED -> loud placeholders (must be real before `flux bootstrap`) ------------
|
||||
# k3s API endpoint Cilium's kube-proxy replacement dials. This is the prod CONTROL-PLANE VIP
|
||||
# (kube-vip), NOT .20 (the ingress VIP) and NOT the single node address .10.
|
||||
#
|
||||
# *** DECIDED 2026-09-06 by the operator: the VIP, as intended. *** (This block previously
|
||||
# read "Not yet decided" while shipping .10 — the founder's own address — and that is what
|
||||
# the LIVE prod cluster is running: all three cilium-agents carry
|
||||
# KUBERNETES_SERVICE_HOST=192.168.2.10 while kube-vip holds .9, verified 2026-09-06. On a
|
||||
# 3-node HA cluster that makes the CNI's control-plane path depend on k3s-prod-1: lose it and
|
||||
# the datapath keeps running off Cilium's cache while no service or policy update lands, with
|
||||
# nothing looking broken. The VIP is a virtual address that reaches the cluster wherever the
|
||||
# control plane is, which is the whole reason kube-vip exists here.)
|
||||
#
|
||||
# It is owned Ansible-side by **k3s_apiserver_endpoint** (group_vars/all.yml, derived from
|
||||
# this environment's kube_vip_address = 192.168.2.9 in group_vars/k3s_prod/vars.yml) and MUST
|
||||
# equal that value — [parity] contract in platform/cilium/helmrelease.yaml. NOT k3s_api_host:
|
||||
# that variable still exists and still holds the FOUNDER's .10, because the cold-bootstrap
|
||||
# `--server` join needs a host that is up before any VIP is floated. Two addresses, two
|
||||
# variables, deliberately.
|
||||
#
|
||||
# .9 sits OUTSIDE the Cilium LB pool (.20-.30) and is not a node address, so kube-vip and
|
||||
# LB-IPAM can never be handed the same IP — asserted by
|
||||
# conformance/tests/l0-static/ha-control-plane-as-code.
|
||||
# (kube-vip), NOT .20 (the ingress VIP) and NOT the single node address .10. It is owned
|
||||
# Ansible-side (prod inventory k3s_api_host) and MUST equal that value — [parity] contract in
|
||||
# platform/cilium/helmrelease.yaml. Not yet decided.
|
||||
- op: replace
|
||||
path: /data/K3S_API_HOST
|
||||
value: "192.168.2.9"
|
||||
# --- still UNDECIDED -> loud placeholders (must be real before `flux bootstrap`) ------------
|
||||
value: "192.168.2.10"
|
||||
# Pinned L4 VIPs, both carved out of the pool by lbipam.cilium.io/ips. Not yet assigned for
|
||||
# prod. Leaving the dev literals in place would hand a prod LoadBalancer a dev-LAN address, so
|
||||
# they stay placeholders; whatever is chosen MUST fall inside [POOL_START, POOL_STOP] and not
|
||||
|
||||
+5
-26
@@ -79,30 +79,11 @@ Ansible installs Cilium; the Flux `platform/cilium` HelmRelease adopts config
|
||||
afterward. The parity-critical values **must match** between this role's
|
||||
`templates/cilium-values.yaml.j2` and that HelmRelease:
|
||||
- `kubeProxyReplacement: true`
|
||||
- `k8sServiceHost: {{ k3s_apiserver_endpoint }}` — **the kube-vip CONTROL-PLANE VIP**
|
||||
(`= kube_vip_address`: dev `192.168.1.49`, prod `192.168.2.9`), **not** `k3s_api_host`
|
||||
- `k8sServicePort: {{ k3s_api_port }}` (6443 in both environments)
|
||||
- `k8sServiceHost: {{ k3s_api_host }}` (= primary node IP, `10.73.0.11` by default)
|
||||
- `k8sServicePort: 6443`
|
||||
|
||||
### Two API addresses, split 2026-09-06 — do not re-merge them
|
||||
`k3s_api_host` and `k3s_apiserver_endpoint` were one variable, and the conflation put every
|
||||
prod `cilium-agent` on `192.168.2.10` (`k3s-prod-1`'s own address) on a cluster whose point is
|
||||
HA — verified live. Lose that node and the datapath keeps serving from Cilium's cache while no
|
||||
service or policy update ever lands, which is a silent failure, not an outage.
|
||||
|
||||
| variable | value | consumed by | why it must be that address |
|
||||
|---|---|---|---|
|
||||
| `k3s_api_host` | the founder's own node IP (dev `192.168.1.50`, prod `192.168.2.10`) | `roles/k3s_server`: the secondary `--server https://…:6443` join URL, and one `k3s_tls_sans` entry | on a **cold** build no VIP exists yet (kube-vip is a k3s auto-deploy manifest, so it needs a cluster first). The founder just ran `--cluster-init` and is up |
|
||||
| `k3s_apiserver_endpoint` | `{{ kube_vip_address }}` (dev `192.168.1.49`, prod `192.168.2.9`) | `roles/cilium` → `k8sServiceHost`; any steady-state client | it must survive losing any single server |
|
||||
|
||||
Consequences already handled: `playbooks/site.yml` imports `kube-vip.yml` **between** `k3s.yml`
|
||||
and `cilium.yml` (the VIP has to be floating before the CNI dials it), and `roles/kube_vip`'s
|
||||
last task blocks until the VIP answers on 6443. `k3s_tls_sans` already carried both addresses,
|
||||
so no cert reissue and no k3s restart are involved.
|
||||
|
||||
The Flux side of the same contract is `${K3S_API_HOST}` in `platform/cilium/helmrelease.yaml`,
|
||||
set per overlay in `clusters/<env>/kustomization.yaml`; `[parity]` means it must equal
|
||||
`k3s_apiserver_endpoint` for the SAME environment. Asserted by
|
||||
`conformance/tests/l0-static/ha-control-plane-as-code`.
|
||||
The HelmRelease currently has `k8sServiceHost: "REPLACE-ME"` — WS-NET must set it
|
||||
to the same value this role uses (surfaced by `tofu output primary_node_ip`).
|
||||
|
||||
## Interface assumed of `bootstrap/` (authored by other workers, NOT here)
|
||||
`roles/flux_bootstrap` invokes two entrypoints via overridable command vars:
|
||||
@@ -167,9 +148,7 @@ ansible-playbook playbooks/site.yml \
|
||||
What the roles then resolve **by themselves** for a prod host, from `group_vars/k3s_prod/vars.yml`:
|
||||
`k3s_primary_group=k3s_prod_primary` (the `--cluster-init` founder, the node-token `delegate_to`
|
||||
target, and the node Cilium fetches the kubeconfig from), `k3s_servers_group=k3s_prod` (the tls-san
|
||||
node-IP list), `k3s_api_host=192.168.2.10` (the founder, bootstrap-join only),
|
||||
`kube_vip_address=192.168.2.9` — and therefore `k3s_apiserver_endpoint=192.168.2.9`, which is what
|
||||
Cilium dials — and
|
||||
node-IP list), `k3s_api_host=192.168.2.10`, `kube_vip_address=192.168.2.9`, and
|
||||
`cluster_kubeconfig_name=prod.kubeconfig` — so a prod run can never overwrite the
|
||||
`.cache/dev.kubeconfig` that `infra/devcontainer/dev.sh` exports as `$KUBECONFIG`.
|
||||
|
||||
|
||||
@@ -27,85 +27,19 @@ k3s_servers_group: k3s_servers
|
||||
# a kubeconfig is a credential.)
|
||||
cluster_kubeconfig_name: dev.kubeconfig
|
||||
|
||||
# --- TWO API ADDRESSES, AND THEY ARE NOT INTERCHANGEABLE (split 2026-09-06) ----------
|
||||
# This used to be ONE variable, k3s_api_host, consumed by BOTH the cold-bootstrap join and
|
||||
# Cilium's k8sServiceHost. That conflation is what pointed every cilium-agent on the LIVE
|
||||
# prod cluster at 192.168.2.10 — k3s-prod-1's own address — on a cluster whose entire
|
||||
# purpose is HA: verified 2026-09-06, all three agents carried
|
||||
# KUBERNETES_SERVICE_HOST=192.168.2.10 while kube-vip held 192.168.2.9. Lose that one node
|
||||
# and no service/policy update reaches the CNI until it returns (the existing datapath
|
||||
# survives on Cilium's cache, so it fails SILENTLY). The two consumers have genuinely
|
||||
# different requirements, so they now have two variables.
|
||||
#
|
||||
# k3s_api_host THE FOUNDER'S OWN ADDRESS. Bootstrap-time only.
|
||||
# k3s_apiserver_endpoint THE HA ENDPOINT (the kube-vip VIP). Steady-state.
|
||||
#
|
||||
# DO NOT "simplify" these back into one. See each block below for why each half must be
|
||||
# the address it is.
|
||||
|
||||
# THE FOUNDER'S OWN NODE IP — a BOOTSTRAP-TIME address, deliberately NOT the VIP.
|
||||
# The kube-apiserver endpoint the whole stack agrees on: the primary's REAL node IP.
|
||||
# NOTE: ansible_host is now the pfSense NAT SSH target (192.168.2.102:225x), NOT the node
|
||||
# IP — so we read the real 192.168.1.x from k3s_node_ip instead.
|
||||
#
|
||||
# EXACTLY TWO CONSUMERS, and both of them need the founder specifically:
|
||||
# 1. roles/k3s_server — the secondary `--server https://<host>:<port>` JOIN URL. On a COLD
|
||||
# build the VIP DOES NOT EXIST YET at join time: kube-vip is a k3s auto-deploy manifest
|
||||
# that only lands once a cluster exists, so a join aimed at the VIP is a join aimed at
|
||||
# an unanswered address and the cold build is unjoinable. The founder is up (it just ran
|
||||
# --cluster-init), which is the whole reason this variable is the founder.
|
||||
# 2. roles/k3s_server — one entry of k3s_tls_sans, so the founder's own address is a SAN
|
||||
# alongside the VIP and every other server's IP.
|
||||
#
|
||||
# ANYTHING THAT RUNS AGAINST A CLUSTER THAT ALREADY EXISTS MUST NOT USE THIS. It is a
|
||||
# single point of failure by construction. Use k3s_apiserver_endpoint below.
|
||||
# IP — so we read the real 192.168.1.x from k3s_node_ip instead. Used for k3s --tls-san /
|
||||
# secondary --server AND for Cilium k8sServiceHost so the Ansible bootstrap and the Flux
|
||||
# HelmRelease stay in PARITY (docs/02 contract).
|
||||
k3s_api_host: "{{ hostvars[groups[k3s_primary_group][0]]['k3s_node_ip'] }}"
|
||||
|
||||
# Shared by both halves: k3s serves the API on 6443 in every environment, so the port is
|
||||
# not a per-environment knob and is not duplicated per address.
|
||||
k3s_api_port: 6443
|
||||
|
||||
# THE HA kube-apiserver ENDPOINT — a VIRTUAL address that reaches the cluster wherever the
|
||||
# control plane currently is, not a machine. OPERATOR DECISION 2026-09-06: in-cluster
|
||||
# components dial the VIP, as kube-vip was always intended to provide.
|
||||
#
|
||||
# DERIVED from kube_vip_address (below), never restated: an environment overrides the VIP in
|
||||
# ONE place (group_vars/k3s_prod/vars.yml sets kube_vip_address: 192.168.2.9) and this
|
||||
# follows automatically, so prod needs no override of its own and a fourth environment
|
||||
# cannot set one of the two and forget the other.
|
||||
#
|
||||
# NAMED FOR THE ROLE, NOT THE MECHANISM. Consumers want "the address that reaches the
|
||||
# apiserver, HA"; that it is currently floated by kube-vip is an implementation detail. If
|
||||
# the mechanism is ever something else (an external LB, a DNS name), this one line moves and
|
||||
# no consumer changes.
|
||||
#
|
||||
# CONSUMERS:
|
||||
# - roles/cilium -> cilium-values.yaml.j2 -> k8sServiceHost. [parity] with the Flux side:
|
||||
# clusters/base/cluster-vars.yaml K3S_API_HOST (and each overlay's patch) MUST equal
|
||||
# THIS value for that environment, or Flux and the bootstrap fight over the CNI's
|
||||
# control-plane path. See platform/cilium/helmrelease.yaml's PARITY CONTRACT header.
|
||||
#
|
||||
# TLS IS ALREADY COVERED — verified, not assumed: k3s_tls_sans (roles/k3s_server/defaults)
|
||||
# has always included kube_vip_address, and on 2026-09-06 the prod apiserver was reached
|
||||
# through https://192.168.2.9:6443 with FULL CA verification and tls-server-name=192.168.2.9
|
||||
# (infra/ansible/.cache/prod.kubeconfig, no insecure-skip-tls-verify). So the serving cert
|
||||
# needs no change and no k3s restart for this switch.
|
||||
#
|
||||
# ORDERING CONSEQUENCE, and it is why playbooks/site.yml changed too: Cilium now needs the
|
||||
# VIP to be UP before it installs. site.yml therefore imports kube-vip.yml BETWEEN k3s.yml
|
||||
# and cilium.yml, and roles/kube_vip waits for the VIP to answer before returning. kube-vip
|
||||
# can precede the CNI because its DaemonSet is hostNetwork with
|
||||
# KUBERNETES_SERVICE_HOST=127.0.0.1 — it needs neither a CNI nor kube-proxy.
|
||||
k3s_apiserver_endpoint: "{{ kube_vip_address }}"
|
||||
|
||||
# Control-plane HA VIP floated by kube-vip (roles/kube_vip). A stable failover
|
||||
# endpoint for the kube-apiserver, fronted by an operator pfSense port-forward at
|
||||
# 192.168.1.49:6443. Shared here so THREE things agree on the one address:
|
||||
# - roles/kube_vip — what to ARP and lease-elect for;
|
||||
# - roles/k3s_server — the tls-san entry (the serving cert must carry this IP or
|
||||
# TLS via the VIP fails to validate);
|
||||
# - k3s_apiserver_endpoint — above; what in-cluster components actually dial.
|
||||
# THIS is the one line an environment overrides (prod: 192.168.2.9, in
|
||||
# group_vars/k3s_prod/vars.yml); all three follow.
|
||||
# 192.168.1.49:6443. Shared here so BOTH kube_vip AND the k3s_server tls-san list
|
||||
# agree on the address (the serving cert must carry this IP or TLS via the VIP
|
||||
# fails to validate).
|
||||
kube_vip_address: "192.168.1.49"
|
||||
|
||||
# Controller (dev container) reaches the kube-API DIRECTLY via the pfSense NAT
|
||||
|
||||
@@ -51,18 +51,12 @@ k3s_servers_group: k3s_prod
|
||||
# than no kubeconfig at all). .cache/ is gitignored; a kubeconfig is a credential.
|
||||
cluster_kubeconfig_name: prod.kubeconfig
|
||||
|
||||
# --- k3s API host — THE FOUNDER'S ADDRESS, bootstrap-time only --------------------
|
||||
# --- k3s API host — safe to set now, the primary's real IP is already known -------
|
||||
# Dev's k3s_api_host (group_vars/all.yml) reads a separate k3s_node_ip hostvar because
|
||||
# dev's ansible_host is the pfSense NAT SSH target, not the node's real address. Prod
|
||||
# has NO NAT hop — ansible_host in hosts.ini IS already the node's real, routable LAN
|
||||
# address (see the KEY DIFFERENCE note there) — so point straight at it instead of
|
||||
# needing a k3s_node_ip split.
|
||||
#
|
||||
# WHAT THIS IS *NOT*, since 2026-09-06: it is not the endpoint anything steady-state
|
||||
# dials. That is k3s_apiserver_endpoint (group_vars/all.yml), which derives from
|
||||
# kube_vip_address below and therefore needs NO override here. k3s_api_host is consumed
|
||||
# only by the secondary `--server` cold-join URL and by one tls-san entry. Read the
|
||||
# TWO API ADDRESSES block at the top of group_vars/all.yml before touching either.
|
||||
k3s_api_host: "{{ hostvars[groups[k3s_primary_group][0]]['ansible_host'] }}"
|
||||
# cluster_controller_tls_server_name (group_vars/all.yml) derives from kube_vip_address
|
||||
# by template reference, so it needs no separate override here — it picks up the real
|
||||
@@ -81,15 +75,8 @@ k3s_api_host: "{{ hostvars[groups[k3s_primary_group][0]]['ansible_host'] }}"
|
||||
# round (control-plane .49, ingress pool .53-.60).
|
||||
#
|
||||
# This is the address kube-vip floats for the API and that k3s tls-san's onto its
|
||||
# serving cert. Dev's equivalent is kube_vip_address: 192.168.1.49 in group_vars/all.yml.
|
||||
#
|
||||
# CORRECTION 2026-09-06: an earlier version of this comment claimed "secondaries use it as
|
||||
# their --server join target". They do not, and they must not — on a COLD build the VIP is
|
||||
# not up when the secondaries join (kube-vip is a k3s auto-deploy manifest and needs a
|
||||
# cluster first), so the join uses k3s_api_host, the founder. What DOES use this address is
|
||||
# k3s_apiserver_endpoint (group_vars/all.yml, derived from this line), i.e. Cilium's
|
||||
# k8sServiceHost and any other steady-state client. Overriding kube_vip_address here moves
|
||||
# BOTH the VIP and prod's Cilium endpoint together, which is the point of deriving it.
|
||||
# serving cert; secondaries use it as their --server join target. Dev's equivalent is
|
||||
# kube_vip_address: 192.168.1.49 in group_vars/all.yml.
|
||||
#
|
||||
# HISTORY: an earlier pass here wrongly set this to 192.168.2.20 — that is the INGRESS
|
||||
# VIP, not the control-plane VIP. Corrected. Keep the two distinct.
|
||||
|
||||
@@ -3,13 +3,6 @@
|
||||
# Control-plane ONLY (svc_enable=false) — Cilium owns Service LoadBalancing.
|
||||
# Writes the manifest into each server's k3s auto-deploy dir; k3s applies it.
|
||||
#
|
||||
# PART OF THE ONE-COMMAND BRING-UP SINCE 2026-09-06, not a separate operator step:
|
||||
# playbooks/site.yml imports this play BETWEEN k3s.yml and cilium.yml, because Cilium's
|
||||
# k8sServiceHost is now this VIP (k3s_apiserver_endpoint, group_vars/all.yml) and the CNI
|
||||
# cannot be installed against an endpoint nothing announces yet. The role's final task waits
|
||||
# for the VIP to answer on 6443, so a failure to float it stops the run here with a named
|
||||
# cause instead of surfacing later as unreachable cilium-agents.
|
||||
#
|
||||
# TARGET (R-ENV-02): `k3s_hosts` defaults to dev's `k3s_servers`; prod is
|
||||
# ansible-playbook playbooks/kube-vip.yml -e k3s_hosts=k3s_prod
|
||||
# (the VIP itself comes from kube_vip_address, already scoped per environment in
|
||||
|
||||
@@ -1,10 +1,7 @@
|
||||
---
|
||||
# Full dev/test cluster bring-up, in order (docs/02 sequencing; run AFTER
|
||||
# `tofu apply` has created and cloud-init has baselined the VMs):
|
||||
# k3s HA install -> kube-vip control-plane VIP -> Cilium CNI -> verify readiness
|
||||
# -> Flux + SOPS hand-off.
|
||||
# (kube-vip joined the chain 2026-09-06 — see the note at its import below; it is a
|
||||
# prerequisite of the CNI now, not a separate operator step.)
|
||||
# k3s HA install -> Cilium CNI -> verify readiness -> Flux + SOPS hand-off.
|
||||
#
|
||||
# TARGET (R-ENV-02): every imported play takes its host pattern from a var that
|
||||
# defaults to the dev group, so `ansible-playbook playbooks/site.yml` with no extra
|
||||
@@ -23,29 +20,6 @@
|
||||
- name: Import - install k3s HA control plane
|
||||
ansible.builtin.import_playbook: k3s.yml
|
||||
|
||||
# ADDED 2026-09-06, AND ITS POSITION IS LOAD-BEARING: kube-vip MUST come up between k3s and
|
||||
# Cilium. Cilium's k8sServiceHost is now the control-plane VIP (k3s_apiserver_endpoint =
|
||||
# kube_vip_address) rather than the founder's own node IP, so installing Cilium before the
|
||||
# VIP is announced would point every agent at an address nothing answers on — the exact
|
||||
# shape of the 2026-09-01 incident, where agents that could not reach the API server
|
||||
# deadlocked on a cold cluster.
|
||||
#
|
||||
# IT CANNOT COME EARLIER. kube-vip ships as a k3s auto-deploy manifest
|
||||
# (<data-dir>/server/manifests), so a cluster must exist before it can be applied — hence
|
||||
# after k3s.yml, not before.
|
||||
#
|
||||
# IT DOES NOT NEED THE CNI, which is what makes this ordering possible at all: the kube-vip
|
||||
# DaemonSet is hostNetwork with KUBERNETES_SERVICE_HOST=127.0.0.1 (roles/kube_vip's template
|
||||
# explains why), so it reaches the node-local apiserver with no CNI and no kube-proxy, and
|
||||
# DaemonSet pods carry the default not-ready/network-unavailable tolerations that let them
|
||||
# schedule onto a still-NotReady node. roles/kube_vip blocks until the VIP actually answers
|
||||
# on 6443, so cilium.yml below cannot start against a VIP that is not up.
|
||||
#
|
||||
# Re-running this against the LIVE dev cluster is a no-op for this play: dev already runs
|
||||
# kube-vip with the same rendered manifest, and the wait finds the VIP already answering.
|
||||
- name: Import - float the control-plane HA VIP (kube-vip) BEFORE the CNI dials it
|
||||
ansible.builtin.import_playbook: kube-vip.yml
|
||||
|
||||
- name: Import - install Cilium CNI
|
||||
ansible.builtin.import_playbook: cilium.yml
|
||||
|
||||
|
||||
@@ -2,17 +2,7 @@
|
||||
# cilium role defaults (ADR-0002, docs/02 Cilium-ownership contract).
|
||||
# This role performs the INITIAL Cilium install; the Flux platform/cilium
|
||||
# HelmRelease adopts config afterward. The parity-critical values
|
||||
# (kubeProxyReplacement, k8sServiceHost/Port) are templated from
|
||||
# k3s_apiserver_endpoint / k3s_api_port (group_vars/all.yml).
|
||||
#
|
||||
# k8sServiceHost IS THE HA VIP, NOT THE FOUNDER (changed 2026-09-06). It reads
|
||||
# k3s_apiserver_endpoint — which derives from kube_vip_address, so it is dev 192.168.1.49 /
|
||||
# prod 192.168.2.9 — and deliberately NOT k3s_api_host, the founder's own node IP. Those two
|
||||
# were one variable until 2026-09-06, and the conflation is what left all three prod
|
||||
# cilium-agents dialing k3s-prod-1 on an HA cluster. Do not re-merge them.
|
||||
#
|
||||
# THIS ROLE THEREFORE DEPENDS ON THE VIP EXISTING. playbooks/site.yml runs kube-vip.yml
|
||||
# before cilium.yml and roles/kube_vip blocks until the VIP answers on 6443.
|
||||
# (kubeProxyReplacement, k8sServiceHost/Port) are templated from k3s_api_host.
|
||||
|
||||
# Exact pin — MUST equal the platform/cilium HelmRelease chart.spec.version (1.20.1, an
|
||||
# exact pin, not a range). Renovate bumps both together.
|
||||
|
||||
@@ -89,7 +89,7 @@
|
||||
run_once: true
|
||||
changed_when: false
|
||||
|
||||
- name: Render the Cilium values file (parity-critical settings from k3s_apiserver_endpoint)
|
||||
- name: Render the Cilium values file (parity-critical settings from k3s_api_host)
|
||||
ansible.builtin.template:
|
||||
src: cilium-values.yaml.j2
|
||||
dest: "{{ cilium_values_path }}"
|
||||
|
||||
@@ -4,24 +4,9 @@
|
||||
---
|
||||
# [parity] k3s kube-proxy is disabled → Cilium replaces it (ADR-0001/0002).
|
||||
kubeProxyReplacement: true
|
||||
# [parity] = k3s_apiserver_endpoint, THE HA CONTROL-PLANE VIP (kube_vip_address) — dev
|
||||
# 192.168.1.49, prod 192.168.2.9. NOT k3s_api_host, which is the FOUNDER'S OWN node IP:
|
||||
# pointing the CNI's kube-proxy replacement at one machine on a 3-server HA cluster means
|
||||
# losing that machine stops every service/policy update reaching the datapath, silently
|
||||
# (the existing datapath keeps working from Cilium's cache, so nothing looks broken). That
|
||||
# was the live state of prod until 2026-09-06 — all three cilium-agents carried
|
||||
# KUBERNETES_SERVICE_HOST=192.168.2.10 while kube-vip held .9. See group_vars/all.yml.
|
||||
#
|
||||
# [parity] The Flux side is platform/cilium/helmrelease.yaml's k8sServiceHost:
|
||||
# "${K3S_API_HOST}", whose per-environment value comes from clusters/<env>/kustomization.yaml.
|
||||
# THAT VALUE MUST EQUAL k3s_apiserver_endpoint for the SAME environment. If it does not,
|
||||
# Flux and this bootstrap render different control-plane paths and fight over the CNI.
|
||||
#
|
||||
# THE VIP MUST BE UP BEFORE THIS ROLE RUNS. playbooks/site.yml imports kube-vip.yml between
|
||||
# k3s.yml and cilium.yml for exactly this reason, and roles/kube_vip blocks until the VIP
|
||||
# answers. Rendering this against a VIP that nothing announces yet reproduces the
|
||||
# 2026-09-01 shape: agents that cannot reach the API server on a cold cluster.
|
||||
k8sServiceHost: "{{ k3s_apiserver_endpoint }}"
|
||||
# [parity] = the k3s API endpoint (primary node IP). The Flux HelmRelease's
|
||||
# k8sServiceHost "REPLACE-ME" must be set to this same value.
|
||||
k8sServiceHost: "{{ k3s_api_host }}"
|
||||
k8sServicePort: {{ k3s_api_port }}
|
||||
|
||||
# Ingress via Gateway API — no Traefik (docs/09, ADR-0002).
|
||||
|
||||
@@ -59,17 +59,6 @@ k3s_server_mode: "secondary"
|
||||
# serving cert on restart when this set changes. Deduplicated (the primary IP is
|
||||
# also k3s_api_host).
|
||||
#
|
||||
# THIS LIST COVERS BOTH API ADDRESSES, WHICH IS WHY THE 2026-09-06 SPLIT NEEDED NO CERT
|
||||
# CHANGE AND NO k3s RESTART. group_vars/all.yml now distinguishes k3s_api_host (the
|
||||
# founder, for the cold join) from k3s_apiserver_endpoint (= kube_vip_address, what Cilium
|
||||
# and every steady-state client dial). Both are already in this set, so repointing a
|
||||
# consumer from one to the other cannot produce a TLS failure. VERIFIED LIVE 2026-09-06,
|
||||
# not assumed: prod's apiserver answered through https://192.168.2.9:6443 with full CA
|
||||
# verification and tls-server-name=192.168.2.9. Do not remove either entry — dropping
|
||||
# kube_vip_address breaks every client of the VIP; dropping k3s_api_host breaks nothing
|
||||
# today only because it is also in the k3s_servers_group node-IP list below, and that
|
||||
# coincidence is not a contract.
|
||||
#
|
||||
# k3s_servers_group — NOT a literal group name — is what makes this role environment-
|
||||
# agnostic (R-ENV-02): group_vars/all.yml sets it to `k3s_servers` (dev) and
|
||||
# group_vars/k3s_prod/vars.yml to `k3s_prod`, so each environment SANs its own servers.
|
||||
|
||||
@@ -155,18 +155,6 @@
|
||||
| default(k3s_primary_token_raw.content | b64decode | trim, true) }}
|
||||
when: k3s_server_mode == 'secondary'
|
||||
|
||||
# THE JOIN TARGET IS k3s_api_host — THE FOUNDER — AND MUST NOT BECOME THE VIP.
|
||||
# group_vars/all.yml splits the two API addresses (k3s_api_host = the founder's own node IP,
|
||||
# k3s_apiserver_endpoint = the kube-vip HA VIP) and this is the canonical founder-only
|
||||
# consumer. On a COLD build the VIP does not exist at this point in the sequence: kube-vip
|
||||
# is a k3s auto-deploy manifest, so it cannot be applied until a cluster exists, and
|
||||
# playbooks/site.yml only imports kube-vip.yml AFTER k3s.yml. A `--server` pointed at the
|
||||
# VIP would therefore dial an address nothing announces and the cold build would be
|
||||
# unjoinable — with no diagnostic beyond a connection timeout.
|
||||
# On a WARM single-node rejoin (`--limit k3s-dev-N`) the founder is up and answering too, so
|
||||
# the founder works in both cases and the VIP works in only one. Cilium's k8sServiceHost is
|
||||
# the opposite trade (steady state, must survive losing the founder) and uses the endpoint
|
||||
# variable instead.
|
||||
- name: Install k3s secondary (join as server)
|
||||
ansible.builtin.command:
|
||||
cmd: >-
|
||||
|
||||
@@ -24,16 +24,6 @@ kube_vip_port: 6443
|
||||
# only if detection is wrong.
|
||||
kube_vip_interface: ""
|
||||
|
||||
# How long to wait for the VIP to actually ANSWER on kube_vip_port after the manifest is
|
||||
# written (seconds). This is a real gate, not cosmetics: since 2026-09-06 Cilium's
|
||||
# k8sServiceHost is this VIP (k3s_apiserver_endpoint), and playbooks/site.yml runs this role
|
||||
# immediately before cilium.yml — so if the VIP never comes up, the run must stop HERE with
|
||||
# a clear message rather than proceed to install a CNI pointed at an address nothing
|
||||
# answers. On a cold cluster the budget covers k3s's deploy controller applying the
|
||||
# manifest, the ghcr.io image pull and leader election; on an already-running cluster the
|
||||
# check returns immediately.
|
||||
kube_vip_wait_timeout: 300
|
||||
|
||||
# Leader-election tuning (seconds) — a snappy failover without thrashing.
|
||||
kube_vip_lease_duration: 15
|
||||
kube_vip_renew_deadline: 10
|
||||
|
||||
@@ -21,14 +21,6 @@
|
||||
executable: /bin/bash
|
||||
register: kube_vip_iface_detect
|
||||
changed_when: false
|
||||
# RUN THIS EVEN UNDER --check. It is `ip -o -4 addr show`: it reads and changes nothing.
|
||||
# Without check_mode: false the shell module is SKIPPED in check mode, kube_vip_iface_detect
|
||||
# has no stdout, and the assert two tasks down aborts the play with "Could not detect the
|
||||
# interface ... set kube_vip_interface explicitly" — a misleading failure that says nothing
|
||||
# about the node. That mattered less while this play was an out-of-band operator step; it
|
||||
# matters now that playbooks/site.yml imports it, because `--check` on the bring-up before a
|
||||
# cold prod build is exactly the rehearsal an operator should be able to run.
|
||||
check_mode: false
|
||||
|
||||
- name: Resolve the kube-vip VIP interface (override wins, else detected)
|
||||
ansible.builtin.set_fact:
|
||||
@@ -60,34 +52,3 @@
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0600"
|
||||
|
||||
# PROVE THE VIP IS ACTUALLY UP BEFORE THIS PLAY RETURNS (added 2026-09-06).
|
||||
#
|
||||
# Templating the manifest is not the same as having a floating endpoint: k3s's deploy
|
||||
# controller still has to apply it, the image still has to pull, and leader election still
|
||||
# has to pick an owner. Nothing used to check, which was tolerable while the VIP was only an
|
||||
# operator convenience. It is not tolerable now — Cilium's k8sServiceHost IS this VIP
|
||||
# (k3s_apiserver_endpoint) and playbooks/site.yml runs cilium.yml straight after this play,
|
||||
# so an unannounced VIP would mean installing a CNI that cannot reach the API server. That
|
||||
# failure is the 2026-09-01 shape and it is very expensive to diagnose from the far side;
|
||||
# failing here instead names the actual cause.
|
||||
#
|
||||
# run_once: the VIP is ONE address for the whole environment, so one probe answers for the
|
||||
# environment. Probing from a server host (not the controller) is deliberate: the servers
|
||||
# are on the VIP's own LAN in both environments, whereas the dev controller reaches dev's
|
||||
# API only through a pfSense NAT — a controller-side probe would have been a property of
|
||||
# where the play was launched from. READ-ONLY: it opens a TCP connection and changes nothing.
|
||||
- name: Wait for the control-plane VIP to answer (the CNI's endpoint depends on it)
|
||||
ansible.builtin.wait_for:
|
||||
host: "{{ kube_vip_address }}"
|
||||
port: "{{ kube_vip_port }}"
|
||||
timeout: "{{ kube_vip_wait_timeout }}"
|
||||
run_once: true
|
||||
changed_when: false
|
||||
vars:
|
||||
ansible_become: false
|
||||
# DELIBERATELY NOT check_mode: false, unlike the interface probe above — so a `--check`
|
||||
# rehearsal SKIPS this and does not sit on a socket for kube_vip_wait_timeout. wait_for
|
||||
# declares no check-mode support, so the skip is the module's own behaviour and this comment
|
||||
# is here only so the skip does not read as an oversight. CONSEQUENCE, stated plainly: a
|
||||
# `--check` run does NOT verify the VIP is up. Only a real run gates the CNI on it.
|
||||
@@ -25,14 +25,6 @@ resources:
|
||||
- ../platform/kyverno/policies/restrict-host-namespaces.yaml
|
||||
- ../platform/kyverno/policies/require-egress-class-label.yaml
|
||||
- ../platform/kyverno/policies/require-auth-mode-label.yaml
|
||||
# COLD-START FIX (2026-09-06, prod): a MUTATE policy that must travel WITH the two label
|
||||
# policies above. CDI generates its own short-lived worker pods (importer-*, uploader-*,
|
||||
# cloner-*) inside app namespaces; nothing in git can label them, so both policies DENIED them
|
||||
# and `DataVolume unifi/unifi-rootdisk` sat in ImportScheduled with no disk. This policy adds
|
||||
# the two labels to pods created by ServiceAccount cdi/cdi-sa, so the requirement is SATISFIED
|
||||
# rather than bypassed. Applying it here (not later) matters: if it lands after the validating
|
||||
# policies, the next import is blocked in the gap.
|
||||
- ../platform/kyverno/policies/label-cdi-worker-pods.yaml
|
||||
- ../platform/kyverno/admission/secret-provenance.yaml
|
||||
# cilium: L2 LB IP pool + announcement policy (cilium.io CRDs already registered by Cilium)
|
||||
- cilium/l2-ip-pool.yaml
|
||||
|
||||
@@ -10,8 +10,7 @@
|
||||
#
|
||||
# SOURCE OF TRUTH (do not diverge without updating both):
|
||||
# infra/ansible/roles/cilium/templates/cilium-values.yaml.j2
|
||||
# infra/ansible/roles/cilium/defaults/main.yml (cilium_version)
|
||||
# infra/ansible/inventory/group_vars/all.yml (k3s_apiserver_endpoint, k3s_api_port)
|
||||
# infra/ansible/roles/cilium/defaults/main.yml (cilium_version, k3s_api_host/port)
|
||||
#
|
||||
# Every value tagged [parity] below is asserted identical to the Ansible render. If you change one
|
||||
# here, change it in the Ansible role in the same PR. (docs/02 Cilium-ownership contract.)
|
||||
@@ -46,35 +45,11 @@ spec:
|
||||
# [parity] MUST equal Ansible: kubeProxyReplacement: true
|
||||
kubeProxyReplacement: true
|
||||
# [parity] MUST equal Ansible: k8sServiceHost/k8sServicePort = the k3s API endpoint
|
||||
# (**k3s_apiserver_endpoint** / k3s_api_port, infra/ansible/inventory/group_vars/all.yml).
|
||||
# (k3s_api_host/k3s_api_port). This is the primary control-plane node IP.
|
||||
# Setting these wrong breaks Cilium's connection to the API server on adopt.
|
||||
#
|
||||
# THIS IS THE CONTROL-PLANE VIP (kube-vip), NOT A NODE ADDRESS — operator decision 2026-09-06.
|
||||
# ${K3S_API_HOST} (cluster-vars, R-ENV-02): dev 192.168.1.49 | prod 192.168.2.9, each equal to
|
||||
# that environment's kube_vip_address. The knob's NAME still says "host"; read it as "the API
|
||||
# host Cilium dials", and see clusters/base/cluster-vars.yaml for why it was not renamed.
|
||||
#
|
||||
# WHY, in one line: this value used to be the PRIMARY NODE's IP, so on a 3-server HA cluster the
|
||||
# CNI's control-plane path depended on one machine. Verified on live prod 2026-09-06 — every
|
||||
# cilium-agent carried KUBERNETES_SERVICE_HOST=192.168.2.10 (k3s-prod-1) while kube-vip held .9.
|
||||
# Losing that node does not break the datapath (Cilium keeps serving from cache); it silently
|
||||
# stops every service and policy update from ever landing, which is worse to detect.
|
||||
#
|
||||
# [parity] IS NOW AGAINST k3s_apiserver_endpoint AND NOT k3s_api_host. Those were one Ansible
|
||||
# variable until 2026-09-06 and the conflation is the defect. k3s_api_host still exists and
|
||||
# still holds the FOUNDER's own address, because roles/k3s_server's secondary
|
||||
# `--server https://<host>:6443` join must target a host that is already up — on a COLD build
|
||||
# no VIP exists yet. So: founder for the join, VIP for the CNI. If you change this line, change
|
||||
# cilium-values.yaml.j2 in the same PR, and point it at the SAME variable.
|
||||
#
|
||||
# The `platform` Flux Kustomization substitutes it; the Ansible side is parameterized from
|
||||
# inventory (k3s_apiserver_endpoint derives from kube_vip_address), so the parity contract holds
|
||||
# per-environment, not per-literal.
|
||||
#
|
||||
# NEEDS NO CERT WORK: k3s_tls_sans has always carried kube_vip_address alongside every node IP,
|
||||
# so both addresses are already SANs on the serving cert (verified live 2026-09-06 — prod
|
||||
# answers through https://192.168.2.9:6443 under full CA verification, no insecure skip). This
|
||||
# switch reissues nothing and restarts no apiserver.
|
||||
# ${K3S_API_HOST} (cluster-vars, R-ENV-02): dev 192.168.1.50 | prod set in clusters/prod.
|
||||
# The `platform` Flux Kustomization substitutes it; the Ansible side is already parameterized
|
||||
# from inventory (k3s_api_host), so the parity contract holds per-environment, not per-literal.
|
||||
k8sServiceHost: "${K3S_API_HOST}"
|
||||
k8sServicePort: 6443
|
||||
# --- Gateway API (ingress; replaces Traefik) — docs/09, ADR-0002 ---
|
||||
|
||||
@@ -6,20 +6,16 @@
|
||||
#
|
||||
# SOURCE OF TRUTH (all [parity] fields MUST equal these):
|
||||
# infra/ansible/roles/cilium/templates/cilium-values.yaml.j2
|
||||
# infra/ansible/roles/cilium/defaults/main.yml (cilium_version: 1.20.1)
|
||||
# infra/ansible/inventory/group_vars/all.yml (k3s_apiserver_endpoint = the kube-vip VIP,
|
||||
# k3s_api_port) — NOT k3s_api_host, which is the
|
||||
# founder's own address and bootstrap-only
|
||||
# infra/ansible/roles/cilium/defaults/main.yml (cilium_version: 1.20.1, k3s_api_host/port)
|
||||
#
|
||||
# chart version (helmrelease.yaml chart.spec.version): 1.20.1 # [parity] = cilium_version
|
||||
|
||||
# [parity] k3s kube-proxy disabled → Cilium replaces it (ADR-0001/0002)
|
||||
kubeProxyReplacement: true
|
||||
# [parity] = the k3s API endpoint + port. THE CONTROL-PLANE VIP (kube-vip), not the primary node —
|
||||
# changed 2026-09-06; the Ansible half is k3s_apiserver_endpoint, NOT k3s_api_host.
|
||||
# [parity] = the k3s API endpoint (primary control-plane node IP) + port.
|
||||
# Mirrors the HelmRelease's ${K3S_API_HOST} cluster-vars token (R-ENV-02). This file is NOT a
|
||||
# kustomize resource, so nothing substitutes it — the token is shown with its dev value beside it.
|
||||
k8sServiceHost: "${K3S_API_HOST}" # the CONTROL-PLANE VIP — dev: 192.168.1.49 | prod: 192.168.2.9
|
||||
k8sServiceHost: "${K3S_API_HOST}" # dev: 192.168.1.50
|
||||
k8sServicePort: 6443
|
||||
# [parity] ingress via Gateway API, no Traefik (docs/09)
|
||||
gatewayAPI:
|
||||
|
||||
@@ -22,19 +22,14 @@ platform/kyverno/
|
||||
│ ├── disallow-plaintext-secrets.yaml # R-SEC-01 (CI-only: sops block required)
|
||||
│ ├── restrict-host-namespaces.yaml # R-SEC-02 (+R-NET-01 bypass guard)
|
||||
│ ├── require-egress-class-label.yaml # R-NET-05
|
||||
│ ├── require-auth-mode-label.yaml # R-IDP-02
|
||||
│ └── label-cdi-worker-pods.yaml # R-NET-05+R-IDP-02 (mutate: CDI's own worker pods)
|
||||
│ └── require-auth-mode-label.yaml # R-IDP-02
|
||||
├── servicedefinition/ # ServiceDefinition shape validation (kyverno-json)
|
||||
│ ├── validate-servicedefinition.yaml # R-GITOPS-02
|
||||
│ ├── payloads/ (valid-* / invalid-*) # pass & fail fixtures
|
||||
│ └── README.md
|
||||
├── tests/ # `kyverno test` suite — runnable L0 tests NOW, no cluster
|
||||
│ ├── kyverno-test.yaml # policies × fixtures × expected results
|
||||
│ ├── resources/ (good-* / bad-* fixtures + generated golden)
|
||||
│ └── controller-worker-pods/ # the CDI-worker mutation, proven BOTH ways
|
||||
│ ├── kyverno-test.yaml # 2 Tests: cdi-sa requester vs ordinary requester
|
||||
│ ├── userinfo-cdi-sa.yaml / userinfo-ordinary.yaml
|
||||
│ └── resources/ + patched/ # real + FORGED importer pods; expected patch
|
||||
│ └── resources/ (good-* / bad-* fixtures + generated golden)
|
||||
├── admission/ # in-cluster enforcing bundle (scaffolding, T-2.7)
|
||||
│ ├── namespace.yaml · helmrelease.yaml · kustomization.yaml
|
||||
│ ├── secret-provenance.yaml # R-SEC-01 admission variant (provenance)
|
||||
@@ -52,7 +47,6 @@ platform/kyverno/
|
||||
| `restrict-host-namespaces` | `disallow-host-namespaces`, `disallow-privileged-containers` | **R-SEC-02** (+R-NET-01) | docs/07 least reach | App pods can't use hostNetwork/PID/IPC or privileged (would bypass default-deny) |
|
||||
| `require-egress-class-label` | `require-egress-class` | **R-NET-05** | docs/03, docs/13 | App pods must declare `egress-class` ∈ {internet,vpn} |
|
||||
| `require-auth-mode-label` | `require-auth-mode` | **R-IDP-02** | docs/13, docs/15 | App pods must declare `auth-mode` ∈ {oidc,forward-auth,native,none} |
|
||||
| `label-cdi-worker-pods` (mutate) | `add-worker-pod-labels` | **R-NET-05**, **R-IDP-02** | docs/03, docs/13 | CDI's **controller-generated** worker pods (`importer-*`/`uploader-*`/`cloner-*`, created by SA `cdi/cdi-sa`) get `egress-class: internet` + `auth-mode: none` **added**, so they satisfy the two policies above instead of being exempted from them. Matched on the **generating ServiceAccount**, never a pod-name prefix |
|
||||
| `validate-servicedefinition` (kyverno-json) | shape / dependencies / storage | **R-GITOPS-02** | docs/13 (+03,15) | ServiceDefinition is schema/invariant valid before render; public⇒not `none` auth |
|
||||
| `admission/secret-provenance` | `secrets-must-be-flux-managed` | **R-SEC-01** | docs/07 | **Admission-only** variant: app-ns Secrets must come from Flux/SOPS (see below) |
|
||||
|
||||
@@ -96,19 +90,6 @@ self-contained — no cluster, no external state (reproducibility contract, R-TE
|
||||
|
||||
- **Label keys are a proposed contract** (`labels.md`) shared with WS-NET / WS-SEC /
|
||||
WS-GITOPS. Values are spec-fixed; keys need owner sign-off (coordination-protocol §2).
|
||||
- **Controller-generated pods are a cold-start hazard.** A policy that matches *any* Pod in an
|
||||
app namespace also matches pods that no manifest in git authors. Prod hit this on 2026-09-06:
|
||||
`require-auth-mode-label` + `require-egress-class-label` denied CDI's own `importer-prime-*`
|
||||
pod, so `DataVolume unifi/unifi-rootdisk` stuck in `ImportScheduled` and the UniFi VM had no
|
||||
disk. Fix: `policies/label-cdi-worker-pods.yaml` **mutates** those pods to carry truthful
|
||||
labels (matched on the generating ServiceAccount `cdi/cdi-sa`), rather than excluding them.
|
||||
`virt-launcher-*` pods need nothing — they inherit the VM pod-template labels. KubeVirt
|
||||
hotplug (`hp-volume-*`) and export (`virt-export-*`) pods do **not** inherit them and are
|
||||
unused here; extend the same policy if either is adopted.
|
||||
- **`PolicyException` is NOT usable in this install**: the Kyverno admission/background/reports
|
||||
controllers all run `--enablePolicyException=false` (chart default, verified on prod
|
||||
2026-09-06). The CRD exists (from the pre-applied CRD bundle) so a `PolicyException` would
|
||||
APPLY cleanly and be silently ignored — a policy hole that looks like a fix and isn't.
|
||||
- **Secrets CI vs admission divergence** is deliberate and documented in
|
||||
`admission/README.md` and `policies/disallow-plaintext-secrets.yaml`.
|
||||
- The ServiceDefinition JSON Schema (WS-GITOPS T-0.3) is the complementary structural
|
||||
|
||||
@@ -13,10 +13,6 @@ resources:
|
||||
- ../policies/restrict-host-namespaces.yaml
|
||||
- ../policies/require-egress-class-label.yaml
|
||||
- ../policies/require-auth-mode-label.yaml
|
||||
# mutate: gives CDI's OWN generated worker pods (importer-*/uploader-*/cloner-*) the two
|
||||
# labels the policies above require -- they are controller-generated, so nothing in git can
|
||||
# label them. Ships with them, never separately.
|
||||
- ../policies/label-cdi-worker-pods.yaml
|
||||
# --- secrets: the admission variant REPLACES the CI-only sops-block policy ---
|
||||
# (../policies/disallow-plaintext-secrets.yaml is CI-only; see secret-provenance.yaml)
|
||||
- secret-provenance.yaml
|
||||
@@ -15,17 +15,6 @@ Shared contract with WS-NET (egress rendering), WS-SEC (auth), and WS-GITOPS (re
|
||||
| `homelab.keithjackson.dev/baseline` | NetworkPolicy | `default-deny` | (convention) identifies the baseline NP | `require-default-deny-baseline` (generate), `validate-default-deny-shape` |
|
||||
| `homelab.keithjackson.dev/secret-managed-by` | Secret | `controller` | (convention) exempts controller-generated Secrets | `disallow-plaintext-secrets` (exclude), admission `secret-provenance` |
|
||||
|
||||
## Controller-generated pods (who sets the labels when git cannot)
|
||||
|
||||
`egress-class` / `auth-mode` are normally projected onto pod templates by the renderer (below).
|
||||
Some pods are generated by a **controller at runtime** and therefore have no manifest in git to
|
||||
label — CDI's `importer-*` / `uploader-*` / `cloner-*` worker pods are the live example: both
|
||||
label policies denied CDI's importer on prod (2026-09-06) and the UniFi VM got no disk. For those,
|
||||
`policies/label-cdi-worker-pods.yaml` **adds** the labels at admission (`+(…)` add-if-absent),
|
||||
matched on the **generating ServiceAccount** (`cdi/cdi-sa`) — never on a pod-name prefix or on
|
||||
CDI's own labels, both of which any pod author in the namespace can copy. `virt-launcher-*` needs
|
||||
nothing: KubeVirt propagates the VM pod-template labels onto it (verified on dev).
|
||||
|
||||
## Rendering expectation (for WS-GITOPS)
|
||||
|
||||
The ServiceDefinition (`docs/13`) declares intent — `egress:` and `auth.mode:` — at the
|
||||
|
||||
@@ -1,136 +0,0 @@
|
||||
# Serves: R-NET-05 (egress is a declared property) + R-IDP-02 (auth mode is declared) —
|
||||
# this policy does NOT weaken either requirement. It makes the one class of pod that
|
||||
# CANNOT carry the labels from git SATISFY them, instead of being exempted from them.
|
||||
#
|
||||
# THE DEFECT IT FIXES (observed on LIVE prod, 2026-09-05/06):
|
||||
# `DataVolume unifi/unifi-rootdisk` sat in ImportScheduled for hours holding a detached
|
||||
# 40Gi `prime-…` scratch PVC. `kubectl -n cdi logs deploy/cdi-deployment` showed:
|
||||
# admission webhook "validate.kyverno.svc-fail" denied the request: resource
|
||||
# Pod/unifi/importer-prime-<uid> was blocked due to the following policies
|
||||
# require-auth-mode-label / require-egress-class-label
|
||||
# CDI generates its own short-lived worker pods (importer-*, uploader-*, cloner-*/
|
||||
# clone-source-*) from inside the cdi-deployment controller. Those pods are not in git,
|
||||
# so nothing can put the platform labels on them, and both label policies match ANY Pod
|
||||
# in a namespace labelled homelab.keithjackson.dev/app-namespace=true. Result: the VM
|
||||
# never gets a disk. This is a COLD-START defect, not a prod quirk — dev's identical
|
||||
# import Succeeded before the policies were enforcing, so dev has never exercised it and
|
||||
# would hit it on any rebuild.
|
||||
#
|
||||
# WHY MUTATE (and not an exception/exclusion):
|
||||
# The honest reading of R-NET-05/R-IDP-02 is "every pod's egress path and auth posture is
|
||||
# DECLARED". A CDI importer pod has truthful values for both:
|
||||
# egress-class: internet — it pulls the image from the public internet over https
|
||||
# (cloud-images.ubuntu.com / quay.io) and must take the
|
||||
# default WAN path, which is exactly what Cilium's egress
|
||||
# gateway does for `internet` (ADR-0013/ADR-0015). It is NOT
|
||||
# a `vpn` workload.
|
||||
# auth-mode: none — the pod serves no UI and terminates when the import ends;
|
||||
# there is nothing to put an IdP in front of. docs/13 permits
|
||||
# `none` for internal, non-public workloads (the
|
||||
# ServiceDefinition policy forbids only public + none).
|
||||
# So the requirement is SATISFIED rather than bypassed: the invariant "every pod in an app
|
||||
# namespace carries both labels" stays literally true cluster-wide, both label policies
|
||||
# stay untouched (no `exclude`, no PolicyException, background scanning still on), and the
|
||||
# labels are functional — the pod is now selectable by the Cilium egress fabric instead of
|
||||
# being an unlabelled pod that silently falls through to the default path.
|
||||
#
|
||||
# THE MATCH IS THE GENERATING ServiceAccount, NOT A NAME PREFIX:
|
||||
# `subjects: ServiceAccount cdi/cdi-sa` is the identity in the AdmissionReview's userInfo —
|
||||
# the controller that issued the CREATE. Verified on prod from the Kyverno admission log of
|
||||
# the blocked request:
|
||||
# user={"username":"system:serviceaccount:cdi:cdi-sa", ...} roles=["cdi:cdi-deployment"]
|
||||
# name=importer-prime-be3f9d1c-… namespace=unifi operation=CREATE
|
||||
# A pod NAME (`importer-*`) or CDI's own labels (`cdi.kubevirt.io`) are attacker-CHOOSABLE:
|
||||
# anyone who can create a pod in the namespace can name it `importer-prime-x` and copy the
|
||||
# labels, and would then inherit the exemption. The requesting ServiceAccount cannot be
|
||||
# chosen by the pod author — only by someone who already holds cdi-sa's token, which already
|
||||
# confers "create any pod in any namespace" cluster-wide. So this adds no reachable
|
||||
# privilege. It is also VERSION-ROBUST: it covers every CDI worker kind (importer, uploader,
|
||||
# cloner/clone-source) without enumerating names that change between CDI releases —
|
||||
# confirmed on prod that cdi-sa is the ONLY cdi-namespace ServiceAccount that can create
|
||||
# pods (`kubectl auth can-i create pods -n unifi --as …`: cdi-sa yes; cdi-apiserver no;
|
||||
# cdi-uploadproxy no).
|
||||
#
|
||||
# DELIBERATELY NOT COVERED — KubeVirt's own generated pods:
|
||||
# virt-launcher-* pods INHERIT the VM's pod-template labels, so a VM authored in this repo
|
||||
# already passes (verified on dev: virt-launcher-unifi-22dg7 carries egress-class=internet,
|
||||
# auth-mode=native from apps/vms/unifi/30-virtualmachine.yaml). Exempting
|
||||
# kubevirt-controller would REMOVE that useful enforcement on VM authors, so it is not done.
|
||||
# KubeVirt's hotplug (`hp-volume-*`) and export (`virt-export-*`) pods do NOT inherit those
|
||||
# labels, but nothing in this repo uses volume hotplug or VirtualMachineExport; if either is
|
||||
# adopted, extend this policy then — with the same ServiceAccount-scoped shape.
|
||||
#
|
||||
# OPERATOR NOTE — this is the cluster's FIRST resource-MUTATING policy. Verified on prod
|
||||
# 2026-09-06 that `kyverno-resource-mutating-webhook-cfg` currently holds 0 webhooks; with the
|
||||
# chart's `autoUpdateWebhooks=true` Kyverno registers `mutate.kyverno.svc-fail` for Pods within
|
||||
# seconds of this policy being applied. Prod runs `forceFailurePolicyIgnore=false`, so that
|
||||
# webhook is failurePolicy: Fail — the same posture as the validating webhook that is ALREADY in
|
||||
# the path for these pods, so no new fail-open surface and no new blocking surface. CDI retries
|
||||
# the importer pod on a loop, so the stuck import proceeds once the policy lands (the detached
|
||||
# `prime-…` scratch PVC left behind by the blocked attempts is a separate operator cleanup).
|
||||
#
|
||||
# RESIDUAL TRADE-OFF (state it plainly, it is reversible by deleting this file):
|
||||
# Anything created by `cdi:cdi-sa` in an app namespace now gets `egress-class: internet` +
|
||||
# `auth-mode: none` supplied for it instead of being rejected for not declaring them. Whoever
|
||||
# holds cdi-sa's token can therefore run a pod in an app namespace without declaring its egress
|
||||
# class. That identity can already create arbitrary pods in every namespace (ClusterRole `cdi`:
|
||||
# pods create/delete cluster-wide), so it is not a new capability — but the declaration is now
|
||||
# assumed rather than demanded for that one identity. Nothing else changes: the pod is still
|
||||
# subject to restrict-host-namespaces, the namespace default-deny baseline, and every other
|
||||
# policy, and `internet` is the LEAST-reach of the two egress classes (it does not grant the VPN
|
||||
# path).
|
||||
#
|
||||
# PROOF (offline, no cluster — platform/kyverno/tests/controller-worker-pods/):
|
||||
# (a) a CDI importer pod created by cdi:cdi-sa is mutated and then PASSES both policies;
|
||||
# (b) the SAME namespace's ordinary pod — including one that FORGES the importer name and
|
||||
# CDI's labels — is NOT mutated and is still DENIED by both policies.
|
||||
apiVersion: kyverno.io/v1
|
||||
kind: ClusterPolicy
|
||||
metadata:
|
||||
name: label-cdi-worker-pods
|
||||
annotations:
|
||||
policies.kyverno.io/title: Label CDI-generated worker pods
|
||||
policies.kyverno.io/category: Network Isolation
|
||||
policies.kyverno.io/severity: medium
|
||||
homelab.keithjackson.dev/requirements: R-NET-05,R-IDP-02
|
||||
policies.kyverno.io/description: >-
|
||||
Adds homelab.keithjackson.dev/egress-class=internet and auth-mode=none to the
|
||||
short-lived worker Pods that the CDI controller (ServiceAccount cdi/cdi-sa)
|
||||
generates in application namespaces, so they satisfy require-egress-class-label
|
||||
and require-auth-mode-label instead of being exempted from them.
|
||||
spec:
|
||||
# No validationFailureAction: this is a MUTATE policy, there is nothing to fail.
|
||||
# background MUST be false: `match.subjects` is userInfo, which exists only on an
|
||||
# AdmissionReview. Kyverno rejects a userInfo match with background scanning enabled
|
||||
# ("invalid variable used at path ... policy contains 'userInfo'"), and a background scan
|
||||
# of an already-created pod has no requester to match anyway.
|
||||
background: false
|
||||
rules:
|
||||
- name: add-worker-pod-labels
|
||||
match:
|
||||
# One filter entry: the resource conditions AND the subject must both match.
|
||||
all:
|
||||
- resources:
|
||||
kinds:
|
||||
- Pod
|
||||
operations:
|
||||
- CREATE
|
||||
namespaceSelector:
|
||||
matchLabels:
|
||||
homelab.keithjackson.dev/app-namespace: "true"
|
||||
subjects:
|
||||
# The CDI controller (cdi-deployment) ServiceAccount — the identity that
|
||||
# CREATEs importer-*/uploader-*/cloner-* pods. Not the pods' own SA.
|
||||
- kind: ServiceAccount
|
||||
namespace: cdi
|
||||
name: cdi-sa
|
||||
mutate:
|
||||
patchStrategicMerge:
|
||||
metadata:
|
||||
labels:
|
||||
# `+(key)` is Kyverno's ADD-IF-ABSENT anchor: an explicit value already on the
|
||||
# pod is never overwritten (so a future CDI worker that legitimately needs
|
||||
# egress-class: vpn can be given it, and virt-launcher-style inherited labels
|
||||
# would survive).
|
||||
+(homelab.keithjackson.dev/egress-class): internet
|
||||
+(homelab.keithjackson.dev/auth-mode): none
|
||||
@@ -1,91 +0,0 @@
|
||||
# L0 policy-as-code suite for the CDI worker-pod label mutation — runs in CI WITHOUT a cluster
|
||||
# (discovered recursively by the gate's `kyverno test platform/kyverno`; ci/l0.sh).
|
||||
#
|
||||
# WHY A SEPARATE SUITE from ../kyverno-test.yaml: a `kyverno test` manifest carries ONE
|
||||
# `userinfo`, and the whole point here is that the SAME pods get different outcomes for
|
||||
# different requesters. Two Test documents, two identities, one file:
|
||||
# 1. cdi-controller-worker-pods-admitted (requester = cdi/cdi-sa) -> mutated, ADMITTED
|
||||
# 2. cdi-worker-exemption-is-narrow (requester = ordinary user) -> not mutated, DENIED
|
||||
# Keeping the mutate policy OUT of ../kyverno-test.yaml is deliberate: that suite declares no
|
||||
# userinfo, and with no userinfo the offline engine does not discriminate on `subjects`, which
|
||||
# would silently turn its negative (result: fail) rows into passes.
|
||||
apiVersion: cli.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: cdi-controller-worker-pods-admitted
|
||||
# Namespace labels for offline namespaceSelector resolution (shared with the main suite).
|
||||
variables: ../values.yaml
|
||||
userinfo: userinfo-cdi-sa.yaml
|
||||
policies:
|
||||
- ../../policies/label-cdi-worker-pods.yaml
|
||||
- ../../policies/require-egress-class-label.yaml
|
||||
- ../../policies/require-auth-mode-label.yaml
|
||||
resources:
|
||||
- ../resources/namespaces.yaml
|
||||
- resources/cdi-worker-pods.yaml
|
||||
results:
|
||||
# ---- the mutation fires for the CDI controller and adds exactly the two labels ----
|
||||
- policy: label-cdi-worker-pods
|
||||
rule: add-worker-pod-labels
|
||||
kind: Pod
|
||||
resources:
|
||||
- importer-prime-be3f9d1c-c3ce-455a-a450-3d9e95c7baa2
|
||||
patchedResources: patched/importer-mutated.yaml
|
||||
result: pass
|
||||
# ---- R-NET-05 + R-IDP-02: the mutated CDI worker pod now SATISFIES both policies, so the
|
||||
# import CDI could not start is admitted. (The engine validates the mutated resource.)
|
||||
- policy: require-egress-class-label
|
||||
rule: require-egress-class
|
||||
kind: Pod
|
||||
resources:
|
||||
- importer-prime-be3f9d1c-c3ce-455a-a450-3d9e95c7baa2
|
||||
result: pass
|
||||
- policy: require-auth-mode-label
|
||||
rule: require-auth-mode
|
||||
kind: Pod
|
||||
resources:
|
||||
- importer-prime-be3f9d1c-c3ce-455a-a450-3d9e95c7baa2
|
||||
result: pass
|
||||
---
|
||||
apiVersion: cli.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: cdi-worker-exemption-is-narrow
|
||||
variables: ../values.yaml
|
||||
userinfo: userinfo-ordinary.yaml
|
||||
policies:
|
||||
- ../../policies/label-cdi-worker-pods.yaml
|
||||
- ../../policies/require-egress-class-label.yaml
|
||||
- ../../policies/require-auth-mode-label.yaml
|
||||
resources:
|
||||
- ../resources/namespaces.yaml
|
||||
- resources/cdi-worker-pods.yaml
|
||||
results:
|
||||
# ---- the mutation does NOT fire for an ordinary requester, not even for a pod that forges
|
||||
# the importer's name and CDI's labels ----
|
||||
- policy: label-cdi-worker-pods
|
||||
rule: add-worker-pod-labels
|
||||
kind: Pod
|
||||
resources:
|
||||
- importer-prime-be3f9d1c-c3ce-455a-a450-3d9e95c7baa2
|
||||
- importer-prime-forged-by-an-app
|
||||
- plain-app-pod-no-labels
|
||||
result: skip
|
||||
# ---- so every unlabelled pod in the SAME app namespace is still DENIED (R-NET-05) ----
|
||||
- policy: require-egress-class-label
|
||||
rule: require-egress-class
|
||||
kind: Pod
|
||||
resources:
|
||||
- importer-prime-be3f9d1c-c3ce-455a-a450-3d9e95c7baa2
|
||||
- importer-prime-forged-by-an-app
|
||||
- plain-app-pod-no-labels
|
||||
result: fail
|
||||
# ---- ... and by R-IDP-02 ----
|
||||
- policy: require-auth-mode-label
|
||||
rule: require-auth-mode
|
||||
kind: Pod
|
||||
resources:
|
||||
- importer-prime-be3f9d1c-c3ce-455a-a450-3d9e95c7baa2
|
||||
- importer-prime-forged-by-an-app
|
||||
- plain-app-pod-no-labels
|
||||
result: fail
|
||||
@@ -1,16 +0,0 @@
|
||||
# Expected result of the mutation: CDI's own labels untouched, the two platform labels ADDED.
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: importer-prime-be3f9d1c-c3ce-455a-a450-3d9e95c7baa2
|
||||
namespace: app-ns
|
||||
labels:
|
||||
app: containerized-data-importer
|
||||
cdi.kubevirt.io: importer
|
||||
homelab.keithjackson.dev/egress-class: internet
|
||||
homelab.keithjackson.dev/auth-mode: none
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
containers:
|
||||
- name: importer
|
||||
image: quay.io/kubevirt/cdi-importer:v1.62.0
|
||||
@@ -1,53 +0,0 @@
|
||||
# Fixtures for the CDI-worker-pod label mutation (policies/label-cdi-worker-pods.yaml).
|
||||
#
|
||||
# All three pods live in `app-ns` (../resources/namespaces.yaml — carries
|
||||
# homelab.keithjackson.dev/app-namespace: "true"), so all three are IN SCOPE of
|
||||
# require-egress-class-label + require-auth-mode-label. What separates them is WHO creates
|
||||
# them, which is the whole point of the mechanism: the two "forged" pods are byte-for-byte
|
||||
# as CDI-looking as the real one.
|
||||
#
|
||||
# ---- (a) the REAL CDI worker pod: shape copied from the pod CDI actually tried to create on
|
||||
# prod (name importer-prime-<pvc-uid>, CDI's own labels, no platform labels). It is
|
||||
# created by ServiceAccount cdi/cdi-sa — see userinfo-cdi-sa.yaml.
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: importer-prime-be3f9d1c-c3ce-455a-a450-3d9e95c7baa2
|
||||
namespace: app-ns
|
||||
labels:
|
||||
app: containerized-data-importer
|
||||
cdi.kubevirt.io: importer
|
||||
# NOTE: no homelab.keithjackson.dev/* labels — CDI generates this pod itself, nothing in
|
||||
# git can label it. This is exactly why the import was blocked.
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
containers:
|
||||
- name: importer
|
||||
image: quay.io/kubevirt/cdi-importer:v1.62.0
|
||||
---
|
||||
# ---- (b) FORGED: an ordinary pod that copies the importer's NAME PREFIX and CDI's labels.
|
||||
# Proves that a name-prefix or label-based exemption would have been walkable, and
|
||||
# that this ServiceAccount-scoped one is not.
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: importer-prime-forged-by-an-app
|
||||
namespace: app-ns
|
||||
labels:
|
||||
app: containerized-data-importer
|
||||
cdi.kubevirt.io: importer
|
||||
spec:
|
||||
containers:
|
||||
- name: app
|
||||
image: registry.example/app:1.0.0
|
||||
---
|
||||
# ---- (c) a plain unlabelled application pod in the SAME namespace. Must still be DENIED.
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: plain-app-pod-no-labels
|
||||
namespace: app-ns
|
||||
spec:
|
||||
containers:
|
||||
- name: app
|
||||
image: registry.example/app:1.0.0
|
||||
@@ -1,21 +0,0 @@
|
||||
# The requester identity Kyverno actually saw on prod when CDI created the importer pod
|
||||
# (from the kyverno-admission-controller log of the blocked request, 2026-09-06):
|
||||
# user={"username":"system:serviceaccount:cdi:cdi-sa","groups":["system:serviceaccounts",
|
||||
# "system:serviceaccounts:cdi","system:authenticated"], ...}
|
||||
# roles=["cdi:cdi-deployment"] clusterroles=["cdi", …]
|
||||
# `kyverno test` feeds this to the engine as the AdmissionReview userInfo so
|
||||
# `match.subjects` can be evaluated offline.
|
||||
apiVersion: cli.kyverno.io/v1alpha1
|
||||
kind: UserInfo
|
||||
metadata:
|
||||
name: cdi-controller
|
||||
clusterRoles:
|
||||
- cdi
|
||||
roles:
|
||||
- cdi:cdi-deployment
|
||||
userInfo:
|
||||
username: system:serviceaccount:cdi:cdi-sa
|
||||
groups:
|
||||
- system:serviceaccounts
|
||||
- system:serviceaccounts:cdi
|
||||
- system:authenticated
|
||||
@@ -1,16 +0,0 @@
|
||||
# An ORDINARY requester in the application namespace — the attacker/mistake case. Anything
|
||||
# that is not cdi/cdi-sa: an app's own ServiceAccount, a human's kubectl, Flux.
|
||||
# The mutation must NOT fire for this identity, so both label policies must still DENY —
|
||||
# even for a pod that forged the importer's name and CDI's labels.
|
||||
apiVersion: cli.kyverno.io/v1alpha1
|
||||
kind: UserInfo
|
||||
metadata:
|
||||
name: ordinary-namespace-user
|
||||
clusterRoles:
|
||||
- edit
|
||||
userInfo:
|
||||
username: system:serviceaccount:app-ns:default
|
||||
groups:
|
||||
- system:serviceaccounts
|
||||
- system:serviceaccounts:app-ns
|
||||
- system:authenticated
|
||||
@@ -1,10 +1,7 @@
|
||||
---
|
||||
# ============================================================================
|
||||
# Cilium Egress Gateway steering for `egress-class=vpn` pods — ADR-0015.
|
||||
# Serves: R-NET-04, R-NET-05, R-NET-06, R-NET-07.
|
||||
# R-NET-04 added 2026-09-06: this file IS the R-NET-04 datapath — the rule that selects between the
|
||||
# two egress paths. Omitting it made `run.sh --status` report R-NET-04 as "TEST-ONLY / impl none"
|
||||
# even though the implementation is right here.
|
||||
# Serves: R-NET-05, R-NET-06, R-NET-07.
|
||||
#
|
||||
# ##########################################################################
|
||||
# ## LIVE (2026-08-24): wired into kustomization.yaml, reconciled by Flux ##
|
||||
@@ -38,7 +35,7 @@ kind: CiliumEgressGatewayPolicy
|
||||
metadata:
|
||||
name: vpn-egress-steering
|
||||
annotations:
|
||||
homelab.keithjackson.dev/requirements: R-NET-04,R-NET-05,R-NET-06,R-NET-07
|
||||
homelab.keithjackson.dev/requirements: R-NET-05,R-NET-06,R-NET-07
|
||||
homelab.keithjackson.dev/adr: ADR-0015
|
||||
spec:
|
||||
# --- Client selection (INVARIANTS 1 & 2) --------------------------------
|
||||
|
||||
Reference in new issue
Block a user