home-network-k3s-impl/infra/ansible/roles/k3s_server/tasks/main.yml

---
# Install k3s as an HA server (embedded etcd) with Cilium-required components
# disabled (ADR-0001, docs/02 stage 3 "Cluster-join"). The primary runs
# --cluster-init; secondaries join it with the shared node-token.

- name: Download the k3s install script
  ansible.builtin.get_url:
    url: "{{ k3s_install_url }}"
    dest: "{{ k3s_install_script }}"
    mode: "0755"

# --node-ip uses k3s_node_ip (the real 192.168.1.x), NOT ansible_host — the latter is
# now the pfSense NAT SSH target (192.168.2.102:225x) and must not be advertised to k3s.
- name: Compose the common k3s server flags
  ansible.builtin.set_fact:
    k3s_common_args: >-
      server
      --flannel-backend=none
      --disable-kube-proxy
      --disable-network-policy
      --disable={{ k3s_disable }}
      --write-kubeconfig-mode=644
      {% for san in k3s_tls_sans %}--tls-san={{ san }} {% endfor %}
      --node-ip={{ k3s_node_ip }}
      --data-dir={{ k3s_data_dir }}

# On an ALREADY-installed cluster the install task above is idempotent (creates:
# k3s.yaml) and won't re-apply new --tls-san flags. Persist the SAN list in
# /etc/rancher/k3s/config.yaml, which a running k3s reads on the next restart —
# it then regenerates the serving cert when the SAN set changes, and regenerates
# its containerd config.toml (so nonroot-devices takes effect). Apply the restart
# ROLLING, one node at a time (kube-vip keeps the API reachable; etcd quorum holds
# at 2/3). See playbooks note: run with `serial: 1`.
- name: Ensure /etc/rancher/k3s exists (a freshly re-provisioned node has no k3s dir until install)
  ansible.builtin.file:
    path: /etc/rancher/k3s
    state: directory
    owner: root
    group: root
    mode: "0755"

- name: Write /etc/rancher/k3s/config.yaml (tls-san + nonroot-devices + oidc)
  ansible.builtin.copy:
    dest: /etc/rancher/k3s/config.yaml
    owner: root
    group: root
    mode: "0600"
    content: |
      # Managed by Ansible (roles/k3s_server). Read by k3s in addition to the
      # systemd unit flags; tls-san entries are merged into the serving cert.
      tls-san:
      {% for san in k3s_tls_sans %}
        - "{{ san }}"
      {% endfor %}

      # Let non-root pods own the raw block devices bind-mounted into them —
      # sets containerd's `device_ownership_from_security_context = true`
      # (k3s >= v1.31.3+k3s1 exposes this as the --nonroot-devices flag).
      #
      # REQUIRED for KubeVirt/CDI on Block-mode PVCs (ADR-0005): CDI's importer,
      # uploader and virt-launcher pods run non-root (UID 107, runAsNonRoot,
      # cap-drop ALL) and open the volume's raw block device (/dev/cdi-block-volume,
      # then the guest disk). Kubernetes only applies fsGroup ownership to
      # *filesystem* volumes — it never chowns a raw block device — and Longhorn's
      # CSIDriver (fsGroupPolicy: ReadWriteOnceWithFSType) skips fsGroup on RWX/Block
      # volumes anyway, so the device stays root:root and the pod fails with
      # `blockdev: cannot open /dev/cdi-block-volume: Permission denied`. This flag
      # makes containerd set the in-container device node's owner from the pod's
      # securityContext, which is the fix the CDI project documents for this case
      # (doc/block_cri_ownership_config.md). Node-scoped and security-neutral: the
      # pod stays non-root, only the device node it already mounts follows its UID/GID.
      # Preserves the longhorn-vm RWX + Block + migratable disk contract unchanged.
      nonroot-devices: true

      # OIDC (Keycloak SSO for kubectl). The API server trusts ID tokens from the homelab realm;
      # the `kubernetes` PUBLIC client (PKCE) + kubelogin mint them, and RBAC binds the `groups`
      # claim (platform/keycloak/k8s-oidc-rbac.yaml: oidc:platform-admins->cluster-admin,
      # oidc:viewers->view). Nodes resolve keycloak.<k3s_base_domain> to the Cilium gateway VIP
      # and trust its public LE-prod cert, so NO oidc-ca-file is needed. The `oidc:`
      # prefixes namespace federated identities away from cert / service-account subjects.
      # ADDITIVE + break-glass-safe: the cert-based admin kubeconfig keeps full access with or
      # without Keycloak, so a bad realm/OIDC state can never lock you out of the cluster.
      #
      # NOT a Flux ${BASE_DOMAIN} token: this is a k3s SERVER FLAG, not a manifest, so
      # postBuild.substituteFrom never touches it. k3s_oidc_issuer_url derives from
      # k3s_base_domain (defaults/main.yml), whose [parity] contract with the cluster's
      # BASE_DOMAIN is documented there. Dev's default renders the value it already runs, so
      # config.yaml is unchanged and the Restart handler does not fire.
      # ── Graceful node shutdown ──────────────────────────────────────────────────────────────
      # Without these the kubelet takes no systemd inhibitor lock, so `shutdown -h` stops
      # containerd and SIGKILLs every pod where it stands -- MEASURED on all three prod nodes
      # 2026-10-17 (defaults/main.yml has the detail). The companion logind drop-in below raises
      # InhibitDelayMaxSec, WITHOUT WHICH systemd caps the drain at ~5s and these values are
      # quietly ignored.
      kubelet-arg:
        - "shutdown-grace-period={{ k3s_kubelet_shutdown_grace_period }}"
        - "shutdown-grace-period-critical-pods={{ k3s_kubelet_shutdown_grace_period_critical_pods }}"

      kube-apiserver-arg:
        - "oidc-issuer-url={{ k3s_oidc_issuer_url }}"
        - "oidc-client-id=kubernetes"
        - "oidc-username-claim=email"
        - "oidc-username-prefix=oidc:"
        - "oidc-groups-claim=groups"
        - "oidc-groups-prefix=oidc:"
  notify: Restart k3s

# --- primary: found the cluster --------------------------------------------
- name: Install k3s primary (cluster-init, embedded etcd)
  ansible.builtin.command:
    cmd: "{{ k3s_install_script }} {{ k3s_common_args }} --cluster-init"
    creates: /etc/rancher/k3s/k3s.yaml
  environment:
    INSTALL_K3S_VERSION: "{{ k3s_version }}"
  when: k3s_server_mode == 'primary'

# PATH FOLLOWS --data-dir. k3s writes the node-token to <data-dir>/server/token, NOT to
# the literal /var/lib/rancher/k3s. On dev k3s_data_dir IS /var/lib/rancher/k3s (the role
# default, restated explicitly above) so this is byte-for-byte the old path; on prod
# --data-dir is /data/k3s (group_vars/k3s_prod) and the hardcoded path did not exist at all
# — this task would have timed out after 180s and the two secondaries could never join.
- name: Wait for the k3s node-token to exist (primary)
  ansible.builtin.wait_for:
    path: "{{ k3s_data_dir }}/server/token"
    timeout: 180
  when: k3s_server_mode == 'primary'

- name: Read the k3s node-token (primary)
  ansible.builtin.slurp:
    src: "{{ k3s_data_dir }}/server/token"
  register: k3s_token_raw
  when: k3s_server_mode == 'primary'

- name: Publish the node-token as a fact for secondaries (primary)
  ansible.builtin.set_fact:
    k3s_node_token: "{{ k3s_token_raw.content | b64decode | trim }}"
  when: k3s_server_mode == 'primary'

# --- secondaries: obtain the join token, then join the existing cluster -----
# Fetch the token straight from the primary via delegate_to, falling back to it
# only when the primary play didn't publish the fact this run. This makes a
# single-node rejoin (`--limit k3s-dev-N`, primary NOT in the batch) self-
# contained — delegate_to still reaches the primary even when it isn't a play
# target, so the join no longer depends on a fact an earlier play had to set.
# T-1.5 finding F2 (see DECISIONS-LOG.md).
#
# "The primary" is resolved through k3s_primary_group (group_vars: `k3s_primary` for
# dev, `k3s_prod_primary` for prod), never a literal group name — so a prod secondary
# delegates to the PROD primary using this same role, unchanged (R-ENV-02).
# k3s_data_dir is a CLUSTER-WIDE value (group_vars for prod, role default for dev), so the
# secondary's own copy of it is also the primary's data dir — correct for the delegated read.
- name: Fetch the k3s node-token directly from the primary (secondary)
  ansible.builtin.slurp:
    src: "{{ k3s_data_dir }}/server/token"
  delegate_to: "{{ groups[k3s_primary_group][0] }}"
  register: k3s_primary_token_raw
  when:
    - k3s_server_mode == 'secondary'
    - hostvars[groups[k3s_primary_group][0]].k3s_node_token is not defined

- name: Resolve the join token (secondary)
  ansible.builtin.set_fact:
    k3s_join_token: >-
      {{ hostvars[groups[k3s_primary_group][0]].k3s_node_token
         | default(k3s_primary_token_raw.content | b64decode | trim, true) }}
  when: k3s_server_mode == 'secondary'

# THE JOIN TARGET IS k3s_api_host — THE FOUNDER — AND MUST NOT BECOME THE VIP.
# group_vars/all.yml splits the two API addresses (k3s_api_host = the founder's own node IP,
# k3s_apiserver_endpoint = the kube-vip HA VIP) and this is the canonical founder-only
# consumer. On a COLD build the VIP does not exist at this point in the sequence: kube-vip
# is a k3s auto-deploy manifest, so it cannot be applied until a cluster exists, and
# playbooks/site.yml only imports kube-vip.yml AFTER k3s.yml. A `--server` pointed at the
# VIP would therefore dial an address nothing announces and the cold build would be
# unjoinable — with no diagnostic beyond a connection timeout.
# On a WARM single-node rejoin (`--limit k3s-dev-N`) the founder is up and answering too, so
# the founder works in both cases and the VIP works in only one. Cilium's k8sServiceHost is
# the opposite trade (steady state, must survive losing the founder) and uses the endpoint
# variable instead.
- name: Install k3s secondary (join as server)
  ansible.builtin.command:
    cmd: >-
      {{ k3s_install_script }} {{ k3s_common_args }}
      --server https://{{ k3s_api_host }}:{{ k3s_api_port }}
    creates: /etc/rancher/k3s/k3s.yaml
  environment:
    INSTALL_K3S_VERSION: "{{ k3s_version }}"
    K3S_TOKEN: "{{ k3s_join_token }}"
  when: k3s_server_mode == 'secondary'

- name: Wait for the k3s API to be serving locally
  ansible.builtin.wait_for:
    host: "127.0.0.1"
    port: "{{ k3s_api_port }}"
    timeout: 180

# ── Graceful node shutdown: the systemd half — tag `shutdown` ──────────────────────────────────
# The kubelet's shutdown-grace-period is ENFORCED BY logind, not by the kubelet: the kubelet holds
# an inhibitor DELAY lock, and systemd will wait at most `InhibitDelayMaxSec` for it (default 5s)
# before proceeding regardless. A grace period above that cap is therefore silently truncated --
# the feature looks configured, logs no error, and still hard-kills pods. Assert the relationship
# instead of documenting it, because the failure is invisible.
- name: Assert the logind inhibitor cap can actually cover the kubelet grace period
  ansible.builtin.assert:
    that:
      - (k3s_logind_inhibit_delay_max_sec | int) >= (k3s_kubelet_shutdown_grace_period | regex_replace('[^0-9]', '') | int)
    fail_msg: >-
      k3s_logind_inhibit_delay_max_sec ({{ k3s_logind_inhibit_delay_max_sec }}s) is below
      k3s_kubelet_shutdown_grace_period ({{ k3s_kubelet_shutdown_grace_period }}). systemd would
      stop waiting for the kubelet's inhibitor lock before the drain finished, so pods would be
      SIGKILLed anyway and graceful node shutdown would be configured but ineffective.
    quiet: true
  tags: [shutdown]

- name: Ensure /etc/systemd/logind.conf.d exists
  ansible.builtin.file:
    path: /etc/systemd/logind.conf.d
    state: directory
    owner: root
    group: root
    mode: "0755"
  tags: [shutdown]

# A drop-in, not an edit of logind.conf: the distro file stays pristine and `state: absent` here
# would fully revert the change.
- name: Raise the logind inhibitor delay cap so the kubelet can finish draining
  ansible.builtin.copy:
    dest: /etc/systemd/logind.conf.d/10-k3s-graceful-shutdown.conf
    owner: root
    group: root
    mode: "0644"
    content: |
      # Managed by Ansible (roles/k3s_server). See defaults/main.yml.
      # MUST stay >= kubelet shutdown-grace-period, or the drain is truncated to this value.
      [Login]
      InhibitDelayMaxSec={{ k3s_logind_inhibit_delay_max_sec }}
  notify: Restart logind
  tags: [shutdown]