--- # KubeVirt VirtualMachine — the self-hosted UniFi OS Server (UOS) controller. # # WHY A VM (PM directive): the classic `unifi` .deb is deprecated. UOS is Ubiquiti's current # self-host product — a single installer that runs on Ubuntu Server and bundles UniFi OS + the # Network application + its OWN embedded datastore inside a Podman container stack. NO separate # MongoDB companion is needed. # # DELIVERY: this VM is NOT on the app "wave" track — it is reconciled by the non-gating `apps-vms` # Flux Kustomization (clusters/base/apps-vms.yaml, wait:false) so the slow first-boot image import # + UOS install can't stall the app waves. UniFi has no in-cluster dependents. # # LIVE MIGRATION: evictionStrategy: LiveMigrate + the RWX (longhorn-vm) root disk let the VM # drain to another node without downtime (node maintenance / rebalancing). # # LABELS on the pod template (propagated to the virt-launcher pod): # - kubevirt.io/vm: unifi -> Service/CNP selector target # - app.kubernetes.io/name: unifi -> convention # - egress-class: internet -> Cilium egress steering (default WAN path) # - auth-mode: native -> UniFi has its OWN login UI (no Keycloak/oauth2-proxy in # front of it). REQUIRED by Kyverno require-auth-mode-label # for any pod in an app-namespace. apiVersion: kubevirt.io/v1 kind: VirtualMachine metadata: name: unifi namespace: unifi labels: app.kubernetes.io/name: unifi spec: running: true template: metadata: labels: kubevirt.io/vm: unifi app.kubernetes.io/name: unifi homelab.keithjackson.dev/egress-class: internet homelab.keithjackson.dev/auth-mode: native spec: # LiveMigrate on eviction — needs the RWX migratable disk (longhorn-vm). Without a shared # disk KubeVirt cannot migrate and eviction would hard-stop the guest. evictionStrategy: LiveMigrate domain: # ===================================================================================== # FIRMWARE UUID == THE CLOUD-INIT INSTANCE-ID. THIS IS THE RE-RUN LEVER. READ BEFORE EDIT. # ===================================================================================== # MEASURED on prod 2026-09-14, not assumed: # * the NoCloud seed KubeVirt builds for this VM carries # {"instance-id":"de5c3338-6d32-4956-89d1-12c8272b2c07","local-hostname":"unifi"} # (read out of noCloud.iso inside virt-launcher-unifi-gp594), and that value was # EXACTLY `spec.template.spec.domain.firmware.uuid`, which KubeVirt's mutating webhook # had stamped onto the VM OBJECT — so it is PERSISTENT, not per-VMI (the VMI's own uid # was b532fd7c-…, a different value). It is not a hash of the VM name either: the four # standard uuid5 namespaces over "unifi" / "unifi.unifi" all produce something else. # * cloud-init's NoCloud datasource decides "is this the same instance?" by comparing the # cached instance-id against the guest's DMI product_uuid — which IS this firmware UUID. # # CONSEQUENCE, and it is the whole reason the previous unattended attempt could never have # worked: EDITING `userData` BELOW AND RESTARTING THE VM CHANGES NOTHING. cloud-init sees an # unchanged instance-id, restores its CACHED datasource, never re-reads the seed, and skips # every per-instance module (write_files, runcmd). The new payload is silently ignored. # # So the UUID is pinned HERE, in git, and BUMPING IT IS THE SUPPORTED WAY to make the guest # process a new cloud-init payload: change the trailing hex, commit, restart the VM. Pinning # it also makes a rebuild reproducible — a fresh apply would otherwise take a random # webhook-generated UUID and give the guest a different identity every time. # # The value below is DELIBERATELY NOT the de5c3338-…c07 the live guest booted with: that # difference is what makes the next boot a NEW cloud-init instance and installs the # `install-uos` helper + its systemd unit. `serial` is pinned to the value the webhook # already stamped, so this change does not also churn the guest's DMI serial. firmware: uuid: "de5c3338-6d32-4956-89d1-12c8272b2c11" serial: "24bb12ac-d311-45f6-a75b-90a9e1b63e18" # FLAG (verify-at-bringup): Ubuntu 24.04 amd64 cloud images are hybrid BIOS+UEFI bootable, # so KubeVirt's default SeaBIOS SHOULD boot cleanly (unlike the Fedora containerdisk, which # is UEFI-only and needed an explicit efi block). RESOLVED at bring-up: boots as shipped; # the guest reports Ubuntu 24.04.4 LTS. cpu: cores: 2 memory: guest: 4Gi # UOS min is 2Gi; 4Gi is comfortable for a home site. Bump to 8Gi # for large device counts / heavy DPI. Keep guest == requests below. devices: disks: - name: rootdisk disk: bus: virtio - name: cloudinit disk: bus: virtio # The operator's installer URL (+ optional SSH key), delivered as a tiny read-only # iso9660 disk — see the `uosdata` volume near the bottom of this file. - name: uosdata disk: bus: virtio interfaces: # MAC PINNED 2026-09-18, and this is a root-cause fix, not cosmetics. # KubeVirt generates a fresh MAC for every VMI when none is pinned. cloud-init writes # /etc/netplan/50-cloud-init.yaml with a `match` on the MAC it saw, and it will NOT rewrite # that file while the NoCloud instance-id is unchanged — and the instance-id here is # domain.firmware.uuid, which is stable across restarts by design. So each VMI recreation # handed the guest a NEW MAC that the retained netplan did not match, leaving enp1s0 # administratively DOWN with no address and no DHCP attempt. # That is exactly what the evidence shows for the boot of 2026-09-12: cloud-init's own # device table reports `enp1s0 | Up: False | Address: .`, no apt failures are logged # (because the per-instance modules were skipped), yet UOS is installed and # qemu-guest-agent runs — both of which required a working network on an EARLIER VMI. # Pinning the MAC to the value the current VMI already holds makes the guest's netplan # match permanently, so this cannot recur on the next recreation. - name: default masquerade: {} macAddress: a6:d7:c0:b3:72:fa # Declaring ports on a masquerade iface (a) restricts pod->VM forwarding to exactly # these and (b) surfaces them as containerPorts on the virt-launcher pod so the # LoadBalancer Services + CiliumNetworkPolicy ingress can target/match them. ports: - { name: inform, port: 8080, protocol: TCP } # device inform / adoption - { name: ui-legacy, port: 8443, protocol: TCP } # legacy Network UI (still served) - { name: ui, port: 11443, protocol: TCP } # UOS web UI (modern default) - { name: portal-http, port: 8880, protocol: TCP } # guest portal HTTP - { name: portal-https, port: 8843, protocol: TCP } # guest portal HTTPS - { name: throughput, port: 6789, protocol: TCP } # mobile speedtest / throughput - { name: stun, port: 3478, protocol: UDP } # STUN (device<->controller) - { name: discovery, port: 10001, protocol: UDP } # L2 device discovery resources: requests: memory: 4Gi networks: - name: default pod: {} volumes: - name: rootdisk dataVolume: name: unifi-rootdisk # ===================================================================================== # THE OPERATOR'S ONE FIELD — SOPS Secret `unifi-uos-installer`, delivered as a DISK. # ===================================================================================== # KubeVirt exposes a Secret as a small read-only iso9660 disk whose volume label we choose; # each Secret key becomes a file on it. The guest mounts it by label (LABEL=UOSDATA) and # reads `installer-url` plus an optional `console-password`. The Secret ships ENCRYPTED with # REPLACE-ME placeholders, and the guest-side script treats anything that is not an https # URL as "not provided yet" and exits 0 — so the VM boots and stays healthy until the # operator fills it in. See unifi-uos-installer.sops.yaml.example next to this file. # # `console-password` exists because THERE IS CURRENTLY NO WAY INTO THIS GUEST. The stock # Ubuntu cloud image ships a locked `ubuntu` account, this cloud-init never installed a key, # and SSH could not help anyway: the masquerade interface above forwards ONLY the declared # ports, and 22 is not one of them — probed from inside virt-launcher on 2026-09-14, the # guest answers RST on 22 exactly as it does on the UniFi ports. The serial console is the # only door, and a console needs a password. Set it and `virtctl -n unifi console unifi` # works; `kubectl -n unifi logs virt-launcher-unifi-… -c guest-console-log` already gives # read-only console output with no virtctl at all. # # WHY A SEPARATE DISK AND NOT `cloudInitNoCloud.secretRef` (which would also lift the # 2048-byte inline cap): secretRef moves the ENTIRE cloud-config into ciphertext. Two # concrete costs. (a) The shell below stops being reviewable in a diff and falls outside the # L0 dollar-collapse guard, which can only read plaintext. (b) Flux runs its postBuild # envsubst over the DECRYPTED resources, so the escape contract in # clusters/base/apps-vms.yaml would still govern text nobody can see in git — the exact # shape of the 2026-09-13 nextcloud-db incident. Keeping the script in plaintext and putting # ONLY the operator's two values behind SOPS keeps both properties. # # A CHANGED SECRET REACHES THE GUEST ON THE NEXT VMI START with NO firmware-UUID bump: the # disk is rebuilt from the Secret every time the VMI starts, and the systemd unit below # re-reads it on EVERY boot instead of relying on cloud-init (which will not re-run — see # the firmware block above). The UUID bump is a ONE-TIME step, to get that unit installed. - name: uosdata secret: secretName: unifi-uos-installer volumeLabel: UOSDATA - name: cloudinit cloudInitNoCloud: # INLINE, and it MUST stay <= 2048 bytes. That ceiling was MEASURED against the # DEPLOYED KubeVirt v1.9.0 rather than quoted: a server-side dry-run apply of an # otherwise-identical VM accepted 2048 bytes of userData and rejected 2049 with # "cloudInitNoCloud userdata exceeds 2048 byte limit. Should use UserDataSecretRef for # larger data." The payload below is ~1.5 KB; keep additions small. # # WHAT IT DOES: installs qemu-guest-agent (the KubeVirt AgentConnected signal, which is # also the proof cloud-init ran) and preps Podman for UOS, then writes `install-uos` # plus a systemd unit that runs it on EVERY boot. The unit — not cloud-init — is what # makes a later URL change take effect, because cloud-init will not re-run per-instance # modules while the instance-id is unchanged. # # `install-uos` IS IDEMPOTENT AND WILL NOT REINSTALL OVER A WORKING CONTROLLER. It bails # out the moment systemd knows a `uosserver.service` — which the live guest ALREADY has: # the serial console log for the current boot shows both `uosserver.service` and # `uosserver-updater.service` starting, and the root filesystem is 6.3 GB of 40 GB used. # UOS is installed here; see the README for what is actually wrong (the guest lost its # IP). A stamp file at /var/lib/uos-installed is the second, weaker guard for the case # where the unit name changes in some future UOS release. # # ENVSUBST: apps-vms declares postBuild.substituteFrom (for ${UNIFI_L4_VIP} in # 50-service-l4.yaml), so Flux's envsubst DOES run over this text. ${BASE_DOMAIN} below # is a deliberate substitution. Everything else is bare `$name` or `$(cmd)`, both of # which envsubst leaves untouched — verified against the live VMI, whose rendered # userData still carries its shell variables verbatim. There is deliberately NO # brace-form shell expansion left in this payload, so nothing here needs escaping; if # you add one, escape it per the contract in clusters/base/apps-vms.yaml, and NEVER # write an indexed expansion. userData: |- #cloud-config hostname: unifi fqdn: unifi.${BASE_DOMAIN} package_update: true package_upgrade: false packages: [qemu-guest-agent, podman, uidmap, slirp4netns, ca-certificates, curl] write_files: - path: /usr/local/sbin/install-uos permissions: '0755' content: | #!/bin/bash set -euo pipefail d=/run/uos; mkdir -p $d mountpoint -q $d || mount -o ro LABEL=UOSDATA $d p=$(tr -d '\r\n' < $d/console-password || true) case "$p" in ""|REPLACE-ME) ;; *) echo "ubuntu:$p" | chpasswd ;; esac systemctl cat uosserver.service >/dev/null 2>&1 && exit 0 s=/var/lib/uos-installed [ -e $s ] && exit 0 u=$(tr -d '\r\n' < $d/installer-url || true) case "$u" in https://*) ;; *) echo "UOS installer-url not set yet"; exit 0;; esac t=$(mktemp) curl -fL --retry 5 --retry-delay 10 -o $t "$u" chmod +x $t yes y | $t install || yes y | $t rm -f $t; date -Is > $s - path: /etc/systemd/system/uos-install.service content: | [Unit] Description=Install UniFi OS Server Wants=network-online.target After=network-online.target [Service] Type=oneshot ExecStart=/usr/local/sbin/install-uos StandardOutput=journal+console [Install] WantedBy=multi-user.target runcmd: - [systemctl, enable, --now, qemu-guest-agent] - [systemctl, enable, --now, podman.socket] - [systemctl, enable, --now, uos-install.service]