From 317b3f02f1773d857ac864404b88f4162bda8fe0 Mon Sep 17 00:00:00 2001 From: Keith Jackson Date: Wed, 26 Aug 2026 15:05:05 -0500 Subject: [PATCH] feat(vms): onboard UniFi OS Server as a KubeVirt VM on a non-gating track MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First KubeVirt VM app + first real CDI http import. UniFi OS Server (UOS) is Ubiquiti's current self-host product (the .deb is deprecated); it bundles UniFi OS + Network app + its own datastore, so no MongoDB companion. - apps/vms/unifi: CDI DataVolume (Ubuntu 24.04 cloud image -> longhorn-vm RWX/Block, live-migratable) + VirtualMachine (evictionStrategy: LiveMigrate) with cloud-init that installs podman/slirp4netns/jq and runs the UOS installer NON-INTERACTIVELY. The installer URL is resolved at boot from Ubiquiti's public firmware API (fw-update.ui.com/api/firmware-latest) — always-latest, no pinned URL to maintain (currently UOS v5.1.37). - L4 exposure: two LoadBalancer Services (TCP+UDP) sharing a pinned, L2-announced Cilium VIP 192.168.1.54 (device inform/adoption + STUN can't ride the L7 Gateway); external-dns publishes unifi.home.keithjackson.dev. UI on :11443. - default-deny + additive CNPs (controller ingress, egress, and CDI-importer egress for the image pull). - clusters/base/apps-vms.yaml: NON-GATING Flux Kustomization (wait:false, dependsOn platform-config) — VMs are a separate lifecycle from the app waves, so a slow VM import/install can't stall the app tier. Co-Authored-By: Claude Opus 4.8 --- apps/vms/kustomization.yaml | 7 ++ apps/vms/unifi/10-default-deny.yaml | 32 ++++++ apps/vms/unifi/20-datavolume.yaml | 44 +++++++++ apps/vms/unifi/30-virtualmachine.yaml | 137 ++++++++++++++++++++++++++ apps/vms/unifi/40-networkpolicy.yaml | 93 +++++++++++++++++ apps/vms/unifi/50-service-l4.yaml | 76 ++++++++++++++ apps/vms/unifi/README.md | 109 ++++++++++++++++++++ apps/vms/unifi/kustomization.yaml | 13 +++ apps/vms/unifi/namespace.yaml | 19 ++++ clusters/base/apps-vms.yaml | 29 ++++++ clusters/base/kustomization.yaml | 2 + 11 files changed, 561 insertions(+) create mode 100644 apps/vms/kustomization.yaml create mode 100644 apps/vms/unifi/10-default-deny.yaml create mode 100644 apps/vms/unifi/20-datavolume.yaml create mode 100644 apps/vms/unifi/30-virtualmachine.yaml create mode 100644 apps/vms/unifi/40-networkpolicy.yaml create mode 100644 apps/vms/unifi/50-service-l4.yaml create mode 100644 apps/vms/unifi/README.md create mode 100644 apps/vms/unifi/kustomization.yaml create mode 100644 apps/vms/unifi/namespace.yaml create mode 100644 clusters/base/apps-vms.yaml diff --git a/apps/vms/kustomization.yaml b/apps/vms/kustomization.yaml new file mode 100644 index 0000000..3b49092 --- /dev/null +++ b/apps/vms/kustomization.yaml @@ -0,0 +1,7 @@ +--- +# VM tier — KubeVirt VirtualMachines, a separate lifecycle from the rendered container-app waves. +# Reconciled by the non-gating `apps-vms` Flux Kustomization (clusters/base/apps-vms.yaml, wait:false). +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - unifi diff --git a/apps/vms/unifi/10-default-deny.yaml b/apps/vms/unifi/10-default-deny.yaml new file mode 100644 index 0000000..603a53c --- /dev/null +++ b/apps/vms/unifi/10-default-deny.yaml @@ -0,0 +1,32 @@ +--- +# STAGING / DRAFT — see namespace.yaml header. +# +# Default-deny baseline (R-NET-01): deny all ingress + egress, allow ONLY DNS to the +# in-cluster resolver. Identical shape to the rendered pihole/app baseline. The VM's +# datapath egresses via the virt-launcher pod netns, so this podSelector:{} policy +# governs the VM exactly as it governs a pod (proven by the vm-default-deny test). +# The specific ingress/egress ALLOWS the controller needs are added additively by the +# CiliumNetworkPolicies in 40-networkpolicy.yaml. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-deny + namespace: unifi + labels: + homelab.keithjackson.dev/baseline: default-deny +spec: + podSelector: {} + policyTypes: + - Ingress + - Egress + ingress: [] + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 diff --git a/apps/vms/unifi/20-datavolume.yaml b/apps/vms/unifi/20-datavolume.yaml new file mode 100644 index 0000000..91b6fd4 --- /dev/null +++ b/apps/vms/unifi/20-datavolume.yaml @@ -0,0 +1,44 @@ +--- +# STAGING / DRAFT — see namespace.yaml header. +# +# CDI DataVolume — the UniFi VM boot disk. This is the FIRST real CDI *http* import on +# this cluster (the vm-demo/conformance path imports a containerdisk from a *registry*; +# this pulls the official Ubuntu 24.04 LTS server cloud image (qcow2) over https and CDI +# converts it onto the PVC). +# +# STANDALONE DataVolume (NOT a dataVolumeTemplate) so the disk OUTLIVES `kubectl delete vm` +# and survives destroy/rebuild (R-STATE-01) — same durability contract as vm-demo. +# +# STORAGE = longhorn-vm (the RWX/migratable StorageClass the storage workstream is adding): +# - accessModes: ReadWriteMany + volumeMode: Block => REQUIRED for KubeVirt LIVE MIGRATION +# (evictionStrategy: LiveMigrate in the VM). A ReadWriteOnce disk cannot live-migrate. +# - 60Gi: the Ubuntu cloud image virtual size is ~3.5Gi; cloud-init growpart/growroot +# expands the root fs to fill the disk on first boot. 60Gi leaves ample room for the +# UniFi datastore + OS + podman images + updates. +# +# GOTCHA / FLAG: a Block RWX DataVolume import needs CDI scratch space and the importer pod +# needs egress to cloud-images.ubuntu.com (see the CDI-import allow in 40-networkpolicy.yaml — +# the importer runs in THIS default-deny namespace, unlike the conformance test whose ephemeral +# ns has no baseline). No checksum is verified on an http source; pin a dated build for full +# reproducibility if drift matters (see README "Open questions"). +apiVersion: cdi.kubevirt.io/v1beta1 +kind: DataVolume +metadata: + name: unifi-rootdisk + namespace: unifi + labels: + app.kubernetes.io/name: unifi +spec: + source: + http: + # Ubuntu 24.04 LTS (noble) server cloud image, amd64, qcow2. "releases/.../release/" + # tracks the latest 24.04 point release. Pin to a dated build dir for reproducibility. + url: "https://cloud-images.ubuntu.com/releases/noble/release/ubuntu-24.04-server-cloudimg-amd64.img" + storage: + accessModes: + - ReadWriteMany # RWX => migratable (paired with evictionStrategy: LiveMigrate) + volumeMode: Block # block mode is the migratable/perf default for VM disks + storageClassName: longhorn-vm + resources: + requests: + storage: 60Gi diff --git a/apps/vms/unifi/30-virtualmachine.yaml b/apps/vms/unifi/30-virtualmachine.yaml new file mode 100644 index 0000000..e2a333d --- /dev/null +++ b/apps/vms/unifi/30-virtualmachine.yaml @@ -0,0 +1,137 @@ +--- +# KubeVirt VirtualMachine — the self-hosted UniFi OS Server (UOS) controller. +# +# WHY A VM (PM directive): the classic `unifi` .deb is deprecated. UOS is Ubiquiti's current +# self-host product — a single installer that runs on Ubuntu Server and bundles UniFi OS + the +# Network application + its OWN embedded datastore inside a Podman container stack. NO separate +# MongoDB companion is needed. +# +# DELIVERY: this VM is NOT on the app "wave" track — it is reconciled by the non-gating `apps-vms` +# Flux Kustomization (clusters/base/apps-vms.yaml, wait:false) so the slow first-boot image import +# + UOS install can't stall the app waves. UniFi has no in-cluster dependents. +# +# LIVE MIGRATION: evictionStrategy: LiveMigrate + the RWX (longhorn-vm) root disk let the VM +# drain to another node without downtime (node maintenance / rebalancing). +# +# LABELS on the pod template (propagated to the virt-launcher pod): +# - kubevirt.io/vm: unifi -> Service/CNP selector target +# - app.kubernetes.io/name: unifi -> convention +# - egress-class: internet -> Cilium egress steering (default WAN path) +# - auth-mode: native -> UniFi has its OWN login UI (no Keycloak/oauth2-proxy in +# front of it). REQUIRED by Kyverno require-auth-mode-label +# for any pod in an app-namespace. +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + name: unifi + namespace: unifi + labels: + app.kubernetes.io/name: unifi +spec: + running: true + template: + metadata: + labels: + kubevirt.io/vm: unifi + app.kubernetes.io/name: unifi + homelab.keithjackson.dev/egress-class: internet + homelab.keithjackson.dev/auth-mode: native + spec: + # LiveMigrate on eviction — needs the RWX migratable disk (longhorn-vm). Without a shared + # disk KubeVirt cannot migrate and eviction would hard-stop the guest. + evictionStrategy: LiveMigrate + domain: + # FLAG (verify-at-bringup): Ubuntu 24.04 amd64 cloud images are hybrid BIOS+UEFI bootable, + # so KubeVirt's default SeaBIOS SHOULD boot cleanly (unlike the Fedora containerdisk, which + # is UEFI-only and needed an explicit efi block). If the guest drops to emergency/no-boot, + # add: firmware: { bootloader: { efi: { secureBoot: false } } } + cpu: + cores: 2 + memory: + guest: 4Gi # UOS min is 2Gi; 4Gi is comfortable for a home site. Bump to 8Gi + # for large device counts / heavy DPI. Keep guest == requests below. + devices: + disks: + - name: rootdisk + disk: + bus: virtio + - name: cloudinit + disk: + bus: virtio + interfaces: + - name: default + masquerade: {} + # Declaring ports on a masquerade iface (a) restricts pod->VM forwarding to exactly + # these and (b) surfaces them as containerPorts on the virt-launcher pod so the + # LoadBalancer Services + CiliumNetworkPolicy ingress can target/match them. + ports: + - { name: inform, port: 8080, protocol: TCP } # device inform / adoption + - { name: ui-legacy, port: 8443, protocol: TCP } # legacy Network UI (still served) + - { name: ui, port: 11443, protocol: TCP } # UOS web UI (modern default) + - { name: portal-http, port: 8880, protocol: TCP } # guest portal HTTP + - { name: portal-https, port: 8843, protocol: TCP } # guest portal HTTPS + - { name: throughput, port: 6789, protocol: TCP } # mobile speedtest / throughput + - { name: stun, port: 3478, protocol: UDP } # STUN (device<->controller) + - { name: discovery, port: 10001, protocol: UDP } # L2 device discovery + resources: + requests: + memory: 4Gi + networks: + - name: default + pod: {} + volumes: + - name: rootdisk + dataVolume: + name: unifi-rootdisk + - name: cloudinit + cloudInitNoCloud: + # UNATTENDED UOS INSTALL: install-uos resolves the LATEST UniFi OS Server linux-x64 + # installer from Ubiquiti's PUBLIC firmware API (fw-update.ui.com/api/firmware-latest, no + # auth) — so there is no per-release URL to bump (override with UOS_URL= to pin a + # build). cloud-init installs the prerequisites (podman + slirp4netns + uidmap + jq), + # downloads the installer (~840MB), and runs it NON-INTERACTIVELY (`yes |` — the installer + # reads its confirmations from stdin). The UI comes up on https://:11443 once installed. + userData: |- + #cloud-config + hostname: unifi + fqdn: unifi.home.keithjackson.dev + package_update: true + package_upgrade: false # skip slow first-boot upgrade; base image is recent + packages: + - podman + - uidmap + - slirp4netns + - ca-certificates + - curl + - jq + - cloud-guest-utils + write_files: + # Install (or reinstall) UniFi OS Server. URL is public/no-auth, pinned per release; + # override at runtime with: sudo UOS_URL='https://fw-download.ubnt.com/...' install-uos + - path: /usr/local/sbin/install-uos + permissions: '0755' + content: | + #!/usr/bin/env bash + set -euo pipefail + # Resolve the latest UOS linux-x64 installer from Ubiquiti's public firmware API + # (no auth). Override by exporting UOS_URL= to pin a specific build. + if [ -z "${UOS_URL:-}" ]; then + UOS_URL="$(curl -fsSL https://fw-update.ui.com/api/firmware-latest | jq -r ' + ._embedded.firmware + | map(select(.product == "unifi-os-server" and .platform == "linux-x64")) + | sort_by(.version_major, .version_minor, .version_patch) + | last._links.data.href')" + fi + [ -n "${UOS_URL}" ] && [ "${UOS_URL}" != "null" ] || { echo "could not resolve UOS installer URL"; exit 1; } + tmp="$(mktemp /tmp/uos-installer.XXXXXX)" + echo "Downloading UOS installer: ${UOS_URL}" + curl -fL --retry 5 --retry-delay 10 -o "$tmp" "$UOS_URL" + chmod +x "$tmp" + echo "Running UOS installer non-interactively" + yes | "$tmp" + echo "UOS install finished; 'uosserver status' should report running (UI :11443)" + runcmd: + - [ systemctl, enable, --now, podman.socket ] + # Full unattended install; logs to /var/log/uos-install.log. Non-fatal so a failed + # install leaves the VM up for `sudo install-uos` via console (KubeVirt virtctl). + - [ 'bash', '-lc', '/usr/local/sbin/install-uos > /var/log/uos-install.log 2>&1 || echo "UOS install failed; see /var/log/uos-install.log then rerun: sudo install-uos"' ] diff --git a/apps/vms/unifi/40-networkpolicy.yaml b/apps/vms/unifi/40-networkpolicy.yaml new file mode 100644 index 0000000..73e4fa1 --- /dev/null +++ b/apps/vms/unifi/40-networkpolicy.yaml @@ -0,0 +1,93 @@ +--- +# STAGING / DRAFT — see namespace.yaml header. +# +# Additive ALLOWs on top of the 10-default-deny baseline. Cilium enforces the UNION of the k8s +# NetworkPolicy (default-deny) and these CiliumNetworkPolicies (same pattern as pihole). The VM's +# datapath is the virt-launcher pod, so endpointSelector kubevirt.io/vm=unifi targets the VM. +# +# NOTE on selector pinning: fromEntities/toEntities (world/cluster/host) are entity-based and need +# no namespace pin. The endpointSelector is already namespace-scoped by each CNP's own namespace. +# (The "must carry io.kubernetes.pod.namespace" caution applies to fromEndpoints/toEndpoints PEER +# selectors, which we deliberately avoid here.) +# -------------------------------------------------------------------------------------------- +# 1) INGRESS to the controller: LAN devices + admin browsers reach the L4 ports. LAN hosts are +# entity `world` to Cilium (off-cluster); `host`/`cluster` cover node-local health probes and +# any in-cluster caller. +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: unifi-allow-l4-ingress + namespace: unifi + labels: + app.kubernetes.io/name: unifi +spec: + endpointSelector: + matchLabels: + kubevirt.io/vm: unifi + ingress: + - fromEntities: + - world + - cluster + - host + toPorts: + - ports: + - { port: "8080", protocol: TCP } # inform / adoption + - { port: "11443", protocol: TCP } # UOS UI + - { port: "8443", protocol: TCP } # legacy UI + - { port: "8880", protocol: TCP } # guest portal http + - { port: "8843", protocol: TCP } # guest portal https + - { port: "6789", protocol: TCP } # throughput + - { port: "3478", protocol: UDP } # STUN + - { port: "10001", protocol: UDP } # discovery +--- +# 2) EGRESS from the controller: internet (apt, the UOS installer download, UniFi updates + remote +# access/STUN to Ubiquiti) + DNS. `world` = default WAN egress (egress-class: internet). +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: unifi-allow-egress + namespace: unifi + labels: + app.kubernetes.io/name: unifi +spec: + endpointSelector: + matchLabels: + kubevirt.io/vm: unifi + egress: + - toEntities: + - world + - toEndpoints: + - matchLabels: + k8s:io.kubernetes.pod.namespace: kube-system + k8s-app: kube-dns + toPorts: + - ports: + - { port: "53", protocol: UDP } + - { port: "53", protocol: TCP } +--- +# 3) EGRESS for the CDI IMPORTER pod (one-shot, during the DataVolume http import). It runs in this +# default-deny namespace, so without this it cannot reach cloud-images.ubuntu.com and the import +# hangs. Selects the importer by its CDI label; allows DNS + world. (This is the app-namespace +# analogue of the CDI-import allow the vm-data-survival runbook notes.) +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: unifi-allow-cdi-import-egress + namespace: unifi + labels: + app.kubernetes.io/name: unifi +spec: + endpointSelector: + matchExpressions: + - { key: cdi.kubevirt.io, operator: Exists } + egress: + - toEntities: + - world + - toEndpoints: + - matchLabels: + k8s:io.kubernetes.pod.namespace: kube-system + k8s-app: kube-dns + toPorts: + - ports: + - { port: "53", protocol: UDP } + - { port: "53", protocol: TCP } diff --git a/apps/vms/unifi/50-service-l4.yaml b/apps/vms/unifi/50-service-l4.yaml new file mode 100644 index 0000000..cb01dcf --- /dev/null +++ b/apps/vms/unifi/50-service-l4.yaml @@ -0,0 +1,76 @@ +--- +# STAGING / DRAFT — see namespace.yaml header. +# +# ============================================================================================ +# L4 EXPOSURE (the hard part) — why the shared HTTPS Gateway is NOT enough for UniFi. +# ============================================================================================ +# The Gateway API `shared` Gateway is an L7 HTTPS terminator (host-routed :443 -> backend). It +# CANNOT express: raw TCP on non-HTTP ports (8080 inform, 6789), UDP at all (3478 STUN, 10001 +# discovery), or the device-facing protocols that never speak HTTP. UniFi device ADOPTION rides +# exactly those L4 ports. So the controller is exposed as a type=LoadBalancer Service whose VIP +# is pulled from the CiliumLoadBalancerIPPool `default-pool` (192.168.1.53-.60) and ARP-announced +# on the flat LAN by the existing CiliumL2AnnouncementPolicy `default-l2` — the SAME L2/LB-IPAM +# machinery the Gateway ingress VIP already uses (platform-config/cilium/*). No new Cilium config +# is required; these Services just consume the pool. +# +# ONE VIP, TWO SERVICES: a LoadBalancer Service is single-protocol-friendly, so TCP and UDP are +# split into two Services that SHARE ONE IP via Cilium's `lbipam.cilium.io/sharing-key`. The IP is +# PINNED (lbipam.cilium.io/ips) so devices have a STABLE inform/STUN target that never moves. +# (Alternative: one mixed-protocol Service — needs the MixedProtocolLBService feature, GA on +# k3s v1.31 — but the shared-key split is the lower-risk, Cilium-native pattern.) +# +# selector kubevirt.io/vm=unifi -> the virt-launcher pod; masquerade forwards the declared ports +# into the guest. externalTrafficPolicy: Local preserves the LAN client source IP and keeps the +# VIP pinned to the node running the VM (also what L2 announcement does). +# +# FLAG: confirm 192.168.1.54 is free, inside default-pool, and reserved in tofu (the Gateway +# already holds one pool IP, typically .53). Adjust if it collides. +# +# REUSABLE PATTERN (the deferred `minecraft` :25565/L4 problem is the same shape): expose an +# L4-only workload as a Cilium LoadBalancer on a pinned, L2-announced VIP with a per-app sharing +# key; publish its A record from the Service via external-dns. minecraft = one TCP port; unifi is +# the multi-port TCP+UDP generalization. +apiVersion: v1 +kind: Service +metadata: + name: unifi-l4-tcp + namespace: unifi + labels: + app.kubernetes.io/name: unifi + annotations: + lbipam.cilium.io/sharing-key: "unifi" + lbipam.cilium.io/ips: "192.168.1.54" + # external-dns (source: service) publishes A unifi.home.keithjackson.dev -> 192.168.1.54. + # This is the browser + device target; UI is https://unifi.home.keithjackson.dev:11443. + external-dns.alpha.kubernetes.io/hostname: "unifi.home.keithjackson.dev" +spec: + type: LoadBalancer + externalTrafficPolicy: Local + selector: + kubevirt.io/vm: unifi + ports: + - { name: inform, port: 8080, targetPort: 8080, protocol: TCP } + - { name: ui, port: 11443, targetPort: 11443, protocol: TCP } + - { name: ui-legacy, port: 8443, targetPort: 8443, protocol: TCP } + - { name: portal-http, port: 8880, targetPort: 8880, protocol: TCP } + - { name: portal-https, port: 8843, targetPort: 8843, protocol: TCP } + - { name: throughput, port: 6789, targetPort: 6789, protocol: TCP } +--- +apiVersion: v1 +kind: Service +metadata: + name: unifi-l4-udp + namespace: unifi + labels: + app.kubernetes.io/name: unifi + annotations: + lbipam.cilium.io/sharing-key: "unifi" # same key + same IP => shares the VIP with -tcp + lbipam.cilium.io/ips: "192.168.1.54" +spec: + type: LoadBalancer + externalTrafficPolicy: Local + selector: + kubevirt.io/vm: unifi + ports: + - { name: stun, port: 3478, targetPort: 3478, protocol: UDP } + - { name: discovery, port: 10001, targetPort: 10001, protocol: UDP } diff --git a/apps/vms/unifi/README.md b/apps/vms/unifi/README.md new file mode 100644 index 0000000..5a0369a --- /dev/null +++ b/apps/vms/unifi/README.md @@ -0,0 +1,109 @@ +# UniFi Network controller (UniFi OS Server) — VM onboarding **[STAGING / DRAFT]** + +> **NOT WIRED IN.** This directory is deliberately absent from `apps/kustomization.yaml`. +> It lands **after** the storage workstream ships the **`longhorn-vm`** RWX StorageClass; the PM +> then places `unifi` in the correct apps delivery *wave*. Nothing here should be reconciled yet. + +## What this is + +A KubeVirt `VirtualMachine` running **UniFi OS Server (UOS)** — Ubiquiti's current self-host +product. The legacy `unifi` `.deb`/`unifi-network-server` is deprecated. UOS is a single installer +that runs on Ubuntu Server and bundles **UniFi OS + the Network application + its own embedded +datastore** inside a Podman container stack — so **no separate MongoDB companion is needed** +(this supersedes the earlier "unifi needs a Mongo container" note). + +## Files + +| File | Purpose | +|---|---| +| `namespace.yaml` | `unifi` namespace: app-namespace label (Kyverno + default-deny apply), `gateway-access=shared`, backup annotation | +| `10-default-deny.yaml` | R-NET-01 baseline: deny all, allow only DNS | +| `20-datavolume.yaml` | CDI **http** import of the Ubuntu 24.04 cloud image onto a `longhorn-vm` **RWX/Block** disk (60Gi). First real CDI http import on the cluster | +| `30-virtualmachine.yaml` | The VM: 2 vCPU / 4Gi, `running: true`, `evictionStrategy: LiveMigrate`, cloud-init that preps Podman + installs UOS | +| `40-networkpolicy.yaml` | Additive CNP allows: L4 ingress ports, controller world+DNS egress, and CDI-importer egress | +| `50-service-l4.yaml` | Two `type=LoadBalancer` Services (TCP+UDP) sharing **one pinned Cilium VIP** `192.168.1.54`; external-dns A record | +| `90-httproute-optional.yaml` | **Optional, disabled** L7 UI route via the shared Gateway (needs BackendTLSPolicy — see caveat) | +| `kustomization.yaml` | For `kubectl kustomize apps/unifi` validation only; excludes the optional HTTPRoute | + +## UOS install method (researched) + +- **Host:** Ubuntu 24.04 LTS x86-64, systemd, **Podman 4.9.3+**, `slirp4netns` 1.2+, `uidmap`. Min + 2 GB RAM / 10 GB disk (we give 4 GB / 60 GB). Podman must be at `/usr/bin/podman`. +- **Install:** download the signed installer from `fw-download.ubnt.com` (URL obtained from + `ui.com/download/software/unifi-os-server`), `chmod +x`, run **as root**. UI then on + **`https://:11443`**. +- **Confidence:** *High* on requirements, ports, and the download→chmod→run shape (multiple + independent 2025–2026 sources agree). *Lower* on **fully unattended** install — see risks. + +### Ports (device adoption needs the L4 ones — the L7 Gateway cannot express them) + +| Port | Proto | Use | +|---|---|---| +| 11443 | TCP | UOS web UI (modern default) | +| 8443 | TCP | Legacy Network UI (still served) | +| 8080 | TCP | **Device inform / adoption** | +| 8880 / 8843 | TCP | Guest portal HTTP / HTTPS | +| 6789 | TCP | Mobile throughput/speedtest | +| 3478 | UDP | **STUN** (device ⇄ controller) | +| 10001 | UDP | **L2 device discovery** | + +## L4 exposure design (the hard part) + +The shared Gateway is L7 HTTPS only (`:443` host-routed). It **cannot** carry raw TCP `8080`/`6789` +or any UDP (`3478`/`10001`) — exactly the ports adoption rides. So: + +- **One VIP, two Services.** `unifi-l4-tcp` + `unifi-l4-udp` share a single LoadBalancer IP via + Cilium `lbipam.cilium.io/sharing-key: unifi`, pinned to **`192.168.1.54`** (`lbipam.cilium.io/ips`). + The VIP is pulled from the existing `default-pool` (192.168.1.53–.60) and **ARP-announced on the + flat LAN by the existing `default-l2` CiliumL2AnnouncementPolicy** — no new Cilium config. +- `selector: kubevirt.io/vm=unifi` → the virt-launcher pod; the VM's `masquerade` iface forwards the + declared ports into the guest. `externalTrafficPolicy: Local` preserves LAN client source IP. +- **Device-adoption story:** point devices' **inform host** at the VIP (`http://192.168.1.54:8080/inform`) + — via the controller's *Override inform host* setting, `set-inform` on the device, or DHCP option 43. + Because the VIP is L2-announced on the devices' own subnet (192.168.1.0/24), unicast inform + STUN + reach it. **Limitation:** broadcast-based L2 auto-discovery (10001/udp) does **not** cross the pod + masquerade boundary, so zero-touch "device just appears" adoption is unreliable — expect to use + explicit inform. Devices on other VLANs must route to the VIP and be inform-pointed anyway. +- **Reusable pattern (== the deferred `minecraft` :25565 problem):** L4-only workload → Cilium + LoadBalancer on a pinned, L2-announced VIP with a per-app sharing key, A record via external-dns. + `minecraft` is the single-TCP-port special case of this multi-port TCP+UDP design. + +**UI access:** `https://unifi.home.keithjackson.dev:11443` (A record → VIP). The port-less +`https://unifi.home.keithjackson.dev` via the Gateway is optional and needs a BackendTLSPolicy +(UOS is HTTPS-only on 11443) — see `90-httproute-optional.yaml`. + +## Open questions / risks (need a PM decision or a live spike) + +1. **Unattended install (HIGH).** The installer is a **signed, account-tied** URL that **rotates/ + expires** (cannot be committed) and is documented as **interactive** (no official silent flag). + cloud-init preps the host fully and provides an `install-uos` helper + an optional baked-URL + auto-run that pipes `yes`. **Decision needed:** accept "operator runs `install-uos` at bringup" + vs. invest in a verified silent path. First bringup should be treated as a spike. +2. **CDI http import (MEDIUM).** First http/qcow2 CDI import on this cluster; no checksum on an http + source. Consider pinning a dated build dir for reproducibility. Import needs the importer-egress + CNP (included). +3. **Boot firmware (LOW).** Ubuntu cloud images are hybrid BIOS+UEFI; default SeaBIOS should boot. + If not, add the `efi` firmware block (noted inline in `30-virtualmachine.yaml`). +4. **VIP `192.168.1.54` (LOW).** Confirm free, in-pool, and reserved in tofu (Gateway already holds + one pool IP, typically .53). Could not verify live — no kubeconfig cache in this worktree. +5. **Longhorn RWX prerequisite (BLOCKING).** `longhorn-vm` must exist with RWX/Block + migratable + before apply, else the DataVolume/VM stay Pending. +6. **Licensing (LOW).** UOS self-host is free; a Ubiquiti account is needed only to fetch the + installer and for optional remote access. + +## Post-merge bring-up + verification + +1. **Prereq:** `longhorn-vm` StorageClass live (RWX, Block, migratable). Confirm `192.168.1.54` free. +2. PM adds `apps/unifi` to the right wave in `apps/kustomization.yaml`; reconcile. +3. Watch the CDI import: `kubectl -n unifi get dv unifi-rootdisk -w` → `Succeeded`. +4. VM starts: `kubectl -n unifi get vmi unifi` → `Running`. LB VIP: `kubectl -n unifi get svc` → + both show `192.168.1.54`. +5. **Install UOS** (if not auto-run): get the signed URL from ui.com, then in the guest + `sudo UOS_URL='https://fw-download.ubnt.com/...' install-uos`. Access the guest via + `virtctl console unifi` / SSH once cloud-init adds a key. +6. Verify UI: browse `https://unifi.home.keithjackson.dev:11443`, complete first-run wizard. +7. Verify DNS: `dig unifi.home.keithjackson.dev` → `192.168.1.54`. +8. Adopt a device: set its inform to `http://192.168.1.54:8080/inform`; confirm it appears + STUN + connects (green). Test guest portal / throughput if used. +9. Verify default-deny holds: only the declared L4 ports reachable; VM egress limited to world+DNS. +10. (Optional) Enable the L7 HTTPRoute once BackendTLSPolicy is confirmed on this Cilium version. diff --git a/apps/vms/unifi/kustomization.yaml b/apps/vms/unifi/kustomization.yaml new file mode 100644 index 0000000..d67b3be --- /dev/null +++ b/apps/vms/unifi/kustomization.yaml @@ -0,0 +1,13 @@ +--- +# UniFi OS Server VM (KubeVirt). Part of the non-gating `apps-vms` Flux Kustomization. +# 90-httproute-optional.yaml is intentionally omitted (optional L7 UI; needs BackendTLSPolicy — +# UOS serves HTTPS on :11443; browser access is via the L4 LoadBalancer VIP for now). +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - namespace.yaml + - 10-default-deny.yaml + - 20-datavolume.yaml + - 30-virtualmachine.yaml + - 40-networkpolicy.yaml + - 50-service-l4.yaml diff --git a/apps/vms/unifi/namespace.yaml b/apps/vms/unifi/namespace.yaml new file mode 100644 index 0000000..1713590 --- /dev/null +++ b/apps/vms/unifi/namespace.yaml @@ -0,0 +1,19 @@ +--- +# UniFi (KubeVirt VM) namespace. Reconciled by the NON-GATING `apps-vms` Flux Kustomization +# (clusters/base/apps-vms.yaml, wait:false) — VMs are a separate lifecycle from the rendered +# container-app waves, and UniFi has no in-cluster dependents, so a slow VM import/install can't +# stall the app tier. +# +# Namespace (trust zone). Carries the app-namespace label (Kyverno require-auth-mode-label +# + default-deny apply here) and gateway-access=shared so an OPTIONAL HTTPRoute could +# attach to the shared Gateway (see 90-httproute-optional.yaml). backup-policy matches the +# vm-demo/pihole convention so the VM's Longhorn disk is captured off-cluster. +apiVersion: v1 +kind: Namespace +metadata: + name: unifi + labels: + homelab.keithjackson.dev/app-namespace: 'true' + gateway-access: shared + annotations: + homelab.keithjackson.dev/backup-policy: daily-offcluster diff --git a/clusters/base/apps-vms.yaml b/clusters/base/apps-vms.yaml new file mode 100644 index 0000000..9a8fd6c --- /dev/null +++ b/clusters/base/apps-vms.yaml @@ -0,0 +1,29 @@ +--- +# Flux Kustomization: VM tier (KubeVirt VirtualMachines). Reconciles ./apps/vms. +# +# NON-GATING (wait:false): unlike the app waves, this does NOT health-wait. A VM's first-boot image +# import (CDI, minutes) + in-guest software install mean a VM can be intentionally long-not-Ready, so +# health-gating it would wedge reconciliation for no benefit. VMs have no in-cluster dependents. +# dependsOn platform-config so KubeVirt/CDI + the longhorn-vm StorageClass + CDI StorageProfiles +# exist first. prune:true — this Kustomization solely owns the VM manifests (no adoption concern). +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: apps-vms + namespace: flux-system +spec: + interval: 10m + retryInterval: 2m + timeout: 5m + prune: true + wait: false + dependsOn: + - name: platform-config + sourceRef: + kind: GitRepository + name: flux-system + path: ./apps/vms + postBuild: + substituteFrom: + - kind: ConfigMap + name: cluster-vars diff --git a/clusters/base/kustomization.yaml b/clusters/base/kustomization.yaml index 8d308b4..2898ed1 100644 --- a/clusters/base/kustomization.yaml +++ b/clusters/base/kustomization.yaml @@ -15,4 +15,6 @@ resources: - apps-wave-0.yaml # Flux Kustomization -> ./apps/wave-0 (dependsOn platform-config) - apps-wave-1.yaml # Flux Kustomization -> ./apps/wave-1 (dependsOn apps-wave-0) - apps-wave-2.yaml # Flux Kustomization -> ./apps/wave-2 (dependsOn apps-wave-1) + # VM tier (KubeVirt) — non-gating (wait:false), separate lifecycle from the app waves. + - apps-vms.yaml # Flux Kustomization -> ./apps/vms (dependsOn platform-config) - cluster-vars.yaml # overlay knob surface (base defaults)