Merge fix/kubevirt-vm-boot: real-KVM DataVolume VMs (R-PLAT-03/04 green; useEmulation:false, EFI, DataVolume roots)

conformance/tests/l2-behavioral/vm-default-deny/chainsaw-test.yaml

@@ -12,9 +12,14 @@
 #       own datapath (its traffic egresses via the virt-launcher pod netns, so a
 #       podSelector:{} NetworkPolicy covers it). R-NET-01 for IaaS.
 #
-# METHOD (virtctl-free): cloud-init runs the two probes and writes a PASS/FAIL
+# METHOD (virtctl-free): the VM's ROOT is a CDI DataVolume on Longhorn (a mounted
-# verdict to a raw Longhorn block disk, then powers off (RerunOnFailure => stays
+# PVC — NOT an ephemeral containerDisk, whose boot path is broken on this k3s cluster:
-# down). A busybox reader pod reads the verdict back. The container default-deny
+# KubeVirt v1.9 serves the container-disk binary via a k8s `image` volume gated by the
+# ALPHA `ImageVolume` feature gate, disabled on k3s v1.31, so the apiserver prunes it
+# and the guest dies at OCI init. DataVolume roots bypass that path and are the
+# spec-required disk model — docs/06). cloud-init runs the two probes and writes a
+# PASS/FAIL verdict to a raw Longhorn block disk, then powers off (RerunOnFailure =>
+# stays down). A busybox reader pod reads the verdict back. The container default-deny
 # test proves the datapath fail-closed (default-deny-failclosed, L3); this asserts
 # the SAME baseline is enforced for a VM.
 #
@@ -41,7 +46,7 @@ metadata:
       an undeclared public TCP dial is blocked. Skips cleanly until KubeVirt + longhorn exist.
 spec:
   timeouts:
-    exec: 20m
+    exec: 30m
   steps:
     - name: default-deny-applies-to-the-vm-datapath
       try:
@@ -60,8 +65,42 @@ spec:
 
               CANARY="vm-deny-$(date +%s%N)"
 
+              # -- GIVEN (disk): CDI imports the Fedora root onto a Longhorn PVC. This MUST happen
+              #    BEFORE the default-deny NetworkPolicy below: default-deny (podSelector {}) also
+              #    matches the CDI importer pod and would sever its egress to quay.io. The import is
+              #    a one-time provisioning step; once the PVC holds the disk the VM boots with NO
+              #    external egress, so applying default-deny AFTER the import still proves the VM's
+              #    own datapath is fail-closed (the runbook's app-namespace equivalent is a scoped
+              #    CiliumNetworkPolicy for the importer — see fixtures/vm.yaml). --
+              kubectl -n "$NS" apply -f - <<'EOF'
+              apiVersion: cdi.kubevirt.io/v1beta1
+              kind: DataVolume
+              metadata:
+                name: prober-rootdisk
+              spec:
+                source:
+                  registry:
+                    url: "docker://quay.io/containerdisks/fedora:41"
+                storage:
+                  accessModes:
+                    - ReadWriteOnce
+                  storageClassName: longhorn
+                  resources:
+                    requests:
+                      storage: 6Gi
+              EOF
+              dv_phase=""
+              for i in $(seq 1 120); do
+                dv_phase="$(kubectl -n "$NS" get dv prober-rootdisk -o jsonpath='{.status.phase}' 2>/dev/null || true)"
+                [ "$dv_phase" = "Succeeded" ] && break
+                sleep 10
+              done
+              [ "$dv_phase" = "Succeeded" ] || { echo "FAIL: CDI import did not Succeed (last='$dv_phase')" >&2; exit 1; }
+              echo "CDI import complete: prober-rootdisk Succeeded (Longhorn PVC)"
+
               # -- GIVEN: the default-deny baseline in this namespace (deny all, allow only DNS to
-              #    kube-system) — the exact shape Kyverno generates for app namespaces (docs/03). --
+              #    kube-system) — the exact shape Kyverno generates for app namespaces (docs/03).
+              #    Applied AFTER the import so it governs the running VM's datapath (the point of the test). --
               kubectl -n "$NS" apply -f - <<'EOF'
               apiVersion: networking.k8s.io/v1
               kind: NetworkPolicy
@@ -111,6 +150,15 @@ spec:
                       homelab.keithjackson.dev/egress-class: internet
                   spec:
                     domain:
+                      # The containerdisks/fedora:41 image is UEFI-native (ESP + /boot mounted via
+                      # systemd GPT auto-discovery). KubeVirt defaults to SeaBIOS, under which the guest
+                      # boots to emergency mode (ESP + /boot by-uuid mounts time out -> local-fs.target
+                      # fails) and cloud-init's runcmd/poweroff never runs. Boot with EFI (OVMF) so the
+                      # guest reaches multi-user and the probe executes. secureBoot:false => no SMM needed.
+                      firmware:
+                        bootloader:
+                          efi:
+                            secureBoot: false
                       cpu: { cores: 1 }
                       memory: { guest: 1024Mi }
                       devices:
@@ -126,8 +174,8 @@ spec:
                           memory: 1024Mi
                     volumes:
                       - name: rootdisk
-                        containerDisk:
+                        dataVolume:
-                          image: quay.io/containerdisks/fedora:41
+                          name: prober-rootdisk
                       - name: verdict
                         persistentVolumeClaim:
                           claimName: vm-verdict
@@ -185,5 +233,6 @@ spec:
             content: |
               kubectl -n "$NAMESPACE" delete vm prober --ignore-not-found --wait=false || true
               kubectl -n "$NAMESPACE" delete pod verdict-reader --ignore-not-found --wait=false || true
+              kubectl -n "$NAMESPACE" delete dv prober-rootdisk --ignore-not-found --wait=false || true
               kubectl -n "$NAMESPACE" delete pvc vm-verdict --ignore-not-found --wait=false || true
               echo "cleanup best-effort; chainsaw namespace teardown finalizes"

conformance/tests/l2-behavioral/vm-lifecycle/chainsaw-test.yaml

@@ -14,13 +14,26 @@
 #       vm-default-deny test — both share this VM model.
 #
 # RUNNABLE / SELF-SKIPPING: KubeVirt (Phase 5) may not be installed when the
-# broader suite runs. This test SKIPS CLEANLY (green) until the
+# broader suite runs. This test SKIPS CLEANLY (green) until BOTH the
-# virtualmachineinstances.kubevirt.io CRD exists, then runs the full lifecycle the
+# virtualmachineinstances.kubevirt.io CRD and a `longhorn` StorageClass exist, then
-# moment it does — mirroring the data-survival self-skip on `longhorn`. Everything
+# runs the full lifecycle the moment they do — mirroring the data-survival self-skip.
-# lives in the chainsaw per-test namespace and is torn down with it (hermetic). A
+# Everything lives in the chainsaw per-test namespace and is torn down with it
-# containerDisk root (kubelet-pulled, no CDI import) keeps the lifecycle proof fast
+# (hermetic).
-# and free of image-import timing; the CDI-onto-Longhorn path is exercised by the
+#
-# representative fixtures/vm.yaml via the deploy runbook and by vm-data-survival.
+# DISK MODEL — CDI DataVolume on Longhorn (NOT an ephemeral containerDisk):
+#   KubeVirt v1.9 supplies the container-disk binary to the `volumerootdisk` init
+#   container via a Kubernetes-native `image` volume (VolumeSource.Image) mounted at
+#   /container-disk-binary. That k8s feature is gated by the ALPHA `ImageVolume`
+#   feature gate, which is DISABLED on this k3s v1.31 cluster — so the apiserver
+#   silently PRUNES the image volume + its mount, the binary is never present, and a
+#   containerDisk VM dies at OCI init with:
+#     exec: "/container-disk-binary/usr/bin/container-disk": no such file or directory
+#   A DataVolume-backed root is a mounted Longhorn PVC (CDI imports the same Fedora
+#   image into it out-of-band via its importer pod) and therefore does NOT touch the
+#   container-disk binary / image-volume path at all — it boots on this cluster. It is
+#   also the disk model the SPEC requires for VMs (docs/06, R-STATE-01: Longhorn-backed
+#   disks that survive rebuild), so this mirrors the representative fixtures/vm.yaml.
+#   Cost: a one-time CDI import per run (covered by the raised exec timeout below).
 # =============================================================================
 apiVersion: chainsaw.kyverno.io/v1alpha1
 kind: Test
@@ -39,9 +52,10 @@ metadata:
       honours each transition (Running on start, gone on delete). Skips cleanly
       until the KubeVirt CRDs exist.
 spec:
-  # VM start pulls a containerDisk + boots a guest — well beyond the default 180s exec window.
+  # CDI imports the Fedora image onto a Longhorn PVC, then the VM boots the guest —
+  # import + boot is well beyond the default 180s exec window.
   timeouts:
-    exec: 20m
+    exec: 30m
   steps:
     - name: vm-lifecycle-through-k8s-api
       try:
@@ -50,14 +64,48 @@ spec:
               set -euo pipefail
               NS="$NAMESPACE"
 
-              # -- Gate: skip cleanly (green) until KubeVirt is installed. --
+              # -- Gate: skip cleanly (green) until BOTH KubeVirt and longhorn are present. --
               if ! kubectl get crd virtualmachineinstances.kubevirt.io >/dev/null 2>&1; then
                 echo "SKIP: KubeVirt CRDs absent — vm-lifecycle not applicable on this cluster yet"
                 echo VM_LIFECYCLE_SKIPPED_OK
                 exit 0
               fi
+              if ! kubectl get storageclass longhorn >/dev/null 2>&1; then
+                echo "SKIP: 'longhorn' StorageClass absent — vm-lifecycle not applicable yet"
+                echo VM_LIFECYCLE_SKIPPED_OK
+                exit 0
+              fi
 
-              # -- GIVEN: a stopped VirtualMachine, defined declaratively (containerDisk root). --
+              # -- GIVEN (disk): CDI imports the Fedora image onto a Longhorn PVC (standalone
+              #    DataVolume — outlives VM start/stop/restart so the disk is reused, not re-imported). --
+              kubectl -n "$NS" apply -f - <<'EOF'
+              apiVersion: cdi.kubevirt.io/v1beta1
+              kind: DataVolume
+              metadata:
+                name: lc-rootdisk
+              spec:
+                source:
+                  registry:
+                    url: "docker://quay.io/containerdisks/fedora:41"
+                storage:
+                  accessModes:
+                    - ReadWriteOnce
+                  storageClassName: longhorn
+                  resources:
+                    requests:
+                      storage: 6Gi
+              EOF
+              # Wait for the CDI import to complete before booting (bounded poll; import pulls quay.io).
+              dv_phase=""
+              for i in $(seq 1 120); do
+                dv_phase="$(kubectl -n "$NS" get dv lc-rootdisk -o jsonpath='{.status.phase}' 2>/dev/null || true)"
+                [ "$dv_phase" = "Succeeded" ] && break
+                sleep 10
+              done
+              [ "$dv_phase" = "Succeeded" ] || { echo "FAIL: CDI import did not Succeed (last='$dv_phase')" >&2; exit 1; }
+              echo "CDI import complete: DataVolume lc-rootdisk Succeeded (Longhorn PVC)"
+
+              # -- GIVEN (VM): a stopped VirtualMachine bound to that Longhorn DataVolume disk. --
               kubectl -n "$NS" apply -f - <<'EOF'
               apiVersion: kubevirt.io/v1
               kind: VirtualMachine
@@ -72,6 +120,14 @@ spec:
                       homelab.keithjackson.dev/egress-class: internet
                   spec:
                     domain:
+                      # EFI (OVMF) boot: the containerdisks/fedora:41 image is UEFI-native; under the
+                      # KubeVirt SeaBIOS default it boots to emergency mode (ESP + /boot mounts fail).
+                      # Not strictly required for this test (it only asserts VMI phase transitions, not
+                      # in-guest state) but kept consistent with the probe tests + fixtures/vm.yaml.
+                      firmware:
+                        bootloader:
+                          efi:
+                            secureBoot: false
                       cpu: { cores: 1 }
                       memory: { guest: 1024Mi }
                       devices:
@@ -89,8 +145,8 @@ spec:
                         pod: {}
                     volumes:
                       - name: rootdisk
-                        containerDisk:
+                        dataVolume:
-                          image: quay.io/containerdisks/fedora:41
+                          name: lc-rootdisk
               EOF
               # No VMI should exist while running=false.
               if kubectl -n "$NS" get vmi lc >/dev/null 2>&1; then
@@ -147,4 +203,5 @@ spec:
         - script:
             content: |
               kubectl -n "$NAMESPACE" delete vm lc --ignore-not-found --wait=false || true
+              kubectl -n "$NAMESPACE" delete dv lc-rootdisk --ignore-not-found --wait=false || true
               echo "cleanup best-effort; chainsaw namespace teardown finalizes"

conformance/tests/l2-behavioral/vm-lifecycle/fixtures/vm.yaml

@@ -47,6 +47,14 @@ spec:
         homelab.keithjackson.dev/egress-class: internet
     spec:
       domain:
+        # EFI (OVMF) boot — the containerdisks/fedora:41 image is UEFI-native (separate ESP + /boot
+        # mounted via systemd GPT auto-discovery). Under KubeVirt's SeaBIOS default the guest drops to
+        # emergency mode (ESP + /boot by-uuid mounts time out -> local-fs.target fails), so cloud-init
+        # (dnf-automatic setup below) never runs. secureBoot:false avoids the SMM requirement.
+        firmware:
+          bootloader:
+            efi:
+              secureBoot: false
         cpu:
           cores: 1
         memory:

conformance/tests/l3-fault-injection/vm-data-survival/chainsaw-test.yaml

@@ -10,9 +10,15 @@
 #       exactly the container proof (data-survival), now for a KubeVirt VM
 #       (R-STATE-01 covers PaaS + IaaS).
 #
-# METHOD (virtctl-free, filesystem-agnostic, deterministic): a containerDisk-root
+# METHOD (virtctl-free, filesystem-agnostic, deterministic): a VM whose ROOT is a
-# VM (ephemeral) with a Longhorn BLOCK data disk. cloud-init writes a unique canary
+# CDI DataVolume on Longhorn (a mounted PVC — NOT an ephemeral containerDisk; the
-# to the raw block device and powers the guest off; runStrategy RerunOnFailure means
+# containerDisk boot path is broken on this k3s cluster because KubeVirt v1.9 serves
+# the container-disk binary through a k8s `image` volume gated by the ALPHA
+# `ImageVolume` feature gate, which k3s v1.31 has disabled → the apiserver prunes the
+# volume and the guest dies at OCI init. DataVolume disks bypass that path and are the
+# spec-required disk model anyway — docs/06/R-STATE-01). The VM also has a SECOND
+# Longhorn BLOCK data disk; cloud-init writes a unique canary to that raw block device
+# and powers the guest off; runStrategy RerunOnFailure means
 # a clean shutdown STAYS down (VMI -> Succeeded), so the write is known-complete and
 # the disk is released. We then DELETE the VM (keeping the PVC) and bind a throwaway
 # busybox reader pod to the SAME Longhorn PVC as a raw block device — if the bytes
@@ -38,9 +44,10 @@ metadata:
       Destroy a VM entirely, keep its Longhorn PVC, and a fresh workload re-bound to
       that PVC still reads the VM-written data. Skips cleanly until KubeVirt + longhorn exist.
 spec:
-  # Boot a guest + run cloud-init + power off, then reattach a Longhorn RWO volume — far beyond 180s.
+  # CDI import onto Longhorn + boot a guest + run cloud-init + power off, then reattach a
+  # Longhorn RWO volume — far beyond 180s.
   timeouts:
-    exec: 20m
+    exec: 30m
   steps:
     - name: vm-data-survives-destroy-and-rebuild
       try:
@@ -77,7 +84,35 @@ spec:
                     storage: 1Gi
               EOF
 
-              # -- The VM: containerDisk root (ephemeral) + the Longhorn block disk (/dev/vdb) + a
+              # -- The VM's ROOT disk: CDI imports Fedora onto a Longhorn PVC (bypasses the broken
+              #    containerDisk/image-volume path; standalone DataVolume). Wait for the import. --
+              kubectl -n "$NS" apply -f - <<'EOF'
+              apiVersion: cdi.kubevirt.io/v1beta1
+              kind: DataVolume
+              metadata:
+                name: survivor-rootdisk
+              spec:
+                source:
+                  registry:
+                    url: "docker://quay.io/containerdisks/fedora:41"
+                storage:
+                  accessModes:
+                    - ReadWriteOnce
+                  storageClassName: longhorn
+                  resources:
+                    requests:
+                      storage: 6Gi
+              EOF
+              dv_phase=""
+              for i in $(seq 1 120); do
+                dv_phase="$(kubectl -n "$NS" get dv survivor-rootdisk -o jsonpath='{.status.phase}' 2>/dev/null || true)"
+                [ "$dv_phase" = "Succeeded" ] && break
+                sleep 10
+              done
+              [ "$dv_phase" = "Succeeded" ] || { echo "FAIL: CDI import did not Succeed (last='$dv_phase')" >&2; exit 1; }
+              echo "CDI import complete: survivor-rootdisk Succeeded (Longhorn PVC)"
+
+              # -- The VM: DataVolume root (Longhorn) + the Longhorn block disk (/dev/vdb) + a
               #    cloud-init disk. cloud-init writes the canary to the raw device and powers off.
               #    RerunOnFailure => a clean poweroff STAYS Succeeded (not restarted), so the write is
               #    known-complete and the disk released. Only $CANARY expands (no other '$' in the body).
@@ -95,6 +130,14 @@ spec:
                       homelab.keithjackson.dev/egress-class: internet
                   spec:
                     domain:
+                      # The containerdisks/fedora:41 image is UEFI-native (ESP + /boot mounted via
+                      # systemd GPT auto-discovery). KubeVirt defaults to SeaBIOS, under which the guest
+                      # boots to emergency mode (ESP + /boot by-uuid mounts time out) and cloud-init's
+                      # runcmd/poweroff never runs. Boot with EFI (OVMF). secureBoot:false => no SMM needed.
+                      firmware:
+                        bootloader:
+                          efi:
+                            secureBoot: false
                       cpu: { cores: 1 }
                       memory: { guest: 1024Mi }
                       devices:
@@ -110,8 +153,8 @@ spec:
                           memory: 1024Mi
                     volumes:
                       - name: rootdisk
-                        containerDisk:
+                        dataVolume:
-                          image: quay.io/containerdisks/fedora:41
+                          name: survivor-rootdisk
                       - name: datadisk
                         persistentVolumeClaim:
                           claimName: vm-survivor
@@ -168,5 +211,6 @@ spec:
             content: |
               kubectl -n "$NAMESPACE" delete vm survivor --ignore-not-found --wait=false || true
               kubectl -n "$NAMESPACE" delete pod reader --ignore-not-found --wait=false || true
+              kubectl -n "$NAMESPACE" delete dv survivor-rootdisk --ignore-not-found --wait=false || true
               kubectl -n "$NAMESPACE" delete pvc vm-survivor --ignore-not-found --wait=false || true
               echo "cleanup best-effort; chainsaw namespace teardown finalizes"

platform/kubevirt/kubevirt-cr.yaml

@@ -3,20 +3,25 @@
 # operator (platform/kubevirt/kustomization.yaml), so this is applied by the `platform-config` Flux
 # Kustomization (dependsOn: platform) — see platform-config/kustomization.yaml + F-MON1.
 #
-# VIRT MODE (prefer real KVM; emulation is only a fallback):
+# VIRT MODE — use real KVM (nested) on this cluster:
-#   developerConfiguration.useEmulation does NOT force emulation — KubeVirt ALWAYS prefers hardware
+#   developerConfiguration.useEmulation was previously `true` on the belief that it "does NOT force
-#   (KVM) when /dev/kvm is present, and this flag only decides the behaviour when it is ABSENT:
+#   emulation, KubeVirt still prefers KVM when /dev/kvm is present." That belief is EMPIRICALLY WRONG
-#     * true  (set here) -> if a node lacks /dev/kvm, fall back to slow software emulation (TCG) so the
+#   on KubeVirt v1.9 (WS-VM verified live): when useEmulation=true, virt-controller does NOT add the
-#                           IaaS path stays PROVABLE even where nested virt is not exposed in-guest. On
+#   `devices.kubevirt.io/kvm` resource to the virt-launcher pod, so /dev/kvm is never mounted into the
-#                           any node that HAS /dev/kvm (prod mini-PCs; dev nodes provisioned
+#   compute container and qemu ALWAYS falls back to software emulation (TCG) — even on nodes that
-#                           cpu_mode=host-passthrough per infra/tofu) VMs still run on real KVM.
+#   advertise `devices.kubevirt.io/kvm=1k` and are fully CPU-labelled (`hypervisor=true`). Proof: on
-#     * false            -> a node without /dev/kvm cannot schedule VMs at all (fail loud).
+#   the dev cluster with useEmulation=true, `ls /dev/kvm` inside a virt-launcher compute container
-#   `true` is a safe universal default (prefer real, emulate only as a last resort) and needs no
+#   returned "No such file or directory" and the qemu accelerator was `tcg`. TCG guests are ~10x slow,
-#   per-environment override. RUNBOOK STEP 1 is to confirm `/dev/kvm` on a node (`virt-host-validate` /
+#   never finish the in-guest conformance probes in time, and saturate node CPUs (a node went NotReady
-#   `ls -l /dev/kvm`); if it is present, dev runs real KVM exactly like prod. If the operator later
+#   and its VMs were evicted -> VMI Failed).
-#   wants prod to HARD-REQUIRE KVM (no silent emulation), flip this to false via a prod-scoped
+#     * false (set here) -> virt-controller REQUIRES + requests the kvm device, /dev/kvm is mounted,
-#   cluster-var — deferred here only to avoid the pre-existing WS-TEST coverage gate that any clusters/
+#                           and VMs run on real (nested) KVM. All three dev nodes have /dev/kvm
-#   overlay edit trips (see the WS-VM report); it is a one-line follow-up, not a design constraint.
+#                           (advertised =1k, vmx/svm present), and prod mini-PCs have native KVM, so
+#                           this is safe cluster-wide. A node genuinely lacking /dev/kvm now fails LOUD
+#                           (correct — we want to know, not silently run 10x-slow emulated VMs).
+#     * true             -> the broken-on-1.9 behaviour above (silent TCG everywhere). Do NOT use while
+#                           the nodes have KVM.
+#   RUNBOOK STEP 1 remains: confirm `/dev/kvm` on a node (`virt-host-validate` / `ls -l /dev/kvm`).
 apiVersion: kubevirt.io/v1
 kind: KubeVirt
 metadata:
@@ -29,8 +34,9 @@ spec:
   # (Live migration across the 3 converged nodes is a nice-to-have per docs/06, not required.)
   configuration:
     developerConfiguration:
-      # Prefer real (nested) KVM; fall back to emulation ONLY when /dev/kvm is absent (see header).
+      # Use real (nested) KVM. `false` makes virt-controller REQUEST the kvm device so /dev/kvm is
-      useEmulation: true
+      # mounted into virt-launcher (KVM acceleration). `true` is broken on v1.9 -> silent TCG. (See header.)
+      useEmulation: false
     # Persistent EFI/TPM VM state (if a VM enables it) lands on the replicated Longhorn tier (ADR-0003),
     # inheriting the same durability as every other stateful volume.
     vmStateStorageClass: longhorn