diff --git a/conformance/tests/l2-behavioral/vm-default-deny/chainsaw-test.yaml b/conformance/tests/l2-behavioral/vm-default-deny/chainsaw-test.yaml index 1586c0a..0e72074 100644 --- a/conformance/tests/l2-behavioral/vm-default-deny/chainsaw-test.yaml +++ b/conformance/tests/l2-behavioral/vm-default-deny/chainsaw-test.yaml @@ -12,9 +12,14 @@ # own datapath (its traffic egresses via the virt-launcher pod netns, so a # podSelector:{} NetworkPolicy covers it). R-NET-01 for IaaS. # -# METHOD (virtctl-free): cloud-init runs the two probes and writes a PASS/FAIL -# verdict to a raw Longhorn block disk, then powers off (RerunOnFailure => stays -# down). A busybox reader pod reads the verdict back. The container default-deny +# METHOD (virtctl-free): the VM's ROOT is a CDI DataVolume on Longhorn (a mounted +# PVC — NOT an ephemeral containerDisk, whose boot path is broken on this k3s cluster: +# KubeVirt v1.9 serves the container-disk binary via a k8s `image` volume gated by the +# ALPHA `ImageVolume` feature gate, disabled on k3s v1.31, so the apiserver prunes it +# and the guest dies at OCI init. DataVolume roots bypass that path and are the +# spec-required disk model — docs/06). cloud-init runs the two probes and writes a +# PASS/FAIL verdict to a raw Longhorn block disk, then powers off (RerunOnFailure => +# stays down). A busybox reader pod reads the verdict back. The container default-deny # test proves the datapath fail-closed (default-deny-failclosed, L3); this asserts # the SAME baseline is enforced for a VM. # @@ -41,7 +46,7 @@ metadata: an undeclared public TCP dial is blocked. Skips cleanly until KubeVirt + longhorn exist. spec: timeouts: - exec: 20m + exec: 30m steps: - name: default-deny-applies-to-the-vm-datapath try: @@ -60,8 +65,42 @@ spec: CANARY="vm-deny-$(date +%s%N)" + # -- GIVEN (disk): CDI imports the Fedora root onto a Longhorn PVC. This MUST happen + # BEFORE the default-deny NetworkPolicy below: default-deny (podSelector {}) also + # matches the CDI importer pod and would sever its egress to quay.io. The import is + # a one-time provisioning step; once the PVC holds the disk the VM boots with NO + # external egress, so applying default-deny AFTER the import still proves the VM's + # own datapath is fail-closed (the runbook's app-namespace equivalent is a scoped + # CiliumNetworkPolicy for the importer — see fixtures/vm.yaml). -- + kubectl -n "$NS" apply -f - <<'EOF' + apiVersion: cdi.kubevirt.io/v1beta1 + kind: DataVolume + metadata: + name: prober-rootdisk + spec: + source: + registry: + url: "docker://quay.io/containerdisks/fedora:41" + storage: + accessModes: + - ReadWriteOnce + storageClassName: longhorn + resources: + requests: + storage: 6Gi + EOF + dv_phase="" + for i in $(seq 1 120); do + dv_phase="$(kubectl -n "$NS" get dv prober-rootdisk -o jsonpath='{.status.phase}' 2>/dev/null || true)" + [ "$dv_phase" = "Succeeded" ] && break + sleep 10 + done + [ "$dv_phase" = "Succeeded" ] || { echo "FAIL: CDI import did not Succeed (last='$dv_phase')" >&2; exit 1; } + echo "CDI import complete: prober-rootdisk Succeeded (Longhorn PVC)" + # -- GIVEN: the default-deny baseline in this namespace (deny all, allow only DNS to - # kube-system) — the exact shape Kyverno generates for app namespaces (docs/03). -- + # kube-system) — the exact shape Kyverno generates for app namespaces (docs/03). + # Applied AFTER the import so it governs the running VM's datapath (the point of the test). -- kubectl -n "$NS" apply -f - <<'EOF' apiVersion: networking.k8s.io/v1 kind: NetworkPolicy @@ -111,6 +150,15 @@ spec: homelab.keithjackson.dev/egress-class: internet spec: domain: + # The containerdisks/fedora:41 image is UEFI-native (ESP + /boot mounted via + # systemd GPT auto-discovery). KubeVirt defaults to SeaBIOS, under which the guest + # boots to emergency mode (ESP + /boot by-uuid mounts time out -> local-fs.target + # fails) and cloud-init's runcmd/poweroff never runs. Boot with EFI (OVMF) so the + # guest reaches multi-user and the probe executes. secureBoot:false => no SMM needed. + firmware: + bootloader: + efi: + secureBoot: false cpu: { cores: 1 } memory: { guest: 1024Mi } devices: @@ -126,8 +174,8 @@ spec: memory: 1024Mi volumes: - name: rootdisk - containerDisk: - image: quay.io/containerdisks/fedora:41 + dataVolume: + name: prober-rootdisk - name: verdict persistentVolumeClaim: claimName: vm-verdict @@ -185,5 +233,6 @@ spec: content: | kubectl -n "$NAMESPACE" delete vm prober --ignore-not-found --wait=false || true kubectl -n "$NAMESPACE" delete pod verdict-reader --ignore-not-found --wait=false || true + kubectl -n "$NAMESPACE" delete dv prober-rootdisk --ignore-not-found --wait=false || true kubectl -n "$NAMESPACE" delete pvc vm-verdict --ignore-not-found --wait=false || true echo "cleanup best-effort; chainsaw namespace teardown finalizes" diff --git a/conformance/tests/l2-behavioral/vm-lifecycle/chainsaw-test.yaml b/conformance/tests/l2-behavioral/vm-lifecycle/chainsaw-test.yaml index e2c0fdc..3042395 100644 --- a/conformance/tests/l2-behavioral/vm-lifecycle/chainsaw-test.yaml +++ b/conformance/tests/l2-behavioral/vm-lifecycle/chainsaw-test.yaml @@ -14,13 +14,26 @@ # vm-default-deny test — both share this VM model. # # RUNNABLE / SELF-SKIPPING: KubeVirt (Phase 5) may not be installed when the -# broader suite runs. This test SKIPS CLEANLY (green) until the -# virtualmachineinstances.kubevirt.io CRD exists, then runs the full lifecycle the -# moment it does — mirroring the data-survival self-skip on `longhorn`. Everything -# lives in the chainsaw per-test namespace and is torn down with it (hermetic). A -# containerDisk root (kubelet-pulled, no CDI import) keeps the lifecycle proof fast -# and free of image-import timing; the CDI-onto-Longhorn path is exercised by the -# representative fixtures/vm.yaml via the deploy runbook and by vm-data-survival. +# broader suite runs. This test SKIPS CLEANLY (green) until BOTH the +# virtualmachineinstances.kubevirt.io CRD and a `longhorn` StorageClass exist, then +# runs the full lifecycle the moment they do — mirroring the data-survival self-skip. +# Everything lives in the chainsaw per-test namespace and is torn down with it +# (hermetic). +# +# DISK MODEL — CDI DataVolume on Longhorn (NOT an ephemeral containerDisk): +# KubeVirt v1.9 supplies the container-disk binary to the `volumerootdisk` init +# container via a Kubernetes-native `image` volume (VolumeSource.Image) mounted at +# /container-disk-binary. That k8s feature is gated by the ALPHA `ImageVolume` +# feature gate, which is DISABLED on this k3s v1.31 cluster — so the apiserver +# silently PRUNES the image volume + its mount, the binary is never present, and a +# containerDisk VM dies at OCI init with: +# exec: "/container-disk-binary/usr/bin/container-disk": no such file or directory +# A DataVolume-backed root is a mounted Longhorn PVC (CDI imports the same Fedora +# image into it out-of-band via its importer pod) and therefore does NOT touch the +# container-disk binary / image-volume path at all — it boots on this cluster. It is +# also the disk model the SPEC requires for VMs (docs/06, R-STATE-01: Longhorn-backed +# disks that survive rebuild), so this mirrors the representative fixtures/vm.yaml. +# Cost: a one-time CDI import per run (covered by the raised exec timeout below). # ============================================================================= apiVersion: chainsaw.kyverno.io/v1alpha1 kind: Test @@ -39,9 +52,10 @@ metadata: honours each transition (Running on start, gone on delete). Skips cleanly until the KubeVirt CRDs exist. spec: - # VM start pulls a containerDisk + boots a guest — well beyond the default 180s exec window. + # CDI imports the Fedora image onto a Longhorn PVC, then the VM boots the guest — + # import + boot is well beyond the default 180s exec window. timeouts: - exec: 20m + exec: 30m steps: - name: vm-lifecycle-through-k8s-api try: @@ -50,14 +64,48 @@ spec: set -euo pipefail NS="$NAMESPACE" - # -- Gate: skip cleanly (green) until KubeVirt is installed. -- + # -- Gate: skip cleanly (green) until BOTH KubeVirt and longhorn are present. -- if ! kubectl get crd virtualmachineinstances.kubevirt.io >/dev/null 2>&1; then echo "SKIP: KubeVirt CRDs absent — vm-lifecycle not applicable on this cluster yet" echo VM_LIFECYCLE_SKIPPED_OK exit 0 fi + if ! kubectl get storageclass longhorn >/dev/null 2>&1; then + echo "SKIP: 'longhorn' StorageClass absent — vm-lifecycle not applicable yet" + echo VM_LIFECYCLE_SKIPPED_OK + exit 0 + fi - # -- GIVEN: a stopped VirtualMachine, defined declaratively (containerDisk root). -- + # -- GIVEN (disk): CDI imports the Fedora image onto a Longhorn PVC (standalone + # DataVolume — outlives VM start/stop/restart so the disk is reused, not re-imported). -- + kubectl -n "$NS" apply -f - <<'EOF' + apiVersion: cdi.kubevirt.io/v1beta1 + kind: DataVolume + metadata: + name: lc-rootdisk + spec: + source: + registry: + url: "docker://quay.io/containerdisks/fedora:41" + storage: + accessModes: + - ReadWriteOnce + storageClassName: longhorn + resources: + requests: + storage: 6Gi + EOF + # Wait for the CDI import to complete before booting (bounded poll; import pulls quay.io). + dv_phase="" + for i in $(seq 1 120); do + dv_phase="$(kubectl -n "$NS" get dv lc-rootdisk -o jsonpath='{.status.phase}' 2>/dev/null || true)" + [ "$dv_phase" = "Succeeded" ] && break + sleep 10 + done + [ "$dv_phase" = "Succeeded" ] || { echo "FAIL: CDI import did not Succeed (last='$dv_phase')" >&2; exit 1; } + echo "CDI import complete: DataVolume lc-rootdisk Succeeded (Longhorn PVC)" + + # -- GIVEN (VM): a stopped VirtualMachine bound to that Longhorn DataVolume disk. -- kubectl -n "$NS" apply -f - <<'EOF' apiVersion: kubevirt.io/v1 kind: VirtualMachine @@ -72,6 +120,14 @@ spec: homelab.keithjackson.dev/egress-class: internet spec: domain: + # EFI (OVMF) boot: the containerdisks/fedora:41 image is UEFI-native; under the + # KubeVirt SeaBIOS default it boots to emergency mode (ESP + /boot mounts fail). + # Not strictly required for this test (it only asserts VMI phase transitions, not + # in-guest state) but kept consistent with the probe tests + fixtures/vm.yaml. + firmware: + bootloader: + efi: + secureBoot: false cpu: { cores: 1 } memory: { guest: 1024Mi } devices: @@ -89,8 +145,8 @@ spec: pod: {} volumes: - name: rootdisk - containerDisk: - image: quay.io/containerdisks/fedora:41 + dataVolume: + name: lc-rootdisk EOF # No VMI should exist while running=false. if kubectl -n "$NS" get vmi lc >/dev/null 2>&1; then @@ -147,4 +203,5 @@ spec: - script: content: | kubectl -n "$NAMESPACE" delete vm lc --ignore-not-found --wait=false || true + kubectl -n "$NAMESPACE" delete dv lc-rootdisk --ignore-not-found --wait=false || true echo "cleanup best-effort; chainsaw namespace teardown finalizes" diff --git a/conformance/tests/l2-behavioral/vm-lifecycle/fixtures/vm.yaml b/conformance/tests/l2-behavioral/vm-lifecycle/fixtures/vm.yaml index 1491946..f6dd57b 100644 --- a/conformance/tests/l2-behavioral/vm-lifecycle/fixtures/vm.yaml +++ b/conformance/tests/l2-behavioral/vm-lifecycle/fixtures/vm.yaml @@ -47,6 +47,14 @@ spec: homelab.keithjackson.dev/egress-class: internet spec: domain: + # EFI (OVMF) boot — the containerdisks/fedora:41 image is UEFI-native (separate ESP + /boot + # mounted via systemd GPT auto-discovery). Under KubeVirt's SeaBIOS default the guest drops to + # emergency mode (ESP + /boot by-uuid mounts time out -> local-fs.target fails), so cloud-init + # (dnf-automatic setup below) never runs. secureBoot:false avoids the SMM requirement. + firmware: + bootloader: + efi: + secureBoot: false cpu: cores: 1 memory: diff --git a/conformance/tests/l3-fault-injection/vm-data-survival/chainsaw-test.yaml b/conformance/tests/l3-fault-injection/vm-data-survival/chainsaw-test.yaml index 249f270..853f0cd 100644 --- a/conformance/tests/l3-fault-injection/vm-data-survival/chainsaw-test.yaml +++ b/conformance/tests/l3-fault-injection/vm-data-survival/chainsaw-test.yaml @@ -10,9 +10,15 @@ # exactly the container proof (data-survival), now for a KubeVirt VM # (R-STATE-01 covers PaaS + IaaS). # -# METHOD (virtctl-free, filesystem-agnostic, deterministic): a containerDisk-root -# VM (ephemeral) with a Longhorn BLOCK data disk. cloud-init writes a unique canary -# to the raw block device and powers the guest off; runStrategy RerunOnFailure means +# METHOD (virtctl-free, filesystem-agnostic, deterministic): a VM whose ROOT is a +# CDI DataVolume on Longhorn (a mounted PVC — NOT an ephemeral containerDisk; the +# containerDisk boot path is broken on this k3s cluster because KubeVirt v1.9 serves +# the container-disk binary through a k8s `image` volume gated by the ALPHA +# `ImageVolume` feature gate, which k3s v1.31 has disabled → the apiserver prunes the +# volume and the guest dies at OCI init. DataVolume disks bypass that path and are the +# spec-required disk model anyway — docs/06/R-STATE-01). The VM also has a SECOND +# Longhorn BLOCK data disk; cloud-init writes a unique canary to that raw block device +# and powers the guest off; runStrategy RerunOnFailure means # a clean shutdown STAYS down (VMI -> Succeeded), so the write is known-complete and # the disk is released. We then DELETE the VM (keeping the PVC) and bind a throwaway # busybox reader pod to the SAME Longhorn PVC as a raw block device — if the bytes @@ -38,9 +44,10 @@ metadata: Destroy a VM entirely, keep its Longhorn PVC, and a fresh workload re-bound to that PVC still reads the VM-written data. Skips cleanly until KubeVirt + longhorn exist. spec: - # Boot a guest + run cloud-init + power off, then reattach a Longhorn RWO volume — far beyond 180s. + # CDI import onto Longhorn + boot a guest + run cloud-init + power off, then reattach a + # Longhorn RWO volume — far beyond 180s. timeouts: - exec: 20m + exec: 30m steps: - name: vm-data-survives-destroy-and-rebuild try: @@ -77,7 +84,35 @@ spec: storage: 1Gi EOF - # -- The VM: containerDisk root (ephemeral) + the Longhorn block disk (/dev/vdb) + a + # -- The VM's ROOT disk: CDI imports Fedora onto a Longhorn PVC (bypasses the broken + # containerDisk/image-volume path; standalone DataVolume). Wait for the import. -- + kubectl -n "$NS" apply -f - <<'EOF' + apiVersion: cdi.kubevirt.io/v1beta1 + kind: DataVolume + metadata: + name: survivor-rootdisk + spec: + source: + registry: + url: "docker://quay.io/containerdisks/fedora:41" + storage: + accessModes: + - ReadWriteOnce + storageClassName: longhorn + resources: + requests: + storage: 6Gi + EOF + dv_phase="" + for i in $(seq 1 120); do + dv_phase="$(kubectl -n "$NS" get dv survivor-rootdisk -o jsonpath='{.status.phase}' 2>/dev/null || true)" + [ "$dv_phase" = "Succeeded" ] && break + sleep 10 + done + [ "$dv_phase" = "Succeeded" ] || { echo "FAIL: CDI import did not Succeed (last='$dv_phase')" >&2; exit 1; } + echo "CDI import complete: survivor-rootdisk Succeeded (Longhorn PVC)" + + # -- The VM: DataVolume root (Longhorn) + the Longhorn block disk (/dev/vdb) + a # cloud-init disk. cloud-init writes the canary to the raw device and powers off. # RerunOnFailure => a clean poweroff STAYS Succeeded (not restarted), so the write is # known-complete and the disk released. Only $CANARY expands (no other '$' in the body). @@ -95,6 +130,14 @@ spec: homelab.keithjackson.dev/egress-class: internet spec: domain: + # The containerdisks/fedora:41 image is UEFI-native (ESP + /boot mounted via + # systemd GPT auto-discovery). KubeVirt defaults to SeaBIOS, under which the guest + # boots to emergency mode (ESP + /boot by-uuid mounts time out) and cloud-init's + # runcmd/poweroff never runs. Boot with EFI (OVMF). secureBoot:false => no SMM needed. + firmware: + bootloader: + efi: + secureBoot: false cpu: { cores: 1 } memory: { guest: 1024Mi } devices: @@ -110,8 +153,8 @@ spec: memory: 1024Mi volumes: - name: rootdisk - containerDisk: - image: quay.io/containerdisks/fedora:41 + dataVolume: + name: survivor-rootdisk - name: datadisk persistentVolumeClaim: claimName: vm-survivor @@ -168,5 +211,6 @@ spec: content: | kubectl -n "$NAMESPACE" delete vm survivor --ignore-not-found --wait=false || true kubectl -n "$NAMESPACE" delete pod reader --ignore-not-found --wait=false || true + kubectl -n "$NAMESPACE" delete dv survivor-rootdisk --ignore-not-found --wait=false || true kubectl -n "$NAMESPACE" delete pvc vm-survivor --ignore-not-found --wait=false || true echo "cleanup best-effort; chainsaw namespace teardown finalizes" diff --git a/platform/kubevirt/kubevirt-cr.yaml b/platform/kubevirt/kubevirt-cr.yaml index e5ca3bd..23fb3be 100644 --- a/platform/kubevirt/kubevirt-cr.yaml +++ b/platform/kubevirt/kubevirt-cr.yaml @@ -3,20 +3,25 @@ # operator (platform/kubevirt/kustomization.yaml), so this is applied by the `platform-config` Flux # Kustomization (dependsOn: platform) — see platform-config/kustomization.yaml + F-MON1. # -# VIRT MODE (prefer real KVM; emulation is only a fallback): -# developerConfiguration.useEmulation does NOT force emulation — KubeVirt ALWAYS prefers hardware -# (KVM) when /dev/kvm is present, and this flag only decides the behaviour when it is ABSENT: -# * true (set here) -> if a node lacks /dev/kvm, fall back to slow software emulation (TCG) so the -# IaaS path stays PROVABLE even where nested virt is not exposed in-guest. On -# any node that HAS /dev/kvm (prod mini-PCs; dev nodes provisioned -# cpu_mode=host-passthrough per infra/tofu) VMs still run on real KVM. -# * false -> a node without /dev/kvm cannot schedule VMs at all (fail loud). -# `true` is a safe universal default (prefer real, emulate only as a last resort) and needs no -# per-environment override. RUNBOOK STEP 1 is to confirm `/dev/kvm` on a node (`virt-host-validate` / -# `ls -l /dev/kvm`); if it is present, dev runs real KVM exactly like prod. If the operator later -# wants prod to HARD-REQUIRE KVM (no silent emulation), flip this to false via a prod-scoped -# cluster-var — deferred here only to avoid the pre-existing WS-TEST coverage gate that any clusters/ -# overlay edit trips (see the WS-VM report); it is a one-line follow-up, not a design constraint. +# VIRT MODE — use real KVM (nested) on this cluster: +# developerConfiguration.useEmulation was previously `true` on the belief that it "does NOT force +# emulation, KubeVirt still prefers KVM when /dev/kvm is present." That belief is EMPIRICALLY WRONG +# on KubeVirt v1.9 (WS-VM verified live): when useEmulation=true, virt-controller does NOT add the +# `devices.kubevirt.io/kvm` resource to the virt-launcher pod, so /dev/kvm is never mounted into the +# compute container and qemu ALWAYS falls back to software emulation (TCG) — even on nodes that +# advertise `devices.kubevirt.io/kvm=1k` and are fully CPU-labelled (`hypervisor=true`). Proof: on +# the dev cluster with useEmulation=true, `ls /dev/kvm` inside a virt-launcher compute container +# returned "No such file or directory" and the qemu accelerator was `tcg`. TCG guests are ~10x slow, +# never finish the in-guest conformance probes in time, and saturate node CPUs (a node went NotReady +# and its VMs were evicted -> VMI Failed). +# * false (set here) -> virt-controller REQUIRES + requests the kvm device, /dev/kvm is mounted, +# and VMs run on real (nested) KVM. All three dev nodes have /dev/kvm +# (advertised =1k, vmx/svm present), and prod mini-PCs have native KVM, so +# this is safe cluster-wide. A node genuinely lacking /dev/kvm now fails LOUD +# (correct — we want to know, not silently run 10x-slow emulated VMs). +# * true -> the broken-on-1.9 behaviour above (silent TCG everywhere). Do NOT use while +# the nodes have KVM. +# RUNBOOK STEP 1 remains: confirm `/dev/kvm` on a node (`virt-host-validate` / `ls -l /dev/kvm`). apiVersion: kubevirt.io/v1 kind: KubeVirt metadata: @@ -29,8 +34,9 @@ spec: # (Live migration across the 3 converged nodes is a nice-to-have per docs/06, not required.) configuration: developerConfiguration: - # Prefer real (nested) KVM; fall back to emulation ONLY when /dev/kvm is absent (see header). - useEmulation: true + # Use real (nested) KVM. `false` makes virt-controller REQUEST the kvm device so /dev/kvm is + # mounted into virt-launcher (KVM acceleration). `true` is broken on v1.9 -> silent TCG. (See header.) + useEmulation: false # Persistent EFI/TPM VM state (if a VM enables it) lands on the replicated Longhorn tier (ADR-0003), # inheriting the same durability as every other stateful volume. vmStateStorageClass: longhorn