Merge fix/kubevirt-vm-boot: real-KVM DataVolume VMs (R-PLAT-03/04 green; useEmulation:false, EFI, DataVolume roots)
This commit is contained in:
5 files changed
+208
-44
No files matched your search
@@ -12,9 +12,14 @@
|
||||
# own datapath (its traffic egresses via the virt-launcher pod netns, so a
|
||||
# podSelector:{} NetworkPolicy covers it). R-NET-01 for IaaS.
|
||||
#
|
||||
# METHOD (virtctl-free): cloud-init runs the two probes and writes a PASS/FAIL
|
||||
# verdict to a raw Longhorn block disk, then powers off (RerunOnFailure => stays
|
||||
# down). A busybox reader pod reads the verdict back. The container default-deny
|
||||
# METHOD (virtctl-free): the VM's ROOT is a CDI DataVolume on Longhorn (a mounted
|
||||
# PVC — NOT an ephemeral containerDisk, whose boot path is broken on this k3s cluster:
|
||||
# KubeVirt v1.9 serves the container-disk binary via a k8s `image` volume gated by the
|
||||
# ALPHA `ImageVolume` feature gate, disabled on k3s v1.31, so the apiserver prunes it
|
||||
# and the guest dies at OCI init. DataVolume roots bypass that path and are the
|
||||
# spec-required disk model — docs/06). cloud-init runs the two probes and writes a
|
||||
# PASS/FAIL verdict to a raw Longhorn block disk, then powers off (RerunOnFailure =>
|
||||
# stays down). A busybox reader pod reads the verdict back. The container default-deny
|
||||
# test proves the datapath fail-closed (default-deny-failclosed, L3); this asserts
|
||||
# the SAME baseline is enforced for a VM.
|
||||
#
|
||||
@@ -41,7 +46,7 @@ metadata:
|
||||
an undeclared public TCP dial is blocked. Skips cleanly until KubeVirt + longhorn exist.
|
||||
spec:
|
||||
timeouts:
|
||||
exec: 20m
|
||||
exec: 30m
|
||||
steps:
|
||||
- name: default-deny-applies-to-the-vm-datapath
|
||||
try:
|
||||
@@ -60,8 +65,42 @@ spec:
|
||||
|
||||
CANARY="vm-deny-$(date +%s%N)"
|
||||
|
||||
# -- GIVEN (disk): CDI imports the Fedora root onto a Longhorn PVC. This MUST happen
|
||||
# BEFORE the default-deny NetworkPolicy below: default-deny (podSelector {}) also
|
||||
# matches the CDI importer pod and would sever its egress to quay.io. The import is
|
||||
# a one-time provisioning step; once the PVC holds the disk the VM boots with NO
|
||||
# external egress, so applying default-deny AFTER the import still proves the VM's
|
||||
# own datapath is fail-closed (the runbook's app-namespace equivalent is a scoped
|
||||
# CiliumNetworkPolicy for the importer — see fixtures/vm.yaml). --
|
||||
kubectl -n "$NS" apply -f - <<'EOF'
|
||||
apiVersion: cdi.kubevirt.io/v1beta1
|
||||
kind: DataVolume
|
||||
metadata:
|
||||
name: prober-rootdisk
|
||||
spec:
|
||||
source:
|
||||
registry:
|
||||
url: "docker://quay.io/containerdisks/fedora:41"
|
||||
storage:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: longhorn
|
||||
resources:
|
||||
requests:
|
||||
storage: 6Gi
|
||||
EOF
|
||||
dv_phase=""
|
||||
for i in $(seq 1 120); do
|
||||
dv_phase="$(kubectl -n "$NS" get dv prober-rootdisk -o jsonpath='{.status.phase}' 2>/dev/null || true)"
|
||||
[ "$dv_phase" = "Succeeded" ] && break
|
||||
sleep 10
|
||||
done
|
||||
[ "$dv_phase" = "Succeeded" ] || { echo "FAIL: CDI import did not Succeed (last='$dv_phase')" >&2; exit 1; }
|
||||
echo "CDI import complete: prober-rootdisk Succeeded (Longhorn PVC)"
|
||||
|
||||
# -- GIVEN: the default-deny baseline in this namespace (deny all, allow only DNS to
|
||||
# kube-system) — the exact shape Kyverno generates for app namespaces (docs/03). --
|
||||
# kube-system) — the exact shape Kyverno generates for app namespaces (docs/03).
|
||||
# Applied AFTER the import so it governs the running VM's datapath (the point of the test). --
|
||||
kubectl -n "$NS" apply -f - <<'EOF'
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
@@ -111,6 +150,15 @@ spec:
|
||||
homelab.keithjackson.dev/egress-class: internet
|
||||
spec:
|
||||
domain:
|
||||
# The containerdisks/fedora:41 image is UEFI-native (ESP + /boot mounted via
|
||||
# systemd GPT auto-discovery). KubeVirt defaults to SeaBIOS, under which the guest
|
||||
# boots to emergency mode (ESP + /boot by-uuid mounts time out -> local-fs.target
|
||||
# fails) and cloud-init's runcmd/poweroff never runs. Boot with EFI (OVMF) so the
|
||||
# guest reaches multi-user and the probe executes. secureBoot:false => no SMM needed.
|
||||
firmware:
|
||||
bootloader:
|
||||
efi:
|
||||
secureBoot: false
|
||||
cpu: { cores: 1 }
|
||||
memory: { guest: 1024Mi }
|
||||
devices:
|
||||
@@ -126,8 +174,8 @@ spec:
|
||||
memory: 1024Mi
|
||||
volumes:
|
||||
- name: rootdisk
|
||||
containerDisk:
|
||||
image: quay.io/containerdisks/fedora:41
|
||||
dataVolume:
|
||||
name: prober-rootdisk
|
||||
- name: verdict
|
||||
persistentVolumeClaim:
|
||||
claimName: vm-verdict
|
||||
@@ -185,5 +233,6 @@ spec:
|
||||
content: |
|
||||
kubectl -n "$NAMESPACE" delete vm prober --ignore-not-found --wait=false || true
|
||||
kubectl -n "$NAMESPACE" delete pod verdict-reader --ignore-not-found --wait=false || true
|
||||
kubectl -n "$NAMESPACE" delete dv prober-rootdisk --ignore-not-found --wait=false || true
|
||||
kubectl -n "$NAMESPACE" delete pvc vm-verdict --ignore-not-found --wait=false || true
|
||||
echo "cleanup best-effort; chainsaw namespace teardown finalizes"
|
||||
@@ -14,13 +14,26 @@
|
||||
# vm-default-deny test — both share this VM model.
|
||||
#
|
||||
# RUNNABLE / SELF-SKIPPING: KubeVirt (Phase 5) may not be installed when the
|
||||
# broader suite runs. This test SKIPS CLEANLY (green) until the
|
||||
# virtualmachineinstances.kubevirt.io CRD exists, then runs the full lifecycle the
|
||||
# moment it does — mirroring the data-survival self-skip on `longhorn`. Everything
|
||||
# lives in the chainsaw per-test namespace and is torn down with it (hermetic). A
|
||||
# containerDisk root (kubelet-pulled, no CDI import) keeps the lifecycle proof fast
|
||||
# and free of image-import timing; the CDI-onto-Longhorn path is exercised by the
|
||||
# representative fixtures/vm.yaml via the deploy runbook and by vm-data-survival.
|
||||
# broader suite runs. This test SKIPS CLEANLY (green) until BOTH the
|
||||
# virtualmachineinstances.kubevirt.io CRD and a `longhorn` StorageClass exist, then
|
||||
# runs the full lifecycle the moment they do — mirroring the data-survival self-skip.
|
||||
# Everything lives in the chainsaw per-test namespace and is torn down with it
|
||||
# (hermetic).
|
||||
#
|
||||
# DISK MODEL — CDI DataVolume on Longhorn (NOT an ephemeral containerDisk):
|
||||
# KubeVirt v1.9 supplies the container-disk binary to the `volumerootdisk` init
|
||||
# container via a Kubernetes-native `image` volume (VolumeSource.Image) mounted at
|
||||
# /container-disk-binary. That k8s feature is gated by the ALPHA `ImageVolume`
|
||||
# feature gate, which is DISABLED on this k3s v1.31 cluster — so the apiserver
|
||||
# silently PRUNES the image volume + its mount, the binary is never present, and a
|
||||
# containerDisk VM dies at OCI init with:
|
||||
# exec: "/container-disk-binary/usr/bin/container-disk": no such file or directory
|
||||
# A DataVolume-backed root is a mounted Longhorn PVC (CDI imports the same Fedora
|
||||
# image into it out-of-band via its importer pod) and therefore does NOT touch the
|
||||
# container-disk binary / image-volume path at all — it boots on this cluster. It is
|
||||
# also the disk model the SPEC requires for VMs (docs/06, R-STATE-01: Longhorn-backed
|
||||
# disks that survive rebuild), so this mirrors the representative fixtures/vm.yaml.
|
||||
# Cost: a one-time CDI import per run (covered by the raised exec timeout below).
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
@@ -39,9 +52,10 @@ metadata:
|
||||
honours each transition (Running on start, gone on delete). Skips cleanly
|
||||
until the KubeVirt CRDs exist.
|
||||
spec:
|
||||
# VM start pulls a containerDisk + boots a guest — well beyond the default 180s exec window.
|
||||
# CDI imports the Fedora image onto a Longhorn PVC, then the VM boots the guest —
|
||||
# import + boot is well beyond the default 180s exec window.
|
||||
timeouts:
|
||||
exec: 20m
|
||||
exec: 30m
|
||||
steps:
|
||||
- name: vm-lifecycle-through-k8s-api
|
||||
try:
|
||||
@@ -50,14 +64,48 @@ spec:
|
||||
set -euo pipefail
|
||||
NS="$NAMESPACE"
|
||||
|
||||
# -- Gate: skip cleanly (green) until KubeVirt is installed. --
|
||||
# -- Gate: skip cleanly (green) until BOTH KubeVirt and longhorn are present. --
|
||||
if ! kubectl get crd virtualmachineinstances.kubevirt.io >/dev/null 2>&1; then
|
||||
echo "SKIP: KubeVirt CRDs absent — vm-lifecycle not applicable on this cluster yet"
|
||||
echo VM_LIFECYCLE_SKIPPED_OK
|
||||
exit 0
|
||||
fi
|
||||
if ! kubectl get storageclass longhorn >/dev/null 2>&1; then
|
||||
echo "SKIP: 'longhorn' StorageClass absent — vm-lifecycle not applicable yet"
|
||||
echo VM_LIFECYCLE_SKIPPED_OK
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# -- GIVEN: a stopped VirtualMachine, defined declaratively (containerDisk root). --
|
||||
# -- GIVEN (disk): CDI imports the Fedora image onto a Longhorn PVC (standalone
|
||||
# DataVolume — outlives VM start/stop/restart so the disk is reused, not re-imported). --
|
||||
kubectl -n "$NS" apply -f - <<'EOF'
|
||||
apiVersion: cdi.kubevirt.io/v1beta1
|
||||
kind: DataVolume
|
||||
metadata:
|
||||
name: lc-rootdisk
|
||||
spec:
|
||||
source:
|
||||
registry:
|
||||
url: "docker://quay.io/containerdisks/fedora:41"
|
||||
storage:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: longhorn
|
||||
resources:
|
||||
requests:
|
||||
storage: 6Gi
|
||||
EOF
|
||||
# Wait for the CDI import to complete before booting (bounded poll; import pulls quay.io).
|
||||
dv_phase=""
|
||||
for i in $(seq 1 120); do
|
||||
dv_phase="$(kubectl -n "$NS" get dv lc-rootdisk -o jsonpath='{.status.phase}' 2>/dev/null || true)"
|
||||
[ "$dv_phase" = "Succeeded" ] && break
|
||||
sleep 10
|
||||
done
|
||||
[ "$dv_phase" = "Succeeded" ] || { echo "FAIL: CDI import did not Succeed (last='$dv_phase')" >&2; exit 1; }
|
||||
echo "CDI import complete: DataVolume lc-rootdisk Succeeded (Longhorn PVC)"
|
||||
|
||||
# -- GIVEN (VM): a stopped VirtualMachine bound to that Longhorn DataVolume disk. --
|
||||
kubectl -n "$NS" apply -f - <<'EOF'
|
||||
apiVersion: kubevirt.io/v1
|
||||
kind: VirtualMachine
|
||||
@@ -72,6 +120,14 @@ spec:
|
||||
homelab.keithjackson.dev/egress-class: internet
|
||||
spec:
|
||||
domain:
|
||||
# EFI (OVMF) boot: the containerdisks/fedora:41 image is UEFI-native; under the
|
||||
# KubeVirt SeaBIOS default it boots to emergency mode (ESP + /boot mounts fail).
|
||||
# Not strictly required for this test (it only asserts VMI phase transitions, not
|
||||
# in-guest state) but kept consistent with the probe tests + fixtures/vm.yaml.
|
||||
firmware:
|
||||
bootloader:
|
||||
efi:
|
||||
secureBoot: false
|
||||
cpu: { cores: 1 }
|
||||
memory: { guest: 1024Mi }
|
||||
devices:
|
||||
@@ -89,8 +145,8 @@ spec:
|
||||
pod: {}
|
||||
volumes:
|
||||
- name: rootdisk
|
||||
containerDisk:
|
||||
image: quay.io/containerdisks/fedora:41
|
||||
dataVolume:
|
||||
name: lc-rootdisk
|
||||
EOF
|
||||
# No VMI should exist while running=false.
|
||||
if kubectl -n "$NS" get vmi lc >/dev/null 2>&1; then
|
||||
@@ -147,4 +203,5 @@ spec:
|
||||
- script:
|
||||
content: |
|
||||
kubectl -n "$NAMESPACE" delete vm lc --ignore-not-found --wait=false || true
|
||||
kubectl -n "$NAMESPACE" delete dv lc-rootdisk --ignore-not-found --wait=false || true
|
||||
echo "cleanup best-effort; chainsaw namespace teardown finalizes"
|
||||
@@ -47,6 +47,14 @@ spec:
|
||||
homelab.keithjackson.dev/egress-class: internet
|
||||
spec:
|
||||
domain:
|
||||
# EFI (OVMF) boot — the containerdisks/fedora:41 image is UEFI-native (separate ESP + /boot
|
||||
# mounted via systemd GPT auto-discovery). Under KubeVirt's SeaBIOS default the guest drops to
|
||||
# emergency mode (ESP + /boot by-uuid mounts time out -> local-fs.target fails), so cloud-init
|
||||
# (dnf-automatic setup below) never runs. secureBoot:false avoids the SMM requirement.
|
||||
firmware:
|
||||
bootloader:
|
||||
efi:
|
||||
secureBoot: false
|
||||
cpu:
|
||||
cores: 1
|
||||
memory:
|
||||
|
||||
@@ -10,9 +10,15 @@
|
||||
# exactly the container proof (data-survival), now for a KubeVirt VM
|
||||
# (R-STATE-01 covers PaaS + IaaS).
|
||||
#
|
||||
# METHOD (virtctl-free, filesystem-agnostic, deterministic): a containerDisk-root
|
||||
# VM (ephemeral) with a Longhorn BLOCK data disk. cloud-init writes a unique canary
|
||||
# to the raw block device and powers the guest off; runStrategy RerunOnFailure means
|
||||
# METHOD (virtctl-free, filesystem-agnostic, deterministic): a VM whose ROOT is a
|
||||
# CDI DataVolume on Longhorn (a mounted PVC — NOT an ephemeral containerDisk; the
|
||||
# containerDisk boot path is broken on this k3s cluster because KubeVirt v1.9 serves
|
||||
# the container-disk binary through a k8s `image` volume gated by the ALPHA
|
||||
# `ImageVolume` feature gate, which k3s v1.31 has disabled → the apiserver prunes the
|
||||
# volume and the guest dies at OCI init. DataVolume disks bypass that path and are the
|
||||
# spec-required disk model anyway — docs/06/R-STATE-01). The VM also has a SECOND
|
||||
# Longhorn BLOCK data disk; cloud-init writes a unique canary to that raw block device
|
||||
# and powers the guest off; runStrategy RerunOnFailure means
|
||||
# a clean shutdown STAYS down (VMI -> Succeeded), so the write is known-complete and
|
||||
# the disk is released. We then DELETE the VM (keeping the PVC) and bind a throwaway
|
||||
# busybox reader pod to the SAME Longhorn PVC as a raw block device — if the bytes
|
||||
@@ -38,9 +44,10 @@ metadata:
|
||||
Destroy a VM entirely, keep its Longhorn PVC, and a fresh workload re-bound to
|
||||
that PVC still reads the VM-written data. Skips cleanly until KubeVirt + longhorn exist.
|
||||
spec:
|
||||
# Boot a guest + run cloud-init + power off, then reattach a Longhorn RWO volume — far beyond 180s.
|
||||
# CDI import onto Longhorn + boot a guest + run cloud-init + power off, then reattach a
|
||||
# Longhorn RWO volume — far beyond 180s.
|
||||
timeouts:
|
||||
exec: 20m
|
||||
exec: 30m
|
||||
steps:
|
||||
- name: vm-data-survives-destroy-and-rebuild
|
||||
try:
|
||||
@@ -77,7 +84,35 @@ spec:
|
||||
storage: 1Gi
|
||||
EOF
|
||||
|
||||
# -- The VM: containerDisk root (ephemeral) + the Longhorn block disk (/dev/vdb) + a
|
||||
# -- The VM's ROOT disk: CDI imports Fedora onto a Longhorn PVC (bypasses the broken
|
||||
# containerDisk/image-volume path; standalone DataVolume). Wait for the import. --
|
||||
kubectl -n "$NS" apply -f - <<'EOF'
|
||||
apiVersion: cdi.kubevirt.io/v1beta1
|
||||
kind: DataVolume
|
||||
metadata:
|
||||
name: survivor-rootdisk
|
||||
spec:
|
||||
source:
|
||||
registry:
|
||||
url: "docker://quay.io/containerdisks/fedora:41"
|
||||
storage:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: longhorn
|
||||
resources:
|
||||
requests:
|
||||
storage: 6Gi
|
||||
EOF
|
||||
dv_phase=""
|
||||
for i in $(seq 1 120); do
|
||||
dv_phase="$(kubectl -n "$NS" get dv survivor-rootdisk -o jsonpath='{.status.phase}' 2>/dev/null || true)"
|
||||
[ "$dv_phase" = "Succeeded" ] && break
|
||||
sleep 10
|
||||
done
|
||||
[ "$dv_phase" = "Succeeded" ] || { echo "FAIL: CDI import did not Succeed (last='$dv_phase')" >&2; exit 1; }
|
||||
echo "CDI import complete: survivor-rootdisk Succeeded (Longhorn PVC)"
|
||||
|
||||
# -- The VM: DataVolume root (Longhorn) + the Longhorn block disk (/dev/vdb) + a
|
||||
# cloud-init disk. cloud-init writes the canary to the raw device and powers off.
|
||||
# RerunOnFailure => a clean poweroff STAYS Succeeded (not restarted), so the write is
|
||||
# known-complete and the disk released. Only $CANARY expands (no other '$' in the body).
|
||||
@@ -95,6 +130,14 @@ spec:
|
||||
homelab.keithjackson.dev/egress-class: internet
|
||||
spec:
|
||||
domain:
|
||||
# The containerdisks/fedora:41 image is UEFI-native (ESP + /boot mounted via
|
||||
# systemd GPT auto-discovery). KubeVirt defaults to SeaBIOS, under which the guest
|
||||
# boots to emergency mode (ESP + /boot by-uuid mounts time out) and cloud-init's
|
||||
# runcmd/poweroff never runs. Boot with EFI (OVMF). secureBoot:false => no SMM needed.
|
||||
firmware:
|
||||
bootloader:
|
||||
efi:
|
||||
secureBoot: false
|
||||
cpu: { cores: 1 }
|
||||
memory: { guest: 1024Mi }
|
||||
devices:
|
||||
@@ -110,8 +153,8 @@ spec:
|
||||
memory: 1024Mi
|
||||
volumes:
|
||||
- name: rootdisk
|
||||
containerDisk:
|
||||
image: quay.io/containerdisks/fedora:41
|
||||
dataVolume:
|
||||
name: survivor-rootdisk
|
||||
- name: datadisk
|
||||
persistentVolumeClaim:
|
||||
claimName: vm-survivor
|
||||
@@ -168,5 +211,6 @@ spec:
|
||||
content: |
|
||||
kubectl -n "$NAMESPACE" delete vm survivor --ignore-not-found --wait=false || true
|
||||
kubectl -n "$NAMESPACE" delete pod reader --ignore-not-found --wait=false || true
|
||||
kubectl -n "$NAMESPACE" delete dv survivor-rootdisk --ignore-not-found --wait=false || true
|
||||
kubectl -n "$NAMESPACE" delete pvc vm-survivor --ignore-not-found --wait=false || true
|
||||
echo "cleanup best-effort; chainsaw namespace teardown finalizes"
|
||||
Reference in new issue
Block a user