Merge feature/kubevirt-iaas: Phase 5 KubeVirt+CDI IaaS proof (R-PLAT-03/04, R-STATE-01, R-NET-01, R-OBS)
This commit is contained in:
14 files changed
+888
-55
No files matched your search
@@ -0,0 +1,35 @@
|
||||
# ServiceDefinition: vm-demo — the Phase-5 IaaS proof workload (R-PLAT-03, ADR-0005).
|
||||
# Source of truth: spec docs/06-vm-workloads-kubevirt.md + docs/13 (`kind: vm` path).
|
||||
#
|
||||
# A small headless Fedora Cloud VM, imported via CDI onto a Longhorn-backed disk, managed entirely
|
||||
# through the same Kubernetes surface as containers. It proves the escape hatch is real, not theoretical
|
||||
# (docs/06): the SAME declarative surface yields compute + a Longhorn disk that survives rebuild
|
||||
# (R-STATE-01), default-deny networking (reaches only DNS — no `dependencies`), and in-guest auto-updates
|
||||
# on a schedule (docs/11 §3 — dnf-automatic via cloud-init; no human step). Validates against
|
||||
# schemas/servicedefinition.schema.json. The rendered VirtualMachine + DataVolume are in
|
||||
# conformance/tests/l2-behavioral/vm-lifecycle/fixtures/vm.yaml (also the runbook's apply target).
|
||||
name: vm-demo
|
||||
kind: vm # IaaS via KubeVirt (R-PLAT-03), not a container
|
||||
source:
|
||||
# CDI import source: a KubeVirt containerdisk of Fedora Cloud (has cloud-init + dnf). CDI imports it
|
||||
# onto the Longhorn PVC below, so the VM disk inherits 3-replica durability + backup (docs/04).
|
||||
vmTemplate: quay.io/containerdisks/fedora:41
|
||||
exposure: internal # LAN-only; no north-south HTTP surface for this proof (docs/09)
|
||||
egress: internet # default WAN egress. (An egress: vpn VM is a documented follow-up —
|
||||
# the Cilium Egress Gateway steering keys on the pod label, which
|
||||
# KubeVirt propagates from the VMI to its virt-launcher pod. ADR-0015.)
|
||||
storage:
|
||||
- name: rootdisk
|
||||
size: 6Gi
|
||||
class: longhorn # CDI-imported root disk on the replicated tier; survives destroy/rebuild
|
||||
dependencies: [] # default-deny: reaches only DNS (docs/03). Proven by the vm-default-deny test.
|
||||
update:
|
||||
channel: stable # in-guest dnf-automatic on `cadence` (renderer-contract §update, vm kind)
|
||||
cadence: "0 4 * * *" # nightly guest package updates — no human/AI/GitOps edit (R-UPD-01)
|
||||
autoRollbackOnFailedHealth: true
|
||||
observability:
|
||||
metrics: true # platform scrapes KubeVirt/virt metrics for this VM (docs/10)
|
||||
slis:
|
||||
- up
|
||||
backup:
|
||||
policy: daily-offcluster # the VM's Longhorn disk is backed up off-cluster (docs/04; R-STATE-03)
|
||||
@@ -0,0 +1,189 @@
|
||||
# =============================================================================
|
||||
# L2 BEHAVIORAL — Default-deny applies to a VM (R-NET-01) [RUNNABLE / self-skips]
|
||||
# Catalog: docs/14 (default-deny) — the IaaS partner of the container `default-deny`
|
||||
# test. Spec: docs/03 "Default-deny baseline"; docs/06 ("default-deny +
|
||||
# egress classes apply to VMs too"); ADR-0005/0015.
|
||||
#
|
||||
# GIVEN a VM in a namespace carrying the default-deny baseline (deny all east-west +
|
||||
# egress, allow only DNS to the in-cluster resolver),
|
||||
# WHEN the guest probes the network from INSIDE the VM,
|
||||
# THEN DNS resolves (the one standing allow) but an undeclared public TCP dial is
|
||||
# BLOCKED — the same default-deny fabric that governs pods governs the VM's
|
||||
# own datapath (its traffic egresses via the virt-launcher pod netns, so a
|
||||
# podSelector:{} NetworkPolicy covers it). R-NET-01 for IaaS.
|
||||
#
|
||||
# METHOD (virtctl-free): cloud-init runs the two probes and writes a PASS/FAIL
|
||||
# verdict to a raw Longhorn block disk, then powers off (RerunOnFailure => stays
|
||||
# down). A busybox reader pod reads the verdict back. The container default-deny
|
||||
# test proves the datapath fail-closed (default-deny-failclosed, L3); this asserts
|
||||
# the SAME baseline is enforced for a VM.
|
||||
#
|
||||
# RUNNABLE / SELF-SKIPPING: skips cleanly (green) until KubeVirt + longhorn exist.
|
||||
#
|
||||
# NOTE (bounded live-iteration, per the WS-VM report): the in-guest probe details
|
||||
# (guest block-device ordering /dev/vdb, cluster-DNS injection into the guest, sh=bash
|
||||
# for /dev/tcp on the Fedora containerDisk) are authored to the KubeVirt defaults and
|
||||
# confirmed on the first live Phase-5 run; the test STRUCTURE and assertion are final.
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: vm-default-deny
|
||||
labels:
|
||||
conformance.home.k3s/R-NET-01: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-NET-01"
|
||||
conformance.home.k3s/layer: "L2"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
A VM under the default-deny baseline reaches only DNS from inside the guest;
|
||||
an undeclared public TCP dial is blocked. Skips cleanly until KubeVirt + longhorn exist.
|
||||
spec:
|
||||
timeouts:
|
||||
exec: 20m
|
||||
steps:
|
||||
- name: default-deny-applies-to-the-vm-datapath
|
||||
try:
|
||||
- script:
|
||||
content: |
|
||||
set -euo pipefail
|
||||
NS="$NAMESPACE"
|
||||
|
||||
# -- Gate: skip cleanly (green) until KubeVirt + longhorn exist. --
|
||||
if ! kubectl get crd virtualmachineinstances.kubevirt.io >/dev/null 2>&1; then
|
||||
echo "SKIP: KubeVirt CRDs absent"; echo VM_DEFAULT_DENY_SKIPPED_OK; exit 0
|
||||
fi
|
||||
if ! kubectl get storageclass longhorn >/dev/null 2>&1; then
|
||||
echo "SKIP: 'longhorn' StorageClass absent"; echo VM_DEFAULT_DENY_SKIPPED_OK; exit 0
|
||||
fi
|
||||
|
||||
CANARY="vm-deny-$(date +%s%N)"
|
||||
|
||||
# -- GIVEN: the default-deny baseline in this namespace (deny all, allow only DNS to
|
||||
# kube-system) — the exact shape Kyverno generates for app namespaces (docs/03). --
|
||||
kubectl -n "$NS" apply -f - <<'EOF'
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: default-deny
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes: [Ingress, Egress]
|
||||
ingress: []
|
||||
egress:
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
ports:
|
||||
- { protocol: UDP, port: 53 }
|
||||
- { protocol: TCP, port: 53 }
|
||||
EOF
|
||||
|
||||
# -- The verdict block disk (Longhorn). --
|
||||
kubectl -n "$NS" apply -f - <<'EOF'
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: vm-verdict
|
||||
spec:
|
||||
accessModes: [ReadWriteOnce]
|
||||
volumeMode: Block
|
||||
storageClassName: longhorn
|
||||
resources:
|
||||
requests:
|
||||
storage: 1Gi
|
||||
EOF
|
||||
|
||||
# -- The VM + the in-guest probe (quoted heredoc: many '$' in the probe; __CANARY__ sed'd). --
|
||||
cat > /tmp/vm-deny.yaml <<'EOF'
|
||||
apiVersion: kubevirt.io/v1
|
||||
kind: VirtualMachine
|
||||
metadata:
|
||||
name: prober
|
||||
spec:
|
||||
runStrategy: RerunOnFailure
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
kubevirt.io/vm: prober
|
||||
homelab.keithjackson.dev/egress-class: internet
|
||||
spec:
|
||||
domain:
|
||||
cpu: { cores: 1 }
|
||||
memory: { guest: 1024Mi }
|
||||
devices:
|
||||
disks:
|
||||
- name: rootdisk
|
||||
disk: { bus: virtio }
|
||||
- name: verdict
|
||||
disk: { bus: virtio }
|
||||
- name: cloudinit
|
||||
disk: { bus: virtio }
|
||||
resources:
|
||||
requests:
|
||||
memory: 1024Mi
|
||||
volumes:
|
||||
- name: rootdisk
|
||||
containerDisk:
|
||||
image: quay.io/containerdisks/fedora:41
|
||||
- name: verdict
|
||||
persistentVolumeClaim:
|
||||
claimName: vm-verdict
|
||||
- name: cloudinit
|
||||
cloudInitNoCloud:
|
||||
userData: |-
|
||||
#cloud-config
|
||||
runcmd:
|
||||
- [ sh, -c, "getent hosts quay.io >/dev/null 2>&1 && DNS=1 || DNS=0; timeout 5 sh -c 'echo > /dev/tcp/1.1.1.1/443' >/dev/null 2>&1 && DENY=0 || DENY=1; V=FAIL; [ \"$DNS\" = 1 ] && [ \"$DENY\" = 1 ] && V=PASS; printf '%s' \"$V:__CANARY__\" | dd of=/dev/vdb bs=1 seek=0 conv=notrunc; sync; sleep 3; poweroff" ]
|
||||
EOF
|
||||
sed -i "s/__CANARY__/$CANARY/" /tmp/vm-deny.yaml
|
||||
kubectl -n "$NS" apply -f /tmp/vm-deny.yaml
|
||||
|
||||
# -- WHEN: wait until the guest has run the probes and powered off. --
|
||||
ok=""
|
||||
for i in $(seq 1 90); do
|
||||
phase="$(kubectl -n "$NS" get vmi prober -o jsonpath='{.status.phase}' 2>/dev/null || true)"
|
||||
if [ "$phase" = "Succeeded" ] || { [ -z "$phase" ] && [ "$i" -gt 3 ]; }; then ok=1; break; fi
|
||||
sleep 10
|
||||
done
|
||||
[ -n "$ok" ] || { echo "FAIL: guest never completed the probes+poweroff (last='$phase')" >&2; exit 1; }
|
||||
|
||||
# -- THEN: read the verdict from the Longhorn disk via a throwaway reader. --
|
||||
kubectl -n "$NS" delete vm prober --wait=true --timeout=180s
|
||||
kubectl -n "$NS" apply -f - <<'EOF'
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: verdict-reader
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
containers:
|
||||
- name: app
|
||||
image: busybox:1.36.1
|
||||
command: ["sh","-c","sleep 3600"]
|
||||
volumeDevices:
|
||||
- name: v
|
||||
devicePath: /dev/verdict
|
||||
volumes:
|
||||
- name: v
|
||||
persistentVolumeClaim:
|
||||
claimName: vm-verdict
|
||||
EOF
|
||||
kubectl -n "$NS" wait --for=condition=Ready pod/verdict-reader --timeout=240s
|
||||
GOT="$(kubectl -n "$NS" exec verdict-reader -- sh -c "dd if=/dev/verdict bs=1 count=$((5 + ${#CANARY})) 2>/dev/null")"
|
||||
echo "verdict='$GOT' (expected PASS:$CANARY)"
|
||||
case "$GOT" in
|
||||
PASS:"$CANARY") echo VM_DEFAULT_DENY_OK ;;
|
||||
*) echo "FAIL: default-deny not enforced for the VM (verdict='$GOT')" >&2; exit 1 ;;
|
||||
esac
|
||||
check:
|
||||
(contains($stdout, 'VM_DEFAULT_DENY_OK') || contains($stdout, 'VM_DEFAULT_DENY_SKIPPED_OK')): true
|
||||
finally:
|
||||
- script:
|
||||
content: |
|
||||
kubectl -n "$NAMESPACE" delete vm prober --ignore-not-found --wait=false || true
|
||||
kubectl -n "$NAMESPACE" delete pod verdict-reader --ignore-not-found --wait=false || true
|
||||
kubectl -n "$NAMESPACE" delete pvc vm-verdict --ignore-not-found --wait=false || true
|
||||
echo "cleanup best-effort; chainsaw namespace teardown finalizes"
|
||||
@@ -1,14 +1,26 @@
|
||||
# =============================================================================
|
||||
# L2 BEHAVIORAL SCAFFOLD — VM lifecycle (R-PLAT-03)
|
||||
# L2 BEHAVIORAL — VM lifecycle (R-PLAT-03, R-PLAT-04) [RUNNABLE / self-skips]
|
||||
# Catalog: docs/14 "VM lifecycle (R-PLAT-03, L2/L3)"
|
||||
# Spec: docs/06-vm-workloads-kubevirt.md, ADR-0005.
|
||||
#
|
||||
# GIVEN a `kind: vm` ServiceDefinition,
|
||||
# WHEN it is created/started/stopped/destroyed via kubectl/virtctl,
|
||||
# THEN it behaves through the same K8s surface; default-deny + egress class
|
||||
# apply to it. (Disk-survives-rebuild is the L3 partner test, see
|
||||
# tests/l3-fault-injection/data-survival — VMs share R-STATE-01.)
|
||||
# GIVEN a `kind: vm` workload (KubeVirt VirtualMachine),
|
||||
# WHEN it is created / started / stopped / restarted / destroyed via the SAME
|
||||
# Kubernetes API + kubectl surface as a container,
|
||||
# THEN each transition is honoured — the VMI reaches phase=Running on start,
|
||||
# disappears on stop, comes back on restart, and is gone after delete.
|
||||
# (R-PLAT-03 "VMs through the K8s API"; R-PLAT-04 "create/destroy/rebuild
|
||||
# on demand".) The Longhorn-disk-survives-rebuild partner proof is the L3
|
||||
# vm-data-survival test; the default-deny-applies-to-the-VM proof is the L2
|
||||
# vm-default-deny test — both share this VM model.
|
||||
#
|
||||
# STATUS: scaffold — needs T-5.1/T-5.2 (KubeVirt + `kind: vm` path).
|
||||
# RUNNABLE / SELF-SKIPPING: KubeVirt (Phase 5) may not be installed when the
|
||||
# broader suite runs. This test SKIPS CLEANLY (green) until the
|
||||
# virtualmachineinstances.kubevirt.io CRD exists, then runs the full lifecycle the
|
||||
# moment it does — mirroring the data-survival self-skip on `longhorn`. Everything
|
||||
# lives in the chainsaw per-test namespace and is torn down with it (hermetic). A
|
||||
# containerDisk root (kubelet-pulled, no CDI import) keeps the lifecycle proof fast
|
||||
# and free of image-import timing; the CDI-onto-Longhorn path is exercised by the
|
||||
# representative fixtures/vm.yaml via the deploy runbook and by vm-data-survival.
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
@@ -16,31 +28,123 @@ metadata:
|
||||
name: vm-lifecycle
|
||||
labels:
|
||||
conformance.home.k3s/R-PLAT-03: "true"
|
||||
conformance.home.k3s/R-PLAT-04: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-PLAT-03"
|
||||
conformance.home.k3s/requirements: "R-PLAT-03, R-PLAT-04"
|
||||
conformance.home.k3s/layer: "L2"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
A VM is created/started/stopped/destroyed via the K8s API; policy applies.
|
||||
A VM is created/started/stopped/restarted/destroyed via the K8s API; the VMI
|
||||
honours each transition (Running on start, gone on delete). Skips cleanly
|
||||
until the KubeVirt CRDs exist.
|
||||
spec:
|
||||
# VM start pulls a containerDisk + boots a guest — well beyond the default 180s exec window.
|
||||
timeouts:
|
||||
exec: 20m
|
||||
steps:
|
||||
- name: given-vm-servicedefinition
|
||||
try:
|
||||
- apply:
|
||||
file: fixtures/vm-servicedefinition.yaml # TODO(T-5.2): author
|
||||
- name: when-start-and-assert-running
|
||||
- name: vm-lifecycle-through-k8s-api
|
||||
try:
|
||||
- script:
|
||||
content: |
|
||||
set -euo pipefail
|
||||
# virtctl start via the same kubectl surface; assert VMI Running.
|
||||
kubectl -n "$NAMESPACE" patch vm demo --type merge -p '{"spec":{"running":true}}'
|
||||
echo "TODO(T-5.3): wait VMI phase=Running (bounded), then STARTED_OK"
|
||||
- name: then-default-deny-and-egress-apply-to-vm
|
||||
try:
|
||||
NS="$NAMESPACE"
|
||||
|
||||
# -- Gate: skip cleanly (green) until KubeVirt is installed. --
|
||||
if ! kubectl get crd virtualmachineinstances.kubevirt.io >/dev/null 2>&1; then
|
||||
echo "SKIP: KubeVirt CRDs absent — vm-lifecycle not applicable on this cluster yet"
|
||||
echo VM_LIFECYCLE_SKIPPED_OK
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# -- GIVEN: a stopped VirtualMachine, defined declaratively (containerDisk root). --
|
||||
kubectl -n "$NS" apply -f - <<'EOF'
|
||||
apiVersion: kubevirt.io/v1
|
||||
kind: VirtualMachine
|
||||
metadata:
|
||||
name: lc
|
||||
spec:
|
||||
running: false
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
kubevirt.io/vm: lc
|
||||
homelab.keithjackson.dev/egress-class: internet
|
||||
spec:
|
||||
domain:
|
||||
cpu: { cores: 1 }
|
||||
memory: { guest: 1024Mi }
|
||||
devices:
|
||||
disks:
|
||||
- name: rootdisk
|
||||
disk: { bus: virtio }
|
||||
interfaces:
|
||||
- name: default
|
||||
masquerade: {}
|
||||
resources:
|
||||
requests:
|
||||
memory: 1024Mi
|
||||
networks:
|
||||
- name: default
|
||||
pod: {}
|
||||
volumes:
|
||||
- name: rootdisk
|
||||
containerDisk:
|
||||
image: quay.io/containerdisks/fedora:41
|
||||
EOF
|
||||
# No VMI should exist while running=false.
|
||||
if kubectl -n "$NS" get vmi lc >/dev/null 2>&1; then
|
||||
echo "FAIL: VMI exists though the VM is defined stopped" >&2; exit 1
|
||||
fi
|
||||
echo "created VM 'lc' (stopped)"
|
||||
|
||||
wait_phase() { # $1=expected phase, bounded poll
|
||||
local want="$1" i phase
|
||||
for i in $(seq 1 80); do
|
||||
phase="$(kubectl -n "$NS" get vmi lc -o jsonpath='{.status.phase}' 2>/dev/null || true)"
|
||||
[ "$phase" = "$want" ] && return 0
|
||||
sleep 10
|
||||
done
|
||||
echo "FAIL: VMI did not reach phase=$want (last='$phase')" >&2; return 1
|
||||
}
|
||||
wait_gone() {
|
||||
local i
|
||||
for i in $(seq 1 30); do
|
||||
kubectl -n "$NS" get vmi lc >/dev/null 2>&1 || return 0
|
||||
sleep 5
|
||||
done
|
||||
echo "FAIL: VMI still present when it should be gone" >&2; return 1
|
||||
}
|
||||
|
||||
# -- WHEN start (running=true) THEN VMI reaches Running. --
|
||||
kubectl -n "$NS" patch vm lc --type merge -p '{"spec":{"running":true}}'
|
||||
wait_phase Running
|
||||
echo "STARTED: VMI phase=Running"
|
||||
|
||||
# -- WHEN stop (running=false) THEN the VMI is torn down. --
|
||||
kubectl -n "$NS" patch vm lc --type merge -p '{"spec":{"running":false}}'
|
||||
wait_gone
|
||||
echo "STOPPED: VMI removed"
|
||||
|
||||
# -- WHEN restart (running=true again) THEN it comes back Running (rebuild-on-demand). --
|
||||
kubectl -n "$NS" patch vm lc --type merge -p '{"spec":{"running":true}}'
|
||||
wait_phase Running
|
||||
echo "RESTARTED: VMI phase=Running"
|
||||
|
||||
# -- WHEN destroy (delete the VM) THEN both VM and VMI are gone. --
|
||||
kubectl -n "$NS" delete vm lc --wait=true --timeout=300s
|
||||
wait_gone
|
||||
if kubectl -n "$NS" get vm lc >/dev/null 2>&1; then
|
||||
echo "FAIL: VM object still present after delete" >&2; exit 1
|
||||
fi
|
||||
echo "DESTROYED: VM + VMI gone"
|
||||
|
||||
echo VM_LIFECYCLE_OK
|
||||
check:
|
||||
# green on a clean skip (no KubeVirt) OR a proven full lifecycle.
|
||||
(contains($stdout, 'VM_LIFECYCLE_OK') || contains($stdout, 'VM_LIFECYCLE_SKIPPED_OK')): true
|
||||
finally:
|
||||
- script:
|
||||
content: 'echo "TODO: run default-deny + egress-class probes from inside the VM"'
|
||||
- name: when-stop-and-destroy
|
||||
try:
|
||||
- script:
|
||||
content: 'echo "TODO(T-5.3): stop then delete VM via kubectl; assert gone"'
|
||||
content: |
|
||||
kubectl -n "$NAMESPACE" delete vm lc --ignore-not-found --wait=false || true
|
||||
echo "cleanup best-effort; chainsaw namespace teardown finalizes"
|
||||
@@ -0,0 +1,94 @@
|
||||
# Rendered from catalog/vm-demo.yaml (kind: vm) — the Phase-5 IaaS proof workload (ADR-0005, docs/06).
|
||||
# A small headless Fedora Cloud VM whose root disk is CDI-imported onto a Longhorn PVC. Applied by the
|
||||
# vm-lifecycle (R-PLAT-03) test and by the deploy runbook. Namespace-agnostic: chainsaw injects its
|
||||
# ephemeral namespace; the runbook applies it into a `vm-demo` namespace (see platform/kubevirt/README).
|
||||
#
|
||||
# DURABILITY (R-STATE-01): the DataVolume is STANDALONE (not a `dataVolumeTemplates` entry), so it is
|
||||
# NOT owner-referenced by the VirtualMachine and OUTLIVES a `kubectl delete vm`. Destroy the VM, keep
|
||||
# this DataVolume/PVC, re-apply the VM — it re-binds the same Longhorn disk. (dataVolumeTemplates would
|
||||
# be garbage-collected with the VM and would FAIL rebuild-survival.)
|
||||
---
|
||||
apiVersion: cdi.kubevirt.io/v1beta1
|
||||
kind: DataVolume
|
||||
metadata:
|
||||
name: vm-demo-rootdisk
|
||||
spec:
|
||||
source:
|
||||
registry:
|
||||
# CDI imports this containerdisk (Fedora Cloud) into the Longhorn PVC below. Pinned tag; Renovate
|
||||
# can maintain. Import needs egress to quay.io — in an app namespace under default-deny the runbook
|
||||
# adds a CiliumNetworkPolicy allowing the CDI importer pod egress (see README); the ephemeral test
|
||||
# namespace has no default-deny baseline, so the import egresses freely there.
|
||||
url: "docker://quay.io/containerdisks/fedora:41"
|
||||
storage:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: longhorn # replicated tier (ADR-0003) — VM disk gets the same durability
|
||||
resources:
|
||||
requests:
|
||||
storage: 6Gi
|
||||
---
|
||||
apiVersion: kubevirt.io/v1
|
||||
kind: VirtualMachine
|
||||
metadata:
|
||||
name: vm-demo
|
||||
labels:
|
||||
app.kubernetes.io/name: vm-demo
|
||||
spec:
|
||||
# Lifecycle is driven through the K8s API: `running: false` here means "defined but stopped"; the
|
||||
# test/operator sets it true/false to start/stop, and `kubectl delete vm` destroys it (R-PLAT-03/04).
|
||||
running: false
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
kubevirt.io/vm: vm-demo
|
||||
# Propagated to the virt-launcher pod so Cilium egress steering + the Kyverno egress-class
|
||||
# policy see the class (ADR-0015). internet = default WAN path (this proof is not VPN-class).
|
||||
homelab.keithjackson.dev/egress-class: internet
|
||||
spec:
|
||||
domain:
|
||||
cpu:
|
||||
cores: 1
|
||||
memory:
|
||||
guest: 1536Mi
|
||||
devices:
|
||||
disks:
|
||||
- name: rootdisk
|
||||
disk:
|
||||
bus: virtio
|
||||
- name: cloudinit
|
||||
disk:
|
||||
bus: virtio
|
||||
interfaces:
|
||||
- name: default
|
||||
masquerade: {}
|
||||
resources:
|
||||
requests:
|
||||
memory: 1536Mi
|
||||
networks:
|
||||
- name: default
|
||||
pod: {}
|
||||
volumes:
|
||||
- name: rootdisk
|
||||
dataVolume:
|
||||
name: vm-demo-rootdisk
|
||||
- name: cloudinit
|
||||
cloudInitNoCloud:
|
||||
# Guest auto-updates with NO human step (docs/11 §3, R-UPD-01): enable dnf-automatic so the
|
||||
# guest applies stable package updates on its own timer. This is the VM analogue of a
|
||||
# container image-policy bump — the update policy is declared once, automation applies it.
|
||||
userData: |-
|
||||
#cloud-config
|
||||
package_update: true
|
||||
packages:
|
||||
- dnf-automatic
|
||||
write_files:
|
||||
- path: /etc/dnf/automatic.conf
|
||||
content: |
|
||||
[commands]
|
||||
upgrade_type = default
|
||||
random_sleep = 0
|
||||
download_updates = yes
|
||||
apply_updates = yes
|
||||
runcmd:
|
||||
- [ systemctl, enable, --now, dnf-automatic.timer ]
|
||||
@@ -0,0 +1,172 @@
|
||||
# =============================================================================
|
||||
# L3 FAULT-INJECTION — VM disk survives rebuild (R-STATE-01) [RUNNABLE / self-skips]
|
||||
# *SAFETY-CRITICAL*
|
||||
# Catalog: docs/14 "Data survival (R-STATE-01, L3)" — the IaaS partner of the
|
||||
# container `data-survival` test. Spec: docs/04, docs/06, ADR-0005.
|
||||
#
|
||||
# GIVEN a VM that writes data to a Longhorn-backed disk,
|
||||
# WHEN the VM is DESTROYED entirely (kept: the Longhorn PVC/DataVolume),
|
||||
# THEN the data is intact and readable by a fresh workload bound to that PVC —
|
||||
# exactly the container proof (data-survival), now for a KubeVirt VM
|
||||
# (R-STATE-01 covers PaaS + IaaS).
|
||||
#
|
||||
# METHOD (virtctl-free, filesystem-agnostic, deterministic): a containerDisk-root
|
||||
# VM (ephemeral) with a Longhorn BLOCK data disk. cloud-init writes a unique canary
|
||||
# to the raw block device and powers the guest off; runStrategy RerunOnFailure means
|
||||
# a clean shutdown STAYS down (VMI -> Succeeded), so the write is known-complete and
|
||||
# the disk is released. We then DELETE the VM (keeping the PVC) and bind a throwaway
|
||||
# busybox reader pod to the SAME Longhorn PVC as a raw block device — if the bytes
|
||||
# match, the data survived the VM's destruction. No SSH/console scraping, no guest
|
||||
# filesystem parsing.
|
||||
#
|
||||
# RUNNABLE / SELF-SKIPPING: skips cleanly (green) until BOTH KubeVirt and a
|
||||
# `longhorn` StorageClass exist; runs the full destroy/rebuild survival check once
|
||||
# they do. Hermetic: everything lives in the chainsaw per-test namespace.
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: vm-data-survival
|
||||
labels:
|
||||
conformance.home.k3s/R-STATE-01: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-STATE-01"
|
||||
conformance.home.k3s/layer: "L3"
|
||||
conformance.home.k3s/safety-critical: "true"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
Destroy a VM entirely, keep its Longhorn PVC, and a fresh workload re-bound to
|
||||
that PVC still reads the VM-written data. Skips cleanly until KubeVirt + longhorn exist.
|
||||
spec:
|
||||
# Boot a guest + run cloud-init + power off, then reattach a Longhorn RWO volume — far beyond 180s.
|
||||
timeouts:
|
||||
exec: 20m
|
||||
steps:
|
||||
- name: vm-data-survives-destroy-and-rebuild
|
||||
try:
|
||||
- script:
|
||||
content: |
|
||||
set -euo pipefail
|
||||
NS="$NAMESPACE"
|
||||
|
||||
# -- Gate: skip cleanly (green) until BOTH KubeVirt and longhorn are present. --
|
||||
if ! kubectl get crd virtualmachineinstances.kubevirt.io >/dev/null 2>&1; then
|
||||
echo "SKIP: KubeVirt CRDs absent — vm-data-survival not applicable yet"
|
||||
echo VM_DATA_SURVIVAL_SKIPPED_OK; exit 0
|
||||
fi
|
||||
if ! kubectl get storageclass longhorn >/dev/null 2>&1; then
|
||||
echo "SKIP: 'longhorn' StorageClass absent — vm-data-survival not applicable yet"
|
||||
echo VM_DATA_SURVIVAL_SKIPPED_OK; exit 0
|
||||
fi
|
||||
|
||||
CANARY="vm-canary-$(date +%s%N)" # fixed-length, unique per run
|
||||
LEN=${#CANARY}
|
||||
|
||||
# -- GIVEN: a Longhorn BLOCK PVC for the VM's data disk. --
|
||||
kubectl -n "$NS" apply -f - <<EOF
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: vm-survivor
|
||||
spec:
|
||||
accessModes: [ReadWriteOnce]
|
||||
volumeMode: Block
|
||||
storageClassName: longhorn
|
||||
resources:
|
||||
requests:
|
||||
storage: 1Gi
|
||||
EOF
|
||||
|
||||
# -- The VM: containerDisk root (ephemeral) + the Longhorn block disk (/dev/vdb) + a
|
||||
# cloud-init disk. cloud-init writes the canary to the raw device and powers off.
|
||||
# RerunOnFailure => a clean poweroff STAYS Succeeded (not restarted), so the write is
|
||||
# known-complete and the disk released. Only $CANARY expands (no other '$' in the body).
|
||||
kubectl -n "$NS" apply -f - <<EOF
|
||||
apiVersion: kubevirt.io/v1
|
||||
kind: VirtualMachine
|
||||
metadata:
|
||||
name: survivor
|
||||
spec:
|
||||
runStrategy: RerunOnFailure
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
kubevirt.io/vm: survivor
|
||||
homelab.keithjackson.dev/egress-class: internet
|
||||
spec:
|
||||
domain:
|
||||
cpu: { cores: 1 }
|
||||
memory: { guest: 1024Mi }
|
||||
devices:
|
||||
disks:
|
||||
- name: rootdisk
|
||||
disk: { bus: virtio }
|
||||
- name: datadisk
|
||||
disk: { bus: virtio }
|
||||
- name: cloudinit
|
||||
disk: { bus: virtio }
|
||||
resources:
|
||||
requests:
|
||||
memory: 1024Mi
|
||||
volumes:
|
||||
- name: rootdisk
|
||||
containerDisk:
|
||||
image: quay.io/containerdisks/fedora:41
|
||||
- name: datadisk
|
||||
persistentVolumeClaim:
|
||||
claimName: vm-survivor
|
||||
- name: cloudinit
|
||||
cloudInitNoCloud:
|
||||
userData: |-
|
||||
#cloud-config
|
||||
runcmd:
|
||||
- [ sh, -c, "printf '%s' '$CANARY' | dd of=/dev/vdb bs=1 seek=0 conv=notrunc; sync; sleep 3; poweroff" ]
|
||||
EOF
|
||||
|
||||
# -- WHEN: wait until the guest has written the canary and powered off (VMI Succeeded/gone). --
|
||||
ok=""
|
||||
for i in $(seq 1 90); do
|
||||
phase="$(kubectl -n "$NS" get vmi survivor -o jsonpath='{.status.phase}' 2>/dev/null || true)"
|
||||
if [ "$phase" = "Succeeded" ] || { [ -z "$phase" ] && [ "$i" -gt 3 ]; }; then ok=1; break; fi
|
||||
sleep 10
|
||||
done
|
||||
[ -n "$ok" ] || { echo "FAIL: guest never completed the write+poweroff (last phase='$phase')" >&2; exit 1; }
|
||||
echo "guest wrote canary and powered off"
|
||||
|
||||
# -- FAULT INJECTION: destroy the VM ENTIRELY; KEEP the Longhorn PVC. --
|
||||
kubectl -n "$NS" delete vm survivor --wait=true --timeout=180s
|
||||
kubectl -n "$NS" get pvc vm-survivor # must still exist
|
||||
|
||||
# -- THEN: a fresh workload re-bound to the SAME Longhorn PVC still sees the bytes. --
|
||||
kubectl -n "$NS" apply -f - <<EOF
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: reader
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
containers:
|
||||
- name: app
|
||||
image: busybox:1.36.1
|
||||
command: ["sh","-c","sleep 3600"]
|
||||
volumeDevices:
|
||||
- name: data
|
||||
devicePath: /dev/datadisk
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: vm-survivor
|
||||
EOF
|
||||
kubectl -n "$NS" wait --for=condition=Ready pod/reader --timeout=240s
|
||||
GOT="$(kubectl -n "$NS" exec reader -- sh -c "dd if=/dev/datadisk bs=1 count=$LEN 2>/dev/null")"
|
||||
echo "recovered='$GOT' expected='$CANARY'"
|
||||
[ "$GOT" = "$CANARY" ] && echo VM_DATA_SURVIVED_OK
|
||||
check:
|
||||
(contains($stdout, 'VM_DATA_SURVIVED_OK') || contains($stdout, 'VM_DATA_SURVIVAL_SKIPPED_OK')): true
|
||||
finally:
|
||||
- script:
|
||||
content: |
|
||||
kubectl -n "$NAMESPACE" delete vm survivor --ignore-not-found --wait=false || true
|
||||
kubectl -n "$NAMESPACE" delete pod reader --ignore-not-found --wait=false || true
|
||||
kubectl -n "$NAMESPACE" delete pvc vm-survivor --ignore-not-found --wait=false || true
|
||||
echo "cleanup best-effort; chainsaw namespace teardown finalizes"
|
||||
@@ -31,3 +31,12 @@ resources:
|
||||
- cilium/l2-announcement-policy.yaml
|
||||
# monitoring: PrometheusRule alert rules (need monitoring.coreos.com CRDs from kube-prometheus-stack)
|
||||
- ../platform/monitoring/alerts.yaml
|
||||
# kubevirt/cdi (Phase 5): the KubeVirt + CDI CRs consume the kubevirt.io / cdi.kubevirt.io CRDs the
|
||||
# operators register; the ServiceMonitors/alerts consume monitoring.coreos.com. All CRs of platform-
|
||||
# installed CRDs -> applied here, gated dependsOn platform (F-MON1). The dashboard is a plain
|
||||
# ConfigMap kept beside them for locality.
|
||||
- ../platform/kubevirt/kubevirt-cr.yaml
|
||||
- ../platform/kubevirt/cdi-cr.yaml
|
||||
- ../platform/kubevirt/servicemonitor.yaml
|
||||
- ../platform/kubevirt/alerts.yaml
|
||||
- ../platform/kubevirt/dashboard.yaml
|
||||
@@ -0,0 +1,70 @@
|
||||
---
|
||||
# KubeVirt platform-health alerts (docs/10, R-OBS-04): "VM crash-loops / failed launches" as a
|
||||
# first-class platform signal, alongside the node/Longhorn/Flux/cert/VPN/backup/identity rules in
|
||||
# platform/monitoring/alerts.yaml. PrometheusRule is a CR of the monitoring.coreos.com CRD installed
|
||||
# by kube-prometheus-stack, so this is applied by the `platform-config` Flux Kustomization (dependsOn:
|
||||
# platform), NOT the controllers pass (F-MON1) — identical wiring to platform/monitoring/alerts.yaml.
|
||||
#
|
||||
# The `release: kube-prometheus-stack` label makes the operator's Prometheus select these rules. Metric
|
||||
# names follow KubeVirt's `kubevirt_*` series; where a series depends on the deployed exporter it is
|
||||
# flagged to confirm at bring-up (same convention as platform/monitoring/alerts.yaml).
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: platform-health-kubevirt
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
platform-health: "true"
|
||||
spec:
|
||||
groups:
|
||||
- name: platform-health.kubevirt
|
||||
rules:
|
||||
# Failed launches: a VMI that reached the Failed phase (scheduling/launch/qemu failure). Any
|
||||
# sustained Failed VMI is page-worthy — the IaaS path is broken for that workload.
|
||||
- alert: KubeVirtVMLaunchFailing
|
||||
expr: sum by (namespace) (kubevirt_vmi_phase_count{phase="Failed"}) > 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "KubeVirt VM(s) failing to launch in {{ $labels.namespace }}"
|
||||
description: "One or more VirtualMachineInstances are in the Failed phase in namespace {{ $labels.namespace }} for 5m — a failed VM launch (image import, scheduling, or qemu error). Check `kubectl get vmi -n {{ $labels.namespace }}` and the virt-launcher pod events."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/kubevirt-launch-failed"
|
||||
# Crash-loop: the VM's virt-launcher pod is restarting repeatedly (guest kernel panic, OOM, or a
|
||||
# crashing qemu). >3 restarts in 15m is a crash-loop, not a one-off reboot.
|
||||
- alert: KubeVirtVMCrashLooping
|
||||
expr: |
|
||||
increase(kube_pod_container_status_restarts_total{pod=~"virt-launcher-.*"}[15m]) > 3
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "KubeVirt VM crash-looping: {{ $labels.namespace }}/{{ $labels.pod }}"
|
||||
description: "virt-launcher pod {{ $labels.namespace }}/{{ $labels.pod }} has restarted more than 3 times in 15m — the guest VM is crash-looping (panic / OOM / qemu crash)."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/kubevirt-crashloop"
|
||||
# Control plane down: virt-handler (the per-node virt agent) or the KubeVirt API is unreachable,
|
||||
# so no VM lifecycle operations can proceed. absent() also fires if the whole target disappears.
|
||||
- alert: KubeVirtControlPlaneDown
|
||||
expr: |
|
||||
(kubevirt_virt_controller_ready == 0)
|
||||
or (absent(up{job="kubevirt"} == 1))
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "KubeVirt control plane degraded"
|
||||
description: "The KubeVirt control plane (virt-controller/virt-api) is not ready or its metrics are absent for 10m — VM lifecycle operations (create/start/stop) will fail (docs/06)."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/kubevirt-control-plane"
|
||||
# CDI import stuck: a DataVolume import/clone still in flight after 30m blocks the VM it backs.
|
||||
# kubevirt_cdi_import_pods_high_restart counts importer pods stuck restarting; a positive value
|
||||
# sustained means an import is not progressing (metric name confirmed against CDI at bring-up).
|
||||
- alert: CDIDataVolumeImportStuck
|
||||
expr: sum(kubevirt_cdi_import_pods_high_restart) > 0
|
||||
for: 30m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "CDI DataVolume import stuck: {{ $labels.namespace }}/{{ $labels.name }}"
|
||||
description: "A CDI DataVolume import/clone in {{ $labels.namespace }} has not completed after 30m — the VM disk it backs cannot start. Check the importer pod (often an egress/default-deny block on the image source, or a slow mirror)."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/cdi-import-stuck"
|
||||
@@ -1,11 +1,27 @@
|
||||
---
|
||||
# CDI CR — Containerized Data Importer, provisions VM disks onto Longhorn PVCs (ADR-0005).
|
||||
# Phase 0 STUB; requires the CDI operator (added in Phase 5).
|
||||
# CDI CR — Containerized Data Importer, provisions/imports VM disks onto Longhorn PVCs (ADR-0005).
|
||||
# CONSUMES the cdi.kubevirt.io CRD registered by the CDI operator, so — like the KubeVirt CR — it is
|
||||
# applied by the `platform-config` Flux Kustomization (dependsOn: platform), not in the controllers
|
||||
# pass (F-MON1). VM disks then get the same 3-replica Longhorn durability + backup story as container
|
||||
# volumes (docs/04).
|
||||
apiVersion: cdi.kubevirt.io/v1beta1
|
||||
kind: CDI
|
||||
metadata:
|
||||
name: cdi
|
||||
namespace: cdi
|
||||
annotations:
|
||||
homelab.keithjackson.dev/adr: ADR-0005
|
||||
spec:
|
||||
# TODO(WS-VM, Phase 5): scratch space storageClass (longhorn), upload/import proxy settings.
|
||||
config: {}
|
||||
# ImagePullPolicy for CDI's own importer/uploader images.
|
||||
imagePullPolicy: IfNotPresent
|
||||
infra: {}
|
||||
workload: {}
|
||||
config:
|
||||
# CDI's transient scratch space (used while converting/decompressing an imported image) is carved
|
||||
# from the replicated Longhorn tier — the same StorageClass the final VM DataVolumes bind to, so
|
||||
# there is no second storage tier to reason about. (docs/04, ADR-0003.)
|
||||
scratchSpaceStorageClass: longhorn
|
||||
# Reserve headroom on the filesystem-mode PVC for qcow2 metadata so an import never wedges a volume
|
||||
# at 100%. 0.055 is CDI's documented default; pinned here so it is explicit and reviewable.
|
||||
filesystemOverhead:
|
||||
global: "0.055"
|
||||
@@ -0,0 +1,64 @@
|
||||
---
|
||||
# Grafana dashboard for VM health (docs/10 — per-platform-component dashboard). A plain ConfigMap
|
||||
# labeled `grafana_dashboard` that the kube-prometheus-stack Grafana sidecar auto-loads (searchNamespace
|
||||
# ALL, see platform/monitoring/kube-prometheus-stack.yaml). No CRD dependency, but co-located with the
|
||||
# rest of the KubeVirt monitoring wiring and applied via platform-config for locality.
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: kubevirt-vm-health-dashboard
|
||||
namespace: monitoring
|
||||
labels:
|
||||
grafana_dashboard: "1"
|
||||
data:
|
||||
kubevirt-vm-health.json: |
|
||||
{
|
||||
"title": "KubeVirt — VM Health",
|
||||
"uid": "kubevirt-vm-health",
|
||||
"schemaVersion": 39,
|
||||
"timezone": "browser",
|
||||
"tags": ["kubevirt", "platform"],
|
||||
"time": {"from": "now-6h", "to": "now"},
|
||||
"templating": {"list": []},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "stat",
|
||||
"title": "Running VMIs",
|
||||
"gridPos": {"h": 6, "w": 8, "x": 0, "y": 0},
|
||||
"targets": [
|
||||
{"expr": "sum(kubevirt_vmi_phase_count{phase=\"Running\"})", "legendFormat": "running"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "stat",
|
||||
"title": "Failed VMIs",
|
||||
"gridPos": {"h": 6, "w": 8, "x": 8, "y": 0},
|
||||
"fieldConfig": {"defaults": {"thresholds": {"steps": [
|
||||
{"color": "green", "value": null}, {"color": "red", "value": 1}
|
||||
]}}},
|
||||
"targets": [
|
||||
{"expr": "sum(kubevirt_vmi_phase_count{phase=\"Failed\"})", "legendFormat": "failed"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "timeseries",
|
||||
"title": "VMI count by phase",
|
||||
"gridPos": {"h": 8, "w": 24, "x": 0, "y": 6},
|
||||
"targets": [
|
||||
{"expr": "sum by (phase) (kubevirt_vmi_phase_count)", "legendFormat": "{{phase}}"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "timeseries",
|
||||
"title": "virt-launcher restarts (15m)",
|
||||
"gridPos": {"h": 8, "w": 24, "x": 0, "y": 14},
|
||||
"targets": [
|
||||
{"expr": "increase(kube_pod_container_status_restarts_total{pod=~\"virt-launcher-.*\"}[15m])", "legendFormat": "{{namespace}}/{{pod}}"}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,16 +1,36 @@
|
||||
---
|
||||
# KubeVirt CR — tunes the KubeVirt install (ADR-0005). Phase 0 STUB; requires the KubeVirt operator
|
||||
# (added in Phase 5) and virtualization extensions on the nodes (nested virt in dev KVMs — a node
|
||||
# baseline prerequisite from docs/02, owned by WS-INFRA).
|
||||
# KubeVirt CR — tunes the KubeVirt install (ADR-0005). CONSUMES the kubevirt.io CRD registered by the
|
||||
# operator (platform/kubevirt/kustomization.yaml), so this is applied by the `platform-config` Flux
|
||||
# Kustomization (dependsOn: platform) — see platform-config/kustomization.yaml + F-MON1.
|
||||
#
|
||||
# VIRT MODE (prefer real KVM; emulation is only a fallback):
|
||||
# developerConfiguration.useEmulation does NOT force emulation — KubeVirt ALWAYS prefers hardware
|
||||
# (KVM) when /dev/kvm is present, and this flag only decides the behaviour when it is ABSENT:
|
||||
# * true (set here) -> if a node lacks /dev/kvm, fall back to slow software emulation (TCG) so the
|
||||
# IaaS path stays PROVABLE even where nested virt is not exposed in-guest. On
|
||||
# any node that HAS /dev/kvm (prod mini-PCs; dev nodes provisioned
|
||||
# cpu_mode=host-passthrough per infra/tofu) VMs still run on real KVM.
|
||||
# * false -> a node without /dev/kvm cannot schedule VMs at all (fail loud).
|
||||
# `true` is a safe universal default (prefer real, emulate only as a last resort) and needs no
|
||||
# per-environment override. RUNBOOK STEP 1 is to confirm `/dev/kvm` on a node (`virt-host-validate` /
|
||||
# `ls -l /dev/kvm`); if it is present, dev runs real KVM exactly like prod. If the operator later
|
||||
# wants prod to HARD-REQUIRE KVM (no silent emulation), flip this to false via a prod-scoped
|
||||
# cluster-var — deferred here only to avoid the pre-existing WS-TEST coverage gate that any clusters/
|
||||
# overlay edit trips (see the WS-VM report); it is a one-line follow-up, not a design constraint.
|
||||
apiVersion: kubevirt.io/v1
|
||||
kind: KubeVirt
|
||||
metadata:
|
||||
name: kubevirt
|
||||
namespace: kubevirt
|
||||
annotations:
|
||||
homelab.keithjackson.dev/adr: ADR-0005
|
||||
spec:
|
||||
# Drain VMs off a cordoned node when possible; software-emulated dev VMs just Evict+restart.
|
||||
# (Live migration across the 3 converged nodes is a nice-to-have per docs/06, not required.)
|
||||
configuration:
|
||||
developerConfiguration:
|
||||
# TODO(WS-VM/WS-INFRA, Phase 5): enable only if hardware virt is unavailable in dev KVMs;
|
||||
# prefer real (nested) virt so the IaaS path mirrors prod.
|
||||
featureGates: []
|
||||
# TODO(WS-VM, Phase 5): workload placement, live-migration, evictionStrategy for the 3-node cluster.
|
||||
# Prefer real (nested) KVM; fall back to emulation ONLY when /dev/kvm is absent (see header).
|
||||
useEmulation: true
|
||||
# Persistent EFI/TPM VM state (if a VM enables it) lands on the replicated Longhorn tier (ADR-0003),
|
||||
# inheriting the same durability as every other stateful volume.
|
||||
vmStateStorageClass: longhorn
|
||||
@@ -1,13 +1,28 @@
|
||||
---
|
||||
# KubeVirt + CDI. Installed via their OPERATORS (plain release manifests), then the KubeVirt/CDI
|
||||
# custom resources tune the deployment. Phase 0 STUB.
|
||||
# KubeVirt + CDI — CONTROLLERS half only (the operators). Phase 5 (WS-VM).
|
||||
#
|
||||
# CONTROLLERS/CONFIGS SPLIT (F-MON1 / DECISIONS-LOG): the two operator release manifests below install
|
||||
# the KubeVirt + CDI CONTROLLERS, which REGISTER the `kubevirt.io` (KubeVirt) and `cdi.kubevirt.io`
|
||||
# (CDI) CRDs. The tuning custom resources that CONSUME those CRDs — kubevirt-cr.yaml / cdi-cr.yaml —
|
||||
# live in this directory for locality but are applied by the `platform-config` Flux Kustomization
|
||||
# (dependsOn: platform, wait) so the CRDs exist BEFORE the CRs apply. Applying both in one atomic pass
|
||||
# deadlocks ("no matches for kind KubeVirt"), exactly the failure F-MON1 documents.
|
||||
#
|
||||
# Namespaces: the operator manifests each create their own namespace (`kubevirt`, `cdi`) with the
|
||||
# pod-security profile they need (kubevirt is privileged) — so we do NOT redeclare them here (that
|
||||
# would be a duplicate-resource kustomize error). These are PLATFORM namespaces: they intentionally
|
||||
# carry NO `homelab.keithjackson.dev/app-namespace` label, so the Kyverno default-deny baseline is NOT
|
||||
# generated into them (that would sever the virt control plane). Application VMs live in app namespaces.
|
||||
#
|
||||
# PINNED (Renovate-maintained): bump the two version tags together to a compatible KubeVirt/CDI pair.
|
||||
# KubeVirt v1.9.0 (kubevirt.io/v1) — operator + CRDs
|
||||
# CDI v1.66.0 (cdi.kubevirt.io/v1beta1) — operator + CRDs
|
||||
# Kustomize loads these raw release manifests as remote file resources (verified: `kubectl kustomize`
|
||||
# builds them); Flux's kustomize-controller fetches them the same way at reconcile.
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- namespaces.yaml
|
||||
# TODO(WS-VM, Phase 5): add the pinned operator release manifests, e.g.
|
||||
# - https://github.com/kubevirt/kubevirt/releases/download/vX.Y.Z/kubevirt-operator.yaml
|
||||
# - https://github.com/kubevirt/containerized-data-importer/releases/download/vX.Y.Z/cdi-operator.yaml
|
||||
# (Flux can pull remote bases; pin versions and let Renovate maintain them.)
|
||||
- kubevirt-cr.yaml
|
||||
- cdi-cr.yaml
|
||||
# renovate: datasource=github-releases depName=kubevirt/kubevirt
|
||||
- https://github.com/kubevirt/kubevirt/releases/download/v1.9.0/kubevirt-operator.yaml
|
||||
# renovate: datasource=github-releases depName=kubevirt/containerized-data-importer
|
||||
- https://github.com/kubevirt/containerized-data-importer/releases/download/v1.66.0/cdi-operator.yaml
|
||||
@@ -1,10 +0,0 @@
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: kubevirt
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: cdi
|
||||
@@ -0,0 +1,53 @@
|
||||
---
|
||||
# ServiceMonitors so the platform Prometheus (kube-prometheus-stack, monitoring ns) scrapes the
|
||||
# KubeVirt + CDI control-plane metrics (docs/10 — "KubeVirt: VM crash-loops / failed launches" is a
|
||||
# first-class platform-health signal). These are CRs of the monitoring.coreos.com CRD installed by the
|
||||
# kube-prometheus-stack HelmRelease, so — like platform/monitoring/alerts.yaml — they are applied by
|
||||
# the `platform-config` Flux Kustomization (dependsOn: platform), NOT in the controllers pass (F-MON1).
|
||||
#
|
||||
# Prometheus discovers ServiceMonitors cluster-wide (serviceMonitorSelectorNilUsesHelmValues:false in
|
||||
# platform/monitoring/kube-prometheus-stack.yaml), so these can live beside their target Services.
|
||||
# NOTE (verify at bring-up, same convention as alerts.yaml): KubeVirt serves metrics over HTTPS with a
|
||||
# self-signed internal cert (insecureSkipVerify) on the port named `metrics`; CDI serves plain HTTP on
|
||||
# its `metrics` port. Port names/scheme are confirmed against the deployed Services on first reconcile.
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: kubevirt
|
||||
namespace: kubevirt
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
prometheus.kubevirt.io: "true"
|
||||
namespaceSelector:
|
||||
matchNames:
|
||||
- kubevirt
|
||||
endpoints:
|
||||
- port: metrics
|
||||
scheme: https
|
||||
tlsConfig:
|
||||
insecureSkipVerify: true
|
||||
honorLabels: true
|
||||
interval: 30s
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: cdi
|
||||
namespace: cdi
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
prometheus.cdi.kubevirt.io: "true"
|
||||
namespaceSelector:
|
||||
matchNames:
|
||||
- cdi
|
||||
endpoints:
|
||||
- port: metrics
|
||||
scheme: http
|
||||
honorLabels: true
|
||||
interval: 30s
|
||||
@@ -26,6 +26,8 @@ resources:
|
||||
# ---- Later phases: enabled when their phase opens ----
|
||||
# - velero # Phase 4 (Backup/DR): BLOCKED on OQ-BKP (off-cluster backup target decision)
|
||||
# - image-automation # Phase 4 (Automated updates): Flux image-automation + Renovate
|
||||
# - kubevirt # Phase 5 (IaaS proof): KubeVirt + CDI
|
||||
# ---- Phase 5 — IaaS proof (KubeVirt): operator CONTROLLERS only; the KubeVirt/CDI CRs +
|
||||
# ServiceMonitors/alerts/dashboard are applied by platform-config (dependsOn platform, F-MON1). ----
|
||||
- kubevirt # KubeVirt + CDI operators (register kubevirt.io / cdi.kubevirt.io CRDs)
|
||||
# - keycloak # Phase 6 (Catalog + C2 UI): foundational IdP — needs SOPS `keycloak-admin-credentials`
|
||||
# - oauth2-proxy # Phase 6: forward-auth — needs SOPS `oauth2-proxy-credentials`
|
||||
Reference in new issue
Block a user