diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/lifecycle.go b/images/virtualization-artifact/pkg/controller/vm/internal/lifecycle.go index bcd0e1ebcf..aaa96a20be 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/lifecycle.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/lifecycle.go @@ -23,8 +23,10 @@ import ( "log/slog" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" virtv1 "kubevirt.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -245,19 +247,65 @@ func (h *LifeCycleHandler) syncRunning(ctx context.Context, vm *v1alpha2.Virtual } func (h *LifeCycleHandler) checkVMPodVolumeErrors(ctx context.Context, vm *v1alpha2.VirtualMachine, log *slog.Logger) error { - var podList corev1.PodList - err := h.client.List(ctx, &podList, &client.ListOptions{ + var pods []corev1.Pod + processedPodNames := make(map[string]struct{}) + + var launcherPods corev1.PodList + err := h.client.List(ctx, &launcherPods, &client.ListOptions{ Namespace: vm.Namespace, LabelSelector: labels.SelectorFromSet(map[string]string{ virtv1.VirtualMachineNameLabel: vm.Name, }), }) if err != nil { - log.Error("Failed to list pods", "error", err) + log.Error("Failed to list launcher pods", "error", err) + return err + } + for _, pod := range launcherPods.Items { + if _, exists := processedPodNames[pod.Name]; exists { + continue + } + processedPodNames[pod.Name] = struct{}{} + pods = append(pods, pod) + } + + kvvmi := &virtv1.VirtualMachineInstance{} + err = h.client.Get(ctx, types.NamespacedName{Name: vm.Name, Namespace: vm.Namespace}, kvvmi) + if err != nil && !apierrors.IsNotFound(err) { + log.Error("Failed to get KVVMI for hotplug pod resolution", "error", err) return err } + if err == nil { + for _, vs := range kvvmi.Status.VolumeStatus { + if vs.HotplugVolume == nil || vs.HotplugVolume.AttachPodName == "" { + continue + } + if _, exists := processedPodNames[vs.HotplugVolume.AttachPodName]; exists { + continue + } + + attachPod := &corev1.Pod{} + err := h.client.Get(ctx, types.NamespacedName{ + Name: vs.HotplugVolume.AttachPodName, + Namespace: vm.Namespace, + }, attachPod) + if err != nil { + if apierrors.IsNotFound(err) { + continue + } + log.Error("Failed to get hotplug pod", "error", err, "pod", vs.HotplugVolume.AttachPodName) + return err + } + if vs.HotplugVolume.AttachPodUID != "" && attachPod.UID != vs.HotplugVolume.AttachPodUID { + continue + } + + processedPodNames[attachPod.Name] = struct{}{} + pods = append(pods, *attachPod) + } + } - for _, pod := range podList.Items { + for _, pod := range pods { if !podutil.IsContainerCreating(&pod) { continue } @@ -266,7 +314,7 @@ func (h *LifeCycleHandler) checkVMPodVolumeErrors(ctx context.Context, vm *v1alp log.Error("Failed to get last pod event", "error", err) return err } - if lastEvent != nil && (lastEvent.Reason == watcher.ReasonFailedAttachVolume || lastEvent.Reason == watcher.ReasonFailedMount) { + if lastEvent != nil && watcher.IsVolumeErrorReason(lastEvent.Reason) { return &VMPodVolumeError{ Reason: lastEvent.Reason, Message: lastEvent.Message, diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/lifecycle_hotplug_pod_errors_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/lifecycle_hotplug_pod_errors_test.go new file mode 100644 index 0000000000..6eea6cd8ab --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/vm/internal/lifecycle_hotplug_pod_errors_test.go @@ -0,0 +1,142 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package internal + +import ( + "context" + "errors" + "log/slog" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + virtv1 "kubevirt.io/api/core/v1" + + "github.com/deckhouse/virtualization-controller/pkg/common/testutil" + "github.com/deckhouse/virtualization-controller/pkg/controller/vm/internal/watcher" + "github.com/deckhouse/virtualization/api/core/v1alpha2" +) + +var _ = Describe("LifeCycleHandler hotplug pod errors", func() { + newContainerCreatingPod := func(vm *v1alpha2.VirtualMachine, name string, labels map[string]string) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: vm.Namespace, + Labels: labels, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodPending, + ContainerStatuses: []corev1.ContainerStatus{ + { + State: corev1.ContainerState{ + Waiting: &corev1.ContainerStateWaiting{ + Reason: "ContainerCreating", + }, + }, + }, + }, + }, + } + } + newVolumeErrorEvent := func(vm *v1alpha2.VirtualMachine, podName string) *corev1.Event { + return &corev1.Event{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName + ".1", + Namespace: vm.Namespace, + }, + InvolvedObject: corev1.ObjectReference{ + Kind: "Pod", + Name: podName, + Namespace: vm.Namespace, + }, + Type: corev1.EventTypeWarning, + Reason: watcher.ReasonFailedMount, + Message: "unable to mount volume", + LastTimestamp: metav1.NewTime(time.Now()), + } + } + + It("should return volume errors for launcher pod label", func() { + vm := &v1alpha2.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vm", + Namespace: "default", + }, + } + pod := newContainerCreatingPod(vm, "vm-pod", map[string]string{ + virtv1.VirtualMachineNameLabel: vm.Name, + }) + event := newVolumeErrorEvent(vm, pod.Name) + + fakeClient, err := testutil.NewFakeClientWithObjects(vm, pod, event) + Expect(err).NotTo(HaveOccurred()) + handler := NewLifeCycleHandler(fakeClient, nil) + + err = handler.checkVMPodVolumeErrors(context.Background(), vm, slog.Default()) + Expect(err).To(HaveOccurred()) + + var volumeErr *VMPodVolumeError + Expect(errors.As(err, &volumeErr)).To(BeTrue()) + Expect(volumeErr.Reason).To(Equal(watcher.ReasonFailedMount)) + Expect(volumeErr.Message).To(Equal("unable to mount volume")) + }) + + It("should return volume errors for hotplug pod resolved by kvvmi status", func() { + vm := &v1alpha2.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vm", + Namespace: "default", + }, + } + hotplugPod := newContainerCreatingPod(vm, "hp-pod", nil) + hotplugPod.UID = types.UID("hp-pod-uid") + event := newVolumeErrorEvent(vm, hotplugPod.Name) + kvvmi := &virtv1.VirtualMachineInstance{ + ObjectMeta: metav1.ObjectMeta{ + Name: vm.Name, + Namespace: vm.Namespace, + }, + Status: virtv1.VirtualMachineInstanceStatus{ + VolumeStatus: []virtv1.VolumeStatus{ + { + Name: "vd-hotplug", + HotplugVolume: &virtv1.HotplugVolumeStatus{ + AttachPodName: hotplugPod.Name, + AttachPodUID: hotplugPod.UID, + }, + }, + }, + }, + } + + fakeClient, err := testutil.NewFakeClientWithObjects(vm, kvvmi, hotplugPod, event) + Expect(err).NotTo(HaveOccurred()) + handler := NewLifeCycleHandler(fakeClient, nil) + + err = handler.checkVMPodVolumeErrors(context.Background(), vm, slog.Default()) + Expect(err).To(HaveOccurred()) + + var volumeErr *VMPodVolumeError + Expect(errors.As(err, &volumeErr)).To(BeTrue()) + Expect(volumeErr.Reason).To(Equal(watcher.ReasonFailedMount)) + Expect(volumeErr.Message).To(Equal("unable to mount volume")) + }) +}) diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/watcher/volumeevent_watcher.go b/images/virtualization-artifact/pkg/controller/vm/internal/watcher/volumeevent_watcher.go index de57362f09..2fee7fb79d 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/watcher/volumeevent_watcher.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/watcher/volumeevent_watcher.go @@ -36,8 +36,18 @@ import ( const ( ReasonFailedAttachVolume = "FailedAttachVolume" ReasonFailedMount = "FailedMount" + ReasonFailedMapVolume = "FailedMapVolume" ) +func IsVolumeErrorReason(reason string) bool { + switch reason { + case ReasonFailedAttachVolume, ReasonFailedMount, ReasonFailedMapVolume: + return true + default: + return false + } +} + func NewVolumeEventWatcher(client client.Client) *VolumeEventWatcher { return &VolumeEventWatcher{ client: client, @@ -48,6 +58,46 @@ type VolumeEventWatcher struct { client client.Client } +func getVirtualMachineNameFromPodLabels(pod *corev1.Pod) (string, bool) { + if pod == nil { + return "", false + } + + if vmName, hasLabel := pod.GetLabels()[virtv1.VirtualMachineNameLabel]; hasLabel { + return vmName, true + } + + return "", false +} + +func (w *VolumeEventWatcher) resolveVMNameByHotplugStatus(ctx context.Context, pod *corev1.Pod) (string, bool) { + if pod == nil { + return "", false + } + + var kvvmiList virtv1.VirtualMachineInstanceList + if err := w.client.List(ctx, &kvvmiList, client.InNamespace(pod.Namespace)); err != nil { + return "", false + } + + for _, kvvmi := range kvvmiList.Items { + for _, volumeStatus := range kvvmi.Status.VolumeStatus { + if volumeStatus.HotplugVolume == nil { + continue + } + + if volumeStatus.HotplugVolume.AttachPodUID != "" && volumeStatus.HotplugVolume.AttachPodUID == pod.UID { + return kvvmi.Name, true + } + if volumeStatus.HotplugVolume.AttachPodName == pod.Name { + return kvvmi.Name, true + } + } + } + + return "", false +} + func (w *VolumeEventWatcher) Watch(mgr manager.Manager, ctr controller.Controller) error { if err := ctr.Watch( source.Kind( @@ -58,7 +108,7 @@ func (w *VolumeEventWatcher) Watch(mgr manager.Manager, ctr controller.Controlle return nil } - if e.Reason != ReasonFailedAttachVolume && e.Reason != ReasonFailedMount { + if !IsVolumeErrorReason(e.Reason) { return nil } @@ -70,8 +120,11 @@ func (w *VolumeEventWatcher) Watch(mgr manager.Manager, ctr controller.Controlle return nil } - vmName, hasLabel := pod.GetLabels()[virtv1.VirtualMachineNameLabel] - if !hasLabel { + vmName, ok := getVirtualMachineNameFromPodLabels(pod) + if !ok { + vmName, ok = w.resolveVMNameByHotplugStatus(ctx, pod) + } + if !ok { return nil } @@ -87,7 +140,7 @@ func (w *VolumeEventWatcher) Watch(mgr manager.Manager, ctr controller.Controlle predicate.TypedFuncs[*corev1.Event]{ CreateFunc: func(e event.TypedCreateEvent[*corev1.Event]) bool { return e.Object.Type == corev1.EventTypeWarning && - (e.Object.Reason == ReasonFailedAttachVolume || e.Object.Reason == ReasonFailedMount) + IsVolumeErrorReason(e.Object.Reason) }, UpdateFunc: func(e event.TypedUpdateEvent[*corev1.Event]) bool { return false