diff --git a/assets/gpu-feature-discovery/0500_daemonset.yaml b/assets/gpu-feature-discovery/0500_daemonset.yaml index 765a84e7a5..b7915d96dc 100644 --- a/assets/gpu-feature-discovery/0500_daemonset.yaml +++ b/assets/gpu-feature-discovery/0500_daemonset.yaml @@ -29,7 +29,7 @@ spec: - name: toolkit-validation image: "FILLED BY THE OPERATOR" command: ['sh', '-c'] - args: ["until [ -f /run/nvidia/validations/toolkit-ready ]; do echo waiting for nvidia container stack to be setup; sleep 5; done"] + args: ["until [ -f /run/nvidia/validations/toolkit-ready ] && { grep -q '^nvidia ' /proc/modules || [ -e /dev/dxg ]; }; do echo waiting for nvidia container stack to be setup; sleep 5; done"] securityContext: privileged: true volumeMounts: diff --git a/assets/state-dcgm-exporter/0800_daemonset.yaml b/assets/state-dcgm-exporter/0800_daemonset.yaml index eb0dab1635..ba16c9a868 100644 --- a/assets/state-dcgm-exporter/0800_daemonset.yaml +++ b/assets/state-dcgm-exporter/0800_daemonset.yaml @@ -29,7 +29,7 @@ spec: - name: toolkit-validation image: "FILLED BY THE OPERATOR" command: ['sh', '-c'] - args: ["until [ -f /run/nvidia/validations/toolkit-ready ]; do echo waiting for nvidia container stack to be setup; sleep 5; done"] + args: ["until [ -f /run/nvidia/validations/toolkit-ready ] && { grep -q '^nvidia ' /proc/modules || [ -e /dev/dxg ]; }; do echo waiting for nvidia container stack to be setup; sleep 5; done"] securityContext: privileged: true volumeMounts: diff --git a/assets/state-dcgm/0400_dcgm.yml b/assets/state-dcgm/0400_dcgm.yml index 0c27fd4e60..6f5b348f6a 100644 --- a/assets/state-dcgm/0400_dcgm.yml +++ b/assets/state-dcgm/0400_dcgm.yml @@ -28,7 +28,7 @@ spec: - name: toolkit-validation image: "FILLED BY THE OPERATOR" command: ['sh', '-c'] - args: ["until [ -f /run/nvidia/validations/toolkit-ready ]; do echo waiting for nvidia container stack to be setup; sleep 5; done"] + args: ["until [ -f /run/nvidia/validations/toolkit-ready ] && { grep -q '^nvidia ' /proc/modules || [ -e /dev/dxg ]; }; do echo waiting for nvidia container stack to be setup; sleep 5; done"] securityContext: privileged: true volumeMounts: diff --git a/assets/state-device-plugin/0500_daemonset.yaml b/assets/state-device-plugin/0500_daemonset.yaml index c4d85adfba..a592d2a59f 100644 --- a/assets/state-device-plugin/0500_daemonset.yaml +++ b/assets/state-device-plugin/0500_daemonset.yaml @@ -28,7 +28,7 @@ spec: - image: "FILLED BY THE OPERATOR" name: toolkit-validation command: ['sh', '-c'] - args: ["until [ -f /run/nvidia/validations/toolkit-ready ]; do echo waiting for nvidia container stack to be setup; sleep 5; done"] + args: ["until [ -f /run/nvidia/validations/toolkit-ready ] && { grep -q '^nvidia ' /proc/modules || [ -e /dev/dxg ]; }; do echo waiting for nvidia container stack to be setup; sleep 5; done"] securityContext: privileged: true volumeMounts: diff --git a/assets/state-mig-manager/0600_daemonset.yaml b/assets/state-mig-manager/0600_daemonset.yaml index 48315b4226..75c7450524 100644 --- a/assets/state-mig-manager/0600_daemonset.yaml +++ b/assets/state-mig-manager/0600_daemonset.yaml @@ -28,7 +28,7 @@ spec: - name: toolkit-validation image: "FILLED BY THE OPERATOR" command: ['sh', '-c'] - args: ["until [ -f /run/nvidia/validations/toolkit-ready ]; do echo waiting for nvidia container toolkit to be setup; sleep 5; done"] + args: ["until [ -f /run/nvidia/validations/toolkit-ready ] && { grep -q '^nvidia ' /proc/modules || [ -e /dev/dxg ]; }; do echo waiting for nvidia container stack to be setup; sleep 5; done"] securityContext: privileged: true volumeMounts: diff --git a/assets/state-mps-control-daemon/0400_daemonset.yaml b/assets/state-mps-control-daemon/0400_daemonset.yaml index 3be58af20f..391c8be03e 100644 --- a/assets/state-mps-control-daemon/0400_daemonset.yaml +++ b/assets/state-mps-control-daemon/0400_daemonset.yaml @@ -30,7 +30,7 @@ spec: - image: "FILLED BY THE OPERATOR" name: toolkit-validation command: ['sh', '-c'] - args: ["until [ -f /run/nvidia/validations/toolkit-ready ]; do echo waiting for nvidia container stack to be setup; sleep 5; done"] + args: ["until [ -f /run/nvidia/validations/toolkit-ready ] && { grep -q '^nvidia ' /proc/modules || [ -e /dev/dxg ]; }; do echo waiting for nvidia container stack to be setup; sleep 5; done"] securityContext: privileged: true volumeMounts: diff --git a/controllers/object_controls_test.go b/controllers/object_controls_test.go index 9c1bc9c7dd..76986542d2 100644 --- a/controllers/object_controls_test.go +++ b/controllers/object_controls_test.go @@ -1260,6 +1260,45 @@ func TestVGPUManagerAssets(t *testing.T) { } } +// TestOperandToolkitReadinessWaitsForLiveDriver verifies that every operand +// launched through the NVIDIA runtime waits for a live Linux NVIDIA module, or +// the supported WSL2 kernel path, as well as the toolkit status file. +func TestOperandToolkitReadinessWaitsForLiveDriver(t *testing.T) { + manifests := []string{ + "assets/gpu-feature-discovery/0500_daemonset.yaml", + "assets/state-dcgm-exporter/0800_daemonset.yaml", + "assets/state-dcgm/0400_dcgm.yml", + "assets/state-device-plugin/0500_daemonset.yaml", + "assets/state-mig-manager/0600_daemonset.yaml", + "assets/state-mps-control-daemon/0400_daemonset.yaml", + } + + for _, manifest := range manifests { + t.Run(filepath.Base(filepath.Dir(manifest)), func(t *testing.T) { + buffer, err := os.ReadFile(filepath.Join(cfg.root, manifest)) + require.NoError(t, err) + + ds := appsv1.DaemonSet{} + ser := json.NewSerializerWithOptions(json.DefaultMetaFactory, scheme.Scheme, scheme.Scheme, + json.SerializerOptions{Yaml: true, Pretty: false, Strict: false}) + _, _, err = ser.Decode(buffer, nil, &ds) + require.NoError(t, err) + + var args string + for _, initContainer := range ds.Spec.Template.Spec.InitContainers { + if initContainer.Name == "toolkit-validation" { + args = strings.Join(initContainer.Args, " ") + break + } + } + require.NotEmpty(t, args, "toolkit-validation init container not found") + expectedGate := "until [ -f /run/nvidia/validations/toolkit-ready ] && { grep -q '^nvidia ' /proc/modules || [ -e /dev/dxg ]; }; do" + require.True(t, strings.HasPrefix(args, expectedGate), + "toolkit readiness and a supported live-driver check must gate operand startup") + }) + } +} + // TestVGPUDeviceManagerReadinessGate verifies that the vGPU Device Manager's // vgpu-manager-validation init container waits for the vGPU Manager readiness // status file written in BOTH deployment modes: vgpu-manager-ready (vGPU