Run validation as part of vgpu device manager init container

assets/state-vgpu-device-manager/0600_daemonset.yaml

@@ -28,14 +28,29 @@ spec:
         - name: vgpu-manager-validation
           image: "FILLED BY THE OPERATOR"
           command: ['sh', '-c']
-          # The validator writes vgpu-manager-ready when the vGPU Manager is
-          # deployed as a container, and host-vgpu-manager-ready when the vGPU
-          # Manager driver is pre-installed on the host. Wait for either so this
-          # operand starts in both cases.
-          args: ["until [ -f /run/nvidia/validations/vgpu-manager-ready ] || [ -f /run/nvidia/validations/host-vgpu-manager-ready ]; do echo waiting for NVIDIA vGPU Manager to be setup; sleep 5; done"]
+          args: ["nvidia-validator"]
+          env:
+          - name: WITH_WAIT
+            value: "true"
+          - name: COMPONENT
+            value: vgpu-manager
+          - name: NODE_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: spec.nodeName
           securityContext:
             privileged: true
+            seLinuxOptions:
+              level: "s0"
           volumeMounts:
+            - name: host-root
+              mountPath: /host
+              readOnly: true
+              mountPropagation: HostToContainer
+            - name: driver-install-dir
+              mountPath: /run/nvidia/driver
+              readOnly: true
+              mountPropagation: HostToContainer
             - name: run-nvidia-validations
               mountPath: /run/nvidia/validations
               mountPropagation: Bidirectional

controllers/object_controls.go

@@ -3935,6 +3935,21 @@ func transformValidationInitContainer(obj *appsv1.DaemonSet, config *gpuv1.Clust
 			}
 		}
 
+		// Pass env for vgpu-manager-validation init container
+		if strings.HasPrefix(initContainer.Name, "vgpu-manager") {
+			setContainerEnv(&(obj.Spec.Template.Spec.InitContainers[i]), "DEFAULT_GPU_WORKLOAD_CONFIG", defaultGPUWorkloadConfig)
+			if len(config.Validator.VGPUManager.Env) > 0 {
+				for _, env := range config.Validator.VGPUManager.Env {
+					// Preserve the required execution settings defined by the manifest.
+					switch env.Name {
+					case "COMPONENT", "NODE_NAME", "WITH_WAIT":
+						continue
+					}
+					setContainerEnv(&(obj.Spec.Template.Spec.InitContainers[i]), env.Name, env.Value)
+				}
+			}
+		}
+
 		// update validation image
 		image, err := gpuv1.ImagePath(&config.Validator)
 		if err != nil {

controllers/object_controls_test.go

@@ -1259,14 +1259,10 @@ func TestVGPUManagerAssets(t *testing.T) {
 	}
 }
 
-// TestVGPUDeviceManagerReadinessGate verifies that the vGPU Device Manager's
-// vgpu-manager-validation init container waits for the vGPU Manager readiness
-// status file written in BOTH deployment modes: vgpu-manager-ready (vGPU
-// Manager deployed as a container) and host-vgpu-manager-ready (vGPU Manager
-// driver pre-installed on the host). The validator writes only the host-
-// prefixed file when the driver is pre-installed, so a gate that waits for the
-// container-managed file alone hangs indefinitely when driver.enabled=false.
-func TestVGPUDeviceManagerReadinessGate(t *testing.T) {
+// TestVGPUDeviceManagerValidationInitContainer verifies that the vGPU Device
+// Manager validates the vGPU Manager directly instead of depending on a status
+// file produced by the sandbox validator.
+func TestVGPUDeviceManagerValidationInitContainer(t *testing.T) {
 	manifestPath := filepath.Join(cfg.root, vGPUDeviceManagerAssetsPath, "0600_daemonset.yaml")
 	buffer, err := os.ReadFile(manifestPath)
 	require.NoError(t, err, "unable to read vGPU Device Manager daemonset asset")
@@ -1286,16 +1282,63 @@ func TestVGPUDeviceManagerReadinessGate(t *testing.T) {
 	}
 	require.NotNil(t, initCtr, "vgpu-manager-validation init container not found")
 
-	args := strings.Join(initCtr.Args, " ")
-	require.Contains(t, args, "/run/nvidia/validations/vgpu-manager-ready",
-		"readiness gate must wait for the container-managed vGPU Manager status file")
-	require.Contains(t, args, "/run/nvidia/validations/host-vgpu-manager-ready",
-		"readiness gate must also wait for the host-installed vGPU Manager status file (driver.enabled=false)")
-	// The two files must be combined with OR: the gate must pass when EITHER
-	// status file exists, since the validator only ever writes one of them.
-	// Guard against an accidental AND, which would re-break both modes.
-	require.Contains(t, args, "|| [ -f /run/nvidia/validations/host-vgpu-manager-ready",
-		"the two status files must be combined with OR, not AND, so the gate passes when either is present")
+	require.Equal(t, []string{"sh", "-c"}, initCtr.Command)
+	require.Equal(t, []string{"nvidia-validator"}, initCtr.Args)
+
+	withWait := findEnvVarByName(initCtr.Env, "WITH_WAIT")
+	require.NotNil(t, withWait)
+	require.Equal(t, "true", withWait.Value)
+	component := findEnvVarByName(initCtr.Env, "COMPONENT")
+	require.NotNil(t, component)
+	require.Equal(t, "vgpu-manager", component.Value)
+
+	nodeName := findEnvVarByName(initCtr.Env, "NODE_NAME")
+	require.NotNil(t, nodeName)
+	require.NotNil(t, nodeName.ValueFrom)
+	require.NotNil(t, nodeName.ValueFrom.FieldRef)
+	require.Equal(t, "spec.nodeName", nodeName.ValueFrom.FieldRef.FieldPath)
+
+	require.NotNil(t, initCtr.SecurityContext)
+	require.NotNil(t, initCtr.SecurityContext.SELinuxOptions)
+	require.Equal(t, "s0", initCtr.SecurityContext.SELinuxOptions.Level)
+
+	hostRoot := findVolumeMountByName(initCtr.VolumeMounts, "host-root")
+	require.NotNil(t, hostRoot)
+	require.Equal(t, "/host", hostRoot.MountPath)
+	require.True(t, hostRoot.ReadOnly)
+	require.NotNil(t, hostRoot.MountPropagation)
+	require.Equal(t, corev1.MountPropagationHostToContainer, *hostRoot.MountPropagation)
+
+	driverInstallDir := findVolumeMountByName(initCtr.VolumeMounts, "driver-install-dir")
+	require.NotNil(t, driverInstallDir)
+	require.Equal(t, "/run/nvidia/driver", driverInstallDir.MountPath)
+	require.True(t, driverInstallDir.ReadOnly)
+	require.NotNil(t, driverInstallDir.MountPropagation)
+	require.Equal(t, corev1.MountPropagationHostToContainer, *driverInstallDir.MountPropagation)
+
+	validations := findVolumeMountByName(initCtr.VolumeMounts, "run-nvidia-validations")
+	require.NotNil(t, validations)
+	require.Equal(t, "/run/nvidia/validations", validations.MountPath)
+	require.NotNil(t, validations.MountPropagation)
+	require.Equal(t, corev1.MountPropagationBidirectional, *validations.MountPropagation)
+}
+
+func findEnvVarByName(envVars []corev1.EnvVar, name string) *corev1.EnvVar {
+	for i := range envVars {
+		if envVars[i].Name == name {
+			return &envVars[i]
+		}
+	}
+	return nil
+}
+
+func findVolumeMountByName(volumeMounts []corev1.VolumeMount, name string) *corev1.VolumeMount {
+	for i := range volumeMounts {
+		if volumeMounts[i].Name == name {
+			return &volumeMounts[i]
+		}
+	}
+	return nil
 }
 
 // getSandboxDevicePluginTestInput return a ClusterPolicy instance for a particular

controllers/transforms_test.go

@@ -2279,6 +2279,12 @@ func TestTransformCCManager(t *testing.T) {
 }
 
 func TestTransformVGPUDeviceManager(t *testing.T) {
+	originalDefaultWorkload := defaultGPUWorkloadConfig
+	defaultGPUWorkloadConfig = "container"
+	t.Cleanup(func() {
+		defaultGPUWorkloadConfig = originalDefaultWorkload
+	})
+
 	resources := corev1.ResourceRequirements{
 		Limits: corev1.ResourceList{
 			corev1.ResourceCPU:    resource.MustParse("100m"),
@@ -2301,9 +2307,38 @@ func TestTransformVGPUDeviceManager(t *testing.T) {
 		{
 			description: "transform vgpu device manager",
 			daemonset: NewDaemonset().
+				WithInitContainer(corev1.Container{
+					Name: "vgpu-manager-validation",
+					Env: []corev1.EnvVar{
+						{Name: "WITH_WAIT", Value: "true"},
+						{Name: "COMPONENT", Value: "vgpu-manager"},
+						{
+							Name: "NODE_NAME",
+							ValueFrom: &corev1.EnvVarSource{
+								FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"},
+							},
+						},
+					},
+				}).
 				WithContainer(corev1.Container{Name: "nvidia-vgpu-device-manager"}).
 				WithContainer(corev1.Container{Name: "sidecar"}),
 			clusterPolicySpec: &gpuv1.ClusterPolicySpec{
+				Validator: gpuv1.ValidatorSpec{
+					Repository:       "nvcr.io/nvidia/cloud-native",
+					Image:            "gpu-operator-validator",
+					Version:          "v1.0.0",
+					ImagePullPolicy:  "IfNotPresent",
+					ImagePullSecrets: []string{secret},
+					Resources:        &gpuv1.ResourceRequirements{Limits: resources.Limits, Requests: resources.Requests},
+					VGPUManager: gpuv1.VGPUManagerValidatorSpec{
+						Env: []gpuv1.EnvVar{
+							{Name: "foo", Value: "bar"},
+							{Name: "COMPONENT", Value: "cuda"},
+							{Name: "NODE_NAME", Value: "invalid-override"},
+							{Name: "WITH_WAIT", Value: "false"},
+						},
+					},
+				},
 				VGPUDeviceManager: gpuv1.VGPUDeviceManagerSpec{
 					Repository:       "nvcr.io/nvidia/cloud-native",
 					Image:            "vgpu-device-manager",
@@ -2320,6 +2355,27 @@ func TestTransformVGPUDeviceManager(t *testing.T) {
 				},
 			},
 			expectedDaemonset: NewDaemonset().
+				WithInitContainer(corev1.Container{
+					Name:            "vgpu-manager-validation",
+					Image:           "nvcr.io/nvidia/cloud-native/gpu-operator-validator:v1.0.0",
+					ImagePullPolicy: corev1.PullIfNotPresent,
+					Env: []corev1.EnvVar{
+						{Name: "WITH_WAIT", Value: "true"},
+						{Name: "COMPONENT", Value: "vgpu-manager"},
+						{
+							Name: "NODE_NAME",
+							ValueFrom: &corev1.EnvVarSource{
+								FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"},
+							},
+						},
+						{Name: "DEFAULT_GPU_WORKLOAD_CONFIG", Value: "container"},
+						{Name: "foo", Value: "bar"},
+					},
+					Resources: resources,
+					SecurityContext: &corev1.SecurityContext{
+						RunAsUser: rootUID,
+					},
+				}).
 				WithContainer(corev1.Container{
 					Name:            "nvidia-vgpu-device-manager",
 					Image:           "nvcr.io/nvidia/cloud-native/vgpu-device-manager:v1.0.0",