Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Feature] Ray container must be the first application container #1379

Merged
merged 3 commits into from
Aug 31, 2023
Merged

[Feature] Ray container must be the first application container #1379

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e330a4a
update
kevin85421 Aug 31, 2023
860765d
update
kevin85421 Aug 31, 2023
d116d78
update
kevin85421 Aug 31, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions ray-operator/controllers/ray/common/constant.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,9 @@ const (
RayClusterServingServiceLabelKey = "ray.io/serve"
RayServiceClusterHashKey = "ray.io/cluster-hash"

// In KubeRay, the Ray container must be the first application container in a head or worker Pod.
RayContainerIndex = 0

// Batch scheduling labels
// TODO(tgaddair): consider making these part of the CRD
RaySchedulerName = "ray.io/scheduler-name"
Expand Down
81 changes: 30 additions & 51 deletions ray-operator/controllers/ray/common/pod.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,6 @@ func DefaultHeadPodTemplate(instance rayv1alpha1.RayCluster, headSpec rayv1alpha
headSpec.RayStartParams = setMissingRayStartParams(headSpec.RayStartParams, rayv1alpha1.HeadNode, headPort, "", instance.Annotations)

initTemplateAnnotations(instance, &podTemplate)
rayContainerIndex := getRayContainerIndex(podTemplate.Spec)

// if in-tree autoscaling is enabled, then autoscaler container should be injected into head pod.
if instance.Spec.EnableInTreeAutoscaling != nil && *instance.Spec.EnableInTreeAutoscaling {
Expand All @@ -116,7 +115,7 @@ func DefaultHeadPodTemplate(instance rayv1alpha1.RayCluster, headSpec rayv1alpha
// set custom service account with proper roles bound.
// utils.CheckName clips the name to match the behavior of reconcileAutoscalerServiceAccount
podTemplate.Spec.ServiceAccountName = utils.CheckName(utils.GetHeadGroupServiceAccountName(&instance))
rayHeadImage := podTemplate.Spec.Containers[rayContainerIndex].Image
rayHeadImage := podTemplate.Spec.Containers[RayContainerIndex].Image
// Determine the default image to use for the Ray container.
autoscalerImage := getAutoscalerImage(rayHeadImage, instance.Spec.RayVersion)
// inject autoscaler container into head pod
Expand All @@ -127,13 +126,13 @@ func DefaultHeadPodTemplate(instance rayv1alpha1.RayCluster, headSpec rayv1alpha
}

// If the metrics port does not exist in the Ray container, add a default one for Promethues.
isMetricsPortExists := utils.FindContainerPort(&podTemplate.Spec.Containers[rayContainerIndex], DefaultMetricsName, -1) != -1
isMetricsPortExists := utils.FindContainerPort(&podTemplate.Spec.Containers[RayContainerIndex], DefaultMetricsName, -1) != -1
if !isMetricsPortExists {
metricsPort := v1.ContainerPort{
Name: DefaultMetricsName,
ContainerPort: int32(DefaultMetricsPort),
}
podTemplate.Spec.Containers[rayContainerIndex].Ports = append(podTemplate.Spec.Containers[rayContainerIndex].Ports, metricsPort)
podTemplate.Spec.Containers[RayContainerIndex].Ports = append(podTemplate.Spec.Containers[RayContainerIndex].Ports, metricsPort)
}

return podTemplate
Expand Down Expand Up @@ -194,18 +193,16 @@ func DefaultWorkerPodTemplate(instance rayv1alpha1.RayCluster, workerSpec rayv1a
}

// The Ray worker should only start once the GCS server is ready.
rayContainerIndex := getRayContainerIndex(podTemplate.Spec)

// only inject init container only when ENABLE_INIT_CONTAINER_INJECTION is true
enableInitContainerInjection := getEnableInitContainerInjection()

if enableInitContainerInjection {
// Do not modify `deepCopyRayContainer` anywhere.
deepCopyRayContainer := podTemplate.Spec.Containers[rayContainerIndex].DeepCopy()
deepCopyRayContainer := podTemplate.Spec.Containers[RayContainerIndex].DeepCopy()
initContainer := v1.Container{
Name: "wait-gcs-ready",
Image: podTemplate.Spec.Containers[rayContainerIndex].Image,
ImagePullPolicy: podTemplate.Spec.Containers[rayContainerIndex].ImagePullPolicy,
Image: podTemplate.Spec.Containers[RayContainerIndex].Image,
ImagePullPolicy: podTemplate.Spec.Containers[RayContainerIndex].ImagePullPolicy,
Command: []string{"/bin/bash", "-lc", "--"},
Args: []string{
fmt.Sprintf(`
Expand All @@ -228,7 +225,7 @@ func DefaultWorkerPodTemplate(instance rayv1alpha1.RayCluster, workerSpec rayv1a
done
`, fqdnRayIP, headPort, fqdnRayIP, headPort),
},
SecurityContext: podTemplate.Spec.Containers[rayContainerIndex].SecurityContext.DeepCopy(),
SecurityContext: podTemplate.Spec.Containers[RayContainerIndex].SecurityContext.DeepCopy(),
// This init container requires certain environment variables to establish a secure connection with the Ray head using TLS authentication.
// Additionally, some of these environment variables may reference files stored in volumes, so we need to include both the `Env` and `VolumeMounts` fields here.
// For more details, please refer to: https://docs.ray.io/en/latest/ray-core/configure.html#tls-authentication.
Expand All @@ -251,13 +248,13 @@ func DefaultWorkerPodTemplate(instance rayv1alpha1.RayCluster, workerSpec rayv1a
initTemplateAnnotations(instance, &podTemplate)

// If the metrics port does not exist in the Ray container, add a default one for Promethues.
isMetricsPortExists := utils.FindContainerPort(&podTemplate.Spec.Containers[rayContainerIndex], DefaultMetricsName, -1) != -1
isMetricsPortExists := utils.FindContainerPort(&podTemplate.Spec.Containers[RayContainerIndex], DefaultMetricsName, -1) != -1
if !isMetricsPortExists {
metricsPort := v1.ContainerPort{
Name: DefaultMetricsName,
ContainerPort: int32(DefaultMetricsPort),
}
podTemplate.Spec.Containers[rayContainerIndex].Ports = append(podTemplate.Spec.Containers[rayContainerIndex].Ports, metricsPort)
podTemplate.Spec.Containers[RayContainerIndex].Ports = append(podTemplate.Spec.Containers[RayContainerIndex].Ports, metricsPort)
}

return podTemplate
Expand Down Expand Up @@ -303,35 +300,34 @@ func BuildPod(podTemplateSpec v1.PodTemplateSpec, rayNodeType rayv1alpha1.RayNod
ObjectMeta: podTemplateSpec.ObjectMeta,
Spec: podTemplateSpec.Spec,
}
rayContainerIndex := getRayContainerIndex(pod.Spec)

// Add /dev/shm volumeMount for the object store to avoid performance degradation.
addEmptyDir(&pod.Spec.Containers[rayContainerIndex], &pod, SharedMemoryVolumeName, SharedMemoryVolumeMountPath, v1.StorageMediumMemory)
addEmptyDir(&pod.Spec.Containers[RayContainerIndex], &pod, SharedMemoryVolumeName, SharedMemoryVolumeMountPath, v1.StorageMediumMemory)
if rayNodeType == rayv1alpha1.HeadNode && enableRayAutoscaler != nil && *enableRayAutoscaler {
// The Ray autoscaler writes logs which are read by the Ray head.
// We need a shared log volume to enable this information flow.
// Specifically, this is required for the event-logging functionality
// introduced in https://github.com/ray-project/ray/pull/13434.
autoscalerContainerIndex := getAutoscalerContainerIndex(pod)
addEmptyDir(&pod.Spec.Containers[rayContainerIndex], &pod, RayLogVolumeName, RayLogVolumeMountPath, v1.StorageMediumDefault)
addEmptyDir(&pod.Spec.Containers[RayContainerIndex], &pod, RayLogVolumeName, RayLogVolumeMountPath, v1.StorageMediumDefault)
addEmptyDir(&pod.Spec.Containers[autoscalerContainerIndex], &pod, RayLogVolumeName, RayLogVolumeMountPath, v1.StorageMediumDefault)
}
cleanupInvalidVolumeMounts(&pod.Spec.Containers[rayContainerIndex], &pod)
if len(pod.Spec.InitContainers) > rayContainerIndex {
cleanupInvalidVolumeMounts(&pod.Spec.InitContainers[rayContainerIndex], &pod)
cleanupInvalidVolumeMounts(&pod.Spec.Containers[RayContainerIndex], &pod)
if len(pod.Spec.InitContainers) > RayContainerIndex {
cleanupInvalidVolumeMounts(&pod.Spec.InitContainers[RayContainerIndex], &pod)
}

var cmd, args string
if len(pod.Spec.Containers[rayContainerIndex].Command) > 0 {
cmd = convertCmdToString(pod.Spec.Containers[rayContainerIndex].Command)
if len(pod.Spec.Containers[RayContainerIndex].Command) > 0 {
cmd = convertCmdToString(pod.Spec.Containers[RayContainerIndex].Command)
}
if len(pod.Spec.Containers[rayContainerIndex].Args) > 0 {
cmd += convertCmdToString(pod.Spec.Containers[rayContainerIndex].Args)
if len(pod.Spec.Containers[RayContainerIndex].Args) > 0 {
cmd += convertCmdToString(pod.Spec.Containers[RayContainerIndex].Args)
}
if !strings.Contains(cmd, "ray start") {
cont := concatenateContainerCommand(rayNodeType, rayStartParams, pod.Spec.Containers[rayContainerIndex].Resources)
cont := concatenateContainerCommand(rayNodeType, rayStartParams, pod.Spec.Containers[RayContainerIndex].Resources)
// replacing the old command
pod.Spec.Containers[rayContainerIndex].Command = []string{"/bin/bash", "-lc", "--"}
pod.Spec.Containers[RayContainerIndex].Command = []string{"/bin/bash", "-lc", "--"}
if cmd != "" {
// If 'ray start' has --block specified, commands after it will not get executed.
// so we need to put cmd before cont.
Expand All @@ -345,43 +341,43 @@ func BuildPod(podTemplateSpec v1.PodTemplateSpec, rayNodeType rayv1alpha1.RayNod
args = args + " && sleep infinity"
}

pod.Spec.Containers[rayContainerIndex].Args = []string{args}
pod.Spec.Containers[RayContainerIndex].Args = []string{args}
}

for index := range pod.Spec.InitContainers {
setInitContainerEnvVars(&pod.Spec.InitContainers[index], fqdnRayIP)
}

setContainerEnvVars(&pod, rayContainerIndex, rayNodeType, rayStartParams, fqdnRayIP, headPort, creator)
setContainerEnvVars(&pod, rayNodeType, rayStartParams, fqdnRayIP, headPort, creator)

// health check only if FT enabled
if podTemplateSpec.Annotations != nil {
if enabledString, ok := podTemplateSpec.Annotations[RayFTEnabledAnnotationKey]; ok {
if strings.ToLower(enabledString) == "true" {
// If users do not specify probes, we will set the default probes.
if pod.Spec.Containers[rayContainerIndex].ReadinessProbe == nil {
if pod.Spec.Containers[RayContainerIndex].ReadinessProbe == nil {
probe := &v1.Probe{
InitialDelaySeconds: DefaultReadinessProbeInitialDelaySeconds,
TimeoutSeconds: DefaultReadinessProbeTimeoutSeconds,
PeriodSeconds: DefaultReadinessProbePeriodSeconds,
SuccessThreshold: DefaultReadinessProbeSuccessThreshold,
FailureThreshold: DefaultReadinessProbeFailureThreshold,
}
pod.Spec.Containers[rayContainerIndex].ReadinessProbe = probe
pod.Spec.Containers[RayContainerIndex].ReadinessProbe = probe
}
initHealthProbe(pod.Spec.Containers[rayContainerIndex].ReadinessProbe, rayNodeType)
initHealthProbe(pod.Spec.Containers[RayContainerIndex].ReadinessProbe, rayNodeType)

if pod.Spec.Containers[rayContainerIndex].LivenessProbe == nil {
if pod.Spec.Containers[RayContainerIndex].LivenessProbe == nil {
probe := &v1.Probe{
InitialDelaySeconds: DefaultLivenessProbeInitialDelaySeconds,
TimeoutSeconds: DefaultLivenessProbeTimeoutSeconds,
PeriodSeconds: DefaultLivenessProbePeriodSeconds,
SuccessThreshold: DefaultLivenessProbeSuccessThreshold,
FailureThreshold: DefaultLivenessProbeFailureThreshold,
}
pod.Spec.Containers[rayContainerIndex].LivenessProbe = probe
pod.Spec.Containers[RayContainerIndex].LivenessProbe = probe
}
initHealthProbe(pod.Spec.Containers[rayContainerIndex].LivenessProbe, rayNodeType)
initHealthProbe(pod.Spec.Containers[RayContainerIndex].LivenessProbe, rayNodeType)
}
}
}
Expand Down Expand Up @@ -487,23 +483,6 @@ func convertCmdToString(cmdArr []string) (cmd string) {
return cmdAggr.String()
}

func getRayContainerIndex(podSpec v1.PodSpec) (rayContainerIndex int) {
// a ray pod can have multiple containers.
// we identify the ray container based on env var: RAY=true
// if the env var is missing, we choose containers[0].
for i, container := range podSpec.Containers {
for _, env := range container.Env {
if env.Name == strings.ToLower("ray") && env.Value == strings.ToLower("true") {
log.Info("Head pod container with index " + strconv.Itoa(i) + " identified as Ray container based on env RAY=true.")
return i
}
}
}
// not found, use first container
log.Info("Head pod container with index 0 identified as Ray container.")
return 0
}

func getAutoscalerContainerIndex(pod v1.Pod) (autoscalerContainerIndex int) {
// we identify the autoscaler container based on its name
for i, container := range pod.Spec.Containers {
Expand Down Expand Up @@ -568,9 +547,9 @@ func setInitContainerEnvVars(container *v1.Container, fqdnRayIP string) {
)
}

func setContainerEnvVars(pod *v1.Pod, rayContainerIndex int, rayNodeType rayv1alpha1.RayNodeType, rayStartParams map[string]string, fqdnRayIP string, headPort string, creator string) {
func setContainerEnvVars(pod *v1.Pod, rayNodeType rayv1alpha1.RayNodeType, rayStartParams map[string]string, fqdnRayIP string, headPort string, creator string) {
// TODO: Audit all environment variables to identify which should not be modified by users.
container := &pod.Spec.Containers[rayContainerIndex]
container := &pod.Spec.Containers[RayContainerIndex]
if container.Env == nil || len(container.Env) == 0 {
container.Env = []v1.EnvVar{}
}
Expand Down
24 changes: 11 additions & 13 deletions ray-operator/controllers/ray/common/pod_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -356,7 +356,7 @@ func TestBuildPod(t *testing.T) {
pod := BuildPod(podTemplateSpec, rayv1alpha1.HeadNode, cluster.Spec.HeadGroupSpec.RayStartParams, "6379", nil, "", "")

// Check environment variables
rayContainer := pod.Spec.Containers[getRayContainerIndex(pod.Spec)]
rayContainer := pod.Spec.Containers[RayContainerIndex]
checkContainerEnv(t, rayContainer, RAY_ADDRESS, "127.0.0.1:6379")
checkContainerEnv(t, rayContainer, RAY_USAGE_STATS_KUBERAY_IN_USE, "1")
checkContainerEnv(t, rayContainer, RAY_CLUSTER_NAME, fmt.Sprintf("metadata.labels['%s']", RayClusterLabelKey))
Expand Down Expand Up @@ -407,7 +407,7 @@ func TestBuildPod(t *testing.T) {
pod = BuildPod(podTemplateSpec, rayv1alpha1.WorkerNode, worker.RayStartParams, "6379", nil, "", fqdnRayIP)

// Check environment variables
rayContainer = pod.Spec.Containers[getRayContainerIndex(pod.Spec)]
rayContainer = pod.Spec.Containers[RayContainerIndex]
checkContainerEnv(t, rayContainer, RAY_ADDRESS, "raycluster-sample-head-svc.default.svc.cluster.local:6379")
checkContainerEnv(t, rayContainer, FQ_RAY_IP, "raycluster-sample-head-svc.default.svc.cluster.local")
checkContainerEnv(t, rayContainer, RAY_IP, "raycluster-sample-head-svc")
Expand All @@ -421,7 +421,7 @@ func TestBuildPod(t *testing.T) {
}

// Check Envs
rayContainer = pod.Spec.Containers[getRayContainerIndex(pod.Spec)]
rayContainer = pod.Spec.Containers[RayContainerIndex]
checkContainerEnv(t, rayContainer, "TEST_ENV_NAME", "TEST_ENV_VALUE")
}

Expand Down Expand Up @@ -519,8 +519,7 @@ func TestBuildPod_WithGcsFtEnabled(t *testing.T) {
pod := BuildPod(podTemplateSpec, rayv1alpha1.HeadNode, cluster.Spec.HeadGroupSpec.RayStartParams, "6379", nil, "", "")

// Check environment variable "RAY_GCS_RPC_SERVER_RECONNECT_TIMEOUT_S"
rayContainerIndex := getRayContainerIndex(pod.Spec)
rayContainer := pod.Spec.Containers[rayContainerIndex]
rayContainer := pod.Spec.Containers[RayContainerIndex]

// "RAY_GCS_RPC_SERVER_RECONNECT_TIMEOUT_S" should not be set on the head Pod by default
assert.True(t, !envVarExists(RAY_GCS_RPC_SERVER_RECONNECT_TIMEOUT_S, rayContainer.Env))
Expand All @@ -532,11 +531,11 @@ func TestBuildPod_WithGcsFtEnabled(t *testing.T) {
}

// Add "RAY_GCS_RPC_SERVER_RECONNECT_TIMEOUT_S" env var in the head group spec.
cluster.Spec.HeadGroupSpec.Template.Spec.Containers[rayContainerIndex].Env = append(cluster.Spec.HeadGroupSpec.Template.Spec.Containers[rayContainerIndex].Env,
cluster.Spec.HeadGroupSpec.Template.Spec.Containers[RayContainerIndex].Env = append(cluster.Spec.HeadGroupSpec.Template.Spec.Containers[RayContainerIndex].Env,
v1.EnvVar{Name: RAY_GCS_RPC_SERVER_RECONNECT_TIMEOUT_S, Value: "60"})
podTemplateSpec = DefaultHeadPodTemplate(*cluster, cluster.Spec.HeadGroupSpec, podName, "6379")
pod = BuildPod(podTemplateSpec, rayv1alpha1.HeadNode, cluster.Spec.HeadGroupSpec.RayStartParams, "6379", nil, "", "")
rayContainer = pod.Spec.Containers[rayContainerIndex]
rayContainer = pod.Spec.Containers[RayContainerIndex]

// Check environment variable "RAY_GCS_RPC_SERVER_RECONNECT_TIMEOUT_S"
checkContainerEnv(t, rayContainer, RAY_GCS_RPC_SERVER_RECONNECT_TIMEOUT_S, "60")
Expand All @@ -555,7 +554,7 @@ func TestBuildPod_WithGcsFtEnabled(t *testing.T) {
pod = BuildPod(podTemplateSpec, rayv1alpha1.WorkerNode, worker.RayStartParams, "6379", nil, "", fqdnRayIP)

// Check the default value of "RAY_GCS_RPC_SERVER_RECONNECT_TIMEOUT_S"
rayContainer = pod.Spec.Containers[rayContainerIndex]
rayContainer = pod.Spec.Containers[RayContainerIndex]
checkContainerEnv(t, rayContainer, RAY_GCS_RPC_SERVER_RECONNECT_TIMEOUT_S, DefaultWorkerRayGcsReconnectTimeoutS)

// Test 4
Expand All @@ -565,14 +564,14 @@ func TestBuildPod_WithGcsFtEnabled(t *testing.T) {
}

// Add "RAY_GCS_RPC_SERVER_RECONNECT_TIMEOUT_S" env var in the worker group spec.
cluster.Spec.WorkerGroupSpecs[0].Template.Spec.Containers[rayContainerIndex].Env = append(cluster.Spec.WorkerGroupSpecs[0].Template.Spec.Containers[rayContainerIndex].Env,
cluster.Spec.WorkerGroupSpecs[0].Template.Spec.Containers[RayContainerIndex].Env = append(cluster.Spec.WorkerGroupSpecs[0].Template.Spec.Containers[RayContainerIndex].Env,
v1.EnvVar{Name: RAY_GCS_RPC_SERVER_RECONNECT_TIMEOUT_S, Value: "120"})
worker = cluster.Spec.WorkerGroupSpecs[0]
podTemplateSpec = DefaultWorkerPodTemplate(*cluster, worker, podName, fqdnRayIP, "6379")
pod = BuildPod(podTemplateSpec, rayv1alpha1.WorkerNode, worker.RayStartParams, "6379", nil, "", fqdnRayIP)

// Check the default value of "RAY_GCS_RPC_SERVER_RECONNECT_TIMEOUT_S"
rayContainer = pod.Spec.Containers[rayContainerIndex]
rayContainer = pod.Spec.Containers[RayContainerIndex]
checkContainerEnv(t, rayContainer, RAY_GCS_RPC_SERVER_RECONNECT_TIMEOUT_S, "120")
}

Expand Down Expand Up @@ -889,7 +888,7 @@ func TestDefaultInitContainer(t *testing.T) {
// with the Ray head using TLS authentication. Currently, we simply copied all environment variables from
// Ray container to the init container. This may be changed in the future.
healthCheckContainer := podTemplateSpec.Spec.InitContainers[numInitContainers-1]
rayContainer := worker.Template.Spec.Containers[getRayContainerIndex(worker.Template.Spec)]
rayContainer := worker.Template.Spec.Containers[RayContainerIndex]

assert.NotEqual(t, len(rayContainer.Env), 0, "The test only makes sense if the Ray container has environment variables.")
assert.Equal(t, len(rayContainer.Env), len(healthCheckContainer.Env))
Expand Down Expand Up @@ -938,8 +937,7 @@ func TestDefaultInitContainerImagePullPolicy(t *testing.T) {
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
// set ray container imagePullPolicy
rayContainerIndex := getRayContainerIndex(worker.Template.Spec)
worker.Template.Spec.Containers[rayContainerIndex].ImagePullPolicy = tc.imagePullPolicy
worker.Template.Spec.Containers[RayContainerIndex].ImagePullPolicy = tc.imagePullPolicy

podTemplateSpec := DefaultWorkerPodTemplate(*cluster, *worker.DeepCopy(), podName, fqdnRayIP, "6379")

Expand Down
3 changes: 1 addition & 2 deletions ray-operator/controllers/ray/common/service.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -317,8 +317,7 @@ func getServicePorts(cluster rayv1alpha1.RayCluster) map[string]int32 {
func getPortsFromCluster(cluster rayv1alpha1.RayCluster) (map[string]int32, error) {
svcPorts := map[string]int32{}

index := utils.FindRayContainerIndex(cluster.Spec.HeadGroupSpec.Template.Spec)
cPorts := cluster.Spec.HeadGroupSpec.Template.Spec.Containers[index].Ports
cPorts := cluster.Spec.HeadGroupSpec.Template.Spec.Containers[RayContainerIndex].Ports
for _, port := range cPorts {
if port.Name == "" {
port.Name = fmt.Sprint(port.ContainerPort) + "-port"
Expand Down
4 changes: 1 addition & 3 deletions ray-operator/controllers/ray/common/service_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import (
"testing"

rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1"
"github.com/ray-project/kuberay/ray-operator/controllers/ray/utils"

"github.com/stretchr/testify/assert"

Expand Down Expand Up @@ -182,8 +181,7 @@ func TestGetPortsFromCluster(t *testing.T) {
svcNames[port] = name
}

index := utils.FindRayContainerIndex(instanceWithWrongSvc.Spec.HeadGroupSpec.Template.Spec)
cPorts := instanceWithWrongSvc.Spec.HeadGroupSpec.Template.Spec.Containers[index].Ports
cPorts := instanceWithWrongSvc.Spec.HeadGroupSpec.Template.Spec.Containers[RayContainerIndex].Ports

for _, cPort := range cPorts {
expectedResult := cPort.Name
Expand Down
Loading
Loading