Support enableIngress for RayCluster (#38)

* Support enableIngress for RayCluster

Add ingress resources in role

Copy more configurations from cluster annotation

Add ingress example

Update ingress version from v1beta1 to v1

* Update to expose dashboard only

docs/guidance/ingress.md

@@ -0,0 +1,24 @@
+## Ingress Usage
+
+### Prerequisite
+
+It's user's responsibility to install ingress controller by themselves. Technically, any ingress controller implementation should work well. 
+
+In order to pass through the customized ingress configuration, you can annotate `RayCluster` object and controller will pass to ingress object.
+
+`kubernetes.io/ingress.class` is recommended. 
+
+> Note: If the ingressClassName is omitted, a default Ingress class should be defined. Please make sure default ingress class is created.
+
+```
+apiVersion: ray.io/v1alpha1
+kind: RayCluster
+metadata:
+  annotations:
+    kubernetes.io/ingress.class: nginx -> this is required
+spec:
+  rayVersion: '1.9.2'
+  headGroupSpec:
+    serviceType: NodePort
+    enableIngress: true -> enables ingress
+```

ray-operator/config/rbac/role.yaml

@@ -74,3 +74,24 @@ rules:
   - get
   - patch
   - update
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - ingressclasses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - extensions
+  - networking.k8s.io
+  resources:
+  - ingresses
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch

ray-operator/config/samples/ray-cluster.ingress.yaml

@@ -0,0 +1,37 @@
+apiVersion: ray.io/v1alpha1
+kind: RayCluster
+metadata:
+  annotations:
+    kubernetes.io/ingress.class: nginx
+  name: raycluster-ingress
+spec:
+  rayVersion: '1.6.0' # should match the Ray version in the image of the containers
+  headGroupSpec:
+    serviceType: NodePort
+    enableIngress: true
+    replicas: 1
+    rayStartParams:
+      port: '6379'
+      redis-password: 'LetMeInRay'
+      dashboard-host: '0.0.0.0'
+      num-cpus: '1' # can be auto-completed from the limits
+      node-ip-address: $MY_POD_IP # auto-completed as the head pod IP
+    #pod template
+    template:
+      spec:
+        containers:
+        - name: ray-head
+          image: rayproject/ray:1.9.2
+          env:
+          - name: MY_POD_IP
+            valueFrom:
+              fieldRef:
+                fieldPath: status.podIP
+          ports:
+          - containerPort: 6379
+            name: redis
+          - containerPort: 8265 # Ray dashboard
+            name: dashboard
+          - containerPort: 10001
+            name: client
+

ray-operator/controllers/common/ingress.go

@@ -0,0 +1,81 @@
+package common
+
+import (
+	"fmt"
+
+	rayiov1alpha1 "github.com/ray-project/kuberay/ray-operator/api/raycluster/v1alpha1"
+	"github.com/ray-project/kuberay/ray-operator/controllers/utils"
+	"github.com/sirupsen/logrus"
+	networkingv1 "k8s.io/api/networking/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const IngressClassAnnotationKey = "kubernetes.io/ingress.class"
+
+// BuildIngressForHeadService Builds the ingress for head service dashboard.
+// This is used to expose dashboard for external traffic.
+func BuildIngressForHeadService(cluster rayiov1alpha1.RayCluster) (*networkingv1.Ingress, error) {
+	labels := map[string]string{
+		RayClusterLabelKey: cluster.Name,
+		RayIDLabelKey:      utils.GenerateIdentifier(cluster.Name, rayiov1alpha1.HeadNode),
+	}
+
+	// Copy other ingress configuration from cluster annotation
+	// This is to provide a generic way for user to customize their ingress settings.
+	annotation := map[string]string{}
+	for key, value := range cluster.Annotations {
+		annotation[key] = value
+	}
+
+	var paths []networkingv1.HTTPIngressPath
+	pathType := networkingv1.PathTypeExact
+	servicePorts := getServicePorts(cluster)
+	dashboardPort := int32(DefaultDashboardPort)
+	if port, ok := servicePorts["dashboard"]; ok {
+		dashboardPort = port
+	}
+	paths = []networkingv1.HTTPIngressPath{
+		networkingv1.HTTPIngressPath{
+			Path:     "/" + cluster.Name,
+			PathType: &pathType,
+			Backend: networkingv1.IngressBackend{
+				Service: &networkingv1.IngressServiceBackend{
+					Name: utils.GenerateServiceName(cluster.Name),
+					Port: networkingv1.ServiceBackendPort{
+						Number: dashboardPort,
+					},
+				},
+			},
+		},
+	}
+
+	ingress := &networkingv1.Ingress{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        utils.GenerateServiceName(cluster.Name),
+			Namespace:   cluster.Namespace,
+			Labels:      labels,
+			Annotations: annotation,
+		},
+		Spec: networkingv1.IngressSpec{
+			Rules: []networkingv1.IngressRule{
+				{
+					IngressRuleValue: networkingv1.IngressRuleValue{
+						HTTP: &networkingv1.HTTPIngressRuleValue{
+							Paths: paths,
+						},
+					},
+				},
+			},
+		},
+	}
+
+	// Get ingress class name from rayCluster annotations. this is a required field to use ingress.
+	ingressClassName, ok := cluster.Annotations[IngressClassAnnotationKey]
+	if !ok {
+		logrus.Warn(fmt.Sprintf("ingress class annotation is not set for cluster %s/%s", cluster.Namespace, cluster.Name))
+	} else {
+		ingress.Spec.IngressClassName = &ingressClassName
+	}
+
+	return ingress, nil
+}

ray-operator/controllers/common/ingress_test.go

@@ -0,0 +1,125 @@
+package common
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/ray-project/kuberay/ray-operator/controllers/utils"
+
+	rayiov1alpha1 "github.com/ray-project/kuberay/ray-operator/api/raycluster/v1alpha1"
+
+	"github.com/stretchr/testify/assert"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/utils/pointer"
+)
+
+var instanceWithIngressEnabled = &rayiov1alpha1.RayCluster{
+	ObjectMeta: metav1.ObjectMeta{
+		Name:      "raycluster-sample",
+		Namespace: "default",
+		Annotations: map[string]string{
+			IngressClassAnnotationKey: "nginx",
+		},
+	},
+	Spec: rayiov1alpha1.RayClusterSpec{
+		RayVersion: "1.0",
+		HeadGroupSpec: rayiov1alpha1.HeadGroupSpec{
+			Replicas: pointer.Int32Ptr(1),
+			Template: corev1.PodTemplateSpec{
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						corev1.Container{
+							Name:    "ray-head",
+							Image:   "rayproject/autoscaler",
+							Command: []string{"python"},
+							Args:    []string{"/opt/code.py"},
+						},
+					},
+				},
+			},
+		},
+	},
+}
+
+var instanceWithIngressEnabledWithoutIngressClass = &rayiov1alpha1.RayCluster{
+	ObjectMeta: metav1.ObjectMeta{
+		Name:      "raycluster-sample",
+		Namespace: "default",
+	},
+	Spec: rayiov1alpha1.RayClusterSpec{
+		RayVersion: "1.0",
+		HeadGroupSpec: rayiov1alpha1.HeadGroupSpec{
+			Replicas: pointer.Int32Ptr(1),
+			Template: corev1.PodTemplateSpec{
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						corev1.Container{
+							Name:    "ray-head",
+							Image:   "rayproject/autoscaler",
+							Command: []string{"python"},
+							Args:    []string{"/opt/code.py"},
+							Env: []corev1.EnvVar{
+								corev1.EnvVar{
+									Name: "MY_POD_IP",
+									ValueFrom: &corev1.EnvVarSource{
+										FieldRef: &corev1.ObjectFieldSelector{
+											FieldPath: "status.podIP",
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	},
+}
+
+// only throw warning message and rely on Kubernetes to assign default ingress class
+func TestBuildIngressForHeadServiceWithoutIngressClass(t *testing.T) {
+	ingress, err := BuildIngressForHeadService(*instanceWithIngressEnabledWithoutIngressClass)
+	assert.NotNil(t, ingress)
+	assert.Nil(t, err)
+}
+
+func TestBuildIngressForHeadService(t *testing.T) {
+	ingress, err := BuildIngressForHeadService(*instanceWithIngressEnabled)
+	assert.Nil(t, err)
+
+	// check ingress.class annotation
+	actualResult := ingress.Labels[RayClusterLabelKey]
+	expectedResult := instanceWithIngressEnabled.Name
+	if !reflect.DeepEqual(expectedResult, actualResult) {
+		t.Fatalf("Expected `%v` but got `%v`", expectedResult, actualResult)
+	}
+
+	actualResult = ingress.Annotations[IngressClassAnnotationKey]
+	expectedResult = instanceWithIngressEnabled.Annotations[IngressClassAnnotationKey]
+	if !reflect.DeepEqual(expectedResult, actualResult) {
+		t.Fatalf("Expected `%v` but got `%v`", expectedResult, actualResult)
+	}
+
+	// rules count
+	assert.Equal(t, 1, len(ingress.Spec.Rules))
+
+	// paths count
+	expectedPaths := 1 // dashboard only
+	actualPaths := len(ingress.Spec.Rules[0].IngressRuleValue.HTTP.Paths)
+	if !reflect.DeepEqual(expectedPaths, actualPaths) {
+		t.Fatalf("Expected `%v` but got `%v`", expectedPaths, actualPaths)
+	}
+
+	// path names
+	paths := ingress.Spec.Rules[0].IngressRuleValue.HTTP.Paths
+	for _, path := range paths {
+		actualResult = path.Backend.Service.Name
+		expectedResult = utils.GenerateServiceName(instanceWithIngressEnabled.Name)
+
+		if !reflect.DeepEqual(expectedResult, actualResult) {
+			t.Fatalf("Expected `%v` but got `%v`", expectedResult, actualResult)
+		}
+	}
+}

ray-operator/controllers/common/service.go

@@ -29,14 +29,7 @@ func BuildServiceForHeadPod(cluster rayiov1alpha1.RayCluster) (*corev1.Service,
 		},
 	}
 
-	ports, _ := getPortsFromCluster(cluster)
-	// Assign default ports
-	if len(ports) == 0 {
-		ports[DefaultClientPortName] = DefaultClientPort
-		ports[DefaultRedisPortName] = DefaultRedisPort
-		ports[DefaultDashboardName] = DefaultDashboardPort
-	}
-
+	ports := getServicePorts(cluster)
 	for name, port := range ports {
 		svcPort := corev1.ServicePort{Name: name, Port: port}
 		service.Spec.Ports = append(service.Spec.Ports, svcPort)
@@ -45,6 +38,17 @@ func BuildServiceForHeadPod(cluster rayiov1alpha1.RayCluster) (*corev1.Service,
 	return service, nil
 }
 
+// getServicePorts will either user passing ports or default ports to create service.
+func getServicePorts(cluster rayiov1alpha1.RayCluster) map[string]int32 {
+	ports, err := getPortsFromCluster(cluster)
+	// Assign default ports
+	if err != nil || len(ports) == 0 {
+		ports = getDefaultPorts()
+	}
+
+	return ports
+}
+
 // getPortsFromCluster get the ports from head container and directly map them in service
 // It's user's responsibility to maintain rayStartParam ports and container ports mapping
 // TODO: Consider to infer ports from rayStartParams (source of truth) in the future.
@@ -59,3 +63,11 @@ func getPortsFromCluster(cluster rayiov1alpha1.RayCluster) (map[string]int32, er
 
 	return svcPorts, nil
 }
+
+func getDefaultPorts() map[string]int32 {
+	return map[string]int32{
+		DefaultClientPortName: DefaultClientPort,
+		DefaultRedisPortName:  DefaultRedisPort,
+		DefaultDashboardName:  DefaultDashboardPort,
+	}
+}