[#827] feat(operator): support generating hpa objects (#828)

### What changes were proposed in this pull request?

In this PR, we will be adding the Autoscaler field for the first time in the CRD, enabling users to support ShuffleServer using the HPA object to achieve horizontal automatic scaling.

In the next PR, we will provide more detailed documentation to explain the format and usage examples of .spec.shuffleServer.autoscaler field.

### Why are the changes needed?

Fix: #827

### Does this PR introduce _any_ user-facing change?

If users need to use the automatic scaling feature, they need to edit the .spec.shuffleServer.autoscaler field, which is completely identical to the .spec field definition of kubernetes' HPA object.

### How was this patch tested?

Manually verified.

deploy/kubernetes/integration-test/e2e/kind-config

@@ -22,3 +22,5 @@ nodes:
 - role: worker
 - role: worker
 - role: worker
+- role: worker
+- role: worker

deploy/kubernetes/integration-test/e2e/set-up-local-cluster.sh

@@ -141,6 +141,7 @@ if [ "$STEP_BUILD_NEW_OPERATOR" == "true" ]; then
   echo "--->>>try to apply rss-operator in cluster"
   kubectl apply -f rss-controller.yaml
   kubectl apply -f rss-webhook.yaml
+  kubectl apply -f template/metrics-server.yaml
   echo "--->>>wait some time for rss-operator to be ready"
   sleep 60
 fi

deploy/kubernetes/integration-test/e2e/template/metrics-server.yaml

@@ -0,0 +1,207 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    k8s-app: metrics-server
+  name: metrics-server
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    k8s-app: metrics-server
+    rbac.authorization.k8s.io/aggregate-to-admin: "true"
+    rbac.authorization.k8s.io/aggregate-to-edit: "true"
+    rbac.authorization.k8s.io/aggregate-to-view: "true"
+  name: system:aggregated-metrics-reader
+rules:
+  - apiGroups:
+      - metrics.k8s.io
+    resources:
+      - pods
+      - nodes
+    verbs:
+      - get
+      - list
+      - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    k8s-app: metrics-server
+  name: system:metrics-server
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - nodes/metrics
+    verbs:
+      - get
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - nodes
+    verbs:
+      - get
+      - list
+      - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    k8s-app: metrics-server
+  name: metrics-server-auth-reader
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+  - kind: ServiceAccount
+    name: metrics-server
+    namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    k8s-app: metrics-server
+  name: metrics-server:system:auth-delegator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:auth-delegator
+subjects:
+  - kind: ServiceAccount
+    name: metrics-server
+    namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    k8s-app: metrics-server
+  name: system:metrics-server
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:metrics-server
+subjects:
+  - kind: ServiceAccount
+    name: metrics-server
+    namespace: kube-system
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    k8s-app: metrics-server
+  name: metrics-server
+  namespace: kube-system
+spec:
+  ports:
+    - name: https
+      port: 443
+      protocol: TCP
+      targetPort: https
+  selector:
+    k8s-app: metrics-server
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    k8s-app: metrics-server
+  name: metrics-server
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      k8s-app: metrics-server
+  strategy:
+    rollingUpdate:
+      maxUnavailable: 0
+  template:
+    metadata:
+      labels:
+        k8s-app: metrics-server
+    spec:
+      containers:
+        - args:
+            - --cert-dir=/tmp
+            - --secure-port=4443
+            - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
+            - --kubelet-use-node-status-port
+            - --kubelet-insecure-tls=true
+            - --metric-resolution=15s
+          image: registry.k8s.io/metrics-server/metrics-server:v0.6.3
+          imagePullPolicy: IfNotPresent
+          livenessProbe:
+            failureThreshold: 3
+            httpGet:
+              path: /livez
+              port: https
+              scheme: HTTPS
+            periodSeconds: 10
+          name: metrics-server
+          ports:
+            - containerPort: 4443
+              name: https
+              protocol: TCP
+          resources:
+            requests:
+              cpu: 100m
+              memory: 200Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            runAsNonRoot: true
+            runAsUser: 1000
+          volumeMounts:
+            - mountPath: /tmp
+              name: tmp-dir
+      nodeSelector:
+        kubernetes.io/os: linux
+      priorityClassName: system-cluster-critical
+      serviceAccountName: metrics-server
+      volumes:
+        - emptyDir: {}
+          name: tmp-dir
+---
+apiVersion: apiregistration.k8s.io/v1
+kind: APIService
+metadata:
+  labels:
+    k8s-app: metrics-server
+  name: v1beta1.metrics.k8s.io
+spec:
+  group: metrics.k8s.io
+  groupPriorityMinimum: 100
+  insecureSkipTLSVerify: true
+  service:
+    name: metrics-server
+    namespace: kube-system
+  version: v1beta1
+  versionPriority: 100

deploy/kubernetes/integration-test/e2e/template/rss-controller-template.yaml

@@ -48,6 +48,9 @@ rules:
   - apiGroups: [ "" ]
     resources: [ "events" ]
     verbs: [ "list", "watch", "create", "update", "patch" ]
+  - apiGroups: [ "autoscaling" ]
+    resources: [ "horizontalpodautoscalers" ]
+    verbs: [ "get", "list", "watch", "update", "create", "delete", "patch" ]
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1

deploy/kubernetes/integration-test/e2e/template/rss-template.yaml

@@ -47,6 +47,21 @@ spec:
     replicas: 1
     image: "${RSS_SERVER_IMAGE}"
     initContainerImage: "busybox:latest"
+    resources:
+      requests:
+        cpu: 500m
+        memory: 1Gi
+    autoscaler:
+      enable: true
+      hpaSpec:
+        maxReplicas: 3
+        metrics:
+          - resource:
+              name: cpu
+              target:
+                averageUtilization: 10
+                type: Utilization
+            type: Resource
     upgradeStrategy:
       type: "FullUpgrade"
     xmxSize: "800M"

deploy/kubernetes/integration-test/e2e/template/rss-webhook-template.yaml

@@ -48,9 +48,15 @@ rules:
   - apiGroups: [ "" ]
     resources: [ "events" ]
     verbs: [ "list", "watch", "create", "update", "patch" ]
-  - apiGroups: [ "node.k8s.io"]
+  - apiGroups: [ "node.k8s.io" ]
     resources: [ "runtimeclasses" ]
     verbs: [ "get", "list", "watch" ]
+  - apiGroups: [ "autoscaling" ]
+    resources: [ "horizontalpodautoscalers" ]
+    verbs: [ "get", "list", "watch" ]
+  - apiGroups: [ "apps" ]
+    resources: [ "statefulsets" ]
+    verbs: [ "get", "list", "watch" ]
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
@@ -108,8 +114,8 @@ spec:
           command:
             - "./webhook"
           args:
-           - "--ignore-rss=false"
-           - "--v=4"
+            - "--ignore-rss=false"
+            - "--v=4"
           ports:
             - containerPort: 9876
               protocol: TCP

deploy/kubernetes/operator/api/uniffle/v1alpha1/remoteshuffleservice_types.go

@@ -18,6 +18,7 @@
 package v1alpha1
 
 import (
+	autoscalingv2 "k8s.io/api/autoscaling/v2beta2"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -168,9 +169,65 @@ type CommonConfig struct {
 	// ConfigDir records the directory where the configuration of coordinators or shuffle servers resides.
 	ConfigDir string `json:"configDir"`
 
-	// Parameters holds the optional parameters used by coordinators or shuffle servers .
-	// optional
+	// Parameters holds the optional parameters used by coordinators or shuffle servers.
+	// +optional
 	Parameters map[string]string `json:"parameters,omitempty"`
+
+	// Autoscaler defines desired functionality of HPA object to be generated.
+	// +optional
+	Autoscaler RSSAutoscaler `json:"autoscaler,omitempty"`
+}
+
+// RSSAutoscaler describes the desired functionality of the HPA object to be generated,
+// which automatically manages the replica count of any resource implementing the scale
+// subresource based on the metrics specified.
+type RSSAutoscaler struct {
+	// Enable indicates whether we need to generate an HPA object.
+	Enable bool `json:"enable"`
+	// HPASpec allows users to configure HPA objects to achieve automatic scaling.
+	// This field is very similar to autoscalingv2.HorizontalPodAutoscalerSpec, but
+	// in autoscalingv2.HorizontalPodAutoscalerSpec, ScaleTargetRef is a required
+	// field, while the rss object does not require users to specify this field.
+	// Therefore, we have redefined a HorizontalPodAutoscalerSpec, removing the
+	// ScaleTargetRef field in autoscalingv2.HorizontalPodAutoscalerSpec.
+	// +optional
+	HPASpec HorizontalPodAutoscalerSpec `json:"hpaSpec,omitempty"`
+}
+
+// HorizontalPodAutoscalerSpec is very similar to autoscalingv2.HorizontalPodAutoscalerSpec,
+// but in autoscalingv2.HorizontalPodAutoscalerSpec, ScaleTargetRef is a required field,
+// while the rss object does not require users to specify this field. Therefore, we have
+// redefined a HorizontalPodAutoscalerSpec, removing the ScaleTargetRef field in
+// autoscalingv2.HorizontalPodAutoscalerSpec.
+type HorizontalPodAutoscalerSpec struct {
+	// +kubebuilder:default:=1
+	// minReplicas is the lower limit for the number of replicas to which the autoscaler
+	// can scale down.  It defaults to 1 pod.  minReplicas is allowed to be 0 if the
+	// alpha feature gate HPAScaleToZero is enabled and at least one Object or External
+	// metric is configured.  Scaling is active as long as at least one metric value is
+	// available.
+	// +optional
+	MinReplicas *int32 `json:"minReplicas,omitempty"`
+
+	// maxReplicas is the upper limit for the number of replicas to which the autoscaler can scale up.
+	// It cannot be less that minReplicas.
+	MaxReplicas int32 `json:"maxReplicas"`
+	// metrics contains the specifications for which to use to calculate the
+	// desired replica count (the maximum replica count across all metrics will
+	// be used).  The desired replica count is calculated multiplying the
+	// ratio between the target value and the current value by the current
+	// number of pods.  Ergo, metrics used must decrease as the pod count is
+	// increased, and vice-versa.  See the individual metric source types for
+	// more information about how each type of metric must respond.
+	// If not set, the default metric will be set to 80% average CPU utilization.
+	// +optional
+	Metrics []autoscalingv2.MetricSpec `json:"metrics,omitempty"`
+
+	// behavior configures the scaling behavior of the target
+	// in both Up and Down directions (scaleUp and scaleDown fields respectively).
+	// If not set, the default HPAScalingRules for scale up and scale down are used.
+	// +optional
+	Behavior *autoscalingv2.HorizontalPodAutoscalerBehavior `json:"behavior,omitempty"`
 }
 
 // RSSPodSpec defines the desired state of coordinators or shuffle servers' pods.