Autoscaling selenium grid on kubernetes with scaledjobs (#1854)

* Autoscaling selenium grid on kubernetes Autoscale selenium browser nodes running in kubernetes based on the request pending in session queue using KEDA. Toggle autoscaling on/off using 'autoscalingEnabled' option in helm charts. * Added all suggested changes * feat: add support for KEDA ScaledJobs and make them the default KEDA scaling type install keda automatically set SE_NODE_GRID_URL and DRAIN_AFTER_SESSION_COUNT automatically Set graphqlurl automatically refactor out pod templates to named template conditionally adding preStop hook for deregistering node Signed-off-by: Mårten Svantesson <Marten.Svantesson@ticket.se> * fix: work around for limitations in helm helm seem to normally install subchart after current chart so trying this work around * fix: format * fix: extracting default node env variables to configmap --------- Signed-off-by: Mårten Svantesson <Marten.Svantesson@ticket.se> Co-authored-by: Prashanth R <prashanth.ramadass@volvocars.com> Co-authored-by: Prashanth R <rprashanth27@gmail.com>
SeleniumHQ · Jun 29, 2023 · f0bbfe0 · f0bbfe0
1 parent 33bd1ab
commit f0bbfe0
Show file tree

Hide file tree

Showing 14 changed files with 587 additions and 417 deletions.
diff --git a/charts/selenium-grid/Chart.yaml b/charts/selenium-grid/Chart.yaml
@@ -5,3 +5,8 @@ type: application
 version: 0.18.1
 appVersion: 4.10.0-20230607
 icon: https://github.com/SeleniumHQ/docker-selenium/raw/trunk/logo.png
+dependencies:
+- repository: https://kedacore.github.io/charts
+  version: 2.10.2
+  name: keda
+  condition: autoscaling.enabled
diff --git a/charts/selenium-grid/README.md b/charts/selenium-grid/README.md
diff --git a/charts/selenium-grid/templates/_helpers.tpl b/charts/selenium-grid/templates/_helpers.tpl
@@ -79,3 +79,130 @@ Ingress fullname
 {{- define "seleniumGrid.ingress.fullname" -}}
 {{- default "selenium-ingress" .Values.ingress.nameOverride | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
+
+{{/*
+Is autoscaling using KEDA enabled
+*/}}
+{{- define "seleniumGrid.useKEDA" -}}
+{{- or .Values.autoscaling.enabled .Values.autoscaling.enableWithExistingKEDA | ternary "true" "" -}}
+{{- end -}}
+
+
+{{/*
+Common pod template
+*/}}
+{{- define "seleniumGrid.podTemplate" -}}
+template:
+  metadata:
+    labels:
+      app: {{.name}}
+      app.kubernetes.io/name: {{.name}}
+      {{- include "seleniumGrid.commonLabels" . | nindent 6 }}
+      {{- with .node.labels }}
+        {{- toYaml . | nindent 6 }}
+      {{- end }}
+      {{- with .Values.customLabels }}
+        {{- toYaml . | nindent 6 }}
+      {{- end }}
+    annotations:
+      checksum/event-bus-configmap: {{ include (print $.Template.BasePath "/event-bus-configmap.yaml") . | sha256sum }}
+      {{- with .node.annotations }}
+        {{ toYaml . | nindent 6 }}
+      {{- end }}
+  spec:
+    restartPolicy: {{ and (eq (include "seleniumGrid.useKEDA" .) "true") (eq .Values.autoscaling.scalingType "job") | ternary "Never" "Always" }}
+  {{- with .node.hostAliases }}
+    hostAliases: {{ toYaml . | nindent 6 }}
+  {{- end }}
+    containers:
+      - name: {{.name}}
+        {{- $imageTag := default .Values.global.seleniumGrid.nodesImageTag .node.imageTag }}
+        image: {{ printf "%s:%s" .node.imageName $imageTag }}
+        imagePullPolicy: {{ .node.imagePullPolicy }}
+      {{- with .node.extraEnvironmentVariables }}
+        env: {{- tpl (toYaml .) $ | nindent 10 }}
+      {{- end }}
+        envFrom:
+          - configMapRef:
+              name: {{ .Values.busConfigMap.name }}
+          - configMapRef:
+              name: {{ .Values.nodeConfigMap.name }}
+          {{- with .node.extraEnvFrom }}
+            {{- toYaml . | nindent 10 }}
+          {{- end }}
+      {{- if gt (len .node.ports) 0 }}
+        ports:
+        {{- range .node.ports }}
+          - containerPort: {{ . }}
+            protocol: TCP
+        {{- end }}
+      {{- end }}
+        volumeMounts:
+          - name: dshm
+            mountPath: /dev/shm
+        {{- if .node.extraVolumeMounts }}
+          {{- toYaml .node.extraVolumeMounts | nindent 10 }}
+        {{- end }}
+      {{- with .node.resources }}
+        resources: {{- toYaml . | nindent 10 }}
+      {{- end }}
+      {{- include "seleniumGrid.lifecycle" . | nindent 8 -}}
+      {{- with .node.startupProbe }}
+        startupProbe: {{- toYaml . | nindent 10 }}
+      {{- end }}
+  {{- if or .Values.global.seleniumGrid.imagePullSecret .node.imagePullSecret }}
+    imagePullSecrets:
+      - name: {{ default .Values.global.seleniumGrid.imagePullSecret .node.imagePullSecret }}
+  {{- end }}
+  {{- with .node.nodeSelector }}
+    nodeSelector: {{- toYaml . | nindent 6 }}
+  {{- end }}
+  {{- with .node.tolerations }}
+    tolerations:
+      {{ toYaml . | nindent 4 }}
+  {{- end }}
+  {{- with .node.priorityClassName }}
+    priorityClassName: {{ . }}
+  {{- end }}
+    terminationGracePeriodSeconds: {{ .node.terminationGracePeriodSeconds }}
+    volumes:
+      - name: dshm
+        emptyDir:
+          medium: Memory
+          sizeLimit: {{ default "1Gi" .node.dshmVolumeSizeLimit }}
+    {{- if .node.extraVolumes }}
+      {{ toYaml .node.extraVolumes | nindent 6 }}
+    {{- end }}
+{{- end -}}
+
+{{/*
+Get the url of the grid. If the external url can be figured out from the ingress use that, otherwise the cluster internal url
+*/}}
+{{- define "seleniumGrid.url" -}}
+{{- if and .Values.ingress.enabled .Values.ingress.hostname (ne .Values.ingress.hostname "selenium-grid.local") -}}
+http{{if .Values.ingress.tls}}s{{end}}://{{.Values.ingress.hostname}}
+{{- else -}}
+http://{{ include ($.Values.isolateComponents | ternary "seleniumGrid.router.fullname" "seleniumGrid.hub.fullname") $ }}.{{ .Release.Namespace }}:{{ $.Values.components.router.port }}
+{{- end }}
+{{- end -}}
+
+{{/*
+Graphql Url of the hub or the router
+*/}}
+{{- define "seleniumGrid.graphqlURL" -}}
+http://{{ include ($.Values.isolateComponents | ternary "seleniumGrid.router.fullname" "seleniumGrid.hub.fullname") $ }}.{{ .Release.Namespace }}:{{ $.Values.components.router.port }}/graphql
+{{- end -}}
+
+{{/*
+Get the lifecycle of the pod. When KEDA is activated and the lifecycle is used for a pod of a
+deployment preStop hook to deregister from the selenium hub.
+*/}}
+{{- define "seleniumGrid.lifecycle" }}
+{{ $lifecycle := tpl (toYaml (default (dict) .node.lifecycle)) $ }}
+{{- if and (eq .Values.autoscaling.scalingType "deployment") (eq (include "seleniumGrid.useKEDA" .) "true") -}}
+{{ $lifecycle = merge ($lifecycle | fromYaml ) .Values.autoscaling.deregisterLifecycle | toYaml }}
+{{- end -}}
+{{ if and $lifecycle (ne $lifecycle "{}") -}}
+lifecycle: {{ $lifecycle | nindent 2 }}
+{{- end -}}
+{{- end -}}
diff --git a/charts/selenium-grid/templates/chrome-node-deployment.yaml b/charts/selenium-grid/templates/chrome-node-deployment.yaml
@@ -1,10 +1,10 @@
-{{- if and .Values.chromeNode.enabled .Values.chromeNode.deploymentEnabled }}
+{{- if and .Values.chromeNode.enabled ((eq (include "seleniumGrid.useKEDA" .) "true") | ternary (eq .Values.autoscaling.scalingType "deployment") .Values.chromeNode.deploymentEnabled) }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: {{ template "seleniumGrid.chromeNode.fullname" . }}
   namespace: {{ .Release.Namespace }}
-  labels: &chrome_node_labels
+  labels:
     app: selenium-chrome-node
     app.kubernetes.io/name: selenium-chrome-node
     {{- include "seleniumGrid.commonLabels" . | nindent 4 }}
@@ -20,81 +20,8 @@ spec:
     matchLabels:
       app: selenium-chrome-node
       app.kubernetes.io/instance: {{ .Release.Name }}
-  template:
-    metadata:
-      labels: *chrome_node_labels
-      annotations:
-        checksum/event-bus-configmap: {{ include (print $.Template.BasePath "/event-bus-configmap.yaml") . | sha256sum }}
-        {{- with .Values.chromeNode.annotations }}
-          {{ toYaml . | nindent 8 }}
-        {{- end }}
-    spec:
-    {{- with .Values.chromeNode.hostAliases }}
-      hostAliases: {{ toYaml . | nindent 8 }}
-    {{- end }}
-      containers:
-        - name: selenium-chrome-node
-          {{- $imageTag := default .Values.global.seleniumGrid.nodesImageTag .Values.chromeNode.imageTag }}
-          image: {{ printf "%s:%s" .Values.chromeNode.imageName $imageTag }}
-          imagePullPolicy: {{ .Values.chromeNode.imagePullPolicy }}
-        {{- with .Values.chromeNode.extraEnvironmentVariables }}
-          env: {{- tpl (toYaml .) $ | nindent 12 }}
-        {{- end }}
-          envFrom:
-            - configMapRef:
-                name: {{ .Values.busConfigMap.name }}
-            {{- with .Values.chromeNode.extraEnvFrom }}
-              {{- toYaml . | nindent 12 }}
-            {{- end }}
-        {{- if gt (len .Values.chromeNode.ports) 0 }}
-          ports:
-          {{- range .Values.chromeNode.ports }}
-            - containerPort: {{ . }}
-              protocol: TCP
-          {{- end }}
-        {{- end }}
-          volumeMounts:
-            - name: dshm
-              mountPath: /dev/shm
-          {{- if .Values.chromeNode.extraVolumeMounts }}
-            {{- toYaml .Values.chromeNode.extraVolumeMounts | nindent 12 }}
-          {{- end }}
-        {{- with .Values.chromeNode.resources }}
-          resources: {{- toYaml . | nindent 12 }}
-        {{- end }}
-        {{- with .Values.chromeNode.securityContext }}
-          securityContext: {{- toYaml . | nindent 12 }}
-        {{- end }}
-        {{- with .Values.chromeNode.lifecycle }}
-          lifecycle: {{- toYaml . | nindent 12 }}
-        {{- end }}
-        {{- with .Values.chromeNode.startupProbe }}
-          startupProbe: {{- toYaml . | nindent 12 }}
-        {{- end }}
-    {{- if or .Values.global.seleniumGrid.imagePullSecret .Values.chromeNode.imagePullSecret }}
-      imagePullSecrets:
-        - name: {{ default .Values.global.seleniumGrid.imagePullSecret .Values.chromeNode.imagePullSecret }}
-    {{- end }}
-    {{- with .Values.chromeNode.nodeSelector }}
-      nodeSelector: {{- toYaml . | nindent 8 }}
-    {{- end }}
-    {{- if or .Values.global.seleniumGrid.affinity .Values.chromeNode.affinity }}
-      {{- $affinityYaml := default .Values.global.seleniumGrid.affinity .Values.chromeNode.affinity }}
-      affinity: {{- toYaml $affinityYaml | nindent 8 }}
-    {{- end }}    
-    {{- with .Values.chromeNode.tolerations }}
-      tolerations: {{ toYaml . | nindent 6 }} 
-    {{- end }}
-    {{- with .Values.chromeNode.priorityClassName }}
-      priorityClassName: {{ . }}
-    {{- end }}
-      terminationGracePeriodSeconds: {{ .Values.chromeNode.terminationGracePeriodSeconds }}
-      volumes:
-        - name: dshm
-          emptyDir:
-            medium: Memory
-            sizeLimit: {{ default "1Gi" .Values.chromeNode.dshmVolumeSizeLimit }}
-      {{- if .Values.chromeNode.extraVolumes }}
-        {{ toYaml .Values.chromeNode.extraVolumes | nindent 8 }}
-      {{- end }}
+{{- $podScope := deepCopy . -}}
+{{- $_ := set $podScope "name" "selenium-chrome-node" -}}
+{{- $_ =  set $podScope "node" .Values.chromeNode  -}}
+{{- include "seleniumGrid.podTemplate" $podScope | nindent 2 }}
 {{- end }}
diff --git a/charts/selenium-grid/templates/chrome-node-hpa.yaml b/charts/selenium-grid/templates/chrome-node-hpa.yaml
@@ -0,0 +1,22 @@
+{{- if and .Values.chromeNode.enabled (eq (include "seleniumGrid.useKEDA" .) "true") (eq .Values.autoscaling.scalingType "deployment") }}
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+  name: selenium-grid-chrome-scaledobject
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    {{- with .Values.autoscaling.annotations }}
+      {{- toYaml . | nindent 4 }}
+    {{- end }}
+  labels:
+    deploymentName: {{ template "seleniumGrid.chromeNode.fullname" . }}
+spec:
+  maxReplicaCount: {{ .Values.chromeNode.maxReplicaCount }}
+  scaleTargetRef:
+    name: {{ template "seleniumGrid.chromeNode.fullname" . }}
+  triggers:
+    - type: selenium-grid
+    {{- with .Values.chromeNode.hpa }}
+      metadata: {{- tpl (toYaml .) $ | nindent 8 }}
+    {{- end }}
+{{- end }}
diff --git a/charts/selenium-grid/templates/chrome-node-scaledjobs.yaml b/charts/selenium-grid/templates/chrome-node-scaledjobs.yaml
@@ -0,0 +1,39 @@
+{{- if and .Values.chromeNode.enabled (include "seleniumGrid.useKEDA" .) (eq .Values.autoscaling.scalingType "job") }}
+apiVersion: keda.sh/v1alpha1
+kind: ScaledJob
+metadata:
+  name: {{ template "seleniumGrid.chromeNode.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    {{- with .Values.autoscaling.annotations }}
+      {{- toYaml . | nindent 4 }}
+    {{- end }}
+  labels:
+    app: selenium-chrome-node
+    app.kubernetes.io/name: selenium-chrome-node
+    {{- include "seleniumGrid.commonLabels" . | nindent 4 }}
+    {{- with .Values.chromeNode.labels }}
+      {{- toYaml . | nindent 4 }}
+    {{- end }}
+    {{- with .Values.customLabels }}
+      {{- toYaml . | nindent 4 }}
+    {{- end }}
+spec:
+  maxReplicaCount: {{ .Values.chromeNode.maxReplicaCount }}
+  {{- with .Values.autoscaling.scaledJobOptions -}}
+    {{ toYaml . | nindent 2 }}
+  {{- end }}
+  triggers:
+    - type: selenium-grid
+    {{- with .Values.chromeNode.hpa }}
+      metadata: {{- tpl (toYaml .) $ | nindent 8 }}
+    {{- end }}
+  jobTargetRef:
+    parallelism: 1
+    completions: 1
+    backoffLimit: 0
+{{- $podScope := deepCopy . -}}
+{{- $_ := set $podScope "name" "selenium-chrome-node" -}}
+{{- $_ =  set $podScope "node" .Values.chromeNode  -}}
+{{- include "seleniumGrid.podTemplate" $podScope | nindent 4 }}
+{{- end }}