SeleniumHQ · VietND96 · Feb 19, 2024 · Feb 18, 2024
diff --git a/.github/workflows/helm-chart-test.yml b/.github/workflows/helm-chart-test.yml
@@ -15,6 +15,9 @@ on:
         description: 'Test parameter for different request timeout'
         required: false
         default: '370'
+      cluster:
+        description: 'Cluster type to test (kind, minikube)'
+        required: false
   schedule:
     - cron: '0 0 * * *'
 
@@ -33,6 +36,8 @@ jobs:
           - job_https
           - deployment
           - deployment_https
+    env:
+      CLUSTER: ${{ github.event.inputs.cluster || 'minikube' }}
     steps:
       - uses: actions/checkout@main
       - name: Output Docker info
@@ -63,7 +68,7 @@ jobs:
         with:
           timeout_minutes: 10
           max_attempts: 3
-          command: make chart_setup_env
+          command: CLUSTER=${CLUSTER} make chart_setup_env
       - name: Build Helm charts
         run: |
           BUILD_DATE=${BUILD_DATE} make chart_build
@@ -76,7 +81,7 @@ jobs:
         with:
           timeout_minutes: 10
           max_attempts: 3
-          command: NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make chart_cluster_setup
+          command: CLUSTER=${CLUSTER} NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make chart_cluster_setup
       - name: Build Docker images
         run: NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make build
       - name: Test set custom CA certificate
@@ -96,7 +101,7 @@ jobs:
             NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make chart_test_autoscaling_${{ matrix.test-strategy }}
       - name: Cleanup Kubernetes cluster
         if: always()
-        run: make chart_cluster_cleanup
+        run: CLUSTER=${CLUSTER} make chart_cluster_cleanup
       - name: Upload Helm chart package
         if: always()
         uses: actions/upload-artifact@main

diff --git a/Makefile b/Makefile
@@ -447,16 +447,22 @@ chart_test_edge:
 	./tests/charts/make/chart_test.sh NodeEdge
 
 chart_test_autoscaling_deployment_https:
-	SELENIUM_GRID_TEST_HEADLESS=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_PORT=443 make chart_test_autoscaling_deployment
+	SE_FULL_DISTRIBUTED_MODE=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_PORT=443 \
+	VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \
+	./tests/charts/make/chart_test.sh DeploymentAutoscaling
 
 chart_test_autoscaling_deployment:
-	SE_ENABLE_TRACING=true VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \
+	SE_ENABLE_TRACING=true SELENIUM_GRID_TEST_HEADLESS=true \
+	VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \
 	./tests/charts/make/chart_test.sh DeploymentAutoscaling
 
 chart_test_autoscaling_job_https:
-	SE_ENABLE_TRACING=true SELENIUM_GRID_TEST_HEADLESS=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_PORT=443 make chart_test_autoscaling_job
+	SELENIUM_GRID_TEST_HEADLESS=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_PORT=443 \
+	VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \
+	./tests/charts/make/chart_test.sh JobAutoscaling
 
 chart_test_autoscaling_job:
+	SE_ENABLE_TRACING=true SE_FULL_DISTRIBUTED_MODE=true \
 	VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \
 	./tests/charts/make/chart_test.sh JobAutoscaling
 

diff --git a/Video/video.sh b/Video/video.sh
@@ -159,7 +159,10 @@ else
   echo Checking if node API responds
   until curl -sk --request GET ${SE_SERVER_PROTOCOL}://${DISPLAY_CONTAINER_NAME}:${SE_NODE_PORT}/status || [[ $attempts = "$max_attempts" ]]
   do
-      echo Waiting before next API check
+      if [ $(($attempts % 60)) -eq 0 ];
+      then
+          echo Waiting before next API check
+      fi
       sleep 0.5
       attempts=$((attempts+1))
   done

diff --git a/charts/selenium-grid/configs/node/nodePreStop.sh b/charts/selenium-grid/configs/node/nodePreStop.sh
@@ -1,13 +1,73 @@
 #!/bin/bash
+
+function on_exit() {
+  rm -rf /tmp/preStopOutput
+}
+trap on_exit EXIT
+
+# Set headers if Node Registration Secret is set
 if [ ! -z "${SE_REGISTRATION_SECRET}" ];
 then
   HEADERS="X-REGISTRATION-SECRET: ${SE_REGISTRATION_SECRET}"
 else
   HEADERS="X-REGISTRATION-SECRET;"
 fi
 
-if curl -sfk ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/status; then
+function is_full_distributed_mode() {
+  if [ -n "${SE_DISTRIBUTOR_HOST}" ] && [ -n "${SE_DISTRIBUTOR_PORT}" ]; then
+    DISTRIBUTED_MODE=true
+    echo "Detected full distributed mode: ${DISTRIBUTED_MODE}. Since SE_DISTRIBUTOR_HOST and SE_DISTRIBUTOR_PORT are set in Node ConfigMap"
+  else
+    DISTRIBUTED_MODE=false
+    echo "Detected full distributed mode: ${DISTRIBUTED_MODE}"
+  fi
+}
+is_full_distributed_mode
+
+function signal_distributor_to_drain_node() {
+  if [ "${DISTRIBUTED_MODE}" = true ]; then
+    echo "Signaling Distributor to drain node"
+    set -x
+    curl -k -X POST ${SE_SERVER_PROTOCOL}://${SE_DISTRIBUTOR_HOST}:${SE_DISTRIBUTOR_PORT}/se/grid/distributor/node/${NODE_ID}/drain --header "${HEADERS}"
+    set +x
+  fi
+}
+
+function signal_hub_to_drain_node() {
+  if [ "${DISTRIBUTED_MODE}" = false ]; then
+    echo "Signaling Hub to drain node"
+    curl -k -X POST ${SE_GRID_URL}/se/grid/distributor/node/${NODE_ID}/drain --header "${HEADERS}"
+  fi
+}
+
+function signal_node_to_drain() {
+    echo "Signaling Node to drain itself"
     curl -k -X POST ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/se/grid/node/drain --header "${HEADERS}"
+}
+
+function replace_localhost_by_service_name() {
+  internal="${SE_HUB_HOST:-$SE_ROUTER_HOST}:${SE_HUB_PORT:-$SE_ROUTER_PORT}"
+  if [[ "${SE_NODE_GRID_URL}" == *"/localhost"* ]]; then
+      SE_GRID_URL=${SE_NODE_GRID_URL//localhost/${internal}}
+  elif [[ "${SE_NODE_GRID_URL}" == *"/127.0.0.1"* ]]; then
+      SE_GRID_URL=${SE_NODE_GRID_URL//127.0.0.1/${internal}}
+  elif [[ "${SE_NODE_GRID_URL}" == *"/0.0.0.0"* ]]; then
+      SE_GRID_URL=${SE_NODE_GRID_URL//0.0.0.0/${internal}}
+  fi
+  echo "SE_GRID_URL: ${SE_GRID_URL}"
+}
+replace_localhost_by_service_name
+
+if curl -sfk ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/status > /tmp/preStopOutput; then
+    NODE_ID=$(jq -r '.value.node.nodeId' /tmp/preStopOutput)
+    if [ -n "${NODE_ID}" ]; then
+      echo "Current Node ID is: ${NODE_ID}"
+      signal_hub_to_drain_node
+      signal_distributor_to_drain_node
+      echo
+    fi
+    signal_node_to_drain
+    # Wait for the current session to be finished if any
     while curl -sfk ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/status -o /tmp/preStopOutput;
     do
       echo "Node preStop is waiting for current session to be finished if any. Node details: message: $(jq -r '.value.message' /tmp/preStopOutput || "unknown"), availability: $(jq -r '.value.node.availability' /tmp/preStopOutput || "unknown")"

diff --git a/charts/selenium-grid/configs/node/nodeProbe.sh b/charts/selenium-grid/configs/node/nodeProbe.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+
+function on_exit() {
+  rm -rf /tmp/nodeProbe${ID}
+  rm -rf /tmp/gridProbe${ID}
+}
+trap on_exit EXIT
+
+ID=$(echo $RANDOM)
+
+function replace_localhost_by_service_name() {
+  internal="${SE_HUB_HOST:-$SE_ROUTER_HOST}:${SE_HUB_PORT:-$SE_ROUTER_PORT}"
+  if [[ "${SE_NODE_GRID_URL}" == *"/localhost"* ]]; then
+      SE_GRID_URL=${SE_NODE_GRID_URL//localhost/${internal}}
+  elif [[ "${SE_NODE_GRID_URL}" == *"/127.0.0.1"* ]]; then
+      SE_GRID_URL=${SE_NODE_GRID_URL//127.0.0.1/${internal}}
+  elif [[ "${SE_NODE_GRID_URL}" == *"/0.0.0.0"* ]]; then
+      SE_GRID_URL=${SE_NODE_GRID_URL//0.0.0.0/${internal}}
+  fi
+  echo "SE_GRID_URL: ${SE_GRID_URL}"
+}
+replace_localhost_by_service_name
+
+if curl -sfk ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/status -o /tmp/nodeProbe${ID}; then
+  NODE_ID=$(jq -r '.value.node.nodeId' /tmp/nodeProbe${ID})
+  NODE_STATUS=$(jq -r '.value.node.availability' /tmp/nodeProbe${ID})
+
+  curl -sfk "${SE_GRID_URL}/status" -o /tmp/gridProbe${ID}
+  GRID_NODE_ID=$(jq -e ".value.nodes[].id|select(. == \"${NODE_ID}\")" /tmp/gridProbe${ID} | tr -d '"' || true)
+
+  if [ "${NODE_STATUS}" = "UP" ] && [ -n "${NODE_ID}" ] && [ -n "${GRID_NODE_ID}" ] && [ "${NODE_ID}" = "${GRID_NODE_ID}" ]; then
+    echo "Node ID: ${NODE_ID} with status: ${NODE_STATUS}"
+    echo "Found in the Grid a matched Node ID: ${GRID_NODE_ID}"
+    exit 0
+  else
+    echo "Node ID: ${NODE_ID} is not found in the Grid. The registration could be in progress."
+    exit 1
+  fi
+else
+  exit 1
+fi
diff --git a/charts/selenium-grid/templates/_helpers.tpl b/charts/selenium-grid/templates/_helpers.tpl
@@ -17,18 +17,20 @@ Probe httpGet schema
 Check user define custom probe method
 */}}
 {{- define "seleniumGrid.probe.fromUserDefine" -}}
+{{- $values := index . "values" -}}
+{{- $root := index . "root" -}}
 {{- $overrideProbe := dict -}}
-{{- with .exec -}}
-{{- $overrideProbe = dict "exec" . -}}
+{{- with $values.exec -}}
+{{- $overrideProbe = dict "exec" (tpl (toYaml .) $root | fromYaml) -}}
 {{- end }}
-{{- with .httpGet -}}
-{{- $overrideProbe = dict "httpGet" . -}}
+{{- with $values.httpGet -}}
+{{- $overrideProbe = dict "httpGet" (tpl (toYaml .) $root | fromYaml) -}}
 {{- end }}
-{{- with .tcpSocket -}}
-{{- $overrideProbe = dict "tcpSocket" . -}}
+{{- with $values.tcpSocket -}}
+{{- $overrideProbe = dict "tcpSocket" (tpl (toYaml .) $root | fromYaml) -}}
 {{- end }}
-{{- with .grpc -}}
-{{- $overrideProbe = dict "grpc" . -}}
+{{- with $values.grpc -}}
+{{- $overrideProbe = dict "grpc" (tpl (toYaml .) $root | fromYaml) -}}
 {{- end -}}
 {{- $overrideProbe | toYaml -}}
 {{- end -}}
@@ -258,8 +260,8 @@ template:
       {{- if .node.startupProbe.enabled }}
       {{- with .node.startupProbe }}
         startupProbe:
-        {{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }}
-          {{- include "seleniumGrid.probe.fromUserDefine" . | nindent 10 }}
+        {{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }}
+          {{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }}
         {{- else }}
           httpGet:
             scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }}
@@ -274,8 +276,8 @@ template:
       {{- if .node.readinessProbe.enabled }}
       {{- with .node.readinessProbe }}
         readinessProbe:
-        {{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }}
-          {{- include "seleniumGrid.probe.fromUserDefine" . | nindent 12 }}
+        {{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }}
+          {{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 12 }}
         {{- else }}
           httpGet:
             scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }}
@@ -290,8 +292,8 @@ template:
       {{- if .node.livenessProbe.enabled }}
       {{- with .node.livenessProbe }}
         livenessProbe:
-        {{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }}
-          {{- include "seleniumGrid.probe.fromUserDefine" . | nindent 10 }}
+        {{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }}
+          {{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }}
         {{- else }}
           httpGet:
             scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }}

diff --git a/charts/selenium-grid/templates/hub-deployment.yaml b/charts/selenium-grid/templates/hub-deployment.yaml
@@ -45,8 +45,8 @@ spec:
         {{- if .Values.hub.startupProbe.enabled }}
           {{- with .Values.hub.startupProbe }}
           startupProbe:
-          {{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }}
-            {{- include "seleniumGrid.probe.fromUserDefine" . | nindent 10 }}
+          {{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }}
+            {{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }}
           {{- else }}
             httpGet:
               scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }}
@@ -61,8 +61,8 @@ spec:
         {{- if .Values.hub.readinessProbe.enabled }}
           {{- with .Values.hub.readinessProbe }}
           readinessProbe:
-          {{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }}
-            {{- include "seleniumGrid.probe.fromUserDefine" . | nindent 10 }}
+          {{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }}
+            {{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }}
           {{- else }}
             httpGet:
               scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }}
@@ -77,8 +77,8 @@ spec:
         {{- if .Values.hub.livenessProbe.enabled }}
           {{- with .Values.hub.livenessProbe }}
           livenessProbe:
-          {{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }}
-            {{- include "seleniumGrid.probe.fromUserDefine" . | nindent 10 }}
+          {{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }}
+            {{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }}
           {{- else }}
             httpGet:
               scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }}

diff --git a/charts/selenium-grid/templates/node-configmap.yaml b/charts/selenium-grid/templates/node-configmap.yaml
@@ -12,9 +12,18 @@ metadata:
       {{- toYaml . | nindent 4 }}
     {{- end }}
 data:
+{{- if .Values.isolateComponents }}
+  SE_DISTRIBUTOR_HOST: '{{ include "seleniumGrid.distributor.fullname" . }}.{{ .Release.Namespace }}'
+  SE_DISTRIBUTOR_PORT: '{{ .Values.components.distributor.port }}'
+  SE_ROUTER_HOST: '{{ include "seleniumGrid.router.fullname" . }}.{{ .Release.Namespace }}'
+  SE_ROUTER_PORT: '{{ .Values.components.router.port }}'
+{{- else }}
+  SE_HUB_HOST: '{{ include "seleniumGrid.hub.fullname" . }}.{{ .Release.Namespace }}'
+  SE_HUB_PORT: '{{ .Values.hub.port }}'
+{{- end }}
   SE_DRAIN_AFTER_SESSION_COUNT: '{{- and (eq (include "seleniumGrid.useKEDA" .) "true") (eq .Values.autoscaling.scalingType "job") | ternary "1" "0" -}}'
   SE_NODE_GRID_URL: '{{ include "seleniumGrid.url" .}}'
-  SE_NODE_GRID_GRAPHQL_URL: '{{ include "seleniumGrid.graphqlURL" .}}'
+  SE_NODE_GRID_GRAPHQL_URL: '{{ include "seleniumGrid.graphqlURL" . }}'
 {{- range $fileName, $value := .Values.nodeConfigMap.extraScripts }}
 {{- if not (empty $value) }}
 {{ $fileName | indent 2 -}}: |

diff --git a/charts/selenium-grid/templates/router-deployment.yaml b/charts/selenium-grid/templates/router-deployment.yaml
@@ -84,8 +84,8 @@ spec:
         {{- if .Values.components.router.startupProbe.enabled }}
           {{- with .Values.components.router.startupProbe }}
           startupProbe:
-          {{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }}
-            {{- include "seleniumGrid.probe.fromUserDefine" . | nindent 10 }}
+          {{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }}
+            {{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }}
           {{- else }}
             httpGet:
               scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }}
@@ -100,8 +100,8 @@ spec:
         {{- if .Values.components.router.readinessProbe.enabled }}
           {{- with .Values.components.router.readinessProbe }}
           readinessProbe:
-          {{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }}
-            {{- include "seleniumGrid.probe.fromUserDefine" . | nindent 10 }}
+          {{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }}
+            {{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }}
           {{- else }}
             httpGet:
               scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }}
@@ -116,8 +116,8 @@ spec:
         {{- if .Values.components.router.livenessProbe.enabled }}
           livenessProbe:
           {{- with .Values.components.router.livenessProbe }}
-          {{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }}
-            {{- include "seleniumGrid.probe.fromUserDefine" . | nindent 10 }}
+          {{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }}
+            {{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }}
           {{- else }}
             httpGet:
               scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }}