Skip to content

Commit

Permalink
update(chart): Node preStop and startupProbe in autoscaling Deployment (
Browse files Browse the repository at this point in the history
#2139)

update(chart): script for Node preStop and startupProbe in autoscaling Deployment

Signed-off-by: Viet Nguyen Duc <nguyenducviet4496@gmail.com>
  • Loading branch information
VietND96 committed Feb 19, 2024
1 parent cd10250 commit 56f3c49
Show file tree
Hide file tree
Showing 14 changed files with 201 additions and 49 deletions.
11 changes: 8 additions & 3 deletions .github/workflows/helm-chart-test.yml
Expand Up @@ -15,6 +15,9 @@ on:
description: 'Test parameter for different request timeout'
required: false
default: '370'
cluster:
description: 'Cluster type to test (kind, minikube)'
required: false
schedule:
- cron: '0 0 * * *'

Expand All @@ -33,6 +36,8 @@ jobs:
- job_https
- deployment
- deployment_https
env:
CLUSTER: ${{ github.event.inputs.cluster || 'minikube' }}
steps:
- uses: actions/checkout@main
- name: Output Docker info
Expand Down Expand Up @@ -63,7 +68,7 @@ jobs:
with:
timeout_minutes: 10
max_attempts: 3
command: make chart_setup_env
command: CLUSTER=${CLUSTER} make chart_setup_env
- name: Build Helm charts
run: |
BUILD_DATE=${BUILD_DATE} make chart_build
Expand All @@ -76,7 +81,7 @@ jobs:
with:
timeout_minutes: 10
max_attempts: 3
command: NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make chart_cluster_setup
command: CLUSTER=${CLUSTER} NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make chart_cluster_setup
- name: Build Docker images
run: NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make build
- name: Test set custom CA certificate
Expand All @@ -96,7 +101,7 @@ jobs:
NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make chart_test_autoscaling_${{ matrix.test-strategy }}
- name: Cleanup Kubernetes cluster
if: always()
run: make chart_cluster_cleanup
run: CLUSTER=${CLUSTER} make chart_cluster_cleanup
- name: Upload Helm chart package
if: always()
uses: actions/upload-artifact@main
Expand Down
12 changes: 9 additions & 3 deletions Makefile
Expand Up @@ -447,16 +447,22 @@ chart_test_edge:
./tests/charts/make/chart_test.sh NodeEdge

chart_test_autoscaling_deployment_https:
SELENIUM_GRID_TEST_HEADLESS=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_PORT=443 make chart_test_autoscaling_deployment
SE_FULL_DISTRIBUTED_MODE=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_PORT=443 \
VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \
./tests/charts/make/chart_test.sh DeploymentAutoscaling

chart_test_autoscaling_deployment:
SE_ENABLE_TRACING=true VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \
SE_ENABLE_TRACING=true SELENIUM_GRID_TEST_HEADLESS=true \
VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \
./tests/charts/make/chart_test.sh DeploymentAutoscaling

chart_test_autoscaling_job_https:
SE_ENABLE_TRACING=true SELENIUM_GRID_TEST_HEADLESS=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_PORT=443 make chart_test_autoscaling_job
SELENIUM_GRID_TEST_HEADLESS=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_PORT=443 \
VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \
./tests/charts/make/chart_test.sh JobAutoscaling

chart_test_autoscaling_job:
SE_ENABLE_TRACING=true SE_FULL_DISTRIBUTED_MODE=true \
VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \
./tests/charts/make/chart_test.sh JobAutoscaling

Expand Down
5 changes: 4 additions & 1 deletion Video/video.sh
Expand Up @@ -159,7 +159,10 @@ else
echo Checking if node API responds
until curl -sk --request GET ${SE_SERVER_PROTOCOL}://${DISPLAY_CONTAINER_NAME}:${SE_NODE_PORT}/status || [[ $attempts = "$max_attempts" ]]
do
echo Waiting before next API check
if [ $(($attempts % 60)) -eq 0 ];
then
echo Waiting before next API check
fi
sleep 0.5
attempts=$((attempts+1))
done
Expand Down
62 changes: 61 additions & 1 deletion charts/selenium-grid/configs/node/nodePreStop.sh
@@ -1,13 +1,73 @@
#!/bin/bash

function on_exit() {
rm -rf /tmp/preStopOutput
}
trap on_exit EXIT

# Set headers if Node Registration Secret is set
if [ ! -z "${SE_REGISTRATION_SECRET}" ];
then
HEADERS="X-REGISTRATION-SECRET: ${SE_REGISTRATION_SECRET}"
else
HEADERS="X-REGISTRATION-SECRET;"
fi

if curl -sfk ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/status; then
function is_full_distributed_mode() {
if [ -n "${SE_DISTRIBUTOR_HOST}" ] && [ -n "${SE_DISTRIBUTOR_PORT}" ]; then
DISTRIBUTED_MODE=true
echo "Detected full distributed mode: ${DISTRIBUTED_MODE}. Since SE_DISTRIBUTOR_HOST and SE_DISTRIBUTOR_PORT are set in Node ConfigMap"
else
DISTRIBUTED_MODE=false
echo "Detected full distributed mode: ${DISTRIBUTED_MODE}"
fi
}
is_full_distributed_mode

function signal_distributor_to_drain_node() {
if [ "${DISTRIBUTED_MODE}" = true ]; then
echo "Signaling Distributor to drain node"
set -x
curl -k -X POST ${SE_SERVER_PROTOCOL}://${SE_DISTRIBUTOR_HOST}:${SE_DISTRIBUTOR_PORT}/se/grid/distributor/node/${NODE_ID}/drain --header "${HEADERS}"
set +x
fi
}

function signal_hub_to_drain_node() {
if [ "${DISTRIBUTED_MODE}" = false ]; then
echo "Signaling Hub to drain node"
curl -k -X POST ${SE_GRID_URL}/se/grid/distributor/node/${NODE_ID}/drain --header "${HEADERS}"
fi
}

function signal_node_to_drain() {
echo "Signaling Node to drain itself"
curl -k -X POST ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/se/grid/node/drain --header "${HEADERS}"
}

function replace_localhost_by_service_name() {
internal="${SE_HUB_HOST:-$SE_ROUTER_HOST}:${SE_HUB_PORT:-$SE_ROUTER_PORT}"
if [[ "${SE_NODE_GRID_URL}" == *"/localhost"* ]]; then
SE_GRID_URL=${SE_NODE_GRID_URL//localhost/${internal}}
elif [[ "${SE_NODE_GRID_URL}" == *"/127.0.0.1"* ]]; then
SE_GRID_URL=${SE_NODE_GRID_URL//127.0.0.1/${internal}}
elif [[ "${SE_NODE_GRID_URL}" == *"/0.0.0.0"* ]]; then
SE_GRID_URL=${SE_NODE_GRID_URL//0.0.0.0/${internal}}
fi
echo "SE_GRID_URL: ${SE_GRID_URL}"
}
replace_localhost_by_service_name

if curl -sfk ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/status > /tmp/preStopOutput; then
NODE_ID=$(jq -r '.value.node.nodeId' /tmp/preStopOutput)
if [ -n "${NODE_ID}" ]; then
echo "Current Node ID is: ${NODE_ID}"
signal_hub_to_drain_node
signal_distributor_to_drain_node
echo
fi
signal_node_to_drain
# Wait for the current session to be finished if any
while curl -sfk ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/status -o /tmp/preStopOutput;
do
echo "Node preStop is waiting for current session to be finished if any. Node details: message: $(jq -r '.value.message' /tmp/preStopOutput || "unknown"), availability: $(jq -r '.value.node.availability' /tmp/preStopOutput || "unknown")"
Expand Down
41 changes: 41 additions & 0 deletions charts/selenium-grid/configs/node/nodeProbe.sh
@@ -0,0 +1,41 @@
#!/bin/bash

function on_exit() {
rm -rf /tmp/nodeProbe${ID}
rm -rf /tmp/gridProbe${ID}
}
trap on_exit EXIT

ID=$(echo $RANDOM)

function replace_localhost_by_service_name() {
internal="${SE_HUB_HOST:-$SE_ROUTER_HOST}:${SE_HUB_PORT:-$SE_ROUTER_PORT}"
if [[ "${SE_NODE_GRID_URL}" == *"/localhost"* ]]; then
SE_GRID_URL=${SE_NODE_GRID_URL//localhost/${internal}}
elif [[ "${SE_NODE_GRID_URL}" == *"/127.0.0.1"* ]]; then
SE_GRID_URL=${SE_NODE_GRID_URL//127.0.0.1/${internal}}
elif [[ "${SE_NODE_GRID_URL}" == *"/0.0.0.0"* ]]; then
SE_GRID_URL=${SE_NODE_GRID_URL//0.0.0.0/${internal}}
fi
echo "SE_GRID_URL: ${SE_GRID_URL}"
}
replace_localhost_by_service_name

if curl -sfk ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/status -o /tmp/nodeProbe${ID}; then
NODE_ID=$(jq -r '.value.node.nodeId' /tmp/nodeProbe${ID})
NODE_STATUS=$(jq -r '.value.node.availability' /tmp/nodeProbe${ID})

curl -sfk "${SE_GRID_URL}/status" -o /tmp/gridProbe${ID}
GRID_NODE_ID=$(jq -e ".value.nodes[].id|select(. == \"${NODE_ID}\")" /tmp/gridProbe${ID} | tr -d '"' || true)

if [ "${NODE_STATUS}" = "UP" ] && [ -n "${NODE_ID}" ] && [ -n "${GRID_NODE_ID}" ] && [ "${NODE_ID}" = "${GRID_NODE_ID}" ]; then
echo "Node ID: ${NODE_ID} with status: ${NODE_STATUS}"
echo "Found in the Grid a matched Node ID: ${GRID_NODE_ID}"
exit 0
else
echo "Node ID: ${NODE_ID} is not found in the Grid. The registration could be in progress."
exit 1
fi
else
exit 1
fi
30 changes: 16 additions & 14 deletions charts/selenium-grid/templates/_helpers.tpl
Expand Up @@ -17,18 +17,20 @@ Probe httpGet schema
Check user define custom probe method
*/}}
{{- define "seleniumGrid.probe.fromUserDefine" -}}
{{- $values := index . "values" -}}
{{- $root := index . "root" -}}
{{- $overrideProbe := dict -}}
{{- with .exec -}}
{{- $overrideProbe = dict "exec" . -}}
{{- with $values.exec -}}
{{- $overrideProbe = dict "exec" (tpl (toYaml .) $root | fromYaml) -}}
{{- end }}
{{- with .httpGet -}}
{{- $overrideProbe = dict "httpGet" . -}}
{{- with $values.httpGet -}}
{{- $overrideProbe = dict "httpGet" (tpl (toYaml .) $root | fromYaml) -}}
{{- end }}
{{- with .tcpSocket -}}
{{- $overrideProbe = dict "tcpSocket" . -}}
{{- with $values.tcpSocket -}}
{{- $overrideProbe = dict "tcpSocket" (tpl (toYaml .) $root | fromYaml) -}}
{{- end }}
{{- with .grpc -}}
{{- $overrideProbe = dict "grpc" . -}}
{{- with $values.grpc -}}
{{- $overrideProbe = dict "grpc" (tpl (toYaml .) $root | fromYaml) -}}
{{- end -}}
{{- $overrideProbe | toYaml -}}
{{- end -}}
Expand Down Expand Up @@ -258,8 +260,8 @@ template:
{{- if .node.startupProbe.enabled }}
{{- with .node.startupProbe }}
startupProbe:
{{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }}
{{- include "seleniumGrid.probe.fromUserDefine" . | nindent 10 }}
{{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }}
{{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }}
{{- else }}
httpGet:
scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }}
Expand All @@ -274,8 +276,8 @@ template:
{{- if .node.readinessProbe.enabled }}
{{- with .node.readinessProbe }}
readinessProbe:
{{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }}
{{- include "seleniumGrid.probe.fromUserDefine" . | nindent 12 }}
{{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }}
{{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 12 }}
{{- else }}
httpGet:
scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }}
Expand All @@ -290,8 +292,8 @@ template:
{{- if .node.livenessProbe.enabled }}
{{- with .node.livenessProbe }}
livenessProbe:
{{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }}
{{- include "seleniumGrid.probe.fromUserDefine" . | nindent 10 }}
{{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }}
{{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }}
{{- else }}
httpGet:
scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }}
Expand Down
12 changes: 6 additions & 6 deletions charts/selenium-grid/templates/hub-deployment.yaml
Expand Up @@ -45,8 +45,8 @@ spec:
{{- if .Values.hub.startupProbe.enabled }}
{{- with .Values.hub.startupProbe }}
startupProbe:
{{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }}
{{- include "seleniumGrid.probe.fromUserDefine" . | nindent 10 }}
{{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }}
{{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }}
{{- else }}
httpGet:
scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }}
Expand All @@ -61,8 +61,8 @@ spec:
{{- if .Values.hub.readinessProbe.enabled }}
{{- with .Values.hub.readinessProbe }}
readinessProbe:
{{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }}
{{- include "seleniumGrid.probe.fromUserDefine" . | nindent 10 }}
{{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }}
{{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }}
{{- else }}
httpGet:
scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }}
Expand All @@ -77,8 +77,8 @@ spec:
{{- if .Values.hub.livenessProbe.enabled }}
{{- with .Values.hub.livenessProbe }}
livenessProbe:
{{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }}
{{- include "seleniumGrid.probe.fromUserDefine" . | nindent 10 }}
{{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }}
{{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }}
{{- else }}
httpGet:
scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }}
Expand Down
11 changes: 10 additions & 1 deletion charts/selenium-grid/templates/node-configmap.yaml
Expand Up @@ -12,9 +12,18 @@ metadata:
{{- toYaml . | nindent 4 }}
{{- end }}
data:
{{- if .Values.isolateComponents }}
SE_DISTRIBUTOR_HOST: '{{ include "seleniumGrid.distributor.fullname" . }}.{{ .Release.Namespace }}'
SE_DISTRIBUTOR_PORT: '{{ .Values.components.distributor.port }}'
SE_ROUTER_HOST: '{{ include "seleniumGrid.router.fullname" . }}.{{ .Release.Namespace }}'
SE_ROUTER_PORT: '{{ .Values.components.router.port }}'
{{- else }}
SE_HUB_HOST: '{{ include "seleniumGrid.hub.fullname" . }}.{{ .Release.Namespace }}'
SE_HUB_PORT: '{{ .Values.hub.port }}'
{{- end }}
SE_DRAIN_AFTER_SESSION_COUNT: '{{- and (eq (include "seleniumGrid.useKEDA" .) "true") (eq .Values.autoscaling.scalingType "job") | ternary "1" "0" -}}'
SE_NODE_GRID_URL: '{{ include "seleniumGrid.url" .}}'
SE_NODE_GRID_GRAPHQL_URL: '{{ include "seleniumGrid.graphqlURL" .}}'
SE_NODE_GRID_GRAPHQL_URL: '{{ include "seleniumGrid.graphqlURL" . }}'
{{- range $fileName, $value := .Values.nodeConfigMap.extraScripts }}
{{- if not (empty $value) }}
{{ $fileName | indent 2 -}}: |
Expand Down
12 changes: 6 additions & 6 deletions charts/selenium-grid/templates/router-deployment.yaml
Expand Up @@ -84,8 +84,8 @@ spec:
{{- if .Values.components.router.startupProbe.enabled }}
{{- with .Values.components.router.startupProbe }}
startupProbe:
{{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }}
{{- include "seleniumGrid.probe.fromUserDefine" . | nindent 10 }}
{{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }}
{{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }}
{{- else }}
httpGet:
scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }}
Expand All @@ -100,8 +100,8 @@ spec:
{{- if .Values.components.router.readinessProbe.enabled }}
{{- with .Values.components.router.readinessProbe }}
readinessProbe:
{{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }}
{{- include "seleniumGrid.probe.fromUserDefine" . | nindent 10 }}
{{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }}
{{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }}
{{- else }}
httpGet:
scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }}
Expand All @@ -116,8 +116,8 @@ spec:
{{- if .Values.components.router.livenessProbe.enabled }}
livenessProbe:
{{- with .Values.components.router.livenessProbe }}
{{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }}
{{- include "seleniumGrid.probe.fromUserDefine" . | nindent 10 }}
{{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }}
{{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }}
{{- else }}
httpGet:
scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }}
Expand Down

0 comments on commit 56f3c49

Please sign in to comment.