From e521d9952adbc746d25f3ef9dea326c7edd4ee67 Mon Sep 17 00:00:00 2001 From: Viet Nguyen Duc Date: Fri, 19 Jan 2024 08:46:15 +0700 Subject: [PATCH] feat(chart): videoRecorder getting scripts from external files (#2095) Signed-off-by: Viet Nguyen Duc --- .github/workflows/build-test.yml | 4 + .github/workflows/helm-chart-test.yml | 8 +- .github/workflows/nightly.yaml | 10 +- .github/workflows/test-video.yml | 4 + Makefile | 10 +- charts/selenium-grid/README.md | 7 + .../selenium-grid/configs/node/nodePreStop.sh | 7 + .../configs/uploader/s3/entry_point.sh | 44 ++++++ charts/selenium-grid/configs/video/video.sh | 136 ++++++++++++++++++ charts/selenium-grid/templates/_helpers.tpl | 32 ++++- .../templates/chrome-node-hpa.yaml | 8 ++ .../templates/edge-node-hpa.yaml | 8 ++ .../templates/firefox-node-hpa.yaml | 8 ++ .../templates/node-configmap.yaml | 9 +- .../templates/server-configmap.yaml | 1 + charts/selenium-grid/templates/video-cm.yaml | 95 +----------- charts/selenium-grid/values.yaml | 40 +++--- tests/SeleniumTests/__init__.py | 11 +- tests/charts/templates/render/dummy.yaml | 13 ++ 19 files changed, 320 insertions(+), 135 deletions(-) create mode 100644 charts/selenium-grid/configs/node/nodePreStop.sh create mode 100644 charts/selenium-grid/configs/uploader/s3/entry_point.sh create mode 100755 charts/selenium-grid/configs/video/video.sh diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index b40db3af6..d0b68d0f5 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -2,7 +2,11 @@ name: Build & test on: push: + paths-ignore: + - '**.md' pull_request: + paths-ignore: + - '**.md' permissions: contents: read diff --git a/.github/workflows/helm-chart-test.yml b/.github/workflows/helm-chart-test.yml index 42408e0e3..5e2bb614f 100644 --- a/.github/workflows/helm-chart-test.yml +++ b/.github/workflows/helm-chart-test.yml @@ -2,7 +2,11 @@ name: Lint and Test Helm Charts on: push: + paths-ignore: + - '**.md' pull_request: + paths-ignore: + - '**.md' workflow_dispatch: permissions: @@ -15,7 +19,7 @@ jobs: strategy: fail-fast: false matrix: - test-strategy: [chart_test, chart_test_parallel_autoscaling, chart_test_https_tls] + test-strategy: [chart_test, chart_test_parallel_autoscaling, chart_test_https, chart_test_parallel_autoscaling_https] steps: - uses: actions/checkout@v4 - name: Output Docker info @@ -49,7 +53,7 @@ jobs: - name: Setup Kubernetes environment run: make chart_setup_env - name: Build Docker images - run: NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make build + run: NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make build_nightly - name: Build and lint charts run: | BUILD_DATE=${BUILD_DATE} make chart_build diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml index 58164e47e..44183bc86 100644 --- a/.github/workflows/nightly.yaml +++ b/.github/workflows/nightly.yaml @@ -87,7 +87,7 @@ jobs: echo "CHART_PACKAGE_PATH=$(cat /tmp/selenium_chart_version)" >> $GITHUB_ENV echo "CHART_FILE_NAME=$(basename $(cat /tmp/selenium_chart_version))" >> $GITHUB_ENV - name: Delete previous nightly tag if any - uses: cb80/delrel@latest + uses: cb80/delrel@main with: tag: ${{ env.BASE_RELEASE }} token: ${{ secrets.GITHUB_TOKEN }} @@ -104,4 +104,10 @@ jobs: generate_release_notes: true draft: false prerelease: true - append_body: true + append_body: false + - name: Update tag nightly + uses: richardsimko/update-tag@v1.0.11 + with: + tag_name: ${{ env.BASE_RELEASE }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/test-video.yml b/.github/workflows/test-video.yml index 02aa7f7c4..04bf156d0 100644 --- a/.github/workflows/test-video.yml +++ b/.github/workflows/test-video.yml @@ -2,7 +2,11 @@ name: Test video files on: push: + paths-ignore: + - '**.md' pull_request: + paths-ignore: + - '**.md' permissions: contents: read diff --git a/Makefile b/Makefile index 46522b156..4984fa37c 100644 --- a/Makefile +++ b/Makefile @@ -426,6 +426,9 @@ chart_build_nightly: chart_build: VERSION=$(TAG_VERSION) ./tests/charts/make/chart_build.sh +chart_test_https: + SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_PORT=443 make chart_test + chart_test: chart_test_template \ chart_test_chrome \ chart_test_firefox \ @@ -443,13 +446,12 @@ chart_test_firefox: chart_test_edge: VERSION=$(TAG_VERSION) NAMESPACE=$(NAMESPACE) ./tests/charts/make/chart_test.sh NodeEdge +chart_test_parallel_autoscaling_https: + SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_PORT=443 make chart_test_parallel_autoscaling + chart_test_parallel_autoscaling: VERSION=$(TAG_VERSION) NAMESPACE=$(NAMESPACE) ./tests/charts/make/chart_test.sh JobAutoscaling -chart_test_https_tls: - VERSION=$(TAG_VERSION) NAMESPACE=$(NAMESPACE) SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_PORT=443 \ - ./tests/charts/make/chart_test.sh JobAutoscaling - .PHONY: \ all \ base \ diff --git a/charts/selenium-grid/README.md b/charts/selenium-grid/README.md index 8655e0a02..a11e3bcd5 100644 --- a/charts/selenium-grid/README.md +++ b/charts/selenium-grid/README.md @@ -524,6 +524,8 @@ This table contains the configuration parameters of the chart and their default | `chromeNode.hpa.url` | `{{ include "seleniumGrid.graphqlURL" . }}` | Graphql Url of the hub or the router | | `chromeNode.hpa.browserName` | `chrome` | BrowserName from the capability | | `chromeNode.hpa.browserVersion` | `` | BrowserVersion from the capability | +| `chromeNode.sidecars` | `[]` | Add a sidecars proxy in the same pod of the browser node | +| `chromeNode.initContainers` | `[]` | Add initContainers in the same pod of the browser node | | `chromeNode.scaledOptions` | See `values.yaml` | Override the global `autoscaling.scaledOptions` with specific scaled options for chrome nodes | | `chromeNode.scaledJobOptions` | See `values.yaml` | Override the global `autoscaling.scaledJobOptions` with specific scaled options for chrome nodes | | `chromeNode.scaledObjectOptions` | See `values.yaml` | Override the global `autoscaling.scaledObjectOptions` with specific scaled options for chrome nodes | @@ -565,6 +567,8 @@ This table contains the configuration parameters of the chart and their default | `firefoxNode.hpa.url` | `{{ include "seleniumGrid.graphqlURL" . }}` | Graphql Url of the hub or the router | | `firefoxNode.hpa.browserName` | `firefox` | BrowserName from the capability | | `firefoxNode.hpa.browserVersion` | `` | BrowserVersion from the capability | +| `firefoxNode.sidecars` | `[]` | Add a sidecars proxy in the same pod of the browser node | +| `firefoxNode.initContainers` | `[]` | Add initContainers in the same pod of the browser node | | `firefoxNode.scaledOptions` | See `values.yaml` | Override the global `autoscaling.scaledOptions` with specific scaled options for firefox nodes | | `firefoxNode.scaledJobOptions` | See `values.yaml` | Override the global `autoscaling.scaledJobOptions` with specific scaled options for firefox nodes | | `firefoxNode.scaledObjectOptions` | See `values.yaml` | Override the global `autoscaling.scaledObjectOptions` with specific scaled options for firefox nodes | @@ -606,6 +610,8 @@ This table contains the configuration parameters of the chart and their default | `edgeNode.hpa.url` | `{{ include "seleniumGrid.graphqlURL" . }}` | Graphql Url of the hub or the router | | `edgeNode.hpa.browserName` | `edge` | BrowserName from the capability | | `edgeNode.hpa.browserVersion` | `` | BrowserVersion from the capability | +| `edgeNode.sidecars` | `[]` | Add a sidecars proxy in the same pod of the browser node | +| `edgeNode.initContainers` | `[]` | Add initContainers in the same pod of the browser node | | `edgeNode.scaledOptions` | See `values.yaml` | Override the global `autoscaling.scaledOptions` with specific scaled options for edge nodes | | `edgeNode.scaledJobOptions` | See `values.yaml` | Override the global `autoscaling.scaledJobOptions` with specific scaled options for edge nodes | | `edgeNode.scaledObjectOptions` | See `values.yaml` | Override the global `autoscaling.scaledObjectOptions` with specific scaled options for edge nodes | @@ -623,6 +629,7 @@ This table contains the configuration parameters of the chart and their default | `videoRecorder.terminationGracePeriodSeconds` | `30` | Time to graceful terminate container (default: 30s) | | `videoRecorder.startupProbe` | `{}` | Probe to check pod is started successfully | | `videoRecorder.livenessProbe` | `{}` | Liveness probe settings | +| `videoRecorder.lifecycle` | `{}` | Define lifecycle events for video recorder | | `videoRecorder.volume.name.folder` | `video` | Name is used to set for the volume to persist and share output video folder in container | | `videoRecorder.volume.name.scripts` | `video-scripts` | Name is used to set for the volume to persist and share video recorder scripts in container | | `videoRecorder.extraVolumeMounts` | `[]` | Extra mounts of declared ExtraVolumes into pod | diff --git a/charts/selenium-grid/configs/node/nodePreStop.sh b/charts/selenium-grid/configs/node/nodePreStop.sh new file mode 100644 index 000000000..dded38676 --- /dev/null +++ b/charts/selenium-grid/configs/node/nodePreStop.sh @@ -0,0 +1,7 @@ +#!/bin/bash +if curl -sfk ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/status; then + curl -k -X POST ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/se/grid/node/drain --header 'X-REGISTRATION-SECRET;' + while curl -sfk ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/status; do sleep 1; done +else + echo "Node is already drained. Shutting down gracefully!" +fi diff --git a/charts/selenium-grid/configs/uploader/s3/entry_point.sh b/charts/selenium-grid/configs/uploader/s3/entry_point.sh new file mode 100644 index 000000000..e2d3d8f5f --- /dev/null +++ b/charts/selenium-grid/configs/uploader/s3/entry_point.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +SE_VIDEO_FOLDER=${SE_VIDEO_FOLDER:-"/videos"} + +if [[ -z "${AWS_REGION}" ]] || [[ -z "${AWS_ACCESS_KEY_ID}" ]] || [[ -z "${AWS_SECRET_ACCESS_KEY}" ]]; +then + echo "AWS credentials needed to provide for configuring AWS CLI" +fi + +aws configure set region ${AWS_REGION} --profile s3-profile +aws configure set aws_access_key_id ${AWS_ACCESS_KEY_ID} --profile s3-profile +aws configure set aws_secret_access_key ${AWS_SECRET_ACCESS_KEY} --profile s3-profile +aws configure --profile s3-profile + +function consume_force_exit() { + rm -f ${SE_VIDEO_FOLDER}/force_exit + echo "Force exit signal consumed" +} +trap consume_force_exit EXIT + +while [ ! -p ${SE_VIDEO_FOLDER}/uploadpipe ]; +do + echo "Waiting for ${SE_VIDEO_FOLDER}/uploadpipe to be created" + sleep 1 +done + +echo "Waiting for video files put into pipe for proceeding to upload" + +while read FILE DESTINATION < ${SE_VIDEO_FOLDER}/uploadpipe +do + if [ "${FILE}" = "exit" ]; + then + exit + else [ "$FILE" != "" ] && [ "$DESTINATION" != "" ]; + echo "Uploading ${FILE} to ${DESTINATION}" + aws s3 cp "${FILE}" "${DESTINATION}" + fi + if [ -f ${SE_VIDEO_FOLDER}/force_exit ] && [ ! -s ${SE_VIDEO_FOLDER}/uploadpipe ]; + then + exit + fi +done + +consume_force_exit diff --git a/charts/selenium-grid/configs/video/video.sh b/charts/selenium-grid/configs/video/video.sh new file mode 100755 index 000000000..921d53399 --- /dev/null +++ b/charts/selenium-grid/configs/video/video.sh @@ -0,0 +1,136 @@ +#!/usr/bin/env bash + +function create_pipe() { + if [[ "$UPLOAD_DESTINATION_PREFIX" != "false" ]]; + then + echo "Create pipe if not exists for video upload stream" + if [ ! -p ${SE_VIDEO_FOLDER}/uploadpipe ]; + then + mkfifo ${SE_VIDEO_FOLDER}/uploadpipe + fi + fi +} +create_pipe + +function wait_util_force_exit_consume() { + if [[ "$UPLOAD_DESTINATION_PREFIX" != "false" ]]; + then + while [[ -f ${SE_VIDEO_FOLDER}/force_exit ]] + do + echo "Waiting for force exit file to be consumed by uploader" + sleep 1 + done + echo "Ready to shutdown the recorder" + fi +} + +function add_exit_signal() { + if [[ "$UPLOAD_DESTINATION_PREFIX" != "false" ]]; + then + echo "exit" > ${SE_VIDEO_FOLDER}/uploadpipe & + echo "exit" > ${SE_VIDEO_FOLDER}/force_exit + fi +} + +function exit_on_max_session_reach() { + if [ $max_recorded_count -gt 0 ] && [ $recorded_count -ge $max_recorded_count ]; + then + echo "Node will be drained since max sessions reached count number ($max_recorded_count)" + exit + fi +} + +function finish { + add_exit_signal + wait_util_force_exit_consume + kill -INT `cat /var/run/supervisor/supervisord.pid` +} +trap finish EXIT + +FRAME_RATE=${FRAME_RATE:-$SE_FRAME_RATE} +CODEC=${CODEC:-$SE_CODEC} +PRESET=${PRESET:-$SE_PRESET} +DISPLAY_CONTAINER_NAME=${DISPLAY_CONTAINER_NAME:-"localhost"} +export DISPLAY=${DISPLAY_CONTAINER_NAME}:${DISPLAY_NUM}.0 + +max_attempts=600 +attempts=0 +if [[ "$UPLOAD_DESTINATION_PREFIX" = "" ]] +then + echo Upload destination not known since UPLOAD_DESTINATION_PREFIX is not set. Exiting video recorder. + exit +fi +echo Checking if the display is open +until xset b off || [[ $attempts = $max_attempts ]] +do + echo Waiting before next display check + sleep 0.5 + attempts=$((attempts+1)) +done +if [[ $attempts = $max_attempts ]] +then + echo Can not open display, exiting. + exit +fi +VIDEO_SIZE=$(xdpyinfo | grep 'dimensions:' | awk '{print $2}') + +recording_started="false" +video_file_name="" +video_file="" +prev_session_id="" +attempts=0 +max_recorded_count=${SE_DRAIN_AFTER_SESSION_COUNT:-0} +recorded_count=0 +echo Checking if node API responds +until curl -sk --request GET ${SE_SERVER_PROTOCOL}://${DISPLAY_CONTAINER_NAME}:${SE_NODE_PORT}/status || [[ $attempts = $max_attempts ]] +do + echo Waiting before next API check + sleep 0.5 + attempts=$((attempts+1)) +done +if [[ $attempts = $max_attempts ]] +then + echo Can not reach node API, exiting. + exit +fi +while curl -sk --request GET ${SE_SERVER_PROTOCOL}://${DISPLAY_CONTAINER_NAME}:${SE_NODE_PORT}/status > /tmp/status.json +do + session_id=$(jq -r '.[]?.node?.slots | .[0]?.session?.sessionId' /tmp/status.json) + echo $session_id + if [[ "$session_id" != "null" && "$session_id" != "" && "$recording_started" = "false" ]] + then + video_file_name="$session_id.mp4" + video_file="${SE_VIDEO_FOLDER}/$video_file_name" + echo "Starting to record video" + ffmpeg -nostdin -y -f x11grab -video_size ${VIDEO_SIZE} -r ${FRAME_RATE} -i ${DISPLAY} -codec:v ${CODEC} ${PRESET} -pix_fmt yuv420p $video_file & + recording_started="true" + echo "Video recording started" + elif [[ "$session_id" != "$prev_session_id" && "$recording_started" = "true" ]] + then + echo "Stopping to record video" + pkill -INT ffmpeg + recorded_count=$((recorded_count+1)) + recording_started="false" + if [[ "$UPLOAD_DESTINATION_PREFIX" != "false" ]] + then + upload_destination=${UPLOAD_DESTINATION_PREFIX}/${video_file_name} + echo "Uploading video to $upload_destination" + echo $video_file $upload_destination >> ${SE_VIDEO_FOLDER}/uploadpipe & + fi + if [ $max_recorded_count -gt 0 ] && [ $recorded_count -ge $max_recorded_count ]; + then + echo "Node will be drained since max sessions reached count number ($max_recorded_count)" + exit + fi + + elif [[ $recording_started = "true" ]] + then + echo "Video recording in progress" + sleep 1 + else + echo "No session in progress" + sleep 1 + fi + prev_session_id=$session_id +done +echo "Node API is not responding, exiting." diff --git a/charts/selenium-grid/templates/_helpers.tpl b/charts/selenium-grid/templates/_helpers.tpl index e033bdac4..a5963d86f 100644 --- a/charts/selenium-grid/templates/_helpers.tpl +++ b/charts/selenium-grid/templates/_helpers.tpl @@ -317,6 +317,10 @@ template: {{- with .node.hostAliases }} hostAliases: {{ toYaml . | nindent 6 }} {{- end }} + initContainers: + {{- if .node.initContainers }} + {{- toYaml .node.initContainers | nindent 6 }} + {{- end }} containers: - name: {{.name}} {{- $imageTag := default .Values.global.seleniumGrid.nodesImageTag .node.imageTag }} @@ -436,6 +440,12 @@ template: image: {{ printf "%s/%s:%s" $imageRegistry .Values.videoRecorder.imageName $imageTag }} imagePullPolicy: {{ .Values.videoRecorder.imagePullPolicy }} env: + - name: SE_NODE_PORT + value: {{ .node.port | quote }} + - name: DISPLAY_CONTAINER_NAME + valueFrom: + fieldRef: + fieldPath: status.podIP - name: UPLOAD_DESTINATION_PREFIX value: {{ .Values.videoRecorder.uploadDestinationPrefix | quote }} {{- with .Values.videoRecorder.extraEnvironmentVariables }} @@ -444,6 +454,10 @@ template: envFrom: - configMapRef: name: {{ .Values.busConfigMap.name }} + - configMapRef: + name: {{ .Values.nodeConfigMap.name }} + - configMapRef: + name: {{ .Values.serverConfigMap.name }} {{- with .Values.videoRecorder.extraEnvFrom }} {{- tpl (toYaml .) $ | nindent 8 }} {{- end }} @@ -467,6 +481,9 @@ template: {{- with .Values.videoRecorder.livenessProbe }} livenessProbe: {{- toYaml . | nindent 10 }} {{- end }} + {{- with .Values.videoRecorder.lifecycle }} + lifecycle: {{- toYaml . | nindent 10 }} + {{- end }} {{- if .uploader }} - name: uploader image: {{ printf "%s:%s" .uploader.imageName .uploader.imageTag }} @@ -687,9 +704,12 @@ Default specs of VolumeMounts and Volumes for video recorder {{- end -}} {{- define "seleniumGrid.video.volumeMounts.default" -}} -- name: {{ include "seleniumGrid.video.volume.name.scripts" . }} - mountPath: /opt/bin/video.sh - subPath: video.sh +{{- $root := . -}} +{{- range $path, $bytes := .Files.Glob "configs/video/*" }} +- name: {{ include "seleniumGrid.video.volume.name.scripts" $ }} + mountPath: /opt/bin/{{ base $path }} + subPath: {{ base $path }} +{{- end }} - name: {{ include "seleniumGrid.video.volume.name.folder" . }} mountPath: /videos {{- end -}} @@ -704,6 +724,12 @@ Default specs of VolumeMounts and Volumes for video recorder {{- end -}} {{- define "seleniumGrid.video.uploader.volumeMounts.default" -}} +{{- $root := . -}} +{{- range $path, $bytes := .Files.Glob (printf "configs/uploader/%s/*" $.Values.videoRecorder.uploader) }} +- name: {{ include "seleniumGrid.video.volume.name.scripts" $ }} + mountPath: /opt/bin/{{ base $path }} + subPath: {{ base $path }} +{{- end }} - name: {{ include "seleniumGrid.video.volume.name.folder" . }} mountPath: /videos {{- end -}} diff --git a/charts/selenium-grid/templates/chrome-node-hpa.yaml b/charts/selenium-grid/templates/chrome-node-hpa.yaml index 74c588198..107e87b8b 100644 --- a/charts/selenium-grid/templates/chrome-node-hpa.yaml +++ b/charts/selenium-grid/templates/chrome-node-hpa.yaml @@ -5,11 +5,19 @@ metadata: name: selenium-grid-chrome-scaledobject namespace: {{ .Release.Namespace }} annotations: + helm.sh/resource-policy: delete {{- with .Values.autoscaling.annotations }} {{- toYaml . | nindent 4 }} {{- end }} labels: deploymentName: {{ template "seleniumGrid.chromeNode.fullname" . }} + {{- include "seleniumGrid.commonLabels" . | nindent 4 }} + {{- with .Values.chromeNode.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.customLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} spec: {{- $podScope := deepCopy . -}} {{- $_ := set $podScope "name" (include "seleniumGrid.chromeNode.fullname" .) -}} diff --git a/charts/selenium-grid/templates/edge-node-hpa.yaml b/charts/selenium-grid/templates/edge-node-hpa.yaml index 0839a3bdd..32bd3968e 100644 --- a/charts/selenium-grid/templates/edge-node-hpa.yaml +++ b/charts/selenium-grid/templates/edge-node-hpa.yaml @@ -5,11 +5,19 @@ metadata: name: selenium-grid-edge-scaledobject namespace: {{ .Release.Namespace }} annotations: + helm.sh/resource-policy: delete {{- with .Values.autoscaling.annotations }} {{- toYaml . | nindent 4 }} {{- end }} labels: deploymentName: {{ template "seleniumGrid.edgeNode.fullname" . }} + {{- include "seleniumGrid.commonLabels" . | nindent 4 }} + {{- with .Values.edgeNode.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.customLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} spec: {{- $podScope := deepCopy . -}} {{- $_ := set $podScope "name" (include "seleniumGrid.edgeNode.fullname" .) -}} diff --git a/charts/selenium-grid/templates/firefox-node-hpa.yaml b/charts/selenium-grid/templates/firefox-node-hpa.yaml index ad18269a5..e331a3475 100644 --- a/charts/selenium-grid/templates/firefox-node-hpa.yaml +++ b/charts/selenium-grid/templates/firefox-node-hpa.yaml @@ -5,11 +5,19 @@ metadata: name: selenium-grid-firefox-scaledobject namespace: {{ .Release.Namespace }} annotations: + helm.sh/resource-policy: delete {{- with .Values.autoscaling.annotations }} {{- toYaml . | nindent 4 }} {{- end }} labels: deploymentName: {{ template "seleniumGrid.firefoxNode.fullname" . }} + {{- include "seleniumGrid.commonLabels" . | nindent 4 }} + {{- with .Values.firefoxNode.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.customLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} spec: {{- $podScope := deepCopy . -}} {{- $_ := set $podScope "name" (include "seleniumGrid.firefoxNode.fullname" .) -}} diff --git a/charts/selenium-grid/templates/node-configmap.yaml b/charts/selenium-grid/templates/node-configmap.yaml index 807e2036c..6f8062d17 100644 --- a/charts/selenium-grid/templates/node-configmap.yaml +++ b/charts/selenium-grid/templates/node-configmap.yaml @@ -14,11 +14,4 @@ metadata: data: SE_DRAIN_AFTER_SESSION_COUNT: '{{- and (eq (include "seleniumGrid.useKEDA" .) "true") (eq .Values.autoscaling.scalingType "job") | ternary "1" "0" -}}' SE_NODE_GRID_URL: '{{ include "seleniumGrid.url" .}}' - {{ .Values.nodeConfigMap.preStopScript }}: | - #!/bin/bash - if curl -sf 127.0.0.1:${SE_NODE_PORT}/status; then - curl -X POST 127.0.0.1:${SE_NODE_PORT}/se/grid/node/drain --header 'X-REGISTRATION-SECRET;' - while curl -sf 127.0.0.1:${SE_NODE_PORT}/status; do sleep 1; done - else - echo "Node is already drained. Shutting down gracefully!" - fi +{{ (.Files.Glob "configs/node/*").AsConfig | indent 2 }} diff --git a/charts/selenium-grid/templates/server-configmap.yaml b/charts/selenium-grid/templates/server-configmap.yaml index 3e4e4b766..dc3d80b73 100644 --- a/charts/selenium-grid/templates/server-configmap.yaml +++ b/charts/selenium-grid/templates/server-configmap.yaml @@ -12,6 +12,7 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} data: + SE_SERVER_PROTOCOL: {{ include "seleniumGrid.server.protocol" . | quote }} {{- if .Values.tls.enabled }} SE_HTTPS_CERTIFICATE: {{ printf "%s/%s" .Values.serverConfigMap.certVolumeMountPath .Values.serverConfigMap.certificateFile | quote }} SE_HTTPS_PRIVATE_KEY: {{ printf "%s/%s" .Values.serverConfigMap.certVolumeMountPath .Values.serverConfigMap.privateKeyFile | quote }} diff --git a/charts/selenium-grid/templates/video-cm.yaml b/charts/selenium-grid/templates/video-cm.yaml index 1b49e026e..34eb24f05 100644 --- a/charts/selenium-grid/templates/video-cm.yaml +++ b/charts/selenium-grid/templates/video-cm.yaml @@ -4,95 +4,8 @@ kind: ConfigMap metadata: name: {{ template "seleniumGrid.video.fullname" . }} data: - video.sh: | - #!/usr/bin/env bash - set -em - function finish { - echo exit > /videos/uploadpipe - kill -s SIGINT `cat /var/run/supervisor/supervisord.pid` - } - trap finish EXIT - FRAME_RATE=${FRAME_RATE:-$SE_FRAME_RATE} - CODEC=${CODEC:-$SE_CODEC} - PRESET=${PRESET:-$SE_PRESET} - export DISPLAY=localhost:${DISPLAY_NUM}.0 - - return_code=1 - max_attempts=600 - attempts=0 - if [ ! -p /videos/uploadpipe ]; - then - mkfifo /videos/uploadpipe - fi - if [[ "$UPLOAD_DESTINATION_PREFIX" = "" ]] - then - echo Upload destination not known since UPLOAD_DESTINATION_PREFIX is not set. Exiting video recorder. - exit - fi - echo Checking if the display is open - until xset b off || [[ $attempts = $max_attempts ]] - do - echo Waiting before next display check - sleep 0.5 - attempts=$((attempts+1)) - done - if [[ $attempts = $max_attempts ]] - then - echo Can not open display, exiting. - exit - fi - VIDEO_SIZE=$(xdpyinfo | grep 'dimensions:' | awk '{print $2}') - - recording_started="false" - video_file_name="" - video_file="" - prev_session_id="" - attempts=0 - echo Checking if node API responds - until curl -s --request GET http://localhost:5555/status || [[ $attempts = $max_attempts ]] - do - echo Waiting before next API check - sleep 0.5 - attempts=$((attempts+1)) - done - if [[ $attempts = $max_attempts ]] - then - echo Can not reach node API, exiting. - exit - fi - while curl -s --request GET http://localhost:5555/status > /tmp/status.json - do - session_id=$(jq -r '.[]?.node?.slots | .[0]?.session?.sessionId' /tmp/status.json) - echo $session_id - if [[ "$session_id" != "null" && "$session_id" != "" && "$recording_started" = "false" ]] - then - video_file_name="$session_id.mp4" - video_file="${SE_VIDEO_FOLDER:-/videos}/$video_file_name" - echo "Starting to record video" - ffmpeg -nostdin -y -f x11grab -video_size ${VIDEO_SIZE} -r ${FRAME_RATE} -i ${DISPLAY} -codec:v ${CODEC} ${PRESET} -pix_fmt yuv420p $video_file & - recording_started="true" - echo "Video recording started" - elif [[ "$session_id" != "$prev_session_id" && "$recording_started" = "true" ]] - then - echo "Stopping to record video" - kill -INT %1 - fg || echo ffmpeg exited with code $? - if [[ "$UPLOAD_DESTINATION_PREFIX" != "false" ]] - then - upload_destination=${UPLOAD_DESTINATION_PREFIX}${video_file_name} - echo "Uploading video to $upload_destination" - echo $video_file $upload_destination > /videos/uploadpipe - fi - recording_started="false" - elif [[ $recording_started = "true" ]] - then - echo "Video recording in progress" - sleep 1 - else - echo "No session in progress" - sleep 1 - fi - prev_session_id=$session_id - done - echo +{{ (.Files.Glob "configs/video/*").AsConfig | indent 2 }} +{{- if and $.Values.videoRecorder.uploadDestinationPrefix $.Values.videoRecorder.uploader }} +{{ (.Files.Glob (printf "configs/uploader/%s/*" $.Values.videoRecorder.uploader)).AsConfig | indent 2 }} +{{- end }} {{- end }} diff --git a/charts/selenium-grid/values.yaml b/charts/selenium-grid/values.yaml index 6f4e9b7c7..79f3eff62 100644 --- a/charts/selenium-grid/values.yaml +++ b/charts/selenium-grid/values.yaml @@ -649,6 +649,10 @@ chromeNode: # browserVersion: '91.0' # Optional. Only required when supporting multiple versions of browser in your Selenium Grid. unsafeSsl: '{{ include "seleniumGrid.graphqlURL.unsafeSsl" . }}' # Optional + # It is used to add initContainers in the same pod of the browser node. + # It should be set using the --set-json option + initContainers: [] + # It is used to add a sidecars proxy in the same pod of the browser node. # It means it will add a new container to the deployment itself. # It should be set using the --set-json option @@ -804,6 +808,10 @@ firefoxNode: sessionBrowserName: 'firefox' unsafeSsl: '{{ include "seleniumGrid.graphqlURL.unsafeSsl" . }}' # Optional + # It is used to add initContainers in the same pod of the browser node. + # It should be set using the --set-json option + initContainers: [] + # It is used to add a sidecars proxy in the same pod of the browser node. # It means it will add a new container to the deployment itself. # It should be set using the --set-json option @@ -958,6 +966,10 @@ edgeNode: sessionBrowserName: 'msedge' unsafeSsl: '{{ include "seleniumGrid.graphqlURL.unsafeSsl" . }}' # Optional + # It is used to add initContainers in the same pod of the browser node. + # It should be set using the --set-json option + initContainers: [] + # It is used to add a sidecars proxy in the same pod of the browser node. # It means it will add a new container to the deployment itself. # It should be set using the --set-json option @@ -1011,6 +1023,8 @@ videoRecorder: # Liveness probe settings livenessProbe: {} + # Define lifecycle events for video recorder + lifecycle: {} volume: # name: @@ -1042,33 +1056,15 @@ videoRecorder: imagePullPolicy: IfNotPresent securityContext: runAsUser: 0 - command: - - /bin/sh - args: - - -c - - | - while ! [ -p /videos/uploadpipe ] - do - echo Waiting for /videos/uploadpipe to be created - sleep 1 - done - echo Waiting for files to upload - while read FILE DESTINATION < /videos/uploadpipe - do - if [ "$FILE" = "exit" ] - then - echo "$FILE" - break - else - echo "Uploading $FILE to $DESTINATION" - aws s3 cp --no-progress $FILE $DESTINATION - fi - done + command: ["/bin/sh"] + args: ["-c", "/opt/bin/entry_point.sh"] extraEnvironmentVariables: # - name: AWS_ACCESS_KEY_ID # value: aws_access_key_id # - name: AWS_SECRET_ACCESS_KEY # value: aws_secret_access_key + # - name: AWS_REGION + # value: region # - name: # valueFrom: # secretKeyRef: diff --git a/tests/SeleniumTests/__init__.py b/tests/SeleniumTests/__init__.py index 9cc62bc36..3c94792b7 100644 --- a/tests/SeleniumTests/__init__.py +++ b/tests/SeleniumTests/__init__.py @@ -1,7 +1,7 @@ import unittest import concurrent.futures import os -import time +import traceback from selenium import webdriver from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import WebDriverWait @@ -137,8 +137,13 @@ def run(self, test_classes): for test in suite: futures.append(executor.submit(test)) for future in concurrent.futures.as_completed(futures): - result = future.result() - if not result.wasSuccessful(): + try: + result = future.result() + if not result.wasSuccessful(): + raise Exception("Test failed") + except Exception as e: + print(f"Test {str(test)} failed with exception: {str(e)}") + print(traceback.format_exc()) raise Exception("Parallel tests failed") class JobAutoscalingTests(unittest.TestCase): diff --git a/tests/charts/templates/render/dummy.yaml b/tests/charts/templates/render/dummy.yaml index a5587451a..c22418d88 100644 --- a/tests/charts/templates/render/dummy.yaml +++ b/tests/charts/templates/render/dummy.yaml @@ -78,3 +78,16 @@ firefoxNode: edgeNode: affinity: *affinity + +videoRecorder: + enabled: true + uploader: s3 + uploadDestinationPrefix: "s3://ndviet" + s3: + extraEnvironmentVariables: + - name: AWS_ACCESS_KEY_ID + value: "xxxx" + - name: AWS_SECRET_ACCESS_KEY + value: "xxxx" + - name: AWS_REGION + value: "ap-southeast-1"