From 771a807feb9c2f3ae7f12d9464bd577aab32cd46 Mon Sep 17 00:00:00 2001 From: Viet Nguyen Duc Date: Sun, 21 Jan 2024 09:31:26 +0700 Subject: [PATCH] feat(chart): Add RCLONE as default video uploader on Kubernetes (#2100) Signed-off-by: Viet Nguyen Duc --- .github/workflows/nightly.yaml | 7 +- .../update-dev-beta-browser-images.yml | 2 +- Makefile | 23 +++- Uploader/Dockerfile | 12 ++ charts/selenium-grid/README.md | 105 ++++++++++++++++-- .../configs/{video => recorder}/video.sh | 22 ++-- .../configs/uploader/rclone/entry_point.sh | 37 ++++++ .../configs/uploader/rclone/rclone.conf | 9 ++ charts/selenium-grid/templates/_helpers.tpl | 55 +++++---- .../templates/chrome-node-deployment.yaml | 2 +- .../templates/chrome-node-scaledjobs.yaml | 2 +- .../templates/edge-node-deployment.yaml | 2 +- .../templates/edge-node-scaledjob.yaml | 2 +- .../templates/firefox-node-deployment.yaml | 2 +- .../templates/firefox-node-scaledjob.yaml | 2 +- .../templates/node-configmap.yaml | 4 + charts/selenium-grid/templates/secrets.yaml | 12 ++ charts/selenium-grid/templates/video-cm.yaml | 9 +- charts/selenium-grid/values.yaml | 74 ++++++++++-- generate_release_notes.sh | 2 + tests/SeleniumTests/__init__.py | 15 ++- tests/charts/make/chart_test.sh | 9 +- tests/charts/templates/render/dummy.yaml | 22 ++-- 23 files changed, 344 insertions(+), 87 deletions(-) create mode 100644 Uploader/Dockerfile rename charts/selenium-grid/configs/{video => recorder}/video.sh (82%) create mode 100644 charts/selenium-grid/configs/uploader/rclone/entry_point.sh create mode 100644 charts/selenium-grid/configs/uploader/rclone/rclone.conf diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml index 3db7ffa42..85e81a071 100644 --- a/.github/workflows/nightly.yaml +++ b/.github/workflows/nightly.yaml @@ -1,11 +1,6 @@ name: Nightly on: workflow_dispatch: - inputs: - NAMESPACE: - description: 'Set image registry' - required: true - default: 'selenium' schedule: - cron: '0 1 * * *' @@ -29,7 +24,7 @@ jobs: echo "PRERELEASE=true" >> $GITHUB_ENV echo "NAME=${NAMESPACE}" >> $GITHUB_ENV env: - NAMESPACE: ${{ github.event.inputs.NAMESPACE || 'selenium' }} + NAMESPACE: ${{ secrets.DOCKER_NAMESPACE || 'selenium' }} - name: Build base image to get Grid version run: VERSION="local" BUILD_DATE=${BUILD_DATE} make base_nightly - name: Get Grid version diff --git a/.github/workflows/update-dev-beta-browser-images.yml b/.github/workflows/update-dev-beta-browser-images.yml index 4bcef2c5b..b3f29527e 100644 --- a/.github/workflows/update-dev-beta-browser-images.yml +++ b/.github/workflows/update-dev-beta-browser-images.yml @@ -17,7 +17,7 @@ jobs: browser: [chrome,firefox,edge] channel: [dev,beta] env: - NAME: selenium + NAME: ${{ secrets.DOCKER_NAMESPACE || 'selenium' }} BROWSER: ${{ matrix.browser }} CHANNEL: ${{ matrix.channel }} diff --git a/Makefile b/Makefile index 4984fa37c..6aa26ae82 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,8 @@ MAJOR_MINOR_PATCH := $(word 1,$(subst -, ,$(TAG_VERSION))) FFMPEG_TAG_VERSION := $(or $(FFMPEG_TAG_VERSION),$(FFMPEG_TAG_VERSION),ffmpeg-6.1) FFMPEG_BASED_NAME := $(or $(FFMPEG_BASED_NAME),$(FFMPEG_BASED_NAME),ndviet) FFMPEG_BASED_TAG := $(or $(FFMPEG_BASED_TAG),$(FFMPEG_BASED_TAG),6.1-ubuntu2204) +RCLONE_BASED_TAG := $(or $(RCLONE_BASED_TAG),$(RCLONE_BASED_TAG),1.65) +RCLONE_TAG_VERSION := $(or $(RCLONE_TAG_VERSION),$(RCLONE_TAG_VERSION),rclone-$(RCLONE_BASED_TAG)) all: hub \ distributor \ @@ -34,6 +36,7 @@ all: hub \ standalone_edge \ standalone_firefox \ standalone_docker \ + uploader \ video build_nightly: @@ -130,6 +133,9 @@ standalone_edge_dev: edge_dev standalone_edge_beta: edge_beta cd ./Standalone && docker build $(BUILD_ARGS) --build-arg NAMESPACE=$(NAME) --build-arg VERSION=beta --build-arg BASE=node-edge -t $(NAME)/standalone-edge:beta . +uploader: + cd ./Uploader && docker build $(BUILD_ARGS) --build-arg BASED_TAG=$(RCLONE_BASED_TAG) -t $(NAME)/uploader:$(RCLONE_TAG_VERSION)-$(BUILD_DATE) . + video: cd ./Video && docker build $(BUILD_ARGS) --build-arg NAMESPACE=$(FFMPEG_BASED_NAME) --build-arg BASED_TAG=$(FFMPEG_BASED_TAG) -t $(NAME)/video:$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) . @@ -165,6 +171,7 @@ tag_latest: docker tag $(NAME)/standalone-firefox:$(TAG_VERSION) $(NAME)/standalone-firefox:latest docker tag $(NAME)/standalone-docker:$(TAG_VERSION) $(NAME)/standalone-docker:latest docker tag $(NAME)/video:$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) $(NAME)/video:latest + docker tag $(NAME)/uploader:$(RCLONE_TAG_VERSION)-$(BUILD_DATE) $(NAME)/uploader:latest release_latest: docker push $(NAME)/base:latest @@ -184,6 +191,7 @@ release_latest: docker push $(NAME)/standalone-firefox:latest docker push $(NAME)/standalone-docker:latest docker push $(NAME)/video:latest + docker push $(NAME)/uploader:latest tag_nightly: docker tag $(NAME)/base:$(TAG_VERSION) $(NAME)/base:nightly @@ -203,6 +211,7 @@ tag_nightly: docker tag $(NAME)/standalone-firefox:$(TAG_VERSION) $(NAME)/standalone-firefox:nightly docker tag $(NAME)/standalone-docker:$(TAG_VERSION) $(NAME)/standalone-docker:nightly docker tag $(NAME)/video:$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) $(NAME)/video:nightly + docker tag $(NAME)/uploader:$(RCLONE_TAG_VERSION)-$(BUILD_DATE) $(NAME)/uploader:nightly release_nightly: docker push $(NAME)/base:nightly @@ -222,6 +231,7 @@ release_nightly: docker push $(NAME)/standalone-firefox:nightly docker push $(NAME)/standalone-docker:nightly docker push $(NAME)/video:nightly + docker push $(NAME)/uploader:nightly tag_major_minor: docker tag $(NAME)/base:$(TAG_VERSION) $(NAME)/base:$(MAJOR) @@ -355,6 +365,7 @@ release: tag_major_minor docker push $(NAME)/standalone-firefox:$(MAJOR_MINOR_PATCH) docker push $(NAME)/standalone-docker:$(MAJOR_MINOR_PATCH) docker push $(NAME)/video:$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) + docker push $(NAME)/uploader:$(RCLONE_TAG_VERSION)-$(BUILD_DATE) test: test_chrome \ test_firefox \ @@ -438,19 +449,23 @@ chart_test_template: ./tests/charts/bootstrap.sh chart_test_chrome: - VERSION=$(TAG_VERSION) NAMESPACE=$(NAMESPACE) ./tests/charts/make/chart_test.sh NodeChrome + VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) UPLOADER_TAG=$(RCLONE_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \ + ./tests/charts/make/chart_test.sh NodeChrome chart_test_firefox: - VERSION=$(TAG_VERSION) NAMESPACE=$(NAMESPACE) ./tests/charts/make/chart_test.sh NodeFirefox + VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) UPLOADER_TAG=$(RCLONE_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \ + ./tests/charts/make/chart_test.sh NodeFirefox chart_test_edge: - VERSION=$(TAG_VERSION) NAMESPACE=$(NAMESPACE) ./tests/charts/make/chart_test.sh NodeEdge + VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) UPLOADER_TAG=$(RCLONE_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \ + ./tests/charts/make/chart_test.sh NodeEdge chart_test_parallel_autoscaling_https: SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_PORT=443 make chart_test_parallel_autoscaling chart_test_parallel_autoscaling: - VERSION=$(TAG_VERSION) NAMESPACE=$(NAMESPACE) ./tests/charts/make/chart_test.sh JobAutoscaling + VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) UPLOADER_TAG=$(RCLONE_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \ + ./tests/charts/make/chart_test.sh JobAutoscaling .PHONY: \ all \ diff --git a/Uploader/Dockerfile b/Uploader/Dockerfile new file mode 100644 index 000000000..c04ebdafb --- /dev/null +++ b/Uploader/Dockerfile @@ -0,0 +1,12 @@ +ARG BASED_TAG +FROM rclone/rclone:${BASED_TAG} + +RUN apk update \ + && apk add --no-cache --update \ + bash \ + curl \ + wget \ + jq \ + && rm -rf /tmp/* /var/cache/apk/* + +USER rclone diff --git a/charts/selenium-grid/README.md b/charts/selenium-grid/README.md index a11e3bcd5..37ccfb347 100644 --- a/charts/selenium-grid/README.md +++ b/charts/selenium-grid/README.md @@ -18,10 +18,13 @@ This chart enables the creation of a Selenium Grid Server in Kubernetes. * [Ingress Configuration](#ingress-configuration) * [Configuration](#configuration) * [Configuration global](#configuration-global) - * [Configuration `global.K8S_PUBLIC_IP`](#configuration-globalk8spublicip) + * [Configuration `global.K8S_PUBLIC_IP`](#configuration-globalk8s_public_ip) * [Configuration of Nodes](#configuration-of-nodes) * [Container ports and Service ports](#container-ports-and-service-ports) * [Probes](#probes) + * [Configuration of video recorder and video uploader](#configuration-of-video-recorder-and-video-uploader) + * [Video recorder](#video-recorder) + * [Video uploader](#video-uploader) * [Configuration of Secure Communication (HTTPS)](#configuration-of-secure-communication-https) * [Secure Communication](#secure-communication) * [Node Registration](#node-registration) @@ -375,6 +378,87 @@ edgeNode: periodSeconds: 5 ``` +### Configuration of video recorder and video uploader + +#### Video recorder + +The video recorder is a sidecar that is deployed with the browser nodes. It is responsible for recording the video of the browser session. The video recorder is disabled by default. To enable it, you need to set the following values: + +```yaml +videoRecorder: + enabled: true +``` + +#### Video uploader + +The uploader is a sidecar that is deployed with the browser nodes. It is responsible for uploading the video to a remote location. The uploader is disabled by default. To enable it, you need to set the following values: + +```yaml +videoRecorder: + uploader: + enabled: true +``` + +By default, the uploader uses [RCLONE](https://rclone.org/) to upload the video to a remote location. RCLONE requires a configuration file to define different remote locations. Refer to [RCLONE docs](https://rclone.org/docs/#config-file) for more details. Config file might contain sensitive information such as access key, secret key, etc. hence it is stored in Secret. + +The uploader requires `destinationPrefix` to be set. It is used to instruct the uploader where to upload the video. The format of destinationPrefix is `remote-name://bucket-name/path`. The `remote-name` is configured in RCLONE. The bucket-name is the name of the bucket in the remote location. The path is the path to the folder in the bucket. + +By default, the config file is loaded from file [configs/uploader/rclone/rclone.conf](configs/uploader/rclone/rclone.conf) to the Secret. You can override the config file via `--set-file videoRecorder.uploader.config=/path/to/config` or set via YAML values. + +For example, to configure an S3 remote hosted on AWS with named `mys3` and the bucket name is `mybucket`, you can set the following values: + +```bash +videoRecorder: + uploader: + destinationPrefix: "mys3://mybucket" + config: | + [mys3] + type = s3 + provider = AWS + env_auth = true + region = ap-southeast-1 + location_constraint = ap-southeast-1 + acl = private + access_key_id = xxx + secret_access_key = xxx +``` + +You can prepare a config file with multiple remotes are defined. Ensure that `[remoteName]` is unique for each remote. +You also can replace your config to default file `configs/uploader/rclone/rclone.conf` in chart. + +Instead of using config file, another way that RCLONE also supports to pass the information via environment variables. ENV variable with format: `RCLONE_CONFIG_ + name of remote + _ + name of config file option` (make it all uppercase). In this case the remote name it can only contain letters, digits, or the _ (underscore) character. All those ENV variables can be set via `videoRecorder.uploader.secrets`, it will be stored in Secret. + +For example, the same above config can be set via ENV vars as below: + +```yaml +videoRecorder: + uploader: + destinationPrefix: "mys3://mybucket" + secrets: + RCLONE_CONFIG_MYS3_TYPE: "s3" + RCLONE_CONFIG_MYS3_PROVIDER: "GCS" + RCLONE_CONFIG_MYS3_ENV_AUTH: "true" + RCLONE_CONFIG_MYS3_REGION: "asia-southeast1" + RCLONE_CONFIG_MYS3_LOCATION_CONSTRAINT: "asia-southeast1" + RCLONE_CONFIG_MYS3_ACL: "private" + RCLONE_CONFIG_MYS3_ACCESS_KEY_ID: "xxx" + RCLONE_CONFIG_MYS3_SECRET_ACCESS_KEY: "xxx" +``` + +Those 2 ways are equivalent. You can choose one of them or combine them together. When both config file and ENV vars are set, value in `rclone.conf` will take precedence. + +Beside the configuration, the script for entry point of uploader container also needed. By default, it is loaded from file [configs/uploader/rclone/entry_point.sh](configs/uploader/rclone/entry_point.sh) to the ConfigMap. You can override the script via `--set-file videoRecorder.uploader.entryPoint=/path/to/script` or set via YAML values. For example: + +```yaml +videoRecorder: + uploader: + entryPoint: | + #!/bin/bash + echo "Your custom script" +``` + +You also can replace your script to default file `configs/uploader/rclone/entry_point.sh` in chart. + ### Configuration of Secure Communication (HTTPS) Selenium Grid supports secure communication between components. Refer to the [instructions](https://github.com/SeleniumHQ/selenium/blob/trunk/java/src/org/openqa/selenium/grid/commands/security.txt) and [options](https://www.selenium.dev/documentation/grid/configuration/cli_options/#server) are able to configure the secure communication. Below is the details on how to enable secure communication in Selenium Grid chart. @@ -620,8 +704,14 @@ This table contains the configuration parameters of the chart and their default | `videoRecorder.imageName` | `video` | Selenium video recorder image name | | `videoRecorder.imageTag` | `ffmpeg-6.1-20231219` | Image tag of video recorder | | `videoRecorder.imagePullPolicy` | `IfNotPresent` | Image pull policy (see https://kubernetes.io/docs/concepts/containers/images/#updating-images) | -| `videoRecorder.uploader` | `false` | Name of the uploader to use. The value `false` is used to disable uploader. Supported default `s3` | -| `videoRecorder.uploadDestinationPrefix` | `false` | Destination URL for uploading video file. The value `false` is used to disable the uploading | +| `videoRecorder.uploader.enabled` | `false` | Enable the uploader for videos | +| `videoRecorder.uploader.destinationPrefix` | `` | Destination for uploading video file. It is following `rclone` config | +| `videoRecorder.uploader.name` | `rclone` | Name of the uploader to use. Supported default `rclone` | +| `videoRecorder.uploader.configFileName` | `rclone.conf` | Config file name for `rclone` in uploader container | +| `videoRecorder.uploader.entryPointFileName` | `entry_point.sh` | Script file name for uploader container entry point | +| `videoRecorder.uploader.config` | `` | Set value to uploader config file via YAML or `--set-file` | +| `videoRecorder.uploader.entryPoint` | `` | Set value to uploader entry point via YAML or `--set-file` | +| `videoRecorder.uploader.secrets` | `` | Environment variables to configure the uploader which store in Secret | | `videoRecorder.ports` | `[9000]` | Port list to enable on video recorder container | | `videoRecorder.resources` | `See values.yaml` | Resources for video recorder | | `videoRecorder.extraEnvironmentVariables` | `nil` | Custom environment variables for video recorder | @@ -634,11 +724,10 @@ This table contains the configuration parameters of the chart and their default | `videoRecorder.volume.name.scripts` | `video-scripts` | Name is used to set for the volume to persist and share video recorder scripts in container | | `videoRecorder.extraVolumeMounts` | `[]` | Extra mounts of declared ExtraVolumes into pod | | `videoRecorder.extraVolumes` | `[]` | Extra Volumes declarations to be used in the pod (can be any supported volume type: ConfigMap, Secret, PVC, NFS, etc.) | -| `videoRecorder.s3` | `See values.yaml` | Container spec for the uploader if `videoRecorder.uploader` is `s3`. Similarly, create for your new uploader | -| `videoRecorder.s3.resources` | `See values.yaml` | Resources for video uploader | -| `videoRecorder.s3.extraEnvironmentVariables` | `` | Custom environment variables for video uploader container | -| `videoRecorder.s3.extraEnvFrom` | `` | Custom environment taken from `configMap` or `secret` variables for video uploader | -| `videoRecorder.s3.extraVolumeMounts` | `[]` | Extra mounts of declared ExtraVolumes into pod of video uploader | +| `videoRecorder.rclone` | `See values.yaml` | Container spec for the uploader if `videoRecorder.uploader` is `s3`. Similarly, create for your new uploader | +| `videoRecorder.rclone.resources | `See values.yaml` | Resources for video uploader | +| `videoRecorder.rclone.extraEnvFrom` | `` | Custom environment taken from `configMap` or `secret` variables for video uploader | +| `videoRecorder.rclone.extraVolumeMounts` | `[]` | Extra mounts of declared ExtraVolumes into pod of video uploader | | `customLabels` | `{}` | Custom labels for k8s resources | | `ingress-nginx.enabled` | `false` | Enable the dependency chart Ingress controller for Kubernetes (https://github.com/kubernetes/ingress-nginx) | diff --git a/charts/selenium-grid/configs/video/video.sh b/charts/selenium-grid/configs/recorder/video.sh similarity index 82% rename from charts/selenium-grid/configs/video/video.sh rename to charts/selenium-grid/configs/recorder/video.sh index 921d53399..fb8d8b3ee 100755 --- a/charts/selenium-grid/configs/video/video.sh +++ b/charts/selenium-grid/configs/recorder/video.sh @@ -1,7 +1,9 @@ #!/usr/bin/env bash +UPLOAD_ENABLED=${UPLOAD_ENABLED:-"false"} + function create_pipe() { - if [[ "$UPLOAD_DESTINATION_PREFIX" != "false" ]]; + if [[ "${UPLOAD_ENABLED}" != "false" ]] && [[ ! -z "${UPLOAD_DESTINATION_PREFIX}" ]]; then echo "Create pipe if not exists for video upload stream" if [ ! -p ${SE_VIDEO_FOLDER}/uploadpipe ]; @@ -13,7 +15,7 @@ function create_pipe() { create_pipe function wait_util_force_exit_consume() { - if [[ "$UPLOAD_DESTINATION_PREFIX" != "false" ]]; + if [[ "${UPLOAD_ENABLED}" != "false" ]] && [[ ! -z "${UPLOAD_DESTINATION_PREFIX}" ]]; then while [[ -f ${SE_VIDEO_FOLDER}/force_exit ]] do @@ -25,9 +27,9 @@ function wait_util_force_exit_consume() { } function add_exit_signal() { - if [[ "$UPLOAD_DESTINATION_PREFIX" != "false" ]]; + if [[ "${UPLOAD_ENABLED}" != "false" ]] && [[ ! -z "${UPLOAD_DESTINATION_PREFIX}" ]]; then - echo "exit" > ${SE_VIDEO_FOLDER}/uploadpipe & + echo "exit" >> ${SE_VIDEO_FOLDER}/uploadpipe & echo "exit" > ${SE_VIDEO_FOLDER}/force_exit fi } @@ -55,11 +57,6 @@ export DISPLAY=${DISPLAY_CONTAINER_NAME}:${DISPLAY_NUM}.0 max_attempts=600 attempts=0 -if [[ "$UPLOAD_DESTINATION_PREFIX" = "" ]] -then - echo Upload destination not known since UPLOAD_DESTINATION_PREFIX is not set. Exiting video recorder. - exit -fi echo Checking if the display is open until xset b off || [[ $attempts = $max_attempts ]] do @@ -111,11 +108,14 @@ do pkill -INT ffmpeg recorded_count=$((recorded_count+1)) recording_started="false" - if [[ "$UPLOAD_DESTINATION_PREFIX" != "false" ]] + if [[ "${UPLOAD_ENABLED}" != "false" ]] && [[ ! -z "${UPLOAD_DESTINATION_PREFIX}" ]]; then upload_destination=${UPLOAD_DESTINATION_PREFIX}/${video_file_name} echo "Uploading video to $upload_destination" - echo $video_file $upload_destination >> ${SE_VIDEO_FOLDER}/uploadpipe & + echo $video_file ${UPLOAD_DESTINATION_PREFIX} >> ${SE_VIDEO_FOLDER}/uploadpipe & + elif [[ "${UPLOAD_ENABLED}" != "false" ]] && [[ -z "${UPLOAD_DESTINATION_PREFIX}" ]]; + then + echo Upload destination not known since UPLOAD_DESTINATION_PREFIX is not set. Continue without uploading. fi if [ $max_recorded_count -gt 0 ] && [ $recorded_count -ge $max_recorded_count ]; then diff --git a/charts/selenium-grid/configs/uploader/rclone/entry_point.sh b/charts/selenium-grid/configs/uploader/rclone/entry_point.sh new file mode 100644 index 000000000..ce9714bf7 --- /dev/null +++ b/charts/selenium-grid/configs/uploader/rclone/entry_point.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +SE_VIDEO_FOLDER=${SE_VIDEO_FOLDER:-"/videos"} +UPLOAD_CONFIG_DIRECTORY=${UPLOAD_CONFIG_DIRECTORY:-"/opt/bin"} +UPLOAD_CONFIG_FILE_NAME=${UPLOAD_CONFIG_FILE_NAME:-"rclone.conf"} +UPLOAD_COMMAND=${UPLOAD_COMMAND:-"copy"} + +function consume_force_exit() { + rm -f ${SE_VIDEO_FOLDER}/force_exit + echo "Force exit signal consumed" +} +trap consume_force_exit EXIT + +while [ ! -p ${SE_VIDEO_FOLDER}/uploadpipe ]; +do + echo "Waiting for ${SE_VIDEO_FOLDER}/uploadpipe to be created" + sleep 1 +done + +echo "Waiting for video files put into pipe for proceeding to upload" + +while read FILE DESTINATION < ${SE_VIDEO_FOLDER}/uploadpipe +do + if [ "${FILE}" = "exit" ]; + then + exit + else [ "$FILE" != "" ] && [ "$DESTINATION" != "" ]; + echo "Uploading ${FILE} to ${DESTINATION}" + rclone --config ${UPLOAD_CONFIG_DIRECTORY}/${UPLOAD_CONFIG_FILE_NAME} ${UPLOAD_COMMAND} ${UPLOAD_OPTS} "${FILE}" "${DESTINATION}" + fi + if [ -f ${SE_VIDEO_FOLDER}/force_exit ] && [ ! -s ${SE_VIDEO_FOLDER}/uploadpipe ]; + then + exit + fi +done + +consume_force_exit diff --git a/charts/selenium-grid/configs/uploader/rclone/rclone.conf b/charts/selenium-grid/configs/uploader/rclone/rclone.conf new file mode 100644 index 000000000..db0337c13 --- /dev/null +++ b/charts/selenium-grid/configs/uploader/rclone/rclone.conf @@ -0,0 +1,9 @@ +[mys3] +type = s3 +provider = AWS +env_auth = true +region = +location_constraint = +acl = private +access_key_id = +secret_access_key = diff --git a/charts/selenium-grid/templates/_helpers.tpl b/charts/selenium-grid/templates/_helpers.tpl index a5963d86f..e3448d22b 100644 --- a/charts/selenium-grid/templates/_helpers.tpl +++ b/charts/selenium-grid/templates/_helpers.tpl @@ -446,8 +446,6 @@ template: valueFrom: fieldRef: fieldPath: status.podIP - - name: UPLOAD_DESTINATION_PREFIX - value: {{ .Values.videoRecorder.uploadDestinationPrefix | quote }} {{- with .Values.videoRecorder.extraEnvironmentVariables }} {{- tpl (toYaml .) $ | nindent 8 }} {{- end }} @@ -475,6 +473,9 @@ template: {{- with .Values.videoRecorder.resources }} resources: {{- toYaml . | nindent 10 }} {{- end }} + {{- with .Values.videoRecorder.securityContext }} + securityContext: {{- toYaml . | nindent 10 }} + {{- end }} {{- with .Values.videoRecorder.startupProbe }} startupProbe: {{- toYaml . | nindent 10 }} {{- end }} @@ -484,21 +485,29 @@ template: {{- with .Values.videoRecorder.lifecycle }} lifecycle: {{- toYaml . | nindent 10 }} {{- end }} - {{- if .uploader }} + {{- if .Values.videoRecorder.uploader.enabled }} - name: uploader - image: {{ printf "%s:%s" .uploader.imageName .uploader.imageTag }} + {{- $imageTag := default .Values.global.seleniumGrid.uploaderImageTag .uploader.imageTag }} + {{- $imageRegistry := default .Values.global.seleniumGrid.imageRegistry .uploader.imageRegistry }} + image: {{ printf "%s/%s:%s" $imageRegistry .uploader.imageName $imageTag }} imagePullPolicy: {{ .uploader.imagePullPolicy }} - {{- with .uploader.command }} - command: {{- tpl (toYaml .) $ | nindent 8 }} + {{- if .uploader.command }} + command: {{- tpl (toYaml .uploader.command) $ | nindent 8 }} + {{- else }} + command: ["/bin/sh"] {{- end }} - {{- with .uploader.args }} - args: {{- tpl (toYaml .) $ | nindent 8 }} + {{- if .uploader.args }} + args: {{- tpl (toYaml .uploader.args) $ | nindent 8 }} + {{- else }} + args: ["-c", "{{ $.Values.videoRecorder.uploader.scriptMountPath }}/{{ $.Values.videoRecorder.uploader.entryPointFileName }}"] {{- end }} {{- with .uploader.extraEnvironmentVariables }} env: {{- tpl (toYaml .) $ | nindent 8 }} {{- end }} - {{- with .uploader.extraEnvFrom }} envFrom: + - secretRef: + name: {{ include "seleniumGrid.common.secrets" $ | quote }} + {{- with .uploader.extraEnvFrom }} {{- tpl (toYaml .) $ | nindent 10 }} {{- end }} volumeMounts: @@ -582,12 +591,12 @@ Get the url of the grid. If the external url can be figured out from the ingress {{- define "seleniumGrid.url.host" -}} {{- $host := printf "%s.%s" (include ($.Values.isolateComponents | ternary "seleniumGrid.router.fullname" "seleniumGrid.hub.fullname") $ ) (.Release.Namespace) -}} {{- if .Values.ingress.enabled -}} - {{- if and ( empty .Values.ingress.hostname) (not (empty .Values.global.K8S_PUBLIC_IP)) -}} + {{- if and (not .Values.ingress.hostname) .Values.global.K8S_PUBLIC_IP -}} {{- $host = .Values.global.K8S_PUBLIC_IP -}} {{- else if and .Values.ingress.hostname (ne .Values.ingress.hostname "selenium-grid.local") -}} {{- $host = .Values.ingress.hostname -}} {{- end -}} -{{- else if not (empty .Values.global.K8S_PUBLIC_IP) -}} +{{- else if .Values.global.K8S_PUBLIC_IP -}} {{- $host = .Values.global.K8S_PUBLIC_IP -}} {{- end -}} {{- $host }} @@ -658,7 +667,7 @@ Get the lifecycle of the pod is used for a Node to deregister from the Hub/Route {{- define "seleniumGrid.node.lifecycle" }} {{- $defaultDeregisterLifecycle := tpl (include "seleniumGrid.node.deregisterLifecycle" .) $ -}} {{- $lifecycle := toYaml (dict) -}} -{{- if and (and (eq .Values.autoscaling.scalingType "deployment") (eq (include "seleniumGrid.useKEDA" .) "true")) (empty .node.deregisterLifecycle) -}} +{{- if and (and (eq .Values.autoscaling.scalingType "deployment") (eq (include "seleniumGrid.useKEDA" .) "true")) (not .node.deregisterLifecycle) -}} {{- $lifecycle = merge ($lifecycle | fromYaml) (tpl (toYaml (default ($defaultDeregisterLifecycle | fromYaml) .Values.autoscaling.deregisterLifecycle)) $ | fromYaml) | toYaml -}} {{- else -}} {{- if eq (.node.deregisterLifecycle | toString | lower) "false" -}} @@ -667,7 +676,7 @@ Get the lifecycle of the pod is used for a Node to deregister from the Hub/Route {{- $lifecycle = (tpl (toYaml (default ($defaultDeregisterLifecycle | fromYaml) .node.deregisterLifecycle) ) $ | fromYaml) | toYaml -}} {{- end -}} {{- end -}} -{{- if not (empty .node.lifecycle) -}} +{{- if not .node.lifecycle -}} {{- $lifecycle = mergeOverwrite ($lifecycle | fromYaml) (tpl (toYaml .node.lifecycle) $ | fromYaml) | toYaml -}} {{- end -}} {{ if and $lifecycle (ne $lifecycle "{}") -}} @@ -705,7 +714,7 @@ Default specs of VolumeMounts and Volumes for video recorder {{- define "seleniumGrid.video.volumeMounts.default" -}} {{- $root := . -}} -{{- range $path, $bytes := .Files.Glob "configs/video/*" }} +{{- range $path, $bytes := .Files.Glob "configs/recorder/*.sh" }} - name: {{ include "seleniumGrid.video.volume.name.scripts" $ }} mountPath: /opt/bin/{{ base $path }} subPath: {{ base $path }} @@ -719,15 +728,23 @@ Default specs of VolumeMounts and Volumes for video recorder configMap: name: {{ template "seleniumGrid.video.fullname" . }} defaultMode: 0500 +- name: {{ template "seleniumGrid.common.secrets" . }} + secret: + secretName: {{ template "seleniumGrid.common.secrets" . }} - name: {{ include "seleniumGrid.video.volume.name.folder" . }} emptyDir: {} {{- end -}} {{- define "seleniumGrid.video.uploader.volumeMounts.default" -}} {{- $root := . -}} -{{- range $path, $bytes := .Files.Glob (printf "configs/uploader/%s/*" $.Values.videoRecorder.uploader) }} +{{- range $path, $bytes := .Files.Glob (printf "configs/uploader/%s/*.sh" $.Values.videoRecorder.uploader.name) }} - name: {{ include "seleniumGrid.video.volume.name.scripts" $ }} - mountPath: /opt/bin/{{ base $path }} + mountPath: {{ $.Values.videoRecorder.uploader.scriptMountPath }}/{{ base $path }} + subPath: {{ base $path }} +{{- end }} +{{- range $path, $bytes := .Files.Glob (printf "configs/uploader/%s/*.conf" $.Values.videoRecorder.uploader.name) }} +- name: {{ include "seleniumGrid.common.secrets" $ }} + mountPath: {{ $.Values.videoRecorder.uploader.scriptMountPath }}/{{ base $path }} subPath: {{ base $path }} {{- end }} - name: {{ include "seleniumGrid.video.volume.name.folder" . }} @@ -744,7 +761,7 @@ Default specs of VolumeMounts and Volumes for video recorder {{- end -}} {{- $defaultVolumeMounts := (include "seleniumGrid.video.volumeMounts.default" . | toString | fromYamlArray ) -}} {{- $videoVolumeMounts = include "utils.appendDefaultIfNotExist" (dict "currentArray" $videoVolumeMounts "defaultArray" $defaultVolumeMounts "uniqueKey" "mountPath") -}} -{{- not (empty $videoVolumeMounts) | ternary $videoVolumeMounts "" -}} +{{- not $videoVolumeMounts | ternary $videoVolumeMounts "" -}} {{- end -}} {{/* Combine videoRecorder.uploader.extraVolumeMounts with the default ones for container video uploader */}} @@ -757,7 +774,7 @@ Default specs of VolumeMounts and Volumes for video recorder {{- end }} {{- $defaultVolumeMounts := (include "seleniumGrid.video.uploader.volumeMounts.default" . | toString | fromYamlArray ) -}} {{- $videoUploaderVolumeMounts = include "utils.appendDefaultIfNotExist" (dict "currentArray" $videoUploaderVolumeMounts "defaultArray" $defaultVolumeMounts "uniqueKey" "mountPath") -}} -{{- not (empty $videoUploaderVolumeMounts) | ternary $videoUploaderVolumeMounts "" -}} +{{- not $videoUploaderVolumeMounts | ternary $videoUploaderVolumeMounts "" -}} {{- end -}} {{/* Combine videoRecorder.extraVolumes with the default ones for the node pod */}} @@ -770,7 +787,7 @@ Default specs of VolumeMounts and Volumes for video recorder {{- end -}} {{- $defaultVolumes := (include "seleniumGrid.video.volumes.default" . | toString | fromYamlArray ) -}} {{- $videoVolumes = include "utils.appendDefaultIfNotExist" (dict "currentArray" $videoVolumes "defaultArray" $defaultVolumes "uniqueKey" "name") -}} -{{- not (empty $videoVolumes) | ternary $videoVolumes "" -}} +{{- not $videoVolumes | ternary $videoVolumes "" -}} {{- end -}} {{/* diff --git a/charts/selenium-grid/templates/chrome-node-deployment.yaml b/charts/selenium-grid/templates/chrome-node-deployment.yaml index e3bf3f1d2..e826d3fbe 100644 --- a/charts/selenium-grid/templates/chrome-node-deployment.yaml +++ b/charts/selenium-grid/templates/chrome-node-deployment.yaml @@ -25,6 +25,6 @@ spec: {{- $podScope := deepCopy . -}} {{- $_ := set $podScope "name" (include "seleniumGrid.chromeNode.fullname" .) -}} {{- $_ = set $podScope "node" .Values.chromeNode -}} -{{- $_ = set $podScope "uploader" (get .Values.videoRecorder (.Values.videoRecorder.uploader | toString)) -}} +{{- $_ = set $podScope "uploader" (get .Values.videoRecorder (.Values.videoRecorder.uploader.name | toString)) -}} {{- include "seleniumGrid.podTemplate" $podScope | nindent 2 }} {{- end }} diff --git a/charts/selenium-grid/templates/chrome-node-scaledjobs.yaml b/charts/selenium-grid/templates/chrome-node-scaledjobs.yaml index 0b0d34443..5654cd5a0 100644 --- a/charts/selenium-grid/templates/chrome-node-scaledjobs.yaml +++ b/charts/selenium-grid/templates/chrome-node-scaledjobs.yaml @@ -22,7 +22,7 @@ spec: {{- $podScope := deepCopy . -}} {{- $_ := set $podScope "name" (include "seleniumGrid.chromeNode.fullname" .) -}} {{- $_ = set $podScope "node" .Values.chromeNode -}} - {{- $_ = set $podScope "uploader" (get .Values.videoRecorder (.Values.videoRecorder.uploader | toString)) -}} + {{- $_ = set $podScope "uploader" (get .Values.videoRecorder (.Values.videoRecorder.uploader.name | toString)) -}} {{- $_ = set $podScope "podTemplate" (include "seleniumGrid.podTemplate" $podScope | fromYaml) }} {{- include "seleniumGrid.autoscalingTemplate" $podScope | nindent 2 }} {{- end }} diff --git a/charts/selenium-grid/templates/edge-node-deployment.yaml b/charts/selenium-grid/templates/edge-node-deployment.yaml index 9802239e8..a518fd678 100644 --- a/charts/selenium-grid/templates/edge-node-deployment.yaml +++ b/charts/selenium-grid/templates/edge-node-deployment.yaml @@ -25,6 +25,6 @@ spec: {{- $podScope := deepCopy . -}} {{- $_ := set $podScope "name" (include "seleniumGrid.edgeNode.fullname" .) -}} {{- $_ = set $podScope "node" .Values.edgeNode -}} -{{- $_ = set $podScope "uploader" (get .Values.videoRecorder (.Values.videoRecorder.uploader | toString)) -}} +{{- $_ = set $podScope "uploader" (get .Values.videoRecorder (.Values.videoRecorder.uploader.name | toString)) -}} {{- include "seleniumGrid.podTemplate" $podScope | nindent 2 }} {{- end }} diff --git a/charts/selenium-grid/templates/edge-node-scaledjob.yaml b/charts/selenium-grid/templates/edge-node-scaledjob.yaml index 132792c79..4a0e4fc51 100644 --- a/charts/selenium-grid/templates/edge-node-scaledjob.yaml +++ b/charts/selenium-grid/templates/edge-node-scaledjob.yaml @@ -22,7 +22,7 @@ spec: {{- $podScope := deepCopy . -}} {{- $_ := set $podScope "name" (include "seleniumGrid.edgeNode.fullname" .) -}} {{- $_ = set $podScope "node" .Values.edgeNode -}} - {{- $_ = set $podScope "uploader" (get .Values.videoRecorder (.Values.videoRecorder.uploader | toString)) -}} + {{- $_ = set $podScope "uploader" (get .Values.videoRecorder (.Values.videoRecorder.uploader.name | toString)) -}} {{- $_ = set $podScope "podTemplate" (include "seleniumGrid.podTemplate" $podScope | fromYaml) }} {{- include "seleniumGrid.autoscalingTemplate" $podScope | nindent 2 }} {{- end }} diff --git a/charts/selenium-grid/templates/firefox-node-deployment.yaml b/charts/selenium-grid/templates/firefox-node-deployment.yaml index 5a1e96b55..f94bd052c 100644 --- a/charts/selenium-grid/templates/firefox-node-deployment.yaml +++ b/charts/selenium-grid/templates/firefox-node-deployment.yaml @@ -25,6 +25,6 @@ spec: {{- $podScope := deepCopy . -}} {{- $_ := set $podScope "name" (include "seleniumGrid.firefoxNode.fullname" .) -}} {{- $_ = set $podScope "node" .Values.firefoxNode -}} -{{- $_ = set $podScope "uploader" (get .Values.videoRecorder (.Values.videoRecorder.uploader | toString)) -}} +{{- $_ = set $podScope "uploader" (get .Values.videoRecorder (.Values.videoRecorder.uploader.name | toString)) -}} {{- include "seleniumGrid.podTemplate" $podScope | nindent 2 }} {{- end }} diff --git a/charts/selenium-grid/templates/firefox-node-scaledjob.yaml b/charts/selenium-grid/templates/firefox-node-scaledjob.yaml index e7992c1fc..732ec8c9b 100644 --- a/charts/selenium-grid/templates/firefox-node-scaledjob.yaml +++ b/charts/selenium-grid/templates/firefox-node-scaledjob.yaml @@ -22,7 +22,7 @@ spec: {{- $podScope := deepCopy . -}} {{- $_ := set $podScope "name" (include "seleniumGrid.firefoxNode.fullname" .) -}} {{- $_ = set $podScope "node" .Values.firefoxNode -}} - {{- $_ = set $podScope "uploader" (get .Values.videoRecorder (.Values.videoRecorder.uploader | toString)) -}} + {{- $_ = set $podScope "uploader" (get .Values.videoRecorder (.Values.videoRecorder.uploader.name | toString)) -}} {{- $_ = set $podScope "podTemplate" (include "seleniumGrid.podTemplate" $podScope | fromYaml) }} {{- include "seleniumGrid.autoscalingTemplate" $podScope | nindent 2 }} {{- end }} diff --git a/charts/selenium-grid/templates/node-configmap.yaml b/charts/selenium-grid/templates/node-configmap.yaml index 6f8062d17..dbb460026 100644 --- a/charts/selenium-grid/templates/node-configmap.yaml +++ b/charts/selenium-grid/templates/node-configmap.yaml @@ -14,4 +14,8 @@ metadata: data: SE_DRAIN_AFTER_SESSION_COUNT: '{{- and (eq (include "seleniumGrid.useKEDA" .) "true") (eq .Values.autoscaling.scalingType "job") | ternary "1" "0" -}}' SE_NODE_GRID_URL: '{{ include "seleniumGrid.url" .}}' + UPLOAD_ENABLED: '{{ .Values.videoRecorder.uploader.enabled }}' + UPLOAD_DESTINATION_PREFIX: '{{ .Values.videoRecorder.uploader.destinationPrefix }}' + UPLOAD_CONFIG_FILE_NAME: '{{ .Values.videoRecorder.uploader.configFileName }}' + UPLOAD_CONFIG_DIRECTORY: '{{ .Values.videoRecorder.uploader.scriptMountPath }}' {{ (.Files.Glob "configs/node/*").AsConfig | indent 2 }} diff --git a/charts/selenium-grid/templates/secrets.yaml b/charts/selenium-grid/templates/secrets.yaml index 06881d15a..932217efc 100644 --- a/charts/selenium-grid/templates/secrets.yaml +++ b/charts/selenium-grid/templates/secrets.yaml @@ -30,4 +30,16 @@ data: {{- if (include "seleniumGrid.tls.registrationSecret.enabled" $) }} SE_REGISTRATION_SECRET: {{ .Values.tls.registrationSecret.value | b64enc }} {{- end }} +{{- if .Values.videoRecorder.uploader.secrets }} +{{- range $name, $value := .Values.videoRecorder.uploader.secrets }} +{{- if not (empty $value) }} + {{ $name }}: {{ tpl ($value) $ | b64enc }} +{{- end }} +{{- end }} +{{- end }} +{{- if and .Values.videoRecorder.uploader.enabled .Values.videoRecorder.uploader.config }} + {{ .Values.videoRecorder.uploader.configFileName }}: {{ .Values.videoRecorder.uploader.config | b64enc }} +{{- else if and .Values.videoRecorder.uploader.enabled (not .Values.videoRecorder.uploader.config) }} +{{ (.Files.Glob (printf "configs/uploader/%s/*.conf" $.Values.videoRecorder.uploader.name)).AsSecrets | indent 2 }} +{{- end }} {{- end }} diff --git a/charts/selenium-grid/templates/video-cm.yaml b/charts/selenium-grid/templates/video-cm.yaml index 34eb24f05..7176dd15d 100644 --- a/charts/selenium-grid/templates/video-cm.yaml +++ b/charts/selenium-grid/templates/video-cm.yaml @@ -4,8 +4,11 @@ kind: ConfigMap metadata: name: {{ template "seleniumGrid.video.fullname" . }} data: -{{ (.Files.Glob "configs/video/*").AsConfig | indent 2 }} -{{- if and $.Values.videoRecorder.uploadDestinationPrefix $.Values.videoRecorder.uploader }} -{{ (.Files.Glob (printf "configs/uploader/%s/*" $.Values.videoRecorder.uploader)).AsConfig | indent 2 }} +{{ (.Files.Glob "configs/recorder/*").AsConfig | indent 2 }} +{{- if and .Values.videoRecorder.uploader.enabled .Values.videoRecorder.uploader.entryPoint }} +{{ .Values.videoRecorder.uploader.entryPointFileName | indent 2 -}}: | +{{ .Values.videoRecorder.uploader.entryPoint | indent 4 }} +{{- else if and .Values.videoRecorder.uploader.enabled (not .Values.videoRecorder.uploader.entryPoint) }} +{{ (.Files.Glob (printf "configs/uploader/%s/*.sh" $.Values.videoRecorder.uploader.name)).AsConfig | indent 2 }} {{- end }} {{- end }} diff --git a/charts/selenium-grid/values.yaml b/charts/selenium-grid/values.yaml index 79f3eff62..864fee8a3 100644 --- a/charts/selenium-grid/values.yaml +++ b/charts/selenium-grid/values.yaml @@ -12,6 +12,8 @@ global: nodesImageTag: 4.16.1-20231219 # Image tag for browser's video recorder videoImageTag: ffmpeg-6.1-20231219 + # Image tag for browser's uploader + uploaderImageTag: rclone-1.65-20231219 # Pull secret for all components, can be overridden individually imagePullSecret: "" # Log level for all components. Possible values describe here: https://www.selenium.dev/documentation/grid/configuration/cli_options/#logging @@ -985,13 +987,39 @@ videoRecorder: # Image pull policy (see https://kubernetes.io/docs/concepts/containers/images/#updating-images) imagePullPolicy: IfNotPresent # Image pull secret (see https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/) - - # What uploader to use. See .videRecorder.s3 for how to create a new one. - # uploader: s3 - uploader: false - # Where to upload the video file. Should be set to something like 's3://myvideobucket/' - uploadDestinationPrefix: false - + uploader: + enabled: false + # Where to upload the video file e.g. remoteName://bucketName/path. Refer to destination syntax of rclone https://rclone.org/docs/ + destinationPrefix: + # What uploader to use. See .videRecorder.rclone for how to create a new one. + name: "rclone" + configFileName: "rclone.conf" + entryPointFileName: "entry_point.sh" + scriptMountPath: "/opt/bin" + # Config file for rclone. + config: + # Script to control the upload process. + entryPoint: + # For environment variables used in uploader which contains sensitive information, store in secret and refer envFrom + # Set config for rclone via ENV var with format: RCLONE_CONFIG_ + name of remote + _ + name of config file option (make it all uppercase) + secrets: + # RCLONE_CONFIG_S3_TYPE: "s3" + # RCLONE_CONFIG_S3_PROVIDER: "AWS" + # RCLONE_CONFIG_S3_ENV_AUTH: "true" + # RCLONE_CONFIG_S3_REGION: "ap-southeast-1" + # RCLONE_CONFIG_S3_LOCATION_CONSTRAINT: "ap-southeast-1" + # RCLONE_CONFIG_S3_ACL: "private" + # RCLONE_CONFIG_S3_ACCESS_KEY_ID: "xxx" + # RCLONE_CONFIG_S3_SECRET_ACCESS_KEY: "xxx" + # RCLONE_CONFIG_GS_TYPE: "s3" + # RCLONE_CONFIG_GS_PROVIDER: "GCS" + # RCLONE_CONFIG_GS_ENV_AUTH: "true" + # RCLONE_CONFIG_GS_REGION: "asia-southeast1" + # RCLONE_CONFIG_GS_LOCATION_CONSTRAINT: "asia-southeast1" + # RCLONE_CONFIG_GS_ACL: "private" + # RCLONE_CONFIG_GS_ACCESS_KEY_ID: "xxx" + # RCLONE_CONFIG_GS_SECRET_ACCESS_KEY: "xxx" + # RCLONE_CONFIG_GS_ENDPOINT: "https://storage.googleapis.com" ports: - 9000 resources: @@ -1001,6 +1029,9 @@ videoRecorder: limits: memory: "1Gi" cpu: "1" + # SecurityContext for recorder container + securityContext: + runAsUser: 0 extraEnvironmentVariables: # - name: SE_VIDEO_FOLDER # value: /videos @@ -1049,15 +1080,34 @@ videoRecorder: # persistentVolumeClaim: # claimName: video-pv-claim - # Container spec for the uploader if above it is defined as "uploader: s3" + # Container spec for the uploader if above it is defined as "uploader.name: rclone" + rclone: + # imageRegistry: selenium + imageName: uploader + # imageTag: "rclone-1.65-20240119" + imagePullPolicy: IfNotPresent + # SecurityContext for uploader container + securityContext: + runAsUser: 0 + command: [] + args: [] + extraEnvironmentVariables: + # Extra options for rclone. Refer to https://rclone.org/flags + - name: UPLOAD_OPTS + value: "--progress" + # - name: UPLOAD_COMMAND + # value: "copy" + + # Container spec for the uploader if above it is defined as "uploader.name: s3" s3: - imageName: public.ecr.aws/bitnami/aws-cli - imageTag: "2" + imageRegistry: public.ecr.aws + imageName: bitnami/aws-cli + imageTag: latest imagePullPolicy: IfNotPresent securityContext: runAsUser: 0 - command: ["/bin/sh"] - args: ["-c", "/opt/bin/entry_point.sh"] + command: [] + args: [] extraEnvironmentVariables: # - name: AWS_ACCESS_KEY_ID # value: aws_access_key_id diff --git a/generate_release_notes.sh b/generate_release_notes.sh index 85f15cc55..caf8f315f 100755 --- a/generate_release_notes.sh +++ b/generate_release_notes.sh @@ -19,6 +19,7 @@ EDGEDRIVER_VERSION=$(docker run --rm ${NAMESPACE}/node-edge:${TAG_VERSION} msedg FIREFOX_VERSION=$(docker run --rm ${NAMESPACE}/node-firefox:${TAG_VERSION} firefox --version | awk '{print $3}') GECKODRIVER_VERSION=$(docker run --rm ${NAMESPACE}/node-firefox:${TAG_VERSION} geckodriver --version | awk 'NR==1{print $2}') FFMPEG_VERSION=$(docker run --entrypoint="" --rm ${NAMESPACE}/video:ffmpeg-6.1-${BUILD_DATE} ffmpeg -version | awk '{print $3}' | head -n 1) +RCLONE_VERSION=$(docker run --entrypoint="" --rm ${NAMESPACE}/uploader:rclone-1.65-${BUILD_DATE} rclone version | head -n 1 | awk '{print $2}') echo "" >> release_notes.md @@ -31,6 +32,7 @@ echo "* EdgeDriver: ${EDGEDRIVER_VERSION}" >> release_notes.md echo "* Firefox: ${FIREFOX_VERSION}" >> release_notes.md echo "* GeckoDriver: ${GECKODRIVER_VERSION}" >> release_notes.md echo "* ffmpeg: ${FFMPEG_VERSION}" >> release_notes.md +echo "* rclone: ${RCLONE_VERSION}" >> release_notes.md echo "" >> release_notes.md echo "### Published Docker images" >> release_notes.md diff --git a/tests/SeleniumTests/__init__.py b/tests/SeleniumTests/__init__.py index 3c94792b7..f4eb967d9 100644 --- a/tests/SeleniumTests/__init__.py +++ b/tests/SeleniumTests/__init__.py @@ -25,8 +25,13 @@ def test_title(self): def test_with_frames(self): driver = self.driver driver.get('http://the-internet.herokuapp.com/nested_frames') - driver.switch_to.frame('frame-top') - driver.switch_to.frame('frame-middle') + wait = WebDriverWait(driver, WEB_DRIVER_WAIT_TIMEOUT) + frame_top = wait.until( + EC.frame_to_be_available_and_switch_to_it('frame-top') + ) + frame_middle = wait.until( + EC.frame_to_be_available_and_switch_to_it('frame-middle') + ) self.assertTrue(driver.find_element(By.ID, 'content').text == "MIDDLE", "content should be MIDDLE") # https://github.com/tourdedave/elemental-selenium-tips/blob/master/05-select-from-a-dropdown/python/dropdown.py @@ -140,11 +145,11 @@ def run(self, test_classes): try: result = future.result() if not result.wasSuccessful(): - raise Exception("Test failed") + raise Exception(f"Test {str(test)} failed") except Exception as e: - print(f"Test {str(test)} failed with exception: {str(e)}") + print(f"{str(test)} failed with exception: {str(e)}") print(traceback.format_exc()) - raise Exception("Parallel tests failed") + raise Exception(f"Parallel tests failed: {str(test)} failed with exception: {str(e)}") class JobAutoscalingTests(unittest.TestCase): def test_parallel_autoscaling(self): diff --git a/tests/charts/make/chart_test.sh b/tests/charts/make/chart_test.sh index 617d3ef6e..7c1984859 100755 --- a/tests/charts/make/chart_test.sh +++ b/tests/charts/make/chart_test.sh @@ -23,6 +23,8 @@ WEB_DRIVER_WAIT_TIMEOUT=${WEB_DRIVER_WAIT_TIMEOUT:-120} SKIP_CLEANUP=${SKIP_CLEANUP:-"false"} # For debugging purposes, retain the cluster after the test run CHART_CERT_PATH=${CHART_CERT_PATH:-"${CHART_PATH}/certs/selenium.pem"} SSL_CERT_DIR=${SSL_CERT_DIR:-"/etc/ssl/certs"} +VIDEO_TAG=${VIDEO_TAG:-"latest"} +UPLOADER_TAG=${UPLOADER_TAG:-"latest"} cleanup() { if [ "${SKIP_CLEANUP}" = "false" ]; then @@ -63,12 +65,15 @@ HELM_COMMAND_ARGS="${RELEASE_NAME} \ ${HELM_COMMAND_SET_AUTOSCALING} \ ${HELM_COMMAND_SET_TLS} \ --values ${TEST_VALUES_PATH}/${MATRIX_BROWSER}-values.yaml \ ---set global.seleniumGrid.imageTag=${VERSION} --set global.seleniumGrid.imageRegistry=${NAMESPACE} \ +--set global.seleniumGrid.imageRegistry=${NAMESPACE} \ +--set global.seleniumGrid.imageTag=${VERSION} \ --set global.seleniumGrid.nodesImageTag=${VERSION} \ +--set global.seleniumGrid.videoImageTag=${VIDEO_TAG} \ +--set global.seleniumGrid.uploaderImageTag=${UPLOADER_TAG} \ ${CHART_PATH} --namespace ${SELENIUM_NAMESPACE} --create-namespace" echo "Render manifests YAML for this deployment" -helm template ${HELM_COMMAND_ARGS} > tests/tests/cluster_deployment_manifests_${MATRIX_BROWSER}.yaml +helm template --debug ${HELM_COMMAND_ARGS} > tests/tests/cluster_deployment_manifests_${MATRIX_BROWSER}.yaml echo "Deploy Selenium Grid Chart" helm upgrade --install ${HELM_COMMAND_ARGS} diff --git a/tests/charts/templates/render/dummy.yaml b/tests/charts/templates/render/dummy.yaml index c22418d88..1e8411c6a 100644 --- a/tests/charts/templates/render/dummy.yaml +++ b/tests/charts/templates/render/dummy.yaml @@ -81,13 +81,15 @@ edgeNode: videoRecorder: enabled: true - uploader: s3 - uploadDestinationPrefix: "s3://ndviet" - s3: - extraEnvironmentVariables: - - name: AWS_ACCESS_KEY_ID - value: "xxxx" - - name: AWS_SECRET_ACCESS_KEY - value: "xxxx" - - name: AWS_REGION - value: "ap-southeast-1" + uploader: + enabled: true + destinationPrefix: "s3://ndviet" + secrets: + RCLONE_CONFIG_S3_TYPE: "s3" + RCLONE_CONFIG_S3_PROVIDER: "AWS" + RCLONE_CONFIG_S3_ENV_AUTH: "true" + RCLONE_CONFIG_S3_REGION: "ap-southeast-1" + RCLONE_CONFIG_S3_LOCATION_CONSTRAINT: "ap-southeast-1" + RCLONE_CONFIG_S3_ACL: "private" + RCLONE_CONFIG_S3_ACCESS_KEY_ID: "xxx" + RCLONE_CONFIG_S3_SECRET_ACCESS_KEY: "xxx"