diff --git a/.github/workflows/helm-chart-test.yml b/.github/workflows/helm-chart-test.yml index 316b2dd14..0cc6e53c1 100644 --- a/.github/workflows/helm-chart-test.yml +++ b/.github/workflows/helm-chart-test.yml @@ -52,7 +52,11 @@ jobs: echo "BUILD_DATE=$(date '+%Y%m%d')" >> $GITHUB_ENV echo "IMAGE_REGISTRY=artifactory/selenium" >> $GITHUB_ENV - name: Setup Kubernetes environment - run: make chart_setup_env + uses: nick-invision/retry@master + with: + timeout_minutes: 10 + max_attempts: 3 + command: make chart_setup_env - name: Build Helm charts run: | BUILD_DATE=${BUILD_DATE} make chart_build diff --git a/Makefile b/Makefile index 587c98db8..ed26be294 100644 --- a/Makefile +++ b/Makefile @@ -19,8 +19,6 @@ MAJOR_MINOR_PATCH := $(word 1,$(subst -, ,$(TAG_VERSION))) FFMPEG_TAG_VERSION := $(or $(FFMPEG_TAG_VERSION),$(FFMPEG_TAG_VERSION),ffmpeg-6.1) FFMPEG_BASED_NAME := $(or $(FFMPEG_BASED_NAME),$(FFMPEG_BASED_NAME),ndviet) FFMPEG_BASED_TAG := $(or $(FFMPEG_BASED_TAG),$(FFMPEG_BASED_TAG),6.1-ubuntu2204) -RCLONE_BASED_TAG := $(or $(RCLONE_BASED_TAG),$(RCLONE_BASED_TAG),1.65) -RCLONE_TAG_VERSION := $(or $(RCLONE_TAG_VERSION),$(RCLONE_TAG_VERSION),rclone-1.65) all: hub \ distributor \ @@ -36,7 +34,6 @@ all: hub \ standalone_edge \ standalone_firefox \ standalone_docker \ - uploader \ video build_nightly: @@ -133,9 +130,6 @@ standalone_edge_dev: edge_dev standalone_edge_beta: edge_beta cd ./Standalone && docker build $(BUILD_ARGS) --build-arg NAMESPACE=$(NAME) --build-arg VERSION=beta --build-arg BASE=node-edge -t $(NAME)/standalone-edge:beta . -uploader: - cd ./Uploader && docker build $(BUILD_ARGS) --build-arg BASED_TAG=$(RCLONE_BASED_TAG) -t $(NAME)/uploader:$(RCLONE_TAG_VERSION)-$(BUILD_DATE) . - video: cd ./Video && docker build $(BUILD_ARGS) --build-arg NAMESPACE=$(FFMPEG_BASED_NAME) --build-arg BASED_TAG=$(FFMPEG_BASED_TAG) -t $(NAME)/video:$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) . @@ -171,7 +165,6 @@ tag_latest: docker tag $(NAME)/standalone-firefox:$(TAG_VERSION) $(NAME)/standalone-firefox:latest docker tag $(NAME)/standalone-docker:$(TAG_VERSION) $(NAME)/standalone-docker:latest docker tag $(NAME)/video:$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) $(NAME)/video:latest - docker tag $(NAME)/uploader:$(RCLONE_TAG_VERSION)-$(BUILD_DATE) $(NAME)/uploader:latest release_latest: docker push $(NAME)/base:latest @@ -191,7 +184,6 @@ release_latest: docker push $(NAME)/standalone-firefox:latest docker push $(NAME)/standalone-docker:latest docker push $(NAME)/video:latest - docker push $(NAME)/uploader:latest tag_nightly: docker tag $(NAME)/base:$(TAG_VERSION) $(NAME)/base:nightly @@ -211,7 +203,6 @@ tag_nightly: docker tag $(NAME)/standalone-firefox:$(TAG_VERSION) $(NAME)/standalone-firefox:nightly docker tag $(NAME)/standalone-docker:$(TAG_VERSION) $(NAME)/standalone-docker:nightly docker tag $(NAME)/video:$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) $(NAME)/video:nightly - docker tag $(NAME)/uploader:$(RCLONE_TAG_VERSION)-$(BUILD_DATE) $(NAME)/uploader:nightly release_nightly: docker push $(NAME)/base:nightly @@ -231,7 +222,6 @@ release_nightly: docker push $(NAME)/standalone-firefox:nightly docker push $(NAME)/standalone-docker:nightly docker push $(NAME)/video:nightly - docker push $(NAME)/uploader:nightly tag_major_minor: docker tag $(NAME)/base:$(TAG_VERSION) $(NAME)/base:$(MAJOR) @@ -365,7 +355,6 @@ release: tag_major_minor docker push $(NAME)/standalone-firefox:$(MAJOR_MINOR_PATCH) docker push $(NAME)/standalone-docker:$(MAJOR_MINOR_PATCH) docker push $(NAME)/video:$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) - docker push $(NAME)/uploader:$(RCLONE_TAG_VERSION)-$(BUILD_DATE) test: test_chrome \ test_firefox \ @@ -443,29 +432,29 @@ chart_test_template: ./tests/charts/bootstrap.sh chart_test_chrome: - VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) UPLOADER_TAG=$(RCLONE_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \ + VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \ ./tests/charts/make/chart_test.sh NodeChrome chart_test_firefox: - VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) UPLOADER_TAG=$(RCLONE_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \ + VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \ ./tests/charts/make/chart_test.sh NodeFirefox chart_test_edge: - VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) UPLOADER_TAG=$(RCLONE_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \ + VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \ ./tests/charts/make/chart_test.sh NodeEdge chart_test_autoscaling_deployment_https: SELENIUM_GRID_TEST_HEADLESS=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_PORT=443 make chart_test_autoscaling_deployment chart_test_autoscaling_deployment: - VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) UPLOADER_TAG=$(RCLONE_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \ + VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \ ./tests/charts/make/chart_test.sh DeploymentAutoscaling chart_test_autoscaling_job_https: SELENIUM_GRID_TEST_HEADLESS=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_PORT=443 make chart_test_autoscaling_job chart_test_autoscaling_job: - VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) UPLOADER_TAG=$(RCLONE_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \ + VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \ ./tests/charts/make/chart_test.sh JobAutoscaling .PHONY: \ diff --git a/README.md b/README.md index 788687a6e..8d65f8bec 100644 --- a/README.md +++ b/README.md @@ -21,45 +21,59 @@ Talk to us at https://www.selenium.dev/support/ ## Contents + +* [Community](#community) +* [Contents](#contents) * [Quick start](#quick-start) + * [Try them out in a ready-to-use GitPod environment!](#try-them-out-in-a-ready-to-use-gitpod-environment) * [Experimental Mult-Arch aarch64/armhf/amd64 Images](#experimental-mult-arch-aarch64armhfamd64-images) * [Nightly Images](#nightly-images) * [Dev and Beta Channel Browser Images](#dev-and-beta-channel-browser-images) - + [Dev and Beta Standalone Mode](#dev-and-beta-standalone-mode) - + [Dev and Beta on the Grid](#dev-and-beta-on-the-grid) + * [Dev and Beta Standalone Mode](#dev-and-beta-standalone-mode) + * [Dev and Beta on the Grid](#dev-and-beta-on-the-grid) * [Execution modes](#execution-modes) - + [Standalone](#standalone) - + [Hub and Nodes](#hub-and-nodes) - - [Docker networking](#docker-networking) - - [Using different machines/VMs](#using-different-machinesvms) - - [Docker Compose](#docker-compose) - + [Fully distributed mode - Router, Queue, Distributor, EventBus, SessionMap and Nodes](#fully-distributed-mode---router-queue-distributor-eventbus-sessionmap-and-nodes) + * [Standalone](#standalone) + * [Hub and Nodes](#hub-and-nodes) + * [Fully distributed mode - Router, Queue, Distributor, EventBus, SessionMap and Nodes](#fully-distributed-mode---router-queue-distributor-eventbus-sessionmap-and-nodes) * [Video recording](#video-recording) +* [Video recording and uploading](#video-recording-and-uploading) * [Dynamic Grid](#dynamic-grid) - + [Configuration example](#configuration-example) - + [Execution with Hub & Node roles](#execution-with-hub--node-roles) - + [Execution with Standalone roles](#execution-with-standalone-roles) - + [Using Dynamic Grid in different machines/VMs](#using-dynamic-grid-in-different-machinesvms) - + [Execution with Docker Compose](#execution-with-docker-compose) - + [Video recording, screen resolution, and time zones in a Dynamic Grid](#video-recording-screen-resolution-and-time-zones-in-a-dynamic-grid) -* [Kubernetes](#deploying-to-kubernetes) + * [Configuration example](#configuration-example) + * [Execution with Hub & Node roles](#execution-with-hub--node-roles) + * [Execution with Standalone roles](#execution-with-standalone-roles) + * [Using Dynamic Grid in different machines/VMs](#using-dynamic-grid-in-different-machinesvms) + * [Execution with Docker Compose](#execution-with-docker-compose) + * [Configuring the child containers](#configuring-the-child-containers) + * [Video recording, screen resolution, and time zones in a Dynamic Grid](#video-recording-screen-resolution-and-time-zones-in-a-dynamic-grid) +* [Deploying to Kubernetes](#deploying-to-kubernetes) * [Configuring the containers](#configuring-the-containers) - + [SE_OPTS Selenium Configuration Options](#se_opts-selenium-configuration-options) - + [SE_JAVA_OPTS Java Environment Options](#se_java_opts-java-environment-options) - + [Node configuration options](#node-configuration-options) - + [Setting Sub Path](#setting-sub-path) - + [Setting Screen Resolution](#setting-screen-resolution) - + [Grid Url and Session Timeout](#grid-url-and-session-timeout) - + [Session request timeout](#session-request-timeout) - + [Increasing session concurrency per container](#increasing-session-concurrency-per-container) - + [Running in Headless mode](#running-in-headless-mode) + * [SE_OPTS Selenium Configuration Options](#se_opts-selenium-configuration-options) + * [SE_JAVA_OPTS Java Environment Options](#se_java_opts-java-environment-options) + * [Node configuration options](#node-configuration-options) + * [Setting Sub Path](#setting-sub-path) + * [Setting Screen Resolution](#setting-screen-resolution) + * [Grid Url and Session Timeout](#grid-url-and-session-timeout) + * [Session request timeout](#session-request-timeout) + * [Increasing session concurrency per container](#increasing-session-concurrency-per-container) + * [Running in Headless mode](#running-in-headless-mode) + * [Stopping the Node/Standalone after N sessions have been executed](#stopping-the-nodestandalone-after-n-sessions-have-been-executed) * [Building the images](#building-the-images) * [Waiting for the Grid to be ready](#waiting-for-the-grid-to-be-ready) + * [Adding a HEALTHCHECK to the Grid](#adding-a-healthcheck-to-the-grid) + * [Using a bash script to wait for the Grid](#using-a-bash-script-to-wait-for-the-grid) +* [Install certificates for Chromium-based browsers](#install-certificates-for-chromium-based-browsers) +* [Alternative method: Add certificates to existing Selenium based images for browsers](#alternative-method-add-certificates-to-existing-selenium-based-images-for-browsers) * [Debugging](#debugging) -* [Install certificates for Chromium based browsers](#install-certificates-for-Chromium-based-browsers) + * [Using a VNC client](#using-a-vnc-client) + * [Using your browser (no VNC client is needed)](#using-your-browser-no-vnc-client-is-needed) + * [Disabling VNC](#disabling-vnc) * [Tracing in Grid](#tracing-in-grid) * [Troubleshooting](#troubleshooting) - + * [`--shm-size="2g"`](#--shm-size2g) + * [Headless](#headless) + * [Mounting volumes to retrieve downloaded files](#mounting-volumes-to-retrieve-downloaded-files) + * [Mounting volumes to retrieve video files](#mounting-volumes-to-retrieve-video-files) + ## Quick start @@ -521,6 +535,47 @@ Here is an example using a Hub and a few Nodes: [`docker-compose-v3-video.yml`](docker-compose-v3-video.yml) +## Video recording and uploading + +[RCLONE](https://rclone.org/) is installed in the video recorder image. You can use it to upload the videos to a cloud storage service. +Besides the video recording mentioned above, you can enable the upload functionality by setting the following environment variables: + +```yaml +version: "3" +services: + chrome_video: + image: selenium/video:nightly + depends_on: + - chrome + environment: + - DISPLAY_CONTAINER_NAME=chrome + - SE_VIDEO_FILE_NAME=auto + - SE_VIDEO_UPLOAD_ENABLED=true + - SE_VIDEO_INTERNAL_UPLOAD=true + - SE_UPLOAD_DESTINATION_PREFIX=s3://mybucket/path + - RCLONE_CONFIG_S3_TYPE=s3 + - RCLONE_CONFIG_S3_PROVIDER=GCS + - RCLONE_CONFIG_S3_ENV_AUTH=true + - RCLONE_CONFIG_S3_REGION=asia-southeast1 + - RCLONE_CONFIG_S3_LOCATION_CONSTRAINT=asia-southeast1 + - RCLONE_CONFIG_S3_ACL=private + - RCLONE_CONFIG_S3_ACCESS_KEY_ID=xxx + - RCLONE_CONFIG_S3_SECRET_ACCESS_KEY=xxx + - RCLONE_CONFIG_S3_ENDPOINT=https://storage.googleapis.com +``` + +`SE_VIDEO_FILE_NAME=auto` will use the session id as the video file name. This ensures that the video file name is unique to upload. + +`SE_VIDEO_UPLOAD_ENABLED=true` will enable the video upload feature. In the background, it will create a pipefile with file and destination for uploader to consume and proceed. + +`SE_VIDEO_INTERNAL_UPLOAD=true` will use RCLONE installed in the container for upload. If you want to use another container for upload, set it to `false`. + +For environment variables with prefix `RCLONE_` is used to pass remote configuration to RCLONE. You can find more information about RCLONE configuration [here](https://rclone.org/docs/). + +[`docker-compose-v3-video-upload.yml`](docker-compose-v3-video-upload.yml) + +Note that upload function is not supported for Dynamic Grid. If you want it, please create a feature request. + ___ ## Dynamic Grid @@ -1336,4 +1391,11 @@ After doing this, you should be able to download files to the mounted directory. If you have a better workaround, please send us a pull request! +### Mounting volumes to retrieve video files + +Similar to mount volumes to retrieve downloaded files. For video files, you might need to do the same +```bash +mkdir /tmp/videos +chown 1200:1201 /tmp/videos +``` diff --git a/Uploader/Dockerfile b/Uploader/Dockerfile deleted file mode 100644 index 5bf8598e7..000000000 --- a/Uploader/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -ARG BASED_TAG -FROM rclone/rclone:${BASED_TAG} - -RUN apk update && apk upgrade \ - && apk add --no-cache --update \ - bash \ - curl \ - wget \ - jq \ - && rm -rf /tmp/* /var/cache/apk/* - -USER rclone diff --git a/Video/Dockerfile b/Video/Dockerfile old mode 100644 new mode 100755 index 2f9b5f9f4..ee21c9230 --- a/Video/Dockerfile +++ b/Video/Dockerfile @@ -3,6 +3,7 @@ ARG BASED_TAG FROM $NAMESPACE/ffmpeg:$BASED_TAG LABEL authors="Selenium " +ARG RCLONE_VERSION=current #Arguments to define the user running the container ARG SEL_USER=seluser ARG SEL_GROUP=${SEL_USER} @@ -29,7 +30,7 @@ ENV DEBIAN_FRONTEND=noninteractive \ RUN apt-get -qqy update \ && apt-get upgrade -yq \ && apt-get -qqy --no-install-recommends install \ - supervisor x11-xserver-utils x11-utils curl jq python3-pip tzdata acl \ + supervisor x11-xserver-utils x11-utils curl jq python3-pip tzdata acl unzip \ && python3 -m pip install --upgrade pip \ && python3 -m pip install --upgrade setuptools \ && python3 -m pip install --upgrade wheel \ @@ -71,7 +72,20 @@ RUN groupadd ${SEL_GROUP} \ # Add Supervisor configuration files #====================================== COPY supervisord.conf /etc -COPY --chown="${SEL_UID}:${SEL_GID}" entry_point.sh video.sh video_ready.py /opt/bin/ +COPY --chown="${SEL_UID}:${SEL_GID}" entry_point.sh video.sh video_ready.py video_graphQLQuery.sh /opt/bin/ + +#====================================== +# Add RCLONE for uploading videos +#====================================== +RUN curl -fLo /tmp/rclone.zip https://downloads.rclone.org/rclone-${RCLONE_VERSION}-linux-amd64.zip \ + && unzip -a /tmp/rclone.zip -d /tmp \ + && mv /tmp/rclone-*-linux-amd64/rclone /usr/local/bin/ \ + && rm -rf /tmp/rclone-* +COPY --chown="${SEL_UID}:${SEL_GID}" upload.sh upload.conf /opt/bin/ +ENV SE_VIDEO_UPLOAD_ENABLED false +ENV SE_UPLOAD_DESTINATION_PREFIX "" +ENV SE_VIDEO_INTERNAL_UPLOAD false +ENV UPLOAD_OPTS "-P" ENV SE_VIDEO_FOLDER /videos RUN mkdir -p /var/run/supervisor /var/log/supervisor $SE_VIDEO_FOLDER \ @@ -82,7 +96,6 @@ RUN mkdir -p /var/run/supervisor /var/log/supervisor $SE_VIDEO_FOLDER \ && setfacl -Rdm u:${SEL_USER}:rwx,g:${SEL_GROUP}:rwx /var/run/supervisor /var/log/supervisor $SE_VIDEO_FOLDER $HOME USER ${SEL_UID} -VOLUME ${SE_VIDEO_FOLDER} ENTRYPOINT ["/opt/bin/entry_point.sh"] CMD ["/opt/bin/entry_point.sh"] @@ -94,6 +107,6 @@ ENV SE_SCREEN_HEIGHT 1020 ENV SE_FRAME_RATE 15 ENV SE_CODEC libx264 ENV SE_PRESET "-preset ultrafast" -ENV FILE_NAME video.mp4 +ENV SE_VIDEO_FILE_NAME video.mp4 EXPOSE 9000 diff --git a/Video/supervisord.conf b/Video/supervisord.conf old mode 100644 new mode 100755 index 68d42b8a2..87a194cf9 --- a/Video/supervisord.conf +++ b/Video/supervisord.conf @@ -34,3 +34,15 @@ stopsignal=INT redirect_stderr=true stdout_logfile=/dev/stdout stdout_logfile_maxbytes=0 + +[program:video-upload] +priority=10 +command=bash -c "if [ ${SE_VIDEO_INTERNAL_UPLOAD} = "true" ]; then /opt/bin/upload.sh; fi" +autostart=%(ENV_SE_VIDEO_INTERNAL_UPLOAD)s +autorestart=%(ENV_SE_VIDEO_INTERNAL_UPLOAD)s +stopsignal=INT + +;Logs (all activity redirected to stdout so it can be seen through "docker logs" +redirect_stderr=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 diff --git a/Video/upload.conf b/Video/upload.conf new file mode 100755 index 000000000..e69de29bb diff --git a/Video/upload.sh b/Video/upload.sh new file mode 100755 index 000000000..15f914691 --- /dev/null +++ b/Video/upload.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash + +SE_VIDEO_FOLDER=${SE_VIDEO_FOLDER:-"/videos"} +UPLOAD_CONFIG_DIRECTORY=${UPLOAD_CONFIG_DIRECTORY:-"/opt/bin"} +UPLOAD_CONFIG_FILE_NAME=${UPLOAD_CONFIG_FILE_NAME:-"upload.conf"} +UPLOAD_COMMAND=${UPLOAD_COMMAND:-"copy"} +UPLOAD_RETAIN_LOCAL_FILE=${SE_UPLOAD_RETAIN_LOCAL_FILE:-"false"} +UPLOAD_PIPE_FILE_NAME=${UPLOAD_PIPE_FILE_NAME:-"uploadpipe"} +SE_VIDEO_INTERNAL_UPLOAD=${SE_VIDEO_INTERNAL_UPLOAD:-"false"} +VIDEO_UPLOAD_ENABLED=${VIDEO_UPLOAD_ENABLED:-SE_VIDEO_UPLOAD_ENABLED} +SE_VIDEO_UPLOAD_BATCH_CHECK=${SE_VIDEO_UPLOAD_BATCH_CHECK:-"10"} + +if [ "${SE_VIDEO_INTERNAL_UPLOAD}" = "true" ]; +then + # If using RCLONE in the same container, write signal to /tmp internally + UPLOAD_PIPE_FILE="/tmp/${UPLOAD_PIPE_FILE_NAME}" + FORCE_EXIT_FILE="/tmp/force_exit" +else + # If using external container for uploading, write signal to the video folder + UPLOAD_PIPE_FILE="${SE_VIDEO_FOLDER}/${UPLOAD_PIPE_FILE_NAME}" + FORCE_EXIT_FILE="${SE_VIDEO_FOLDER}/force_exit" +fi + +if [ "${UPLOAD_RETAIN_LOCAL_FILE}" = "false" ]; +then + echo "UPLOAD_RETAIN_LOCAL_FILE is set to false, force to use RCLONE command: move" + UPLOAD_COMMAND="move" +fi + +function rename_rclone_env() { + # This script is used to support passing environment variables for RCLONE configuration in Dynamic Grid + # Dynamic Grid accepts environment variables with the prefix SE_* + # RCLONE accepts environment variables with the prefix RCLONE_* + # To pass the ENV vars to Dynamic Grid then to RCLONE, we need to rename the ENV vars from SE_RCLONE_* to RCLONE_* + for var in $(env | cut -d= -f1); do + if [[ "$var" == SE_RCLONE_* ]]; + then + suffix="${var#SE_RCLONE_}" + new_var="RCLONE_$suffix" + export "$new_var=${!var}" + fi + done +} + +function graceful_exit() { + for pid in "${list_rclone_pid[@]}"; + do + wait ${pid} + done + rm -rf ${FORCE_EXIT_FILE} + echo "Uploader is ready to shutdown" +} +trap graceful_exit EXIT + +while [ ! -p ${UPLOAD_PIPE_FILE} ]; +do + echo "Waiting for ${UPLOAD_PIPE_FILE} to be created" + sleep 1 +done + +echo "Waiting for video files put into pipe for proceeding to upload" + +rename_rclone_env + +list_rclone_pid=() +while read FILE DESTINATION < ${UPLOAD_PIPE_FILE} +do + if [ "${FILE}" = "exit" ]; + then + exit + elif [ "$FILE" != "" ] && [ "$DESTINATION" != "" ]; + then + echo "Uploading ${FILE} to ${DESTINATION}" + rclone --config ${UPLOAD_CONFIG_DIRECTORY}/${UPLOAD_CONFIG_FILE_NAME} ${UPLOAD_COMMAND} ${UPLOAD_OPTS} "${FILE}" "${DESTINATION}" & + list_rclone_pid+=($!) + else + # Wait for a batch rclone processes to finish + if [ ${#list_rclone_pid[@]} -eq ${SE_VIDEO_UPLOAD_BATCH_CHECK} ] + then + for pid in "${list_rclone_pid[@]}"; + do + wait ${pid} + done + list_rclone_pid=() + fi + fi +done + +graceful_exit diff --git a/Video/video.sh b/Video/video.sh index 7be462a80..4d7ddf812 100755 --- a/Video/video.sh +++ b/Video/video.sh @@ -3,26 +3,208 @@ VIDEO_SIZE="${SE_SCREEN_WIDTH}""x""${SE_SCREEN_HEIGHT}" DISPLAY_CONTAINER_NAME=${DISPLAY_CONTAINER_NAME} DISPLAY_NUM=${DISPLAY_NUM} -FILE_NAME=${FILE_NAME} +VIDEO_FILE_NAME=${FILE_NAME:-$SE_VIDEO_FILE_NAME} FRAME_RATE=${FRAME_RATE:-$SE_FRAME_RATE} CODEC=${CODEC:-$SE_CODEC} PRESET=${PRESET:-$SE_PRESET} VIDEO_FOLDER=${SE_VIDEO_FOLDER} +VIDEO_UPLOAD_ENABLED=${VIDEO_UPLOAD_ENABLED:-$SE_VIDEO_UPLOAD_ENABLED} +VIDEO_CONFIG_DIRECTORY=${VIDEO_CONFIG_DIRECTORY:-"/opt/bin"} +UPLOAD_DESTINATION_PREFIX=${UPLOAD_DESTINATION_PREFIX:-$SE_UPLOAD_DESTINATION_PREFIX} +UPLOAD_PIPE_FILE_NAME=${UPLOAD_PIPE_FILE_NAME:-"uploadpipe"} +SE_VIDEO_INTERNAL_UPLOAD=${SE_VIDEO_INTERNAL_UPLOAD:-"false"} +SE_SERVER_PROTOCOL=${SE_SERVER_PROTOCOL:-"http"} +SE_NODE_PORT=${SE_NODE_PORT:-"5555"} -return_code=1 -max_attempts=50 -attempts=0 -echo 'Checking if the display is open...' -until [ $return_code -eq 0 -o $attempts -eq $max_attempts ]; do - xset -display ${DISPLAY_CONTAINER_NAME}:${DISPLAY_NUM} b off > /dev/null 2>&1 - return_code=$? - if [ $return_code -ne 0 ]; then - echo 'Waiting before next display check...' - sleep 0.5 - fi - attempts=$((attempts+1)) -done - -# exec replaces the video.sh process with ffmpeg, this makes easier to pass the process termination signal -exec ffmpeg -y -f x11grab -video_size ${VIDEO_SIZE} -r ${FRAME_RATE} -i ${DISPLAY_CONTAINER_NAME}:${DISPLAY_NUM}.0 -codec:v ${CODEC} ${PRESET} -pix_fmt yuv420p "$VIDEO_FOLDER/$FILE_NAME" +if [ "${SE_VIDEO_INTERNAL_UPLOAD}" = "true" ]; +then + # If using RCLONE in the same container, write signal to /tmp internally + UPLOAD_PIPE_FILE="/tmp/${UPLOAD_PIPE_FILE_NAME}" + FORCE_EXIT_FILE="/tmp/force_exit" +else + # If using external container for uploading, write signal to the video folder + UPLOAD_PIPE_FILE="${SE_VIDEO_FOLDER}/${UPLOAD_PIPE_FILE_NAME}" + FORCE_EXIT_FILE="${SE_VIDEO_FOLDER}/force_exit" +fi +function create_pipe() { + if [[ "${VIDEO_UPLOAD_ENABLED}" != "false" ]] && [[ -n "${UPLOAD_DESTINATION_PREFIX}" ]]; + then + echo "Create pipe if not exists for video upload stream" + if [[ ! -p ${UPLOAD_PIPE_FILE} ]]; + then + mkfifo ${UPLOAD_PIPE_FILE} + fi + fi +} + +function wait_util_uploader_shutdown() { + max_wait=5 + wait=0 + if [[ "${VIDEO_UPLOAD_ENABLED}" != "false" ]] && [[ -n "${UPLOAD_DESTINATION_PREFIX}" ]]; + then + while [[ -f ${FORCE_EXIT_FILE} ]] && [[ ${wait} -lt ${max_wait} ]]; + do + echo "Waiting for force exit file to be consumed by external upload container" + sleep 1 + wait=$((wait+1)) + done + fi + if [[ "${VIDEO_UPLOAD_ENABLED}" != "false" ]] && [[ -n "${UPLOAD_DESTINATION_PREFIX}" ]] && [[ "${SE_VIDEO_INTERNAL_UPLOAD}" = "true" ]]; + then + while [[ $(pgrep rclone | wc -l) -gt 0 ]] + do + echo "Recorder is waiting for RCLONE to finish" + sleep 1 + done + fi + echo "Ready to shutdown the recorder" +} + +function send_exit_signal_to_uploader() { + if [[ "${VIDEO_UPLOAD_ENABLED}" != "false" ]] && [[ -n "${UPLOAD_DESTINATION_PREFIX}" ]]; + then + echo "exit" >> ${UPLOAD_PIPE_FILE} & + echo "exit" > ${FORCE_EXIT_FILE} + fi +} + +function exit_on_max_session_reach() { + if [[ $max_recorded_count -gt 0 ]] && [[ $recorded_count -ge $max_recorded_count ]]; + then + echo "Node will be drained since max sessions reached count number ($max_recorded_count)" + exit + fi +} + +function stop_recording() { + echo "Stopping to record video" + pkill -INT ffmpeg + recorded_count=$((recorded_count+1)) + recording_started="false" + if [[ "${VIDEO_UPLOAD_ENABLED}" != "false" ]] && [[ -n "${UPLOAD_DESTINATION_PREFIX}" ]]; + then + upload_destination=${UPLOAD_DESTINATION_PREFIX}/${video_file_name} + echo "Add to pipe a signal Uploading video to $upload_destination" + echo $video_file ${UPLOAD_DESTINATION_PREFIX} >> ${UPLOAD_PIPE_FILE} & + elif [[ "${VIDEO_UPLOAD_ENABLED}" != "false" ]] && [[ -z "${UPLOAD_DESTINATION_PREFIX}" ]]; + then + echo Upload destination not known since UPLOAD_DESTINATION_PREFIX is not set. Continue without uploading. + fi +} + +function check_if_recording_inprogress() { + if [[ "$recording_started" = "true" ]] + then + stop_recording + fi +} + +function graceful_exit() { + check_if_recording_inprogress + send_exit_signal_to_uploader + wait_util_uploader_shutdown + kill -INT "$(cat /var/run/supervisor/supervisord.pid)" +} + +if [[ "${VIDEO_UPLOAD_ENABLED}" != "true" ]] && [[ "${VIDEO_FILE_NAME}" != "auto" ]] && [[ -n "${VIDEO_FILE_NAME}" ]]; then + return_code=1 + max_attempts=50 + attempts=0 + echo 'Checking if the display is open...' + until [[ $return_code -eq 0 ]] || [[ $attempts -eq $max_attempts ]]; do + xset -display ${DISPLAY_CONTAINER_NAME}:${DISPLAY_NUM} b off > /dev/null 2>&1 + return_code=$? + if [[ $return_code -ne 0 ]]; then + echo 'Waiting before next display check...' + sleep 0.5 + fi + attempts=$((attempts+1)) + done + + # exec replaces the video.sh process with ffmpeg, this makes easier to pass the process termination signal + exec ffmpeg -y -f x11grab -video_size ${VIDEO_SIZE} -r ${FRAME_RATE} -i ${DISPLAY_CONTAINER_NAME}:${DISPLAY_NUM}.0 -codec:v ${CODEC} ${PRESET} -pix_fmt yuv420p "$VIDEO_FOLDER/$VIDEO_FILE_NAME" + +else + create_pipe + trap graceful_exit SIGTERM SIGINT EXIT + export DISPLAY=${DISPLAY_CONTAINER_NAME}:${DISPLAY_NUM}.0 + + max_attempts=600 + attempts=0 + + echo Checking if the display is open + until xset b off || [[ $attempts = "$max_attempts" ]] + do + echo Waiting before next display check + sleep 0.5 + attempts=$((attempts+1)) + done + if [[ $attempts = "$max_attempts" ]] + then + echo Can not open display, exiting. + exit + fi + + VIDEO_SIZE=$(xdpyinfo | grep 'dimensions:' | awk '{print $2}') + + recording_started="false" + video_file_name="" + video_file="" + prev_session_id="" + attempts=0 + max_recorded_count=${SE_DRAIN_AFTER_SESSION_COUNT:-0} + recorded_count=0 + + echo Checking if node API responds + until curl -sk --request GET ${SE_SERVER_PROTOCOL}://${DISPLAY_CONTAINER_NAME}:${SE_NODE_PORT}/status || [[ $attempts = "$max_attempts" ]] + do + echo Waiting before next API check + sleep 0.5 + attempts=$((attempts+1)) + done + if [[ $attempts = "$max_attempts" ]] + then + echo Can not reach node API, exiting. + exit + fi + while curl -sk --request GET ${SE_SERVER_PROTOCOL}://${DISPLAY_CONTAINER_NAME}:${SE_NODE_PORT}/status > /tmp/status.json + do + session_id=$(jq -r '.[]?.node?.slots | .[0]?.session?.sessionId' /tmp/status.json) + if [[ "$session_id" != "null" && "$session_id" != "" && "$recording_started" = "false" ]] + then + echo "Session: $session_id is created" + caps_se_video_record=$(bash ${VIDEO_CONFIG_DIRECTORY}/video_graphQLQuery.sh "$session_id") + if [[ -f "/tmp/graphQL_$session_id.json" ]]; then + cat "/tmp/graphQL_$session_id.json"; echo + fi + echo "se:recordVideo value is: $caps_se_video_record" + fi + if [[ "$session_id" != "null" && "$session_id" != "" && "$recording_started" = "false" && "$caps_se_video_record" = "true" ]] + then + video_file_name="$session_id.mp4" + video_file="${SE_VIDEO_FOLDER}/$video_file_name" + echo "Starting to record video" + ffmpeg -nostdin -y -f x11grab -video_size ${VIDEO_SIZE} -r ${FRAME_RATE} -i ${DISPLAY} -codec:v ${CODEC} ${PRESET} -pix_fmt yuv420p $video_file & + recording_started="true" + echo "Video recording started" + elif [[ "$session_id" != "$prev_session_id" && "$recording_started" = "true" ]] + then + stop_recording + if [[ $max_recorded_count -gt 0 ]] && [[ $recorded_count -ge $max_recorded_count ]]; + then + echo "Node will be drained since max sessions reached count number ($max_recorded_count)" + exit + fi + elif [[ $recording_started = "true" ]] + then + echo "Video recording in progress " + sleep 1 + else + sleep 1 + fi + prev_session_id=$session_id + done + echo "Node API is not responding, exiting." + exit +fi diff --git a/Video/video_graphQLQuery.sh b/Video/video_graphQLQuery.sh new file mode 100755 index 000000000..54ed011d1 --- /dev/null +++ b/Video/video_graphQLQuery.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +# Define parameters +SESSION_ID=$1 +GRAPHQL_ENDPOINT=${2:-$SE_NODE_GRID_GRAPHQL_URL} +VIDEO_CAP_NAME=${VIDEO_CAP_NAME:-"se:recordVideo"} + +if [ -z "${GRAPHQL_ENDPOINT}" ] && [ -n "${SE_NODE_GRID_URL}" ]; then + GRAPHQL_ENDPOINT="${SE_NODE_GRID_URL}/graphql" +fi + +if [ -n "${GRAPHQL_ENDPOINT}" ]; then + # Send GraphQL query + curl --retry 3 -k -X POST \ + -H "Content-Type: application/json" \ + --data '{"query":"{ session (id: \"'${SESSION_ID}'\") { id, capabilities, startTime, uri, nodeId, nodeUri, sessionDurationMillis, slot { id, stereotype, lastStarted } } } "}' \ + -s "${GRAPHQL_ENDPOINT}" > /tmp/graphQL_${SESSION_ID}.json + + RECORD_VIDEO=$(jq -r '.data.session.capabilities | fromjson | ."'${VIDEO_CAP_NAME}'"' /tmp/graphQL_${SESSION_ID}.json) +fi + +if [ "${RECORD_VIDEO,,}" = "false" ]; then + echo "${RECORD_VIDEO,,}" +else + echo true +fi diff --git a/Video/video_ready.py b/Video/video_ready.py old mode 100644 new mode 100755 diff --git a/charts/selenium-grid/README.md b/charts/selenium-grid/README.md index 0902e7107..f7c1af3bb 100644 --- a/charts/selenium-grid/README.md +++ b/charts/selenium-grid/README.md @@ -18,7 +18,7 @@ This chart enables the creation of a Selenium Grid Server in Kubernetes. * [Ingress Configuration](#ingress-configuration) * [Configuration](#configuration) * [Configuration global](#configuration-global) - * [Configuration `global.K8S_PUBLIC_IP`](#configuration-globalk8spublicip) + * [Configuration `global.K8S_PUBLIC_IP`](#configuration-globalk8s_public_ip) * [Configuration of Nodes](#configuration-of-nodes) * [Container ports and Service ports](#container-ports-and-service-ports) * [Probes](#probes) @@ -392,23 +392,23 @@ nodeConfigMap: echo "Your custom script" recorderConfigMap: - extraScriptsDirectory: "/opt/selenium" + extraScriptsDirectory: "/opt/bin" extraScripts: video.sh: | #!/bin/bash echo "Your custom script" - videoPreStop.sh: | + video_graphQLQuery.sh: | #!/bin/bash echo "My new script" uploaderConfigMap: extraScriptsDirectory: "/opt/bin" extraScripts: - entry_point.sh: | + upload.sh: | #!/bin/bash echo "Your custom entry point" secretFiles: - config.conf: | + upload.conf: | [myremote] type = s3 ``` @@ -448,11 +448,11 @@ driver = webdriver.Remote(options=options, command_executor="http://localhost:44 ) ``` -In Node will perform query GraphQL and extract the value of `se:recordVideo` in capabilities before deciding to start video recording process or not. By default, the script is loaded from file [configs/recorder/graphQLRecordVideo.sh](configs/recorder/graphQLRecordVideo.sh) to the ConfigMap. You can customize by reading on section [Configuration extra scripts mount to container](#configuration-extra-scripts-mount-to-container). +In Node will perform query GraphQL in Hub based on Node SessionId and extract the value of `se:recordVideo` in capabilities before deciding to start video recording process or not. You can customize by reading on section [Configuration extra scripts mount to container](#configuration-extra-scripts-mount-to-container). #### Video uploader -The uploader is a sidecar that is deployed with the browser nodes. It is responsible for uploading the video to a remote location. The uploader is disabled by default. To enable it, you need to set the following values: +The uploader is extra utility in the video container. It is responsible for uploading the video to a remote location. The uploader is disabled by default. To enable it, you need to set the following values: ```yaml videoRecorder: @@ -464,14 +464,14 @@ By default, the uploader uses [RCLONE](https://rclone.org/) to upload the video The uploader requires `destinationPrefix` to be set. It is used to instruct the uploader where to upload the video. The format of destinationPrefix is `remote-name://bucket-name/path`. The `remote-name` is configured in RCLONE. The `bucket-name` is the name of the bucket in the remote location. The `path` is the path to the folder in the bucket. -By default, the config file is loaded from file [configs/uploader/rclone/config.conf](configs/uploader/rclone/config.conf) to the Secret. You can override the config file via `--set-file uploaderConfigMap.secretFiles.config\.conf=/path/to/your_config.conf` or set via YAML values. +By default, the config file is empty. You can override the config file via `--set-file uploaderConfigMap.secretFiles.upload\.conf=/path/to/your_config.conf` or set via YAML values. For example, to configure an S3 remote hosted on AWS with named `mys3` and the bucket name is `mybucket`, you can set the following values: ```bash uploaderConfigMap: secretFiles: - config.conf: | + upload.conf: | [mys3] type = s3 provider = AWS @@ -488,7 +488,6 @@ videoRecorder: ``` You can prepare a config file with multiple remotes are defined. Ensure that `[remoteName]` is unique for each remote. -You also can replace your config to default file `configs/uploader/rclone/config.conf` in chart. Instead of using config file, another way that RCLONE also supports to pass the information via environment variables. ENV variable with format: `RCLONE_CONFIG_ + name of remote + _ + name of config file option` (make it all uppercase). In this case the remote name it can only contain letters, digits, or the _ (underscore) character. All those ENV variables can be set via `videoRecorder.uploader.secrets`, it will be stored in Secret. @@ -507,21 +506,46 @@ videoRecorder: RCLONE_CONFIG_MYS3_ACL: "private" RCLONE_CONFIG_MYS3_ACCESS_KEY_ID: "xxx" RCLONE_CONFIG_MYS3_SECRET_ACCESS_KEY: "xxx" + RCLONE_CONFIG_MYS3_ENDPOINT: "https://storage.googleapis.com" ``` -Those 2 ways are equivalent. You can choose one of them or combine them together. When both config file and ENV vars are set, value in `config.conf` will take precedence. +Those 2 ways are equivalent. You can choose one of them or combine them together. When both config file and ENV vars are set, value in `upload.conf` will take precedence. -Beside the configuration, the script for entry point of uploader container also needed. By default, it is loaded from file [configs/uploader/rclone/entry_point.sh](configs/uploader/rclone/entry_point.sh) to the ConfigMap. You can override the script via `--set-file uploaderConfigMap.extraScripts.entry_point\.sh=/path/to/your_script.sh` or set via YAML values. For example: +Beside the configuration, the script for entry point of uploader container also needed. You can override the script via `--set-file uploaderConfigMap.extraScripts.upload\.sh=/path/to/your_script.sh` or set via YAML values. For example: ```yaml uploaderConfigMap: extraScripts: - entry_point.sh: | + upload.sh: | #!/bin/bash echo "Your custom entry point" ``` -You also can replace your script to default file `configs/uploader/rclone/entry_point.sh` in chart. +In case you want to configure another sidecar container for uploader, you can set a name for `videoRecorder.uploader.name` and create a config key with the same name under `videoRecorder` with all the settings for your container. Set name of `videoRecorder.uploader.entryPointFileName` if your container start by a different entry point. For example: + +```yaml +uploaderConfigMap: + extraScripts: + upload.sh: | + #!/bin/bash + echo "Script control the uploader process" + +videoRecorder: + enabled: true + uploader: + enabled: true + name: "s3" + entryPointFileName: "upload.sh" + destinationPrefix: "s3://mybucket" + secrets: + AWS_REGION: "ap-southeast-1" + AWS_ACCESS_KEY_ID: "xxxx" + AWS_SECRET_ACCESS_KEY: "xxxx" + s3: + imageRegistry: public.ecr.aws + imageName: bitnami/aws-cli + imageTag: latest +``` ### Configuration of Secure Communication (HTTPS) diff --git a/charts/selenium-grid/configs/recorder/graphQLRecordVideo.sh b/charts/selenium-grid/configs/recorder/graphQLRecordVideo.sh deleted file mode 100755 index 28cb15eb5..000000000 --- a/charts/selenium-grid/configs/recorder/graphQLRecordVideo.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# Define parameters -SESSION_ID=$1 -GRAPHQL_ENDPOINT=${2:-$SE_NODE_GRID_GRAPHQL_URL} - -# Send GraphQL query -curl --retry 3 -k -X POST \ - -H "Content-Type: application/json" \ - --data '{"query":"{ session (id: \"'${SESSION_ID}'\") { id, capabilities, startTime, uri, nodeId, nodeUri, sessionDurationMillis, slot { id, stereotype, lastStarted } } } "}' \ - -s "${GRAPHQL_ENDPOINT}" > /tmp/graphQL_$SESSION_ID.json - -RECORD_VIDEO=$(jq -r '.data.session.capabilities | fromjson | ."se:recordVideo"' /tmp/graphQL_$SESSION_ID.json) - -if [ "${RECORD_VIDEO,,}" = "false" ]; then - echo "${RECORD_VIDEO}" -else - echo true -fi diff --git a/charts/selenium-grid/configs/recorder/video.sh b/charts/selenium-grid/configs/recorder/video.sh deleted file mode 100755 index e4648cada..000000000 --- a/charts/selenium-grid/configs/recorder/video.sh +++ /dev/null @@ -1,143 +0,0 @@ -#!/usr/bin/env bash - -VIDEO_UPLOAD_ENABLED=${VIDEO_UPLOAD_ENABLED:-"false"} -VIDEO_CONFIG_DIRECTORY=${VIDEO_CONFIG_DIRECTORY:-"/opt/bin"} - -function create_pipe() { - if [[ "${VIDEO_UPLOAD_ENABLED}" != "false" ]] && [[ ! -z "${UPLOAD_DESTINATION_PREFIX}" ]]; - then - echo "Create pipe if not exists for video upload stream" - if [ ! -p ${SE_VIDEO_FOLDER}/uploadpipe ]; - then - mkfifo ${SE_VIDEO_FOLDER}/uploadpipe - fi - fi -} -create_pipe - -function wait_util_force_exit_consume() { - if [[ "${VIDEO_UPLOAD_ENABLED}" != "false" ]] && [[ ! -z "${UPLOAD_DESTINATION_PREFIX}" ]]; - then - while [[ -f ${SE_VIDEO_FOLDER}/force_exit ]] - do - echo "Waiting for force exit file to be consumed by uploader" - sleep 1 - done - echo "Ready to shutdown the recorder" - fi -} - -function add_exit_signal() { - if [[ "${VIDEO_UPLOAD_ENABLED}" != "false" ]] && [[ ! -z "${UPLOAD_DESTINATION_PREFIX}" ]]; - then - echo "exit" >> ${SE_VIDEO_FOLDER}/uploadpipe & - echo "exit" > ${SE_VIDEO_FOLDER}/force_exit - fi -} - -function exit_on_max_session_reach() { - if [ $max_recorded_count -gt 0 ] && [ $recorded_count -ge $max_recorded_count ]; - then - echo "Node will be drained since max sessions reached count number ($max_recorded_count)" - exit - fi -} - -function finish { - add_exit_signal - wait_util_force_exit_consume - kill -INT `cat /var/run/supervisor/supervisord.pid` -} -trap finish EXIT - -FRAME_RATE=${FRAME_RATE:-$SE_FRAME_RATE} -CODEC=${CODEC:-$SE_CODEC} -PRESET=${PRESET:-$SE_PRESET} -DISPLAY_CONTAINER_NAME=${DISPLAY_CONTAINER_NAME:-"localhost"} -export DISPLAY=${DISPLAY_CONTAINER_NAME}:${DISPLAY_NUM}.0 - -max_attempts=600 -attempts=0 -echo Checking if the display is open -until xset b off || [[ $attempts = $max_attempts ]] -do - echo Waiting before next display check - sleep 0.5 - attempts=$((attempts+1)) -done -if [[ $attempts = $max_attempts ]] -then - echo Can not open display, exiting. - exit -fi -VIDEO_SIZE=$(xdpyinfo | grep 'dimensions:' | awk '{print $2}') - -recording_started="false" -video_file_name="" -video_file="" -prev_session_id="" -attempts=0 -max_recorded_count=${SE_DRAIN_AFTER_SESSION_COUNT:-0} -recorded_count=0 -echo Checking if node API responds -until curl -sk --request GET ${SE_SERVER_PROTOCOL}://${DISPLAY_CONTAINER_NAME}:${SE_NODE_PORT}/status || [[ $attempts = $max_attempts ]] -do - echo Waiting before next API check - sleep 0.5 - attempts=$((attempts+1)) -done -if [[ $attempts = $max_attempts ]] -then - echo Can not reach node API, exiting. - exit -fi -while curl -sk --request GET ${SE_SERVER_PROTOCOL}://${DISPLAY_CONTAINER_NAME}:${SE_NODE_PORT}/status > /tmp/status.json -do - session_id=$(jq -r '.[]?.node?.slots | .[0]?.session?.sessionId' /tmp/status.json) - echo "Session: $session_id is created" - if [[ "$session_id" != "null" && "$session_id" != "" && "$recording_started" = "false" ]] - then - caps_se_video_record=$(bash ${VIDEO_CONFIG_DIRECTORY}/graphQLRecordVideo.sh "$session_id") - cat "/tmp/graphQL_$session_id.json"; echo - echo "se:recordVideo value is: $caps_se_video_record" - fi - if [[ "$session_id" != "null" && "$session_id" != "" && "$recording_started" = "false" && "$caps_se_video_record" = "true" ]] - then - video_file_name="$session_id.mp4" - video_file="${SE_VIDEO_FOLDER}/$video_file_name" - echo "Starting to record video" - ffmpeg -nostdin -y -f x11grab -video_size ${VIDEO_SIZE} -r ${FRAME_RATE} -i ${DISPLAY} -codec:v ${CODEC} ${PRESET} -pix_fmt yuv420p $video_file & - recording_started="true" - echo "Video recording started" - elif [[ "$session_id" != "$prev_session_id" && "$recording_started" = "true" ]] - then - echo "Stopping to record video" - pkill -INT ffmpeg - recorded_count=$((recorded_count+1)) - recording_started="false" - if [[ "${VIDEO_UPLOAD_ENABLED}" != "false" ]] && [[ ! -z "${UPLOAD_DESTINATION_PREFIX}" ]]; - then - upload_destination=${UPLOAD_DESTINATION_PREFIX}/${video_file_name} - echo "Uploading video to $upload_destination" - echo $video_file ${UPLOAD_DESTINATION_PREFIX} >> ${SE_VIDEO_FOLDER}/uploadpipe & - elif [[ "${VIDEO_UPLOAD_ENABLED}" != "false" ]] && [[ -z "${UPLOAD_DESTINATION_PREFIX}" ]]; - then - echo Upload destination not known since UPLOAD_DESTINATION_PREFIX is not set. Continue without uploading. - fi - if [ $max_recorded_count -gt 0 ] && [ $recorded_count -ge $max_recorded_count ]; - then - echo "Node will be drained since max sessions reached count number ($max_recorded_count)" - exit - fi - - elif [[ $recording_started = "true" ]] - then - echo "Video recording in progress" - sleep 1 - else - echo "No session in progress" - sleep 1 - fi - prev_session_id=$session_id -done -echo "Node API is not responding, exiting." diff --git a/charts/selenium-grid/configs/uploader/rclone/config.conf b/charts/selenium-grid/configs/uploader/rclone/config.conf deleted file mode 100644 index db0337c13..000000000 --- a/charts/selenium-grid/configs/uploader/rclone/config.conf +++ /dev/null @@ -1,9 +0,0 @@ -[mys3] -type = s3 -provider = AWS -env_auth = true -region = -location_constraint = -acl = private -access_key_id = -secret_access_key = diff --git a/charts/selenium-grid/configs/uploader/rclone/entry_point.sh b/charts/selenium-grid/configs/uploader/rclone/entry_point.sh deleted file mode 100644 index 211601353..000000000 --- a/charts/selenium-grid/configs/uploader/rclone/entry_point.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash - -SE_VIDEO_FOLDER=${SE_VIDEO_FOLDER:-"/videos"} -UPLOAD_CONFIG_DIRECTORY=${UPLOAD_CONFIG_DIRECTORY:-"/opt/bin"} -UPLOAD_CONFIG_FILE_NAME=${UPLOAD_CONFIG_FILE_NAME:-"config.conf"} -UPLOAD_COMMAND=${UPLOAD_COMMAND:-"copy"} - -function consume_force_exit() { - rm -f ${SE_VIDEO_FOLDER}/force_exit - echo "Force exit signal consumed" -} -trap consume_force_exit EXIT - -while [ ! -p ${SE_VIDEO_FOLDER}/uploadpipe ]; -do - echo "Waiting for ${SE_VIDEO_FOLDER}/uploadpipe to be created" - sleep 1 -done - -echo "Waiting for video files put into pipe for proceeding to upload" - -while read FILE DESTINATION < ${SE_VIDEO_FOLDER}/uploadpipe -do - if [ "${FILE}" = "exit" ]; - then - exit - else [ "$FILE" != "" ] && [ "$DESTINATION" != "" ]; - echo "Uploading ${FILE} to ${DESTINATION}" - rclone --config ${UPLOAD_CONFIG_DIRECTORY}/${UPLOAD_CONFIG_FILE_NAME} ${UPLOAD_COMMAND} ${UPLOAD_OPTS} "${FILE}" "${DESTINATION}" - if [ $? -eq 0 ]; - then - rm -rf $FILE - fi - fi - if [ -f ${SE_VIDEO_FOLDER}/force_exit ] && [ ! -s ${SE_VIDEO_FOLDER}/uploadpipe ]; - then - exit - fi -done - -consume_force_exit diff --git a/charts/selenium-grid/configs/uploader/s3/entry_point.sh b/charts/selenium-grid/configs/uploader/s3/upload.sh similarity index 55% rename from charts/selenium-grid/configs/uploader/s3/entry_point.sh rename to charts/selenium-grid/configs/uploader/s3/upload.sh index e2d3d8f5f..4cb72546f 100644 --- a/charts/selenium-grid/configs/uploader/s3/entry_point.sh +++ b/charts/selenium-grid/configs/uploader/s3/upload.sh @@ -1,6 +1,9 @@ #!/usr/bin/env bash SE_VIDEO_FOLDER=${SE_VIDEO_FOLDER:-"/videos"} +UPLOAD_COMMAND=${UPLOAD_COMMAND:-"cp"} +UPLOAD_RETAIN_LOCAL_FILE=${SE_UPLOAD_RETAIN_LOCAL_FILE:-"false"} +SE_VIDEO_UPLOAD_BATCH_CHECK=${SE_VIDEO_UPLOAD_BATCH_CHECK:-"10"} if [[ -z "${AWS_REGION}" ]] || [[ -z "${AWS_ACCESS_KEY_ID}" ]] || [[ -z "${AWS_SECRET_ACCESS_KEY}" ]]; then @@ -13,11 +16,21 @@ aws configure set aws_secret_access_key ${AWS_SECRET_ACCESS_KEY} --profile s3-pr aws configure --profile s3-profile function consume_force_exit() { - rm -f ${SE_VIDEO_FOLDER}/force_exit + for pid in "${list_upload_pid[@]}"; + do + wait ${pid} + done + rm -rf ${SE_VIDEO_FOLDER}/force_exit echo "Force exit signal consumed" } trap consume_force_exit EXIT +if [ "${UPLOAD_RETAIN_LOCAL_FILE}" = "false" ]; +then + echo "UPLOAD_RETAIN_LOCAL_FILE is set to false, force to use command: move" + UPLOAD_COMMAND="mv" +fi + while [ ! -p ${SE_VIDEO_FOLDER}/uploadpipe ]; do echo "Waiting for ${SE_VIDEO_FOLDER}/uploadpipe to be created" @@ -26,15 +39,29 @@ done echo "Waiting for video files put into pipe for proceeding to upload" +list_upload_pid=() while read FILE DESTINATION < ${SE_VIDEO_FOLDER}/uploadpipe do if [ "${FILE}" = "exit" ]; then exit - else [ "$FILE" != "" ] && [ "$DESTINATION" != "" ]; + elif [ "$FILE" != "" ] && [ "$DESTINATION" != "" ]; + then echo "Uploading ${FILE} to ${DESTINATION}" - aws s3 cp "${FILE}" "${DESTINATION}" + aws s3 ${UPLOAD_COMMAND} "${FILE}" "${DESTINATION}" & + list_upload_pid+=($!) + else + # Wait for a batch of processes to finish + if [ ${#list_upload_pid[@]} -eq ${SE_VIDEO_UPLOAD_BATCH_CHECK} ] + then + for pid in "${list_upload_pid[@]}"; + do + wait ${pid} + done + list_upload_pid=() + fi fi + if [ -f ${SE_VIDEO_FOLDER}/force_exit ] && [ ! -s ${SE_VIDEO_FOLDER}/uploadpipe ]; then exit diff --git a/charts/selenium-grid/templates/_helpers.tpl b/charts/selenium-grid/templates/_helpers.tpl index f11a133f4..e8f4f7270 100644 --- a/charts/selenium-grid/templates/_helpers.tpl +++ b/charts/selenium-grid/templates/_helpers.tpl @@ -467,6 +467,10 @@ template: name: {{ tpl (toYaml .Values.recorderConfigMap.name) $ }} - configMapRef: name: {{ tpl (toYaml .Values.serverConfigMap.name) $ }} + {{- if and .Values.videoRecorder.uploader.enabled (not .Values.videoRecorder.uploader.name) }} + - secretRef: + name: {{ tpl (toYaml .Values.uploaderConfigMap.secretVolumeMountName) $ }} + {{- end }} {{- with .Values.videoRecorder.extraEnvFrom }} {{- tpl (toYaml .) $ | nindent 8 }} {{- end }} @@ -496,10 +500,10 @@ template: {{- with .Values.videoRecorder.lifecycle }} lifecycle: {{- toYaml . | nindent 10 }} {{- end }} - {{- if .Values.videoRecorder.uploader.enabled }} + {{- if and .Values.videoRecorder.uploader.enabled (not (empty .Values.videoRecorder.uploader.name)) }} - name: uploader - {{- $imageTag := default .Values.global.seleniumGrid.uploaderImageTag .uploader.imageTag }} - {{- $imageRegistry := default .Values.global.seleniumGrid.imageRegistry .uploader.imageRegistry }} + {{- $imageTag := .uploader.imageTag }} + {{- $imageRegistry := .uploader.imageRegistry }} image: {{ printf "%s/%s:%s" $imageRegistry .uploader.imageName $imageTag }} imagePullPolicy: {{ .uploader.imagePullPolicy }} {{- if .uploader.command }} @@ -726,6 +730,13 @@ Define terminationGracePeriodSeconds of the node pod. mountPath: {{ $.Values.recorderConfigMap.extraScriptsDirectory }}/{{ $fileName }} subPath: {{ $fileName }} {{- end }} +{{- if and .Values.videoRecorder.uploader.enabled (not .Values.videoRecorder.uploader.name) }} +{{- range $fileName, $value := .Values.uploaderConfigMap.secretFiles }} +- name: {{ tpl (toYaml $.Values.uploaderConfigMap.secretVolumeMountName) $ }} + mountPath: {{ $.Values.uploaderConfigMap.extraScriptsDirectory }}/{{ $fileName }} + subPath: {{ $fileName }} +{{- end }} +{{- end }} - name: {{ tpl (toYaml $.Values.recorderConfigMap.videoVolumeMountName) $ }} mountPath: {{ $.Values.videoRecorder.targetFolder }} {{- end -}} diff --git a/charts/selenium-grid/templates/recorder-configmap.yaml b/charts/selenium-grid/templates/recorder-configmap.yaml index d11dd3fd5..41611cdbd 100644 --- a/charts/selenium-grid/templates/recorder-configmap.yaml +++ b/charts/selenium-grid/templates/recorder-configmap.yaml @@ -13,9 +13,13 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} data: - VIDEO_UPLOAD_ENABLED: '{{ .Values.videoRecorder.uploader.enabled }}' + SE_VIDEO_UPLOAD_ENABLED: '{{ .Values.videoRecorder.uploader.enabled }}' + SE_VIDEO_FILE_NAME: 'auto' +{{- if and .Values.videoRecorder.uploader.enabled (empty .Values.videoRecorder.uploader.name) }} + SE_VIDEO_INTERNAL_UPLOAD: 'true' +{{- end }} VIDEO_CONFIG_DIRECTORY: '{{ .Values.recorderConfigMap.extraScriptsDirectory }}' - UPLOAD_DESTINATION_PREFIX: '{{ .Values.videoRecorder.uploader.destinationPrefix }}' + SE_UPLOAD_DESTINATION_PREFIX: '{{ .Values.videoRecorder.uploader.destinationPrefix }}' {{- range $fileName, $value := .Values.recorderConfigMap.extraScripts }} {{- if not (empty $value) }} {{ $fileName | indent 2 -}}: | diff --git a/charts/selenium-grid/templates/uploader-configmap.yaml b/charts/selenium-grid/templates/uploader-configmap.yaml index bf36f9600..5f623c58c 100644 --- a/charts/selenium-grid/templates/uploader-configmap.yaml +++ b/charts/selenium-grid/templates/uploader-configmap.yaml @@ -13,8 +13,12 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} data: - UPLOAD_CONFIG_DIRECTORY: '{{ .Values.uploaderConfigMap.extraScriptsDirectory }}' - UPLOAD_CONFIG_FILE_NAME: '{{ .Values.videoRecorder.uploader.configFileName }}' +{{- with .Values.uploaderConfigMap.extraScriptsDirectory }} + UPLOAD_CONFIG_DIRECTORY: '{{ . }}' +{{- end }} +{{- with .Values.videoRecorder.uploader.configFileName }} + UPLOAD_CONFIG_FILE_NAME: '{{ . }}' +{{- end }} {{- range $fileName, $value := .Values.uploaderConfigMap.extraScripts }} {{- if not (empty $value) }} {{ $fileName | indent 2 -}}: | diff --git a/charts/selenium-grid/values.yaml b/charts/selenium-grid/values.yaml index 36bf31f49..78054d468 100644 --- a/charts/selenium-grid/values.yaml +++ b/charts/selenium-grid/values.yaml @@ -12,8 +12,6 @@ global: nodesImageTag: 4.17.0-20240123 # Image tag for browser's video recorder videoImageTag: ffmpeg-6.1-20240123 - # Image tag for browser's uploader - uploaderImageTag: rclone-1.65-20240123 # Pull secret for all components, can be overridden individually imagePullSecret: "" # Log level for all components. Possible values describe here: https://www.selenium.dev/documentation/grid/configuration/cli_options/#logging @@ -120,8 +118,8 @@ recorderConfigMap: extraScriptsDirectory: "/opt/bin" # List of extra scripts to be mounted to the container. Format as `filename: content` extraScripts: - video.sh: - graphQLRecordVideo.sh: + # video.sh: + # video_graphQLQuery.sh: # Name of volume mount is used to mount scripts in the ConfigMap scriptVolumeMountName: recorder-helper-scripts videoVolumeMountName: videos @@ -136,10 +134,10 @@ uploaderConfigMap: extraScriptsDirectory: "/opt/bin" # List of extra scripts to be mounted to the container. Format as `filename: content` extraScripts: - entry_point.sh: + upload.sh: # Extra files stored in Secret to be mounted to the container. secretFiles: - config.conf: + upload.conf: "[sample]" # Name of volume mount is used to mount scripts in the ConfigMap scriptVolumeMountName: uploader-helper-scripts # Name of Secret is used to store the `secretFiles` @@ -1031,10 +1029,9 @@ videoRecorder: # Where to upload the video file e.g. remoteName://bucketName/path. Refer to destination syntax of rclone https://rclone.org/docs/ destinationPrefix: # What uploader to use. See .videRecorder.rclone for how to create a new one. - name: "rclone" - configFileName: "config.conf" - entryPointFileName: "entry_point.sh" - scriptMountPath: "/opt/bin" + name: + configFileName: upload.conf + entryPointFileName: upload.sh # For environment variables used in uploader which contains sensitive information, store in secret and refer envFrom # Set config for rclone via ENV var with format: RCLONE_CONFIG_ + name of remote + _ + name of config file option (make it all uppercase) secrets: @@ -1114,24 +1111,6 @@ videoRecorder: # persistentVolumeClaim: # claimName: video-pv-claim - # Container spec for the uploader if above it is defined as "uploader.name: rclone" - rclone: - # imageRegistry: selenium - imageName: uploader - # imageTag: "rclone-1.65-20240119" - imagePullPolicy: IfNotPresent - # SecurityContext for uploader container - securityContext: - runAsUser: 0 - command: [] - args: [] - extraEnvironmentVariables: - # Extra options for rclone. Refer to https://rclone.org/flags - - name: UPLOAD_OPTS - value: "--progress" - # - name: UPLOAD_COMMAND - # value: "copy" - # Container spec for the uploader if above it is defined as "uploader.name: s3" s3: imageRegistry: public.ecr.aws diff --git a/docker-compose-v3-video-upload.yml b/docker-compose-v3-video-upload.yml new file mode 100644 index 000000000..fbeca1ef7 --- /dev/null +++ b/docker-compose-v3-video-upload.yml @@ -0,0 +1,106 @@ +# To execute this docker-compose yml file use `docker-compose -f docker-compose-v3-video-upload.yml up` +# Add the `-d` flag at the end for detached execution +# To stop the execution, hit Ctrl+C, and then `docker-compose -f docker-compose-v3-video-upload.yml down` +# ${variable_pattern} get value from .env in the same directory +version: "3" +services: + chrome: + image: selenium/node-chrome:nightly + shm_size: 2gb + depends_on: + - selenium-hub + environment: + - SE_EVENT_BUS_HOST=selenium-hub + - SE_EVENT_BUS_PUBLISH_PORT=4442 + - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 + - SE_OPTS=--enable-managed-downloads true + + edge: + image: selenium/node-edge:nightly + shm_size: 2gb + depends_on: + - selenium-hub + environment: + - SE_EVENT_BUS_HOST=selenium-hub + - SE_EVENT_BUS_PUBLISH_PORT=4442 + - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 + - SE_OPTS=--enable-managed-downloads true + + firefox: + image: selenium/node-firefox:nightly + shm_size: 2gb + depends_on: + - selenium-hub + environment: + - SE_EVENT_BUS_HOST=selenium-hub + - SE_EVENT_BUS_PUBLISH_PORT=4442 + - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 + - SE_OPTS=--enable-managed-downloads true + + chrome_video: + image: selenium/video:nightly + depends_on: + - chrome + environment: + - DISPLAY_CONTAINER_NAME=chrome + - SE_VIDEO_FILE_NAME=auto + - SE_VIDEO_UPLOAD_ENABLED=true + - SE_VIDEO_INTERNAL_UPLOAD=true + - SE_UPLOAD_DESTINATION_PREFIX=s3://${BUCKET_NAME} + - RCLONE_CONFIG_S3_TYPE=s3 + - RCLONE_CONFIG_S3_PROVIDER=GCS + - RCLONE_CONFIG_S3_ENV_AUTH=true + - RCLONE_CONFIG_S3_REGION=asia-southeast1 + - RCLONE_CONFIG_S3_LOCATION_CONSTRAINT=asia-southeast1 + - RCLONE_CONFIG_S3_ACL=private + - RCLONE_CONFIG_S3_ACCESS_KEY_ID=${GS_ACCESS_KEY_ID} + - RCLONE_CONFIG_S3_SECRET_ACCESS_KEY=${GS_SECRET_ACCESS_KEY} + - RCLONE_CONFIG_S3_ENDPOINT=https://storage.googleapis.com + + edge_video: + image: selenium/video:nightly + depends_on: + - edge + environment: + - DISPLAY_CONTAINER_NAME=edge + - SE_VIDEO_FILE_NAME=auto + - SE_VIDEO_UPLOAD_ENABLED=true + - SE_VIDEO_INTERNAL_UPLOAD=true + - SE_UPLOAD_DESTINATION_PREFIX=s3://${BUCKET_NAME} + - RCLONE_CONFIG_S3_TYPE=s3 + - RCLONE_CONFIG_S3_PROVIDER=GCS + - RCLONE_CONFIG_S3_ENV_AUTH=true + - RCLONE_CONFIG_S3_REGION=asia-southeast1 + - RCLONE_CONFIG_S3_LOCATION_CONSTRAINT=asia-southeast1 + - RCLONE_CONFIG_S3_ACL=private + - RCLONE_CONFIG_S3_ACCESS_KEY_ID=${GS_ACCESS_KEY_ID} + - RCLONE_CONFIG_S3_SECRET_ACCESS_KEY=${GS_SECRET_ACCESS_KEY} + - RCLONE_CONFIG_S3_ENDPOINT=https://storage.googleapis.com + + firefox_video: + image: selenium/video:nightly + depends_on: + - firefox + environment: + - DISPLAY_CONTAINER_NAME=firefox + - SE_VIDEO_FILE_NAME=auto + - SE_VIDEO_UPLOAD_ENABLED=true + - SE_VIDEO_INTERNAL_UPLOAD=true + - SE_UPLOAD_DESTINATION_PREFIX=s3://${BUCKET_NAME} + - RCLONE_CONFIG_S3_TYPE=s3 + - RCLONE_CONFIG_S3_PROVIDER=GCS + - RCLONE_CONFIG_S3_ENV_AUTH=true + - RCLONE_CONFIG_S3_REGION=asia-southeast1 + - RCLONE_CONFIG_S3_LOCATION_CONSTRAINT=asia-southeast1 + - RCLONE_CONFIG_S3_ACL=private + - RCLONE_CONFIG_S3_ACCESS_KEY_ID=${GS_ACCESS_KEY_ID} + - RCLONE_CONFIG_S3_SECRET_ACCESS_KEY=${GS_SECRET_ACCESS_KEY} + - RCLONE_CONFIG_S3_ENDPOINT=https://storage.googleapis.com + + selenium-hub: + image: selenium/hub:nightly + container_name: selenium-hub + ports: + - "4442:4442" + - "4443:4443" + - "4444:4444" diff --git a/generate_release_notes.sh b/generate_release_notes.sh index f3fe9b954..e64dfcdda 100755 --- a/generate_release_notes.sh +++ b/generate_release_notes.sh @@ -21,7 +21,7 @@ EDGEDRIVER_VERSION=$(docker run --rm ${NAMESPACE}/node-edge:${TAG_VERSION} msedg FIREFOX_VERSION=$(docker run --rm ${NAMESPACE}/node-firefox:${TAG_VERSION} firefox --version | awk '{print $3}') GECKODRIVER_VERSION=$(docker run --rm ${NAMESPACE}/node-firefox:${TAG_VERSION} geckodriver --version | awk 'NR==1{print $2}') FFMPEG_VERSION=$(docker run --entrypoint="" --rm ${NAMESPACE}/video:${FFMPEG_TAG_VERSION}-${BUILD_DATE} ffmpeg -version | awk '{print $3}' | head -n 1) -RCLONE_VERSION=$(docker run --entrypoint="" --rm ${NAMESPACE}/uploader:${RCLONE_TAG_VERSION}-${BUILD_DATE} rclone version | head -n 1 | awk '{print $2}') +RCLONE_VERSION=$(docker run --entrypoint="" --rm ${NAMESPACE}/video:${FFMPEG_TAG_VERSION}-${BUILD_DATE} rclone version | head -n 1 | awk '{print $2}') echo "" >> release_notes.md diff --git a/tests/charts/bootstrap.sh b/tests/charts/bootstrap.sh index 33721c44b..276c7c22e 100755 --- a/tests/charts/bootstrap.sh +++ b/tests/charts/bootstrap.sh @@ -15,10 +15,10 @@ cd .. helm template dummy --values tests/charts/templates/render/dummy.yaml \ --set-file 'nodeConfigMap.extraScripts.nodePreStop\.sh=tests/charts/templates/render/dummy_external.sh' \ --set-file 'recorderConfigMap.extraScripts.video\.sh=tests/charts/templates/render/dummy_external.sh' \ - --set-file 'recorderConfigMap.extraScripts.graphQLRecordVideo\.sh=tests/charts/templates/render/dummy_external.sh' \ + --set-file 'recorderConfigMap.extraScripts.video_graphQLQuery\.sh=tests/charts/templates/render/dummy_external.sh' \ --set-file 'recorderConfigMap.extraScripts.newInsertScript\.sh=tests/charts/templates/render/dummy_external.sh' \ - --set-file 'uploaderConfigMap.extraScripts.entry_point\.sh=tests/charts/templates/render/dummy_external.sh' \ - --set-file 'uploaderConfigMap.secretFiles.config\.conf=tests/charts/templates/render/dummy_external.sh' \ + --set-file 'uploaderConfigMap.extraScripts.upload\.sh=tests/charts/templates/render/dummy_external.sh' \ + --set-file 'uploaderConfigMap.secretFiles.upload\.conf=tests/charts/templates/render/dummy_external.sh' \ charts/selenium-grid > ./tests/tests/dummy_template_manifests.yaml python tests/charts/templates/test.py "./tests/tests/dummy_template_manifests.yaml" diff --git a/tests/charts/ci/JobAutoscaling-values.yaml b/tests/charts/ci/JobAutoscaling-values.yaml index 6b08b06fb..248c2f7cc 100644 --- a/tests/charts/ci/JobAutoscaling-values.yaml +++ b/tests/charts/ci/JobAutoscaling-values.yaml @@ -4,6 +4,7 @@ autoscaling: enableWithExistingKEDA: true scalingType: job scaledJobOptions: + successfulJobsHistoryLimit: 100 scalingStrategy: strategy: default scaledOptions: diff --git a/tests/charts/ci/base-recorder-values.yaml b/tests/charts/ci/base-recorder-values.yaml index a87852866..32efa50ed 100644 --- a/tests/charts/ci/base-recorder-values.yaml +++ b/tests/charts/ci/base-recorder-values.yaml @@ -1,9 +1,20 @@ +# Use different container sidecar for uploader +#videoRecorder: +# enabled: true +# uploader: +# enabled: "${UPLOAD_ENABLED}" +# name: s3 +# destinationPrefix: "s3://${BUCKET_NAME}" +# secrets: +# AWS_REGION: "ap-southeast-1" +# AWS_ACCESS_KEY_ID: "${AWS_ACCESS_KEY_ID}" +# AWS_SECRET_ACCESS_KEY: "${AWS_SECRET_ACCESS_KEY}" + videoRecorder: enabled: true uploader: - enabled: false - name: rclone - destinationPrefix: "gs://ndviet" + enabled: "${UPLOAD_ENABLED}" + destinationPrefix: "gs://${BUCKET_NAME}" secrets: RCLONE_CONFIG_S3_TYPE: "s3" RCLONE_CONFIG_S3_PROVIDER: "AWS" @@ -11,17 +22,14 @@ videoRecorder: RCLONE_CONFIG_S3_REGION: "ap-southeast-1" RCLONE_CONFIG_S3_LOCATION_CONSTRAINT: "ap-southeast-1" RCLONE_CONFIG_S3_ACL: "private" - RCLONE_CONFIG_S3_ACCESS_KEY_ID: "****" - RCLONE_CONFIG_S3_SECRET_ACCESS_KEY: "****" + RCLONE_CONFIG_S3_ACCESS_KEY_ID: "${AWS_ACCESS_KEY_ID}" + RCLONE_CONFIG_S3_SECRET_ACCESS_KEY: "${AWS_SECRET_ACCESS_KEY}" RCLONE_CONFIG_GS_TYPE: "s3" RCLONE_CONFIG_GS_PROVIDER: "GCS" RCLONE_CONFIG_GS_ENV_AUTH: "true" RCLONE_CONFIG_GS_REGION: "asia-southeast1" RCLONE_CONFIG_GS_LOCATION_CONSTRAINT: "asia-southeast1" RCLONE_CONFIG_GS_ACL: "private" - RCLONE_CONFIG_GS_ACCESS_KEY_ID: "****" - RCLONE_CONFIG_GS_SECRET_ACCESS_KEY: "****" + RCLONE_CONFIG_GS_ACCESS_KEY_ID: "${GS_ACCESS_KEY_ID}" + RCLONE_CONFIG_GS_SECRET_ACCESS_KEY: "${GS_SECRET_ACCESS_KEY}" RCLONE_CONFIG_GS_ENDPOINT: "https://storage.googleapis.com" - AWS_REGION: "ap-southeast-1" - AWS_ACCESS_KEY_ID: "****" - AWS_SECRET_ACCESS_KEY: "****" diff --git a/tests/charts/make/chart_setup_env.sh b/tests/charts/make/chart_setup_env.sh index 7164029f7..4043516a4 100755 --- a/tests/charts/make/chart_setup_env.sh +++ b/tests/charts/make/chart_setup_env.sh @@ -108,4 +108,10 @@ if [ "$(uname -m)" = "x86_64" ]; then rm -rf ct.tar.gz ct version echo "===============================" + echo "Installing envsubst for AMD64 / x86_64" + curl -L https://github.com/a8m/envsubst/releases/download/v1.4.2/envsubst-`uname -s`-`uname -m` -o envsubst + chmod +x envsubst + sudo mv envsubst /usr/local/bin + sudo ln -sf /usr/local/bin/envsubst /usr/bin/envsubst + echo "===============================" fi diff --git a/tests/charts/make/chart_test.sh b/tests/charts/make/chart_test.sh index 0d1a74f0f..b6cc075e9 100755 --- a/tests/charts/make/chart_test.sh +++ b/tests/charts/make/chart_test.sh @@ -25,7 +25,6 @@ SKIP_CLEANUP=${SKIP_CLEANUP:-"false"} # For debugging purposes, retain the clust CHART_CERT_PATH=${CHART_CERT_PATH:-"${CHART_PATH}/certs/selenium.pem"} SSL_CERT_DIR=${SSL_CERT_DIR:-"/etc/ssl/certs"} VIDEO_TAG=${VIDEO_TAG:-"latest"} -UPLOADER_TAG=${UPLOADER_TAG:-"latest"} cleanup() { if [ "${SKIP_CLEANUP}" = "false" ]; then @@ -48,12 +47,20 @@ on_failure() { # Trap ERR signal and call on_failure function trap 'on_failure' ERR +RECORDER_VALUES_FILE=${TEST_VALUES_PATH}/base-recorder-values.yaml +if [ -f .env ] +then + export $(cat .env | xargs) + echo "Render values files with .env" + envsubst < ${RECORDER_VALUES_FILE} > ./tests/tests/base-recorder-values.yaml + RECORDER_VALUES_FILE=./tests/tests/base-recorder-values.yaml +fi + HELM_COMMAND_SET_IMAGES=" \ --set global.seleniumGrid.imageRegistry=${NAMESPACE} \ --set global.seleniumGrid.imageTag=${VERSION} \ --set global.seleniumGrid.nodesImageTag=${VERSION} \ --set global.seleniumGrid.videoImageTag=${VIDEO_TAG} \ ---set global.seleniumGrid.uploaderImageTag=${UPLOADER_TAG} \ " if [ "${SELENIUM_GRID_AUTOSCALING}" = "true" ]; then @@ -66,7 +73,7 @@ fi HELM_COMMAND_SET_BASE_VALUES=" \ --values ${TEST_VALUES_PATH}/base-auth-ingress-values.yaml \ --values ${TEST_VALUES_PATH}/base-tracing-values.yaml \ ---values ${TEST_VALUES_PATH}/base-recorder-values.yaml \ +--values ${RECORDER_VALUES_FILE} \ --values ${TEST_VALUES_PATH}/base-resources-values.yaml \ " diff --git a/tests/charts/templates/render/dummy.yaml b/tests/charts/templates/render/dummy.yaml index 1e8411c6a..e41cac575 100644 --- a/tests/charts/templates/render/dummy.yaml +++ b/tests/charts/templates/render/dummy.yaml @@ -83,7 +83,7 @@ videoRecorder: enabled: true uploader: enabled: true - destinationPrefix: "s3://ndviet" + destinationPrefix: "s3://bucket-name" secrets: RCLONE_CONFIG_S3_TYPE: "s3" RCLONE_CONFIG_S3_PROVIDER: "AWS"