From a065ede3c2c9ef90385985b3b3cf4c583f170a58 Mon Sep 17 00:00:00 2001 From: Viet Nguyen Duc Date: Tue, 6 Feb 2024 15:32:51 +0530 Subject: [PATCH] test(chart): parallel with autoscalingType deployment & job Signed-off-by: Viet Nguyen Duc --- .github/workflows/build-test.yml | 7 +- .github/workflows/deploy.yml | 3 +- .github/workflows/helm-chart-release.yml | 6 +- .github/workflows/helm-chart-test.yml | 33 +++---- .github/workflows/label-commenter.yml | 2 +- .github/workflows/nightly.yaml | 4 +- .github/workflows/scan-dockerfile.yml | 4 +- .github/workflows/test-video.yml | 13 +-- .github/workflows/update-chart-changelog.yaml | 9 +- .../update-dev-beta-browser-images.yml | 2 +- Makefile | 23 +++-- charts/selenium-grid/TESTING.md | 4 +- charts/selenium-grid/templates/_helpers.tpl | 10 ++- tests/Dockerfile | 2 +- tests/SeleniumTests/__init__.py | 56 +++++++----- ...yaml => DeploymentAutoscaling-values.yaml} | 7 +- tests/charts/ci/JobAutoscaling-values.yaml | 2 +- ...ues.yaml => base-auth-ingress-values.yaml} | 28 ------ tests/charts/ci/base-recorder-values.yaml | 27 ++++++ tests/charts/ci/base-resources-values.yaml | 86 +++++++++++++++++++ .../{tls-values.yaml => base-tls-values.yaml} | 0 ...g-values.yaml => base-tracing-values.yaml} | 0 tests/charts/make/chart_cluster_cleanup.sh | 8 +- tests/charts/make/chart_cluster_setup.sh | 40 +++++++-- tests/charts/make/chart_setup_env.sh | 77 +++++++++++++++-- tests/charts/make/chart_test.sh | 41 ++++++--- tests/test.py | 3 +- 27 files changed, 364 insertions(+), 133 deletions(-) rename tests/charts/ci/{DeploymentAutoScaling-values.yaml => DeploymentAutoscaling-values.yaml} (96%) rename tests/charts/ci/{auth-ingress-values.yaml => base-auth-ingress-values.yaml} (51%) create mode 100644 tests/charts/ci/base-recorder-values.yaml create mode 100644 tests/charts/ci/base-resources-values.yaml rename tests/charts/ci/{tls-values.yaml => base-tls-values.yaml} (100%) rename tests/charts/ci/{tracing-values.yaml => base-tracing-values.yaml} (100%) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index ef9d3166c..1c7727499 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -1,6 +1,9 @@ name: Build & test +concurrency: + group: ${{ github.workflow }} on: + workflow_dispatch: push: paths-ignore: - '**.md' @@ -19,11 +22,11 @@ jobs: matrix: use-random-user: [false, true] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@main - name: Output Docker info run: docker info - name: Set up Python - uses: actions/setup-python@v5.0.0 + uses: actions/setup-python@main with: python-version: '3.11' check-latest: true diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index f0792ee22..13cec5977 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,6 +1,7 @@ name: Deploys on: + workflow_dispatch: push: branches: - trunk @@ -14,7 +15,7 @@ jobs: permissions: write-all steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@main with: persist-credentials: false fetch-depth: 0 diff --git a/.github/workflows/helm-chart-release.yml b/.github/workflows/helm-chart-release.yml index 1f2b1f7c4..4c7dafc52 100644 --- a/.github/workflows/helm-chart-release.yml +++ b/.github/workflows/helm-chart-release.yml @@ -1,4 +1,6 @@ name: Release Charts +concurrency: + group: ${{ github.workflow }} on: push: @@ -14,7 +16,7 @@ jobs: permissions: write-all steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@main with: fetch-depth: 0 @@ -24,7 +26,7 @@ jobs: git config user.email "$GITHUB_ACTOR@users.noreply.github.com" - name: Run chart-releaser - uses: helm/chart-releaser-action@v1.6.0 + uses: helm/chart-releaser-action@main with: mark_as_latest: false skip_existing: true diff --git a/.github/workflows/helm-chart-test.yml b/.github/workflows/helm-chart-test.yml index 1061bf80b..316b2dd14 100644 --- a/.github/workflows/helm-chart-test.yml +++ b/.github/workflows/helm-chart-test.yml @@ -1,4 +1,6 @@ name: Test Helm Charts +concurrency: + group: ${{ github.workflow }} on: push: @@ -19,21 +21,20 @@ jobs: strategy: fail-fast: false matrix: - test-strategy: [chart_test, chart_test_parallel_autoscaling, chart_test_https, chart_test_parallel_autoscaling_https] + test-strategy: + - job + - job_https + - deployment + - deployment_https steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@main - name: Output Docker info run: docker info - name: Set up Python - uses: actions/setup-python@v5.0.0 + uses: actions/setup-python@main with: python-version: '3.11' check-latest: true - - name: Install CA certificates - run: | - sudo apt install openssl -y - sudo apt install ca-certificates -y - sudo update-ca-certificates --fresh - name: Get branch name (only for push to branch) if: github.event_name == 'push' run: echo "BRANCH=$(echo ${PUSH_BRANCH##*/})" >> $GITHUB_ENV @@ -52,34 +53,36 @@ jobs: echo "IMAGE_REGISTRY=artifactory/selenium" >> $GITHUB_ENV - name: Setup Kubernetes environment run: make chart_setup_env - - name: Build Docker images - run: NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make build - - name: Build and lint charts + - name: Build Helm charts run: | BUILD_DATE=${BUILD_DATE} make chart_build echo "CHART_PACKAGE_PATH=$(cat /tmp/selenium_chart_version)" >> $GITHUB_ENV echo "CHART_FILE_NAME=$(basename $(cat /tmp/selenium_chart_version))" >> $GITHUB_ENV + - name: Test chart template + run: NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make chart_test_template - name: Setup Kubernetes cluster run: NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make chart_cluster_setup - - name: Test Selenium Grid on Kubernetes + - name: Build Docker images + run: NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make build + - name: Test Selenium Grid on Kubernetes (Autoscaling ${{ matrix.test-strategy }}) uses: nick-invision/retry@master with: timeout_minutes: 30 max_attempts: 3 command: | - NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make ${{ matrix.test-strategy }} + NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make chart_test_autoscaling_${{ matrix.test-strategy }} - name: Cleanup Kubernetes cluster if: always() run: make chart_cluster_cleanup - name: Upload Helm chart package if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@main with: name: ${{ matrix.test-strategy }}_${{ env.CHART_FILE_NAME }} path: ${{ env.CHART_PACKAGE_PATH }} - name: Upload chart test artifacts if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@main with: name: ${{ matrix.test-strategy }}-artifacts path: ./tests/tests/ diff --git a/.github/workflows/label-commenter.yml b/.github/workflows/label-commenter.yml index bf2e02b25..9df715b63 100644 --- a/.github/workflows/label-commenter.yml +++ b/.github/workflows/label-commenter.yml @@ -13,6 +13,6 @@ jobs: comment: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@main - name: Label Commenter uses: peaceiris/actions-label-commenter@v1 diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml index 9c2319bba..fb5fdd1f9 100644 --- a/.github/workflows/nightly.yaml +++ b/.github/workflows/nightly.yaml @@ -11,7 +11,7 @@ jobs: permissions: write-all steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@main with: persist-credentials: false fetch-depth: 0 @@ -74,7 +74,7 @@ jobs: - name: Create release notes (release_notes.md) run: ./generate_release_notes.sh ${LATEST_TAG} origin/trunk ${GRID_VERSION} ${BUILD_DATE} - name: Set up Python - uses: actions/setup-python@v5.0.0 + uses: actions/setup-python@main with: python-version: '3.11' check-latest: true diff --git a/.github/workflows/scan-dockerfile.yml b/.github/workflows/scan-dockerfile.yml index 259a900dd..0d65fb6dc 100644 --- a/.github/workflows/scan-dockerfile.yml +++ b/.github/workflows/scan-dockerfile.yml @@ -1,4 +1,6 @@ name: Scan Dockerfile vulnerabilities +concurrency: + group: ${{ github.workflow }} on: push: @@ -17,7 +19,7 @@ jobs: permissions: write-all runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@main - name: Set severity for PRs if: github.event_name == 'pull_request' || github.event_name == 'push' run: | diff --git a/.github/workflows/test-video.yml b/.github/workflows/test-video.yml index 04bf156d0..2ea1b5c7d 100644 --- a/.github/workflows/test-video.yml +++ b/.github/workflows/test-video.yml @@ -1,6 +1,9 @@ name: Test video files +concurrency: + group: ${{ github.workflow }} on: + workflow_dispatch: push: paths-ignore: - '**.md' @@ -16,11 +19,11 @@ jobs: name: Test video recorded through Docker Selenium runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@main - name: Output Docker info run: docker info - name: Set up Python - uses: actions/setup-python@v5.0.0 + uses: actions/setup-python@main with: python-version: '3.11' check-latest: true @@ -41,17 +44,17 @@ jobs: - name: Run Docker Compose to record video run: USE_RANDOM_USER_ID=${USE_RANDOM_USER} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make test_video - name: Upload recorded Chrome video - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@main with: name: chrome_video path: ./tests/videos/chrome_video.mp4 - name: Upload recorded Edge video - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@main with: name: edge_video path: ./tests/videos/edge_video.mp4 - name: Upload recorded Firefox video - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@main with: name: firefox_video path: ./tests/videos/firefox_video.mp4 diff --git a/.github/workflows/update-chart-changelog.yaml b/.github/workflows/update-chart-changelog.yaml index 3ac45a214..a360a405f 100644 --- a/.github/workflows/update-chart-changelog.yaml +++ b/.github/workflows/update-chart-changelog.yaml @@ -1,18 +1,19 @@ name: Update Chart CHANGELOG on: - release: - types: [published] + push: + branches: + - trunk workflow_dispatch: jobs: release: runs-on: ubuntu-latest permissions: write-all - if: ${{ contains(github.event.release.tag_name, 'selenium-grid') || github.event_name == 'workflow_dispatch' }} + if: contains(toJson(github.event.commits), 'Update tag in docs and files') == true steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@main with: persist-credentials: false fetch-depth: 0 diff --git a/.github/workflows/update-dev-beta-browser-images.yml b/.github/workflows/update-dev-beta-browser-images.yml index 1a0036574..b640b51a2 100644 --- a/.github/workflows/update-dev-beta-browser-images.yml +++ b/.github/workflows/update-dev-beta-browser-images.yml @@ -22,7 +22,7 @@ jobs: CHANNEL: ${{ matrix.channel }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@main - name: Setup environment variables run: | export SELENIUM_VERSION=$(grep BASE_VERSION Makefile | sed 's/.*,\([^)]*\))/\1/p' | head -n 1) diff --git a/Makefile b/Makefile index c029b095f..587c98db8 100644 --- a/Makefile +++ b/Makefile @@ -428,7 +428,7 @@ chart_setup_env: ./tests/charts/make/chart_setup_env.sh chart_cluster_setup: - VERSION=$(TAG_VERSION) NAMESPACE=$(NAMESPACE) ./tests/charts/make/chart_cluster_setup.sh + VERSION=$(TAG_VERSION) NAMESPACE=$(NAMESPACE) BUILD_DATE=$(BUILD_DATE) ./tests/charts/make/chart_cluster_setup.sh chart_cluster_cleanup: ./tests/charts/make/chart_cluster_cleanup.sh @@ -439,14 +439,6 @@ chart_build_nightly: chart_build: VERSION=$(TAG_VERSION) ./tests/charts/make/chart_build.sh -chart_test_https: - SELENIUM_GRID_TEST_HEADLESS=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_PORT=443 make chart_test - -chart_test: chart_test_template \ - chart_test_chrome \ - chart_test_firefox \ - chart_test_edge - chart_test_template: ./tests/charts/bootstrap.sh @@ -462,10 +454,17 @@ chart_test_edge: VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) UPLOADER_TAG=$(RCLONE_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \ ./tests/charts/make/chart_test.sh NodeEdge -chart_test_parallel_autoscaling_https: - SELENIUM_GRID_TEST_HEADLESS=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_PORT=443 make chart_test_parallel_autoscaling +chart_test_autoscaling_deployment_https: + SELENIUM_GRID_TEST_HEADLESS=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_PORT=443 make chart_test_autoscaling_deployment + +chart_test_autoscaling_deployment: + VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) UPLOADER_TAG=$(RCLONE_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \ + ./tests/charts/make/chart_test.sh DeploymentAutoscaling + +chart_test_autoscaling_job_https: + SELENIUM_GRID_TEST_HEADLESS=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_PORT=443 make chart_test_autoscaling_job -chart_test_parallel_autoscaling: +chart_test_autoscaling_job: VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) UPLOADER_TAG=$(RCLONE_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \ ./tests/charts/make/chart_test.sh JobAutoscaling diff --git a/charts/selenium-grid/TESTING.md b/charts/selenium-grid/TESTING.md index fd29f09c1..b8864b63d 100644 --- a/charts/selenium-grid/TESTING.md +++ b/charts/selenium-grid/TESTING.md @@ -70,9 +70,9 @@ make chart_build make chart_cluster_setup # Test Selenium Grid on Kubernetes -make chart_test +make chart_test_autoscaling_deployment -# make chart_test_parallel_autoscaling +make chart_test_autoscaling_job # Cleanup Kubernetes cluster make chart_cluster_cleanup diff --git a/charts/selenium-grid/templates/_helpers.tpl b/charts/selenium-grid/templates/_helpers.tpl index 449dacd07..f11a133f4 100644 --- a/charts/selenium-grid/templates/_helpers.tpl +++ b/charts/selenium-grid/templates/_helpers.tpl @@ -320,7 +320,7 @@ template: spec: serviceAccountName: {{ template "seleniumGrid.serviceAccount.fullname" . }} serviceAccount: {{ template "seleniumGrid.serviceAccount.fullname" . }} - restartPolicy: {{ and (eq (include "seleniumGrid.useKEDA" .) "true") (eq .Values.autoscaling.scalingType "job") | ternary "Never" "Always" }} + restartPolicy: {{ template "seleniumGrid.node.restartPolicy" . }} {{- with .node.hostAliases }} hostAliases: {{ toYaml . | nindent 6 }} {{- end }} @@ -712,6 +712,14 @@ Define terminationGracePeriodSeconds of the node pod. {{- $period -}} {{- end -}} +{{- define "seleniumGrid.node.restartPolicy" -}} +{{- $restartPolicy := "Always" -}} +{{- if and (eq (include "seleniumGrid.useKEDA" .) "true") (eq .Values.autoscaling.scalingType "job") -}} + {{- $restartPolicy = "Never" -}} +{{- end -}} +{{- $restartPolicy -}} +{{- end -}} + {{- define "seleniumGrid.video.volumeMounts.default" -}} {{- range $fileName, $value := .Values.recorderConfigMap.extraScripts }} - name: {{ tpl (toYaml $.Values.recorderConfigMap.scriptVolumeMountName) $ }} diff --git a/tests/Dockerfile b/tests/Dockerfile index f79b99203..ba837fa97 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10-slim +FROM python:3.11-slim WORKDIR /usr/src/app diff --git a/tests/SeleniumTests/__init__.py b/tests/SeleniumTests/__init__.py index 7341e82ed..bd2f1e4e5 100644 --- a/tests/SeleniumTests/__init__.py +++ b/tests/SeleniumTests/__init__.py @@ -77,19 +77,15 @@ def test_download_file(self): driver.get('https://the-internet.herokuapp.com/download') file_name = 'some-file.txt' is_continue = True - try: - wait = WebDriverWait(driver, 30) - file_link = wait.until( - EC.element_to_be_clickable((By.LINK_TEXT, file_name)) - ) - except: - is_continue = False - if is_continue: - file_link.click() - wait.until( - lambda d: str(d.get_downloadable_files()[0]).endswith(file_name) - ) - self.assertTrue(str(driver.get_downloadable_files()[0]).endswith(file_name)) + wait = WebDriverWait(driver, 30) + file_link = wait.until( + EC.element_to_be_clickable((By.LINK_TEXT, file_name)) + ) + file_link.click() + wait.until( + lambda d: str(d.get_downloadable_files()[0]).endswith(file_name) + ) + self.assertTrue(str(driver.get_downloadable_files()[0]).endswith(file_name)) def tearDown(self): self.driver.quit() @@ -143,25 +139,43 @@ def test_title_and_maximize_window(self): self.driver.maximize_window() self.assertTrue(self.driver.title == 'The Internet') -class JobAutoscaling(): +class Autoscaling(): def run(self, test_classes): with concurrent.futures.ThreadPoolExecutor() as executor: futures = [] + tests = [] for test_class in test_classes: suite = unittest.TestLoader().loadTestsFromTestCase(test_class) for test in suite: futures.append(executor.submit(test)) - for future in concurrent.futures.as_completed(futures): + tests.append(test) + failed_tests = [] + for future, test in zip(concurrent.futures.as_completed(futures), tests): try: - result = future.result() - if not result.wasSuccessful(): - raise Exception(f"Test {str(test)} failed") + if not future.result().wasSuccessful(): + raise Exception except Exception as e: + failed_tests.append(test) + print(traceback.format_exc()) print(f"{str(test)} failed with exception: {str(e)}") + print(f"Original exception: {e.__cause__}") + print(f"Number of failed tests: {len(failed_tests)}") + for test in failed_tests: + try: + print(f"Rerunning test: {str(test)}") + test.run() + except Exception as e: print(traceback.format_exc()) - raise Exception(f"Parallel tests failed: {str(test)} failed with exception: {str(e)}") + print(f"Test {str(test)} failed again with exception: {str(e)}") + print(f"Original exception: {e.__cause__}") + raise Exception(f"Rerun test failed: {str(test)} failed with exception: {str(e)}") + +class DeploymentAutoscalingTests(unittest.TestCase): + def test_parallel_autoscaling(self): + runner = Autoscaling() + runner.run([FirefoxTests, EdgeTests, ChromeTests]) class JobAutoscalingTests(unittest.TestCase): def test_parallel_autoscaling(self): - runner = JobAutoscaling() - runner.run([ChromeTests, EdgeTests, FirefoxTests]) + runner = Autoscaling() + runner.run([FirefoxTests, EdgeTests, ChromeTests]) diff --git a/tests/charts/ci/DeploymentAutoScaling-values.yaml b/tests/charts/ci/DeploymentAutoscaling-values.yaml similarity index 96% rename from tests/charts/ci/DeploymentAutoScaling-values.yaml rename to tests/charts/ci/DeploymentAutoscaling-values.yaml index 08dcf1cb0..ac04713ba 100644 --- a/tests/charts/ci/DeploymentAutoScaling-values.yaml +++ b/tests/charts/ci/DeploymentAutoscaling-values.yaml @@ -3,10 +3,12 @@ autoscaling: scalingType: deployment scaledOptions: minReplicaCount: 0 - maxReplicaCount: 3 + maxReplicaCount: 2 + pollingInterval: 20 scaledObjectOptions: cooldownPeriod: 30 terminationGracePeriodSeconds: 360 + # Configuration for chrome nodes chromeNode: # (test): user is able to change `SE_NODE_PORT` @@ -39,6 +41,7 @@ chromeNode: enabled: &readinessProbe true livenessProbe: enabled: &livenessProbe true + # Configuration for edge nodes edgeNode: port: 8888 @@ -61,8 +64,10 @@ edgeNode: enabled: *readinessProbe livenessProbe: enabled: *livenessProbe + # Configuration for firefox nodes firefoxNode: + port: 7777 nameOverride: my-firefox-name extraEnvironmentVariables: *extraEnvironmentVariables readinessProbe: diff --git a/tests/charts/ci/JobAutoscaling-values.yaml b/tests/charts/ci/JobAutoscaling-values.yaml index 2e689b57c..3913ad535 100644 --- a/tests/charts/ci/JobAutoscaling-values.yaml +++ b/tests/charts/ci/JobAutoscaling-values.yaml @@ -8,7 +8,7 @@ autoscaling: strategy: default scaledOptions: minReplicaCount: 0 - maxReplicaCount: 5 + maxReplicaCount: 6 pollingInterval: 20 # Configuration for chrome nodes chromeNode: diff --git a/tests/charts/ci/auth-ingress-values.yaml b/tests/charts/ci/base-auth-ingress-values.yaml similarity index 51% rename from tests/charts/ci/auth-ingress-values.yaml rename to tests/charts/ci/base-auth-ingress-values.yaml index c597ebc2a..9d4ab3055 100644 --- a/tests/charts/ci/auth-ingress-values.yaml +++ b/tests/charts/ci/base-auth-ingress-values.yaml @@ -42,31 +42,3 @@ ingress-nginx: kind: DaemonSet service: type: ClusterIP - -videoRecorder: - enabled: true - uploader: - enabled: false - name: rclone - destinationPrefix: "gs://ndviet" - secrets: - RCLONE_CONFIG_S3_TYPE: "s3" - RCLONE_CONFIG_S3_PROVIDER: "AWS" - RCLONE_CONFIG_S3_ENV_AUTH: "true" - RCLONE_CONFIG_S3_REGION: "ap-southeast-1" - RCLONE_CONFIG_S3_LOCATION_CONSTRAINT: "ap-southeast-1" - RCLONE_CONFIG_S3_ACL: "private" - RCLONE_CONFIG_S3_ACCESS_KEY_ID: "****" - RCLONE_CONFIG_S3_SECRET_ACCESS_KEY: "****" - RCLONE_CONFIG_GS_TYPE: "s3" - RCLONE_CONFIG_GS_PROVIDER: "GCS" - RCLONE_CONFIG_GS_ENV_AUTH: "true" - RCLONE_CONFIG_GS_REGION: "asia-southeast1" - RCLONE_CONFIG_GS_LOCATION_CONSTRAINT: "asia-southeast1" - RCLONE_CONFIG_GS_ACL: "private" - RCLONE_CONFIG_GS_ACCESS_KEY_ID: "****" - RCLONE_CONFIG_GS_SECRET_ACCESS_KEY: "****" - RCLONE_CONFIG_GS_ENDPOINT: "https://storage.googleapis.com" - AWS_REGION: "ap-southeast-1" - AWS_ACCESS_KEY_ID: "****" - AWS_SECRET_ACCESS_KEY: "****" diff --git a/tests/charts/ci/base-recorder-values.yaml b/tests/charts/ci/base-recorder-values.yaml new file mode 100644 index 000000000..a87852866 --- /dev/null +++ b/tests/charts/ci/base-recorder-values.yaml @@ -0,0 +1,27 @@ +videoRecorder: + enabled: true + uploader: + enabled: false + name: rclone + destinationPrefix: "gs://ndviet" + secrets: + RCLONE_CONFIG_S3_TYPE: "s3" + RCLONE_CONFIG_S3_PROVIDER: "AWS" + RCLONE_CONFIG_S3_ENV_AUTH: "true" + RCLONE_CONFIG_S3_REGION: "ap-southeast-1" + RCLONE_CONFIG_S3_LOCATION_CONSTRAINT: "ap-southeast-1" + RCLONE_CONFIG_S3_ACL: "private" + RCLONE_CONFIG_S3_ACCESS_KEY_ID: "****" + RCLONE_CONFIG_S3_SECRET_ACCESS_KEY: "****" + RCLONE_CONFIG_GS_TYPE: "s3" + RCLONE_CONFIG_GS_PROVIDER: "GCS" + RCLONE_CONFIG_GS_ENV_AUTH: "true" + RCLONE_CONFIG_GS_REGION: "asia-southeast1" + RCLONE_CONFIG_GS_LOCATION_CONSTRAINT: "asia-southeast1" + RCLONE_CONFIG_GS_ACL: "private" + RCLONE_CONFIG_GS_ACCESS_KEY_ID: "****" + RCLONE_CONFIG_GS_SECRET_ACCESS_KEY: "****" + RCLONE_CONFIG_GS_ENDPOINT: "https://storage.googleapis.com" + AWS_REGION: "ap-southeast-1" + AWS_ACCESS_KEY_ID: "****" + AWS_SECRET_ACCESS_KEY: "****" diff --git a/tests/charts/ci/base-resources-values.yaml b/tests/charts/ci/base-resources-values.yaml new file mode 100644 index 000000000..4ec462921 --- /dev/null +++ b/tests/charts/ci/base-resources-values.yaml @@ -0,0 +1,86 @@ +components: + router: + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 2500Mi + distributor: + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 200m + memory: 2500Mi + eventBus: + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 200m + memory: 2500Mi + sessionMap: + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 200m + memory: 512Mi + sessionQueue: + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 200m + memory: 512Mi + +hub: + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 2500Mi + +chromeNode: + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 250m + memory: 2500Mi + +firefoxNode: + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 250m + memory: 2500Mi + +edgeNode: + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 250m + memory: 2500Mi + +videoRecorder: + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 200m + memory: 512Mi diff --git a/tests/charts/ci/tls-values.yaml b/tests/charts/ci/base-tls-values.yaml similarity index 100% rename from tests/charts/ci/tls-values.yaml rename to tests/charts/ci/base-tls-values.yaml diff --git a/tests/charts/ci/tracing-values.yaml b/tests/charts/ci/base-tracing-values.yaml similarity index 100% rename from tests/charts/ci/tracing-values.yaml rename to tests/charts/ci/base-tracing-values.yaml diff --git a/tests/charts/make/chart_cluster_cleanup.sh b/tests/charts/make/chart_cluster_cleanup.sh index 972a9e8b6..b0210a7c1 100755 --- a/tests/charts/make/chart_cluster_cleanup.sh +++ b/tests/charts/make/chart_cluster_cleanup.sh @@ -1,11 +1,17 @@ #!/bin/bash echo "Set ENV variables" +CLUSTER=${CLUSTER:-"minikube"} CLUSTER_NAME=${CLUSTER_NAME:-"chart-testing"} cleanup() { - echo "Clean up kind cluster" + if [ "${CLUSTER}" = "kind" ]; then + echo "Clean up Kind cluster" kind delete clusters ${CLUSTER_NAME} + elif [ "${CLUSTER}" = "minikube" ]; then + echo "Clean up Minikube cluster" + sudo -SE minikube delete + fi } cleanup diff --git a/tests/charts/make/chart_cluster_setup.sh b/tests/charts/make/chart_cluster_setup.sh index e413bd984..c34bd522c 100755 --- a/tests/charts/make/chart_cluster_setup.sh +++ b/tests/charts/make/chart_cluster_setup.sh @@ -1,6 +1,8 @@ #!/bin/bash +set -o xtrace echo "Set ENV variables" +CLUSTER=${CLUSTER:-"minikube"} CLUSTER_NAME=${CLUSTER_NAME:-"chart-testing"} RELEASE_NAME=${RELEASE_NAME:-"test"} SELENIUM_NAMESPACE=${SELENIUM_NAMESPACE:-"selenium"} @@ -32,14 +34,36 @@ on_failure() { # Trap ERR signal and call on_failure function trap 'on_failure' ERR -echo "Create Kind cluster" -kind create cluster --wait ${WAIT_TIMEOUT} --name ${CLUSTER_NAME} --config tests/charts/config/kind-cluster.yaml +# Limit the number of resources to avoid host OOM +CPUs=$(grep -c ^processor /proc/cpuinfo) +if [ "${CPUs}" -gt 1 ]; then + CPUs=$((CPUs-1)) +fi + +MEMORY=$(free -m | awk '/^Mem:/{print $7}') +if [ "${MEMORY}" = "" ]; then + MEMORY=$(free -m | awk '/^Mem:/{print $2}') +fi + +if [ "${CLUSTER}" = "kind" ]; then + echo "Start Kind cluster" + kind create cluster --wait ${WAIT_TIMEOUT} --name ${CLUSTER_NAME} --config tests/charts/config/kind-cluster.yaml +elif [ "${CLUSTER}" = "minikube" ]; then + echo "Start Minikube cluster" + sudo chmod 777 /tmp + export CHANGE_MINIKUBE_NONE_USER=true + sudo -SE minikube start --vm-driver=none --cpus ${CPUs} --memory ${MEMORY} \ + --kubernetes-version=$(curl -L -s https://dl.k8s.io/release/stable.txt) --network-plugin=cni --cni=calico + sudo chown -R $USER $HOME/.kube $HOME/.minikube +fi echo "Install KEDA core on kind kubernetes cluster" -kubectl apply --server-side -f https://github.com/kedacore/keda/releases/download/v2.12.1/keda-2.12.1-core.yaml +helm upgrade -i ${KEDA_NAMESPACE} -n ${KEDA_NAMESPACE} --create-namespace --set webhooks.enabled=false kedacore/keda -echo "Load built local Docker Images into Kind Cluster" -image_list=$(docker images --format "{{.Repository}}:{{.Tag}}" | grep ${NAMESPACE} | grep ${BUILD_DATE:-$VERSION}) -for image in $image_list; do - kind load docker-image --name ${CLUSTER_NAME} "$image" -done +if [ "${CLUSTER}" = "kind" ]; then + echo "Load built local Docker Images into Kind Cluster" + image_list=$(docker images --format "{{.Repository}}:{{.Tag}}" | grep ${NAMESPACE} | grep ${BUILD_DATE:-$VERSION}) + for image in $image_list; do + kind load docker-image --name ${CLUSTER_NAME} "$image" + done +fi diff --git a/tests/charts/make/chart_setup_env.sh b/tests/charts/make/chart_setup_env.sh index 8c77101bf..7164029f7 100755 --- a/tests/charts/make/chart_setup_env.sh +++ b/tests/charts/make/chart_setup_env.sh @@ -1,4 +1,8 @@ #!/bin/bash + +echo "Set ENV variables" +CLUSTER=${CLUSTER:-"minikube"} + # Function to be executed on command failure on_failure() { local exit_status=$? @@ -10,14 +14,71 @@ on_failure() { trap 'on_failure' ERR if [ "$(uname -m)" = "x86_64" ]; then - echo "Installing kind for AMD64 / x86_64" - curl -fsSL -o ./kind https://kind.sigs.k8s.io/dl/latest/kind-linux-amd64 - chmod +x ./kind - sudo cp -frp ./kind /usr/local/bin/kind - sudo ln -sf /usr/local/bin/kind /usr/bin/kind - rm -rf kind - kind version - echo "===============================" + if [ "${CLUSTER}" = "kind" ]; then + echo "Installing kind for AMD64 / x86_64" + curl -fsSL -o ./kind https://kind.sigs.k8s.io/dl/latest/kind-linux-amd64 + chmod +x ./kind + sudo cp -frp ./kind /usr/local/bin/kind + sudo ln -sf /usr/local/bin/kind /usr/bin/kind + rm -rf kind + kind version + echo "===============================" + elif [ "${CLUSTER}" = "minikube" ]; then + echo "Installing additional dependencies for running Minikube on none driver CRI-dockerd" + echo "===============================" + echo "Installing conntrack" + sudo apt-get install -y conntrack + echo "===============================" + echo "Installing Minikube" + curl -sLO https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 + sudo install minikube-linux-amd64 /usr/local/bin/minikube + minikube version + rm -rf minikube-linux-amd64 + echo "===============================" + echo "Installing Go" + GO_VERSION="1.21.6" + curl -sLO https://go.dev/dl/go$GO_VERSION.linux-amd64.tar.gz + sudo tar -xf go$GO_VERSION.linux-amd64.tar.gz -C /usr/local + rm -rf go$GO_VERSION.linux-amd64.tar.gz* + echo 'export PATH=$PATH:/usr/local/go/bin' >> ~/.bashrc + source ~/.bashrc || true + go version + echo "===============================" + echo "Installing CRI-CTL (CLI for CRI-compatible container runtimes)" + CRICTL_VERSION="v1.26.0" + curl -sLO https://github.com/kubernetes-sigs/cri-tools/releases/download/$CRICTL_VERSION/crictl-$CRICTL_VERSION-linux-amd64.tar.gz + sudo tar -xf crictl-$CRICTL_VERSION-linux-amd64.tar.gz -C /usr/local/bin + rm -rf crictl-$CRICTL_VERSION-linux-amd64.tar.gz + crictl --version || true + echo "===============================" + echo "Installing CRI-Dockerd" + CRI_DOCKERD_VERSION="v0.3.9" + rm -rf cri-dockerd + git clone -q https://github.com/Mirantis/cri-dockerd.git --branch $CRI_DOCKERD_VERSION --single-branch -c advice.detachedHead=false + cd cri-dockerd || true + sudo go get -v + sudo go build -v -o /usr/local/bin/cri-dockerd + sudo mkdir -p /etc/systemd/system + sudo cp -a -f packaging/systemd/* /etc/systemd/system + sudo sed -i -e 's,/usr/bin/cri-dockerd,/usr/local/bin/cri-dockerd,' /etc/systemd/system/cri-docker.service + sudo systemctl daemon-reload + sudo systemctl enable cri-docker.service + sudo systemctl enable cri-docker.socket + sudo systemctl status --no-pager cri-docker.socket || true + cd .. || true + rm -rf cri-dockerd + cri-dockerd --version + echo "===============================" + echo "Installing CNI-Plugins (Container Network Interface)" + CNI_PLUGIN_VERSION="v1.4.0" + CNI_PLUGIN_TAR="cni-plugins-linux-amd64-$CNI_PLUGIN_VERSION.tgz" + CNI_PLUGIN_INSTALL_DIR="/opt/cni/bin" + curl -sLO "https://github.com/containernetworking/plugins/releases/download/$CNI_PLUGIN_VERSION/$CNI_PLUGIN_TAR" + sudo mkdir -p "$CNI_PLUGIN_INSTALL_DIR" + sudo tar -xf "$CNI_PLUGIN_TAR" -C "$CNI_PLUGIN_INSTALL_DIR" + rm -rf "$CNI_PLUGIN_TAR" + echo "===============================" + fi echo "Installing kubectl for AMD64 / x86_64" curl -fsSL -o ./kubectl "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" diff --git a/tests/charts/make/chart_test.sh b/tests/charts/make/chart_test.sh index d453ed1c2..0d1a74f0f 100755 --- a/tests/charts/make/chart_test.sh +++ b/tests/charts/make/chart_test.sh @@ -48,29 +48,42 @@ on_failure() { # Trap ERR signal and call on_failure function trap 'on_failure' ERR -HELM_COMMAND_SET_AUTOSCALING="" +HELM_COMMAND_SET_IMAGES=" \ +--set global.seleniumGrid.imageRegistry=${NAMESPACE} \ +--set global.seleniumGrid.imageTag=${VERSION} \ +--set global.seleniumGrid.nodesImageTag=${VERSION} \ +--set global.seleniumGrid.videoImageTag=${VIDEO_TAG} \ +--set global.seleniumGrid.uploaderImageTag=${UPLOADER_TAG} \ +" + if [ "${SELENIUM_GRID_AUTOSCALING}" = "true" ]; then - HELM_COMMAND_SET_AUTOSCALING="--values ${TEST_VALUES_PATH}/DeploymentAutoScaling-values.yaml \ + HELM_COMMAND_SET_AUTOSCALING=" \ --set autoscaling.enableWithExistingKEDA=${SELENIUM_GRID_AUTOSCALING} \ - --set autoscaling.scaledOptions.minReplicaCount=${SELENIUM_GRID_AUTOSCALING_MIN_REPLICA}" + --set autoscaling.scaledOptions.minReplicaCount=${SELENIUM_GRID_AUTOSCALING_MIN_REPLICA} \ + " fi -HELM_COMMAND_SET_TLS="" +HELM_COMMAND_SET_BASE_VALUES=" \ +--values ${TEST_VALUES_PATH}/base-auth-ingress-values.yaml \ +--values ${TEST_VALUES_PATH}/base-tracing-values.yaml \ +--values ${TEST_VALUES_PATH}/base-recorder-values.yaml \ +--values ${TEST_VALUES_PATH}/base-resources-values.yaml \ +" + if [ "${SELENIUM_GRID_PROTOCOL}" = "https" ]; then - HELM_COMMAND_SET_TLS="--values ${TEST_VALUES_PATH}/tls-values.yaml" + HELM_COMMAND_SET_BASE_VALUES="${HELM_COMMAND_SET_BASE_VALUES} \ + --values ${TEST_VALUES_PATH}/base-tls-values.yaml \ + " fi +HELM_COMMAND_SET_BASE_VALUES="${HELM_COMMAND_SET_BASE_VALUES} \ +--values ${TEST_VALUES_PATH}/${MATRIX_BROWSER}-values.yaml \ +" + HELM_COMMAND_ARGS="${RELEASE_NAME} \ ---values ${TEST_VALUES_PATH}/auth-ingress-values.yaml \ ---values ${TEST_VALUES_PATH}/tracing-values.yaml \ +${HELM_COMMAND_SET_BASE_VALUES} \ ${HELM_COMMAND_SET_AUTOSCALING} \ -${HELM_COMMAND_SET_TLS} \ ---values ${TEST_VALUES_PATH}/${MATRIX_BROWSER}-values.yaml \ ---set global.seleniumGrid.imageRegistry=${NAMESPACE} \ ---set global.seleniumGrid.imageTag=${VERSION} \ ---set global.seleniumGrid.nodesImageTag=${VERSION} \ ---set global.seleniumGrid.videoImageTag=${VIDEO_TAG} \ ---set global.seleniumGrid.uploaderImageTag=${UPLOADER_TAG} \ +${HELM_COMMAND_SET_IMAGES} \ ${CHART_PATH} --namespace ${SELENIUM_NAMESPACE} --create-namespace" echo "Render manifests YAML for this deployment" diff --git a/tests/test.py b/tests/test.py index 5be6fbe8c..5482c0a89 100644 --- a/tests/test.py +++ b/tests/test.py @@ -57,7 +57,8 @@ 'StandaloneFirefox': 'FirefoxTests', # Chart Parallel Test - 'JobAutoscaling': 'JobAutoscalingTests' + 'JobAutoscaling': 'JobAutoscalingTests', + 'DeploymentAutoscaling': 'DeploymentAutoscalingTests', } FROM_IMAGE_ARGS = {