From c323bedb883a2523d0b386f3cca353ea2d4f0f1f Mon Sep 17 00:00:00 2001 From: Viet Nguyen Duc Date: Thu, 29 Feb 2024 20:09:04 +0530 Subject: [PATCH] test(chart): autoscaling as job Signed-off-by: Viet Nguyen Duc --- NodeChrome/Dockerfile | 2 +- NodeEdge/Dockerfile | 2 +- Video/video.sh | 3 +- charts/selenium-grid/Chart.yaml | 5 ++ charts/selenium-grid/values.yaml | 57 +++++++++++++++++-- tests/charts/ci/JobAutoscaling-values.yaml | 7 ++- tests/charts/ci/base-auth-ingress-values.yaml | 3 + tests/charts/config/ct.yaml | 1 + tests/charts/make/chart_test.sh | 14 ++++- 9 files changed, 81 insertions(+), 13 deletions(-) diff --git a/NodeChrome/Dockerfile b/NodeChrome/Dockerfile index 1f52e9052..5c31e6e5e 100644 --- a/NodeChrome/Dockerfile +++ b/NodeChrome/Dockerfile @@ -59,7 +59,7 @@ USER ${SEL_UID} # Dumping Browser information for config #============================================ RUN echo "chrome" > /opt/selenium/browser_name -RUN google-chrome --version | awk '{print $3}' > /opt/selenium/browser_version +RUN google-chrome --version | awk '{print $3}' | cut -d'.' -f1,2 > /opt/selenium/browser_version RUN echo "\"goog:chromeOptions\": {\"binary\": \"/usr/bin/google-chrome\"}" > /opt/selenium/browser_binary_location ENV SE_OTEL_SERVICE_NAME "selenium-node-chrome" diff --git a/NodeEdge/Dockerfile b/NodeEdge/Dockerfile index fbe335c6e..8140e7406 100644 --- a/NodeEdge/Dockerfile +++ b/NodeEdge/Dockerfile @@ -52,7 +52,7 @@ USER ${SEL_UID} # Dumping Browser information for config #============================================ RUN echo "MicrosoftEdge" > /opt/selenium/browser_name -RUN microsoft-edge --version | awk '{print $3}' > /opt/selenium/browser_version +RUN microsoft-edge --version | awk '{print $3}' | cut -d'.' -f1,2 > /opt/selenium/browser_version RUN echo "\"ms:edgeOptions\": {\"binary\": \"/usr/bin/microsoft-edge\"}" > /opt/selenium/browser_binary_location ENV SE_OTEL_SERVICE_NAME "selenium-node-edge" diff --git a/Video/video.sh b/Video/video.sh index ca06910a3..63aee7956 100755 --- a/Video/video.sh +++ b/Video/video.sh @@ -15,6 +15,7 @@ UPLOAD_PIPE_FILE_NAME=${UPLOAD_PIPE_FILE_NAME:-"uploadpipe"} SE_VIDEO_INTERNAL_UPLOAD=${SE_VIDEO_INTERNAL_UPLOAD:-"false"} SE_SERVER_PROTOCOL=${SE_SERVER_PROTOCOL:-"http"} SE_NODE_PORT=${SE_NODE_PORT:-"5555"} +max_attempts=${SE_VIDEO_WAIT_ATTEMPTS:-50} if [ "${SE_VIDEO_INTERNAL_UPLOAD}" = "true" ]; then @@ -109,7 +110,6 @@ function graceful_exit() { if [[ "${VIDEO_UPLOAD_ENABLED}" != "true" ]] && [[ "${VIDEO_FILE_NAME}" != "auto" ]] && [[ -n "${VIDEO_FILE_NAME}" ]]; then return_code=1 - max_attempts=50 attempts=0 echo 'Checking if the display is open...' until [[ $return_code -eq 0 ]] || [[ $attempts -eq $max_attempts ]]; do @@ -130,7 +130,6 @@ else trap graceful_exit SIGTERM SIGINT EXIT export DISPLAY=${DISPLAY_CONTAINER_NAME}:${DISPLAY_NUM}.0 - max_attempts=600 attempts=0 echo Checking if the display is open diff --git a/charts/selenium-grid/Chart.yaml b/charts/selenium-grid/Chart.yaml index 156e3ae85..2856f03ac 100644 --- a/charts/selenium-grid/Chart.yaml +++ b/charts/selenium-grid/Chart.yaml @@ -18,6 +18,11 @@ dependencies: version: 1.0.0 name: jaeger condition: tracing.enabled +- repository: https://prometheus-community.github.io/helm-charts + version: 56.15.0 + name: kube-prometheus-stack + condition: monitoring.enabled + alias: prometheus-stack maintainers: - name: SeleniumHQ email: selenium-developers@googlegroups.com diff --git a/charts/selenium-grid/values.yaml b/charts/selenium-grid/values.yaml index 4b82a61df..543375e19 100644 --- a/charts/selenium-grid/values.yaml +++ b/charts/selenium-grid/values.yaml @@ -546,7 +546,7 @@ autoscaling: # Number of Completed jobs should be kept successfulJobsHistoryLimit: 0 # Number of Failed jobs should be kept (for troubleshooting purposes) - failedJobsHistoryLimit: 100 + failedJobsHistoryLimit: 0 jobTargetRef: parallelism: 1 completions: 1 @@ -709,7 +709,7 @@ chromeNode: # scaledObjectOptions: hpa: url: '{{ template "seleniumGrid.graphqlURL" . }}' - browserName: chrome + browserName: 'chrome' sessionBrowserName: 'chrome' # browserVersion: '91.0' # Optional. Only required when supporting multiple versions of browser in your Selenium Grid. unsafeSsl: '{{ template "seleniumGrid.graphqlURL.unsafeSsl" . }}' # Optional @@ -869,7 +869,7 @@ firefoxNode: # scaledObjectOptions: hpa: url: '{{ template "seleniumGrid.graphqlURL" . }}' - browserName: firefox + browserName: 'firefox' sessionBrowserName: 'firefox' unsafeSsl: '{{ template "seleniumGrid.graphqlURL.unsafeSsl" . }}' # Optional @@ -1027,8 +1027,8 @@ edgeNode: # scaledObjectOptions: hpa: url: '{{ template "seleniumGrid.graphqlURL" . }}' - browserName: MicrosoftEdge - sessionBrowserName: 'msedge' + browserName: 'MicrosoftEdge' + sessionBrowserName: 'MicrosoftEdge' unsafeSsl: '{{ template "seleniumGrid.graphqlURL.unsafeSsl" . }}' # Optional # It is used to add initContainers in the same pod of the browser node. @@ -1191,3 +1191,50 @@ jaeger: enabled: false query: enabled: false + +prometheus-stack: + defaultRules: + annotations: + "helm.sh/hook": post-install,post-upgrade,post-rollback + alertmanager: + annotations: + "helm.sh/hook": post-install,post-upgrade,post-rollback + enabled: true + ingress: + enabled: true + ingressClassName: nginx + hosts: + - alertmanager.selenium-grid.prod + grafana: + enabled: true + adminPassword: admin + forceDeployDatasources: true + forceDeployDashboards: true + annotations: + "helm.sh/hook": post-install,post-upgrade,post-rollback + ingress: + enabled: true + ingressClassName: nginx + hosts: + - grafana.selenium-grid.prod + prometheus: + annotations: + "helm.sh/hook": post-install,post-upgrade,post-rollback + enabled: true + ingress: + enabled: true + ingressClassName: nginx + hosts: + - pts.selenium-grid.prod + prometheusSpec: + storageSpec: + volumeClaimTemplate: + spec: + storageClassName: "standard" + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 5Gi + selector: + matchLabels: + app.kubernetes.io/name: "prometheus" diff --git a/tests/charts/ci/JobAutoscaling-values.yaml b/tests/charts/ci/JobAutoscaling-values.yaml index c6eabfd5d..1946a183b 100644 --- a/tests/charts/ci/JobAutoscaling-values.yaml +++ b/tests/charts/ci/JobAutoscaling-values.yaml @@ -2,12 +2,13 @@ autoscaling: enableWithExistingKEDA: true scalingType: job scaledJobOptions: - successfulJobsHistoryLimit: 100 + successfulJobsHistoryLimit: 0 + failedJobsHistoryLimit: 0 scalingStrategy: - strategy: default + strategy: accurate scaledOptions: minReplicaCount: 0 - maxReplicaCount: 6 + maxReplicaCount: 30 pollingInterval: 20 # Configuration for chrome nodes chromeNode: diff --git a/tests/charts/ci/base-auth-ingress-values.yaml b/tests/charts/ci/base-auth-ingress-values.yaml index 9e02237c1..ce889b0fe 100644 --- a/tests/charts/ci/base-auth-ingress-values.yaml +++ b/tests/charts/ci/base-auth-ingress-values.yaml @@ -2,6 +2,9 @@ global: seleniumGrid: logLevel: INFO +monitoring: + enabled: true + ingress: annotations: nginx.ingress.kubernetes.io/use-regex: "true" diff --git a/tests/charts/config/ct.yaml b/tests/charts/config/ct.yaml index bad585538..e07685901 100755 --- a/tests/charts/config/ct.yaml +++ b/tests/charts/config/ct.yaml @@ -7,6 +7,7 @@ chart-repos: - kedacore=https://kedacore.github.io/charts - ingressNginx=https://kubernetes.github.io/ingress-nginx - jaegertracing=https://jaegertracing.github.io/helm-charts + - prometheusCommunity=https://prometheus-community.github.io/helm-charts upgrade: false helm-extra-args: --timeout 600s check-version-increment: false diff --git a/tests/charts/make/chart_test.sh b/tests/charts/make/chart_test.sh index 2f35c1edc..60769c00d 100755 --- a/tests/charts/make/chart_test.sh +++ b/tests/charts/make/chart_test.sh @@ -28,7 +28,7 @@ SSL_CERT_DIR=${SSL_CERT_DIR:-"/etc/ssl/certs"} VIDEO_TAG=${VIDEO_TAG:-"latest"} CHART_ENABLE_TRACING=${CHART_ENABLE_TRACING:-"false"} CHART_FULL_DISTRIBUTED_MODE=${CHART_FULL_DISTRIBUTED_MODE:-"false"} -HOSTNAME_ADDRESS=${HOSTNAME_ADDRESS:-"selenium-grid.local"} +HOSTNAME_ADDRESS=${HOSTNAME_ADDRESS:-"selenium-grid.prod"} CHART_ENABLE_INGRESS_HOSTNAME=${CHART_ENABLE_INGRESS_HOSTNAME:-"false"} CHART_ENABLE_BASIC_AUTH=${CHART_ENABLE_BASIC_AUTH:-"false"} BASIC_AUTH_USERNAME=${BASIC_AUTH_USERNAME:-"sysAdminUser"} @@ -115,6 +115,18 @@ else " fi +if [[ ! $(cat /etc/hosts) == *"alertmanager.selenium-grid.prod"* ]]; then + sudo -- sh -c -e "echo \"$(hostname -i) alertmanager.selenium-grid.prod\" >> /etc/hosts" +fi + +if [[ ! $(cat /etc/hosts) == *"grafana.selenium-grid.prod"* ]]; then + sudo -- sh -c -e "echo \"$(hostname -i) grafana.selenium-grid.prod\" >> /etc/hosts" +fi + +if [[ ! $(cat /etc/hosts) == *"pts.selenium-grid.prod"* ]]; then + sudo -- sh -c -e "echo \"$(hostname -i) pts.selenium-grid.prod\" >> /etc/hosts" +fi + if [ "${CHART_ENABLE_BASIC_AUTH}" = "true" ]; then HELM_COMMAND_SET_IMAGES="${HELM_COMMAND_SET_IMAGES} \ --set basicAuth.enabled=${CHART_ENABLE_BASIC_AUTH} \