feat(chart): Simplify to access Selenium Grid from outside of Kuberne…

…tes (#2073)

[deploy]
Signed-off-by: Viet Nguyen Duc <nguyenducviet4496@gmail.com>

charts/selenium-grid/Chart.yaml

@@ -10,6 +10,10 @@ dependencies:
   version: 2.12.1
   name: keda
   condition: autoscaling.enabled
+- repository: https://kubernetes.github.io/ingress-nginx
+  version: 4.8.3
+  name: ingress-nginx
+  condition: ingress-nginx.enabled
 maintainers:
   - name: SeleniumHQ
     email: selenium-developers@googlegroups.com

charts/selenium-grid/templates/NOTES.txt

@@ -9,20 +9,27 @@ Selenium Grid Server deployed successfully.
 {{- if .Values.ingress.enabled }}
     {{- if .Values.ingress.hostname }}
 1. Ingress is enabled, and it exposes the Grid Hub or Grid Router with the hostname you supplied.
-    To access Selenium from outside of Kubernetes, simply open http://{{ .Values.ingress.hostname }}.
-    {{- else}}
+    To access Selenium from outside of Kubernetes, simply open {{ include "seleniumGrid.url" .}}.
+    {{- else if and (empty .Values.ingress.hostname) .Values.global.K8S_PUBLIC_IP }}
+1. Ingress is enabled, but hostname doesn't set, and it exposes the Grid Hub or Grid Router with the K8S_PUBLIC_IP you supplied.
+    To access Selenium from outside of Kubernetes, simply open {{ include "seleniumGrid.url" .}}.
+    {{- else }}
 1. Ingress is enabled, but hostname doesn't set. All inbound HTTP traffic will be routed to the Grid by matching any host.
     Please keep in mind that it is rarely necessary, and in most cases, you shall provide `ingress.hostname` in values.yaml.
     To access Selenium from outside of Kubernetes:
-        - open IP of the any node with Ingress, or
+        - open the IP of any node with Ingress, or
         - any hostname pointing to the node with Ingress
     {{- end}}
 {{- else}}
-1. Ingress is disabled. To access Selenium from outside of Kubernetes, simply run these commands:
     {{- if contains "NodePort" $serviceType }}
-    export NODE_PORT=$(kubectl get -n {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" svc {{ $svcName }})
+      {{- if .Values.global.K8S_PUBLIC_IP }}
+1. Ingress is disabled, and it exposes the Grid Hub or Grid Router with NodePort and the K8S_PUBLIC_IP you supplied
+    To access Selenium from outside of Kubernetes with NodePort and K8S_PUBLIC_IP you supplied, simply open {{ include "seleniumGrid.url" .}}.
+      {{- else }}
+1. Ingress is disabled. To access Selenium from outside of Kubernetes, simply run these commands:
     export NODE_IP=$(kubectl get nodes -n {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
-    echo http://$NODE_IP:$NODE_PORT
+    echo http://$NODE_IP:{{ include "seleniumGrid.url.port" .}}{{ include "seleniumGrid.url.subPath" .}}
+      {{- end }}
     {{- else if contains "LoadBalancer" $serviceType }}
         NOTE: It may take a few minutes for the LoadBalancer IP to be available.
             You can watch the status of by running 'kubectl get -n {{ .Release.Namespace }} svc -w {{ $svcName }}'

charts/selenium-grid/templates/_helpers.tpl

@@ -341,28 +341,87 @@ template:
 Get the url of the grid. If the external url can be figured out from the ingress use that, otherwise the cluster internal url
 */}}
 {{- define "seleniumGrid.url" -}}
-{{- if and .Values.ingress.enabled .Values.ingress.hostname (ne .Values.ingress.hostname "selenium-grid.local") -}}
-http{{if .Values.ingress.tls}}s{{end}}://{{- if eq .Values.basicAuth.enabled true}}{{ .Values.basicAuth.username}}:{{ .Values.basicAuth.password}}@{{- end}}{{.Values.ingress.hostname}}
+{{- $url := printf "%s://%s%s%s%s" (include "seleniumGrid.url.schema" .) (include "seleniumGrid.url.basicAuth" .) (include "seleniumGrid.url.host" .) (include "seleniumGrid.url.port" .) (include "seleniumGrid.url.subPath" .) -}}
+{{- $url }}
+{{- end -}}
+
+{{- define "seleniumGrid.url.schema" -}}
+{{- $schema := "http" -}}
+{{- if .Values.ingress.enabled -}}
+  {{- if .Values.ingress.tls -}}
+    {{- $schema = "https" -}}
+  {{- end -}}
+{{- end -}}
+{{- $schema }}
+{{- end -}}
+
+{{- define "seleniumGrid.url.basicAuth" -}}
+{{- $basicAuth := "" -}}
+{{- if eq .Values.basicAuth.enabled true -}}
+  {{- $basicAuth = printf "%s:%s@" .Values.basicAuth.username (.Values.basicAuth.password | toString) -}}
+{{- end -}}
+{{- $basicAuth }}
+{{- end -}}
+
+{{- define "seleniumGrid.url.host" -}}
+{{- $host := printf "%s.%s" (include ($.Values.isolateComponents | ternary "seleniumGrid.router.fullname" "seleniumGrid.hub.fullname") $ ) (.Release.Namespace) -}}
+{{- if .Values.ingress.enabled -}}
+  {{- if and ( empty .Values.ingress.hostname) (not (empty .Values.global.K8S_PUBLIC_IP)) -}}
+    {{- $host = .Values.global.K8S_PUBLIC_IP -}}
+  {{- else if and .Values.ingress.hostname (ne .Values.ingress.hostname "selenium-grid.local") -}}
+    {{- $host = .Values.ingress.hostname -}}
+  {{- end -}}
+{{- else if not (empty .Values.global.K8S_PUBLIC_IP) -}}
+  {{- $host = .Values.global.K8S_PUBLIC_IP -}}
+{{- end -}}
+{{- $host }}
+{{- end -}}
+
+{{- define "seleniumGrid.url.port" -}}
+{{- $port := ":4444" -}}
+{{- if .Values.ingress.enabled -}}
+  {{- if or (ne (.Values.ingress.ports.http | toString) "80") (ne (.Values.ingress.ports.https | toString) "443") -}}
+    {{- $port = printf ":%s" (ternary (.Values.ingress.ports.http | toString) (.Values.ingress.ports.https | toString) (eq (include "seleniumGrid.url.schema" .) "http")) -}}
+  {{- else -}}
+    {{- $port = "" -}}
+  {{- end -}}
 {{- else -}}
-http://{{- if eq .Values.basicAuth.enabled true}}{{ .Values.basicAuth.username}}:{{ .Values.basicAuth.password}}@{{- end}}{{ include ($.Values.isolateComponents | ternary "seleniumGrid.router.fullname" "seleniumGrid.hub.fullname") $ }}.{{ .Release.Namespace }}:{{ $.Values.components.router.port }}
-{{- end }}
+  {{- if .Values.isolateComponents -}}
+    {{- if and (eq .Values.components.router.serviceType "NodePort") .Values.components.router.nodePort -}}
+      {{- $port = printf ":%s" (.Values.components.router.nodePort | toString) -}}
+    {{- end -}}
+  {{- else -}}
+    {{- if and (eq .Values.hub.serviceType "NodePort") .Values.hub.nodePort -}}
+      {{- $port = printf ":%s" (.Values.hub.nodePort | toString) -}}
+    {{- end -}}
+  {{- end -}}
+{{- end -}}
+{{- $port }}
 {{- end -}}
 
 {{- define "seleniumGrid.url.subPath" -}}
 {{- $subPath := "/" -}}
-{{ if $.Values.isolateComponents }}
+{{- if $.Values.isolateComponents -}}
   {{- $subPath = default $subPath $.Values.components.subPath -}}
 {{- else -}}
   {{- $subPath = default $subPath $.Values.hub.subPath -}}
 {{- end -}}
-{{ $subPath }}
+{{- $subPath }}
 {{- end -}}
 
 {{/*
 Graphql Url of the hub or the router
 */}}
 {{- define "seleniumGrid.graphqlURL" -}}
-http://{{- if eq .Values.basicAuth.enabled true}}{{ .Values.basicAuth.username}}:{{ .Values.basicAuth.password}}@{{- end}}{{ include ($.Values.isolateComponents | ternary "seleniumGrid.router.fullname" "seleniumGrid.hub.fullname") $ }}.{{ .Release.Namespace }}:{{ $.Values.components.router.port }}/graphql
+{{- printf "http://%s%s%s/graphql" (include "seleniumGrid.url.basicAuth" .) (printf "%s.%s" (include ($.Values.isolateComponents | ternary "seleniumGrid.router.fullname" "seleniumGrid.hub.fullname") $) (.Release.Namespace)) (printf ":%s" ($.Values.isolateComponents | ternary ($.Values.components.router.port | toString) ($.Values.hub.port | toString))) -}}
+{{- end -}}
+
+{{/*
+Graphql unsafeSsl of the hub or the router
+*/}}
+{{- define "seleniumGrid.graphqlURL.unsafeSsl" -}}
+{{- $unsafeSsl := printf "%s" (ternary "false" "true" (contains (include "seleniumGrid.graphqlURL" .) "https")) -}}
+{{- $unsafeSsl }}
 {{- end -}}
 
 {{/*

charts/selenium-grid/templates/distributor-service.yaml

@@ -23,4 +23,7 @@ spec:
       protocol: TCP
       port: {{ .Values.components.distributor.port }}
       targetPort: {{ .Values.components.distributor.port }}
+      {{- if and (eq .Values.components.distributor.serviceType "NodePort") .Values.components.distributor.nodePort }}
+      nodePort: {{ .Values.components.distributor.nodePort }}
+      {{- end }}
 {{- end }}

charts/selenium-grid/templates/event-bus-service.yaml

@@ -23,12 +23,21 @@ spec:
       protocol: TCP
       port: {{ .Values.components.eventBus.port }}
       targetPort: {{ .Values.components.eventBus.port }}
+      {{- if and (eq .Values.components.eventBus.serviceType "NodePort") .Values.components.eventBus.nodePort }}
+      nodePort: {{ .Values.components.eventBus.nodePort }}
+      {{- end }}
     - name: tcp-evtbus-pub
       protocol: TCP
       port: {{ .Values.components.eventBus.publishPort }}
       targetPort: {{ .Values.components.eventBus.publishPort }}
+      {{- if and (eq .Values.components.eventBus.serviceType "NodePort") .Values.components.eventBus.publishNodePort }}
+      nodePort: {{ .Values.components.eventBus.publishNodePort }}
+      {{- end }}
     - name: tcp-evtbus-sub
       protocol: TCP
       port: {{ .Values.components.eventBus.subscribePort }}
       targetPort: {{ .Values.components.eventBus.subscribePort }}
+      {{- if and (eq .Values.components.eventBus.serviceType "NodePort") .Values.components.eventBus.subscribeNodePort }}
+      nodePort: {{ .Values.components.eventBus.subscribeNodePort }}
+      {{- end }}
 {{- end }}

charts/selenium-grid/templates/hub-service.yaml

@@ -26,12 +26,21 @@ spec:
       protocol: TCP
       port: {{ .Values.hub.port }}
       targetPort: {{ .Values.hub.port }}
+      {{- if and (eq .Values.hub.serviceType "NodePort") .Values.hub.nodePort }}
+      nodePort: {{ .Values.hub.nodePort }}
+      {{- end }}
     - name: tcp-hub-pub
       protocol: TCP
       port: {{ .Values.hub.publishPort }}
       targetPort: {{ .Values.hub.publishPort }}
+      {{- if and (eq .Values.hub.serviceType "NodePort") .Values.hub.publishNodePort }}
+      nodePort: {{ .Values.hub.publishNodePort }}
+      {{- end }}
     - name: tcp-hub-sub
       protocol: TCP
       port: {{ .Values.hub.subscribePort }}
       targetPort: {{ .Values.hub.subscribePort }}
+      {{- if and (eq .Values.hub.serviceType "NodePort") .Values.hub.subscribeNodePort }}
+      nodePort: {{ .Values.hub.subscribeNodePort }}
+      {{- end }}
 {{- end }}

charts/selenium-grid/templates/node-configmap.yaml

@@ -13,4 +13,4 @@ metadata:
     {{- end }}
 data:
   SE_DRAIN_AFTER_SESSION_COUNT: '{{- and (eq (include "seleniumGrid.useKEDA" .) "true") (eq .Values.autoscaling.scalingType "job") | ternary "1" "0" -}}'
-  SE_NODE_GRID_URL: '{{ include "seleniumGrid.url" .}}{{ include "seleniumGrid.url.subPath" .}}'
+  SE_NODE_GRID_URL: '{{ include "seleniumGrid.url" .}}'

charts/selenium-grid/templates/router-service.yaml

@@ -26,4 +26,7 @@ spec:
       protocol: TCP
       port: {{ .Values.components.router.port }}
       targetPort: {{ .Values.components.router.port }}
+      {{- if and (eq $.Values.components.router.serviceType "NodePort") $.Values.components.router.nodePort }}
+      nodePort: {{ $.Values.components.router.nodePort }}
+      {{- end }}
 {{- end }}

charts/selenium-grid/templates/session-queuer-service.yaml

@@ -23,4 +23,7 @@ spec:
       protocol: TCP
       port: {{ .Values.components.sessionQueue.port }}
       targetPort: {{ .Values.components.sessionQueue.port }}
+      {{- if and (eq .Values.components.sessionQueue.serviceType "NodePort") .Values.components.sessionQueue.nodePort }}
+      nodePort: {{ .Values.components.sessionQueue.nodePort }}
+      {{- end }}
 {{- end }}

charts/selenium-grid/values.yaml

@@ -1,4 +1,8 @@
 global:
+  # Public IP of the host running Kubernetes cluster.
+  # This is used to access the Selenium Grid from outside the cluster when ingress is disabled or enabled without a hostname is set.
+  # This is part of constructing SE_NODE_GRID_URL and rewrite URL of `se:vnc`, `se:cdp` in the capabilities when `ingress.hostname` is unset
+  K8S_PUBLIC_IP: ""
   seleniumGrid:
     # Image registry for all selenium components
     imageRegistry: selenium
@@ -44,6 +48,9 @@ ingress:
     proxyBuffer:
       size: 512M
       number: 4
+  ports:
+    http: 80
+    https: 443
   # Custom annotations for ingress resource
   annotations: {}
   # Default host for the ingress resource
@@ -96,6 +103,7 @@ components:
     annotations: {}
     # Router port
     port: 4444
+    nodePort: 30444
     # Liveness probe settings
     livenessProbe:
       enabled: true
@@ -148,6 +156,7 @@ components:
     annotations: {}
     # Distributor port
     port: 5553
+    nodePort: 30553
     # Resources for Distributor container
     resources: {}
     # SecurityContext for Distributor container
@@ -180,10 +189,13 @@ components:
     annotations: {}
     # Event Bus port
     port: 5557
+    nodePort: 30557
     # Port where events are published
     publishPort: 4442
+    publishNodePort: 30442
     # Port where to subscribe for events
     subscribePort: 4443
+    subscribeNodePort: 30443
     # Resources for event-bus container
     resources: {}
     # SecurityContext for event-bus container
@@ -246,6 +258,7 @@ components:
     # Custom annotations for Session Queue pods
     annotations: {}
     port: 5559
+    nodePort: 30559
     # Resources for Session Queue container
     resources: {}
     # SecurityContext for Session Queue container
@@ -299,10 +312,13 @@ hub:
   labels: {}
   # Port where events are published
   publishPort: 4442
+  publishNodePort: 31442
   # Port where to subscribe for events
   subscribePort: 4443
+  subscribeNodePort: 31443
   # Selenium Hub port
   port: 4444
+  nodePort: 31444
   # Liveness probe settings
   livenessProbe:
     enabled: true
@@ -387,7 +403,8 @@ autoscaling:
   # see https://keda.sh/docs/latest/concepts/scaling-jobs/#scaledjob-spec
   scaledJobOptions:
     scalingStrategy:
-      strategy: accurate
+      # Change this to "accurate" when the calculation problem is fixed
+      strategy: default
     # Number of Completed jobs should be kept
     successfulJobsHistoryLimit: 0
     # Number of Failed jobs should be kept (for troubleshooting purposes)
@@ -535,8 +552,9 @@ chromeNode:
   hpa:
     url: '{{ include "seleniumGrid.graphqlURL" . }}'
     browserName: chrome
+    sessionBrowserName: 'chrome'
     # browserVersion: '91.0' # Optional. Only required when supporting multiple versions of browser in your Selenium Grid.
-    unsafeSsl: 'true'  # Optional
+    unsafeSsl: '{{ include "seleniumGrid.graphqlURL.unsafeSsl" . }}'  # Optional
 
   # It is used to add a sidecars proxy in the same pod of the browser node.
   # It means it will add a new container to the deployment itself.
@@ -668,6 +686,8 @@ firefoxNode:
   hpa:
     url: '{{ include "seleniumGrid.graphqlURL" . }}'
     browserName: firefox
+    sessionBrowserName: 'firefox'
+    unsafeSsl: '{{ include "seleniumGrid.graphqlURL.unsafeSsl" . }}'  # Optional
 
   # It is used to add a sidecars proxy in the same pod of the browser node.
   # It means it will add a new container to the deployment itself.
@@ -798,6 +818,7 @@ edgeNode:
     url: '{{ include "seleniumGrid.graphqlURL" . }}'
     browserName: MicrosoftEdge
     sessionBrowserName: 'msedge'
+    unsafeSsl: '{{ include "seleniumGrid.graphqlURL.unsafeSsl" . }}'  # Optional
 
   # It is used to add a sidecars proxy in the same pod of the browser node.
   # It means it will add a new container to the deployment itself.
@@ -918,3 +939,17 @@ videoRecorder:
 
 # Custom labels for k8s resources
 customLabels: {}
+
+# Configuration for dependency chart keda
+keda:
+  http:
+    timeout: 60000
+  webhooks:
+    enabled: false
+
+# Configuration for dependency chart ingress-nginx
+ingress-nginx:
+  enabled: false
+  controller:
+    admissionWebhooks:
+      enabled: false

tests/charts/ci/ParallelAutoscaling-values.yaml

@@ -31,10 +31,3 @@ ingress:
           name: '{{ template "seleniumGrid.hub.fullname" $ }}'
           port:
             number: 4444
-    - path: /(/?)(session/.*/se/vnc)
-      pathType: ImplementationSpecific
-      backend:
-        service:
-          name: '{{ template "seleniumGrid.hub.fullname" $ }}'
-          port:
-            number: 4444

tests/charts/ci/auth-ingress-values.yaml

@@ -1,11 +1,16 @@
+global:
+  K8S_PUBLIC_IP: localhost
+
 ingress:
   annotations:
+    kubernetes.io/ingress.class: nginx
     nginx.ingress.kubernetes.io/use-regex: "true"
     nginx.ingress.kubernetes.io/rewrite-target: /$2
     nginx.ingress.kubernetes.io/app-root: &gridAppRoot "/selenium"
     nginx.ingress.kubernetes.io/proxy-connect-timeout: "360"
     nginx.ingress.kubernetes.io/proxy-read-timeout: "360"
     nginx.ingress.kubernetes.io/proxy-send-timeout: "360"
+  ingressClassName: nginx
   hostname: ""
   paths:
     - path: /selenium(/|$)(.*)
@@ -15,20 +20,22 @@ ingress:
           name: '{{ template "seleniumGrid.router.fullname" $ }}'
           port:
             number: 4444
-    - path: /(/?)(session/.*/se/vnc)
-      pathType: ImplementationSpecific
-      backend:
-        service:
-          name: '{{ template "seleniumGrid.router.fullname" $ }}'
-          port:
-            number: 4444
 
 basicAuth:
   enabled: false
+
 isolateComponents: true
 
 hub:
   subPath: *gridAppRoot
 
 components:
   subPath: *gridAppRoot
+
+ingress-nginx:
+  enabled: true
+  controller:
+    hostNetwork: true
+    kind: DaemonSet
+    service:
+      type: ClusterIP