-
-
Notifications
You must be signed in to change notification settings - Fork 2.6k
Closed
Labels
Description
What happened?
On version 0.42.1 of the helm chart I can't fully auto-scale my nodes. If I start 30 long running requests some are always queued and never get allocated a new node.
kubectl get --raw \
"/apis/external.metrics.k8s.io/v1beta1/namespaces/default/s0-seleniumgrid-chrome?labelSelector=scaledobject.keda.sh/name=selenium-grid-selenium-node-chrome" \
| jq
Checking the keda external metric is only returning the value for the number of sessions in the queue:
{
"kind": "ExternalMetricValueList",
"apiVersion": "external.metrics.k8s.io/v1beta1",
"metadata": {},
"items": [
{
"metricName": "s0-seleniumgrid-chrome",
"metricLabels": null,
"timestamp": "2025-04-26T18:12:38Z",
"value": "7"
}
]
}
I believe this is meant to be the number of sessions in the queue + ongoing sessions so that it scales to the required number of nodes? Is my understanding correct or am i missing something. I have tried exactly matching the browsers:
- browserName = "chrome"
- browserVersion = "" # Can be empty for any version according to the docs
- platformName = "linux"
- enableManagedDownloads = "true"
helm install selenium-grid docker-selenium/selenium-grid
--version 0.42.1 \
--values helm-chart.yaml
# Helm values.yaml for Selenium Grid
# Selenium Hub configuration
hub:
# Resource requests/limits for Hub (1 CPU, 2816 MB RAM)
resources:
requests:
cpu: "1"
memory: "2816Mi"
limits:
cpu: "1"
memory: "2816Mi"
# Increase session request timeout to 600 seconds
extraEnvironmentVariables:
- name: SE_SESSION_REQUEST_TIMEOUT
value: "600"
# Enable KEDA-based autoscaling for browser nodes
autoscaling:
enableWithExistingKEDA: true
scalingType: deployment # Scale browser Deployments (not one-shot Jobs)
scaledOptions:
minReplicaCount: 2 # Minimum 2 Chrome nodes
maxReplicaCount: 50 # Maximum 50 Chrome nodes
terminationGracePeriodSeconds: 900 # 15 minute grace period for node shutdown
scaledObjectOptions:
platformName: linux
# Chrome Node (browser) configuration
chromeNode:
deploymentEnabled: true # Ensure Chrome node Deployment is created
replicas: 2 # Start with 2 Chrome nodes (initial replicas)
# Resource requests/limits for each Chrome node (0.8 CPU, 2048 MB RAM)
resources:
requests:
cpu: "0.8"
memory: "2048Mi"
limits:
cpu: "0.8"
memory: "2048Mi"
# Environment variables for Chrome node containers
extraEnvironmentVariables:
- name: SE_NODE_BROWSER_VERSION
value: ""
- name: SE_NODE_ENABLE_CDP
value: "true"
- name: SE_NODE_ENABLE_MANAGED_DOWNLOADS
value: "true"
- name: SE_NODE_MAX_SESSIONS
value: "1"
- name: SE_NODE_PLATFORM_NAME
value: "linux"
- name: SE_NODE_PORT
value: "4444"
- name: SE_VNC_NO_PASSWORD
value: "true"
# Disable other browser nodes (only Chrome is used)
firefoxNode:
enabled: false
edgeNode:
enabled: false
Test Script in python:
import asyncio
import time
import concurrent.futures
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
"""
Test script to run 100 browsers over time in parallel - see if selenium grid / porter scales up
"""
async def selenium_task(task_id):
try:
print(f"Starting task {task_id}")
# Set up Chrome options
chrome_options = Options()
chrome_options.set_capability('platformName', 'linux')
# chrome_options.set_capability('browserVersion', '135.0')
chrome_options.enable_downloads = True
# This will run in a separate thread to not block the event loop
with concurrent.futures.ThreadPoolExecutor() as executor:
# We'll use run_in_executor to run the blocking Selenium code in a separate threadpoetry run which python
await asyncio.get_event_loop().run_in_executor(
executor,
selenium_function,
task_id,
chrome_options
)
print(f"Completed task {task_id}")
return f"Result from task {task_id}"
except Exception as e:
print(f"Task {task_id} encountered an error: {e}")
def selenium_function(task_id, chrome_options):
"""The original selenium function, slightly modified to show which task is running"""
print(f"Task {task_id}: Creating driver")
# Connect to Selenium Grid at the provided URL
driver = webdriver.Remote(
command_executor="http://localhost:4444/wd/hub",
options=chrome_options,
keep_alive=True # Maintain the session
)
try:
# Navigate to Google
print(f"Task {task_id}: Navigating to Google")
driver.get("https://www.google.com")
print(f"Task {task_id}: Sleeping at google...")
for i in range(20):
driver.get("https://www.google.com")
time.sleep(100)
# Print the page title
print(f"Task {task_id}: Page title is: {driver.title}")
return driver.title
finally:
print(f"Task {task_id}: Closing driver")
driver.quit()
async def main():
# Create a list to hold the tasks
tasks = []
# Start 6 tasks, one every 2 seconds
for i in range(35):
task = asyncio.create_task(selenium_task(i + 1))
tasks.append(task)
print(f"Scheduled task {i + 1}")
# Wait 2 seconds before starting the next task
await asyncio.sleep(1)
# Wait for all tasks to complete
print("Waiting for all tasks to complete...")
results = await asyncio.gather(*tasks)
print("All tasks completed!")
print("Results:", results)
if __name__ == "__main__":
# Run the async main function
asyncio.run(main())
Example output from a curl:
curl --location 'http://localhost:4444/graphql' \
--header 'Content-Type: application/json' \
--header 'Authorization: ••••••' \
--data '{"query":"{ grid { sessionCount, maxSession, totalSlots }, nodesInfo { nodes { id, status, sessionCount, maxSession, slotCount, stereotypes, sessions { id, capabilities, slot { stereotype } } } }, sessionsInfo { sessionQueueRequests } }","variables":{}}'
{
"data": {
"grid": {
"sessionCount": 2,
"maxSession": 2,
"totalSlots": 2
},
"nodesInfo": {
"nodes": [
{
"id": "93060358-a27b-4a58-807e-aab18f557abb",
"status": "UP",
"sessionCount": 1,
"maxSession": 1,
"slotCount": 1,
"stereotypes": "[\n {\n \"slots\": 1,\n \"stereotype\": {\n \"browserName\": \"chrome\",\n \"browserVersion\": \"\",\n \"container:hostname\": \"selenium-grid-selenium-node-chrome-6c476c74cb-rmw7s\",\n \"goog:chromeOptions\": {\n \"binary\": \"\/usr\/bin\/google-chrome\"\n },\n \"platformName\": \"linux\",\n \"se:containerName\": \"selenium-grid-selenium-node-chrome-6c476c74cb-rmw7s\",\n \"se:downloadsEnabled\": true,\n \"se:noVncPort\": 7900,\n \"se:vncEnabled\": true\n }\n }\n]",
"sessions": [
{
"id": "c13d218a5d807e688953afee0a340935",
"capabilities": "{\n \"acceptInsecureCerts\": false,\n \"browserName\": \"chrome\",\n \"browserVersion\": \"135.0.7049.84\",\n \"chrome\": {\n \"chromedriverVersion\": \"135.0.7049.84 (6c019e56001911b3fd467e03bf68c435924d62f4-refs\/branch-heads\/7049@{#1778})\",\n \"userDataDir\": \"\/tmp\/.org.chromium.Chromium.QmrcjW\"\n },\n \"fedcm:accounts\": true,\n \"goog:chromeOptions\": {\n \"debuggerAddress\": \"localhost:37491\"\n },\n \"networkConnectionEnabled\": false,\n \"pageLoadStrategy\": \"normal\",\n \"platformName\": \"linux\",\n \"proxy\": {\n },\n \"se:bidiEnabled\": false,\n \"se:cdp\": \"ws:\/\/selenium-grid-selenium-hub.default\/session\/c13d218a5d807e688953afee0a340935\/se\/cdp\",\n \"se:cdpVersion\": \"135.0.7049.84\",\n \"se:containerName\": \"selenium-grid-selenium-node-chrome-6c476c74cb-rmw7s\",\n \"se:downloadsEnabled\": true,\n \"se:noVncPort\": 7900,\n \"se:vnc\": \"ws:\/\/selenium-grid-selenium-hub.default\/session\/c13d218a5d807e688953afee0a340935\/se\/vnc\",\n \"se:vncEnabled\": true,\n \"se:vncLocalAddress\": \"ws:\/\/10.76.20.19:7900\",\n \"setWindowRect\": true,\n \"strictFileInteractability\": false,\n \"timeouts\": {\n \"implicit\": 0,\n \"pageLoad\": 300000,\n \"script\": 30000\n },\n \"unhandledPromptBehavior\": \"dismiss and notify\",\n \"webauthn:extension:credBlob\": true,\n \"webauthn:extension:largeBlob\": true,\n \"webauthn:extension:minPinLength\": true,\n \"webauthn:extension:prf\": true,\n \"webauthn:virtualAuthenticators\": true\n}",
"slot": {
"stereotype": "{\n \"browserName\": \"chrome\",\n \"browserVersion\": \"\",\n \"container:hostname\": \"selenium-grid-selenium-node-chrome-6c476c74cb-rmw7s\",\n \"goog:chromeOptions\": {\n \"binary\": \"\/usr\/bin\/google-chrome\"\n },\n \"platformName\": \"linux\",\n \"se:containerName\": \"selenium-grid-selenium-node-chrome-6c476c74cb-rmw7s\",\n \"se:downloadsEnabled\": true,\n \"se:noVncPort\": 7900,\n \"se:vncEnabled\": true\n}"
}
}
]
},
{
"id": "d551c4b4-1628-4d8c-889a-1d51ed2a6b09",
"status": "UP",
"sessionCount": 1,
"maxSession": 1,
"slotCount": 1,
"stereotypes": "[\n {\n \"slots\": 1,\n \"stereotype\": {\n \"browserName\": \"chrome\",\n \"browserVersion\": \"\",\n \"container:hostname\": \"selenium-grid-selenium-node-chrome-6c476c74cb-r6blq\",\n \"goog:chromeOptions\": {\n \"binary\": \"\/usr\/bin\/google-chrome\"\n },\n \"platformName\": \"linux\",\n \"se:containerName\": \"selenium-grid-selenium-node-chrome-6c476c74cb-r6blq\",\n \"se:downloadsEnabled\": true,\n \"se:noVncPort\": 7900,\n \"se:vncEnabled\": true\n }\n }\n]",
"sessions": [
{
"id": "9bc102271cc7b223c9773140ff17c7cd",
"capabilities": "{\n \"acceptInsecureCerts\": false,\n \"browserName\": \"chrome\",\n \"browserVersion\": \"135.0.7049.84\",\n \"chrome\": {\n \"chromedriverVersion\": \"135.0.7049.84 (6c019e56001911b3fd467e03bf68c435924d62f4-refs\/branch-heads\/7049@{#1778})\",\n \"userDataDir\": \"\/tmp\/.org.chromium.Chromium.ODftnE\"\n },\n \"fedcm:accounts\": true,\n \"goog:chromeOptions\": {\n \"debuggerAddress\": \"localhost:38639\"\n },\n \"networkConnectionEnabled\": false,\n \"pageLoadStrategy\": \"normal\",\n \"platformName\": \"linux\",\n \"proxy\": {\n },\n \"se:bidiEnabled\": false,\n \"se:cdp\": \"ws:\/\/selenium-grid-selenium-hub.default\/session\/9bc102271cc7b223c9773140ff17c7cd\/se\/cdp\",\n \"se:cdpVersion\": \"135.0.7049.84\",\n \"se:containerName\": \"selenium-grid-selenium-node-chrome-6c476c74cb-r6blq\",\n \"se:downloadsEnabled\": true,\n \"se:noVncPort\": 7900,\n \"se:vnc\": \"ws:\/\/selenium-grid-selenium-hub.default\/session\/9bc102271cc7b223c9773140ff17c7cd\/se\/vnc\",\n \"se:vncEnabled\": true,\n \"se:vncLocalAddress\": \"ws:\/\/10.76.20.20:7900\",\n \"setWindowRect\": true,\n \"strictFileInteractability\": false,\n \"timeouts\": {\n \"implicit\": 0,\n \"pageLoad\": 300000,\n \"script\": 30000\n },\n \"unhandledPromptBehavior\": \"dismiss and notify\",\n \"webauthn:extension:credBlob\": true,\n \"webauthn:extension:largeBlob\": true,\n \"webauthn:extension:minPinLength\": true,\n \"webauthn:extension:prf\": true,\n \"webauthn:virtualAuthenticators\": true\n}",
"slot": {
"stereotype": "{\n \"browserName\": \"chrome\",\n \"browserVersion\": \"\",\n \"container:hostname\": \"selenium-grid-selenium-node-chrome-6c476c74cb-r6blq\",\n \"goog:chromeOptions\": {\n \"binary\": \"\/usr\/bin\/google-chrome\"\n },\n \"platformName\": \"linux\",\n \"se:containerName\": \"selenium-grid-selenium-node-chrome-6c476c74cb-r6blq\",\n \"se:downloadsEnabled\": true,\n \"se:noVncPort\": 7900,\n \"se:vncEnabled\": true\n}"
}
}
]
}
]
},
"sessionsInfo": {
"sessionQueueRequests": [
"{\n \"browserName\": \"chrome\",\n \"goog:chromeOptions\": {\n \"extensions\": [\n ],\n \"args\": [\n ]\n },\n \"pageLoadStrategy\": \"normal\",\n \"platformName\": \"linux\",\n \"se:downloadsEnabled\": true\n}",
"{\n \"browserName\": \"chrome\",\n \"goog:chromeOptions\": {\n \"extensions\": [\n ],\n \"args\": [\n ]\n },\n \"pageLoadStrategy\": \"normal\",\n \"platformName\": \"linux\",\n \"se:downloadsEnabled\": true\n}",
"{\n \"browserName\": \"chrome\",\n \"goog:chromeOptions\": {\n \"extensions\": [\n ],\n \"args\": [\n ]\n },\n \"pageLoadStrategy\": \"normal\",\n \"platformName\": \"linux\",\n \"se:downloadsEnabled\": true\n}",
"{\n \"browserName\": \"chrome\",\n \"goog:chromeOptions\": {\n \"extensions\": [\n ],\n \"args\": [\n ]\n },\n \"pageLoadStrategy\": \"normal\",\n \"platformName\": \"linux\",\n \"se:downloadsEnabled\": true\n}",
"{\n \"browserName\": \"chrome\",\n \"goog:chromeOptions\": {\n \"extensions\": [\n ],\n \"args\": [\n ]\n },\n \"pageLoadStrategy\": \"normal\",\n \"platformName\": \"linux\",\n \"se:downloadsEnabled\": true\n}",
"{\n \"browserName\": \"chrome\",\n \"goog:chromeOptions\": {\n \"extensions\": [\n ],\n \"args\": [\n ]\n },\n \"pageLoadStrategy\": \"normal\",\n \"platformName\": \"linux\",\n \"se:downloadsEnabled\": true\n}",
"{\n \"browserName\": \"chrome\",\n \"goog:chromeOptions\": {\n \"extensions\": [\n ],\n \"args\": [\n ]\n },\n \"pageLoadStrategy\": \"normal\",\n \"platformName\": \"linux\",\n \"se:downloadsEnabled\": true\n}",
"{\n \"browserName\": \"chrome\",\n \"goog:chromeOptions\": {\n \"extensions\": [\n ],\n \"args\": [\n ]\n },\n \"pageLoadStrategy\": \"normal\",\n \"platformName\": \"linux\",\n \"se:downloadsEnabled\": true\n}",
"{\n \"browserName\": \"chrome\",\n \"goog:chromeOptions\": {\n \"extensions\": [\n ],\n \"args\": [\n ]\n },\n \"pageLoadStrategy\": \"normal\",\n \"platformName\": \"linux\",\n \"se:downloadsEnabled\": true\n}"
]
}
}
}
Relevant log output
No log output
Operating System
GKE - Linux Machines
Docker Selenium version (image tag)
4.31.0-20250414
Selenium Grid chart version (chart version)
0.42.1