Skip to content

GH-50424: [CI] install Chrome latest for test-conda-python-emscripten #50425

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
tadeja wants to merge 11 commits into
base: main
Choose a base branch
from
+91 −41
Draft

GH-50424: [CI] install Chrome latest for test-conda-python-emscripten #50425

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
fb5d8e5
Verify fix
tadeja Jul 8, 2026
95de4e1
Disable zlib
tadeja Jul 8, 2026
c3346c0
Install newer pytest veersion
tadeja Jul 8, 2026
0100982
Keep pytest8.1.1 plus patch for exec_type
tadeja Jul 9, 2026
e00ae74
Fix script timeout chrome 150
tadeja Jul 9, 2026
5f42bde
Add logging
tadeja Jul 9, 2026
195a1c9
Revert browser timeout to 1200s
tadeja Jul 9, 2026
81d9ac0
Further improve new logging
tadeja Jul 9, 2026
22585a3
Re-add logging to debug hang
tadeja Jul 9, 2026
98d40f2
Re-add chrome opt --disable-dev-shm-usage
tadeja Jul 9, 2026
f2aeb7d
Test retries for Chrome worker/Pyodide init hang
tadeja Jul 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion compose.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -982,7 +982,7 @@ services:
clang_tools: ${CLANG_TOOLS}
llvm: ${LLVM}
pyodide_version: "0.26.0"
chrome_version: "148"
chrome_version: "latest"
selenium_version: "4.41.0"
required_python_min: "(3,12)"
python: ${PYTHON}
Expand Down
4 changes: 3 additions & 1 deletion cpp/cmake_modules/ThirdpartyToolchain.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2073,7 +2073,9 @@ function(build_protobuf)
"-DCMAKE_C_FLAGS="
"-DCMAKE_INSTALL_PREFIX=${PROTOBUF_HOST_PREFIX}"
-Dprotobuf_BUILD_TESTS=OFF
-Dprotobuf_DEBUG_POSTFIX=)
-Dprotobuf_DEBUG_POSTFIX=
# OFF to avoid conda include dirs leaking into vendored Abseil
-Dprotobuf_WITH_ZLIB=OFF)
if(ABSL_VENDORED)
# Force protobuf to reuse Arrow's already-extracted absl source
# so we don't re-download and we don't have issues with multiple abseil.
Expand Down
126 changes: 87 additions & 39 deletions python/scripts/run_emscripten_tests.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,20 @@ def do_GET(self) -> bytes | None:
evt.data.print.forEach((x)=>{window.python_logs.push(x)});
}
}
window.pyworker = new Worker("worker.js");
window.pyworker.onmessage = capturelogs;
function startPyWorker() {
window.pyworker = new Worker("worker.js");
window.pyworker.onmessage = capturelogs;
window.pyworker.onerror = (evt) => {
window.python_logs.push(...new TextEncoder().encode(
`Worker error: ${evt.message}\n`));
if (window.python_done_callback) {
let callback = window.python_done_callback;
window.python_done_callback = undefined;
callback({result: 1});
}
};
}
startPyWorker();
</script>
</head>
<body></body>
Expand All @@ -82,22 +94,36 @@ def do_GET(self) -> bytes | None:
elif self.path.endswith("/worker.js"):
body = b"""
importScripts("./pyodide.js");
function do_print(arg) {
let databytes = Array.from(arg);
self.postMessage({print:databytes});
return databytes.length;
}
function log(message) {
do_print(new TextEncoder().encode(`[worker] ${message}\n`));
}
onmessage = async function (e) {
const data = e.data;
if (!self.pyodide) {
self.pyodide = await loadPyodide();
}
function do_print(arg) {
let databytes = Array.from(arg);
self.postMessage({print:databytes});
return databytes.length;
try {
if (!self.pyodide) {
log("loading Pyodide");
self.pyodide = await loadPyodide();
log("loaded Pyodide");
}
self.pyodide.setStdout({write:do_print,isatty:data.isatty});
self.pyodide.setStderr({write:do_print,isatty:data.isatty});

log("loading packages from imports");
await self.pyodide.loadPackagesFromImports(data.python);
log("running Python");
let results = await self.pyodide.runPythonAsync(data.python);
log("Python completed");
self.postMessage({results});
} catch (error) {
do_print(new TextEncoder().encode(
`Worker failed: ${error && error.stack || error}\n`));
self.postMessage({results: 1});
}
self.pyodide.setStdout({write:do_print,isatty:data.isatty});
self.pyodide.setStderr({write:do_print,isatty:data.isatty});

await self.pyodide.loadPackagesFromImports(data.python);
let results = await self.pyodide.runPythonAsync(data.python);
self.postMessage({results});
}
"""
self.send_response(200)
Expand Down Expand Up @@ -209,36 +235,42 @@ def load_pyodide(self, dist_dir):
pass

def load_arrow(self):
self.execute_python(
f"import pyodide_js as pjs\n"
f"await pjs.loadPackage('{PYARROW_WHEEL_PATH.name}')\n"
code = (f"import pyodide_js as pjs\n"
f"await pjs.loadPackage('{PYARROW_WHEEL_PATH.name}')\n")
for attempt in range(3):
try:
return self.execute_python(code, timeout=300)
except TimeoutError:
if attempt == 2:
raise TimeoutError("Timed out loading PyArrow in browser")
print("Timed out loading PyArrow in browser; restarting worker",
flush=True)
self.restart_worker()

def execute_python(self, code, wait_for_terminate=True, timeout=None):
self.driver.execute_script(
f"""
let python = `{code}`;
delete window.python_script_done;
window.python_done_callback= (x) => {{window.python_script_done=x;}};
window.pyworker.postMessage(
{{python,isatty:{'true' if sys.stdout.isatty() else 'false'}}});
"""
)

def execute_python(self, code, wait_for_terminate=True):
if wait_for_terminate:
self.driver.execute_async_script(
f"""
let callback = arguments[arguments.length-1];
python = `{code}`;
window.python_done_callback = callback;
window.pyworker.postMessage(
{{python, isatty: {'true' if sys.stdout.isatty() else 'false'}}});
"""
)
else:
self.driver.execute_script(
f"""
let python = `{code}`;
window.python_done_callback= (x) => {{window.python_script_done=x;}};
window.pyworker.postMessage(
{{python,isatty:{'true' if sys.stdout.isatty() else 'false'}}});
"""
)
return self.wait_for_done(timeout=timeout)

def restart_worker(self):
self.driver.execute_script(
"window.pyworker.terminate(); startPyWorker(); "
"delete window.python_script_done; window.python_done_callback = undefined;"
)

def clear_logs(self):
self.driver.execute_script("window.python_logs = [];")

def wait_for_done(self):
def wait_for_done(self, timeout=None):
start = last_heartbeat = time.monotonic()
while True:
# poll for console.log messages from our webworker
# which are the output of pytest
Expand All @@ -247,11 +279,18 @@ def wait_for_done(self):
)
if len(lines) > 0:
sys.stdout.buffer.write(bytes(lines))
sys.stdout.buffer.flush()
done = self.driver.execute_script("return window.python_script_done;")
if done is not None:
value = done["result"]
self.driver.execute_script("delete window.python_script_done;")
return value
now = time.monotonic()
if timeout is not None and now - start > timeout:
raise TimeoutError
if now - last_heartbeat > 30:
print("Waiting for browser-side Python to finish...", flush=True)
last_heartbeat = now
time.sleep(0.1)


Expand All @@ -262,6 +301,7 @@ def __init__(self, hostname, port):
options = Options()
options.add_argument("--headless")
options.add_argument("--no-sandbox")
options.add_argument("--disable-dev-shm-usage")
driver = webdriver.Chrome(options=options)
driver.command_executor._client_config.timeout = 1200
super().__init__(hostname, port, driver)
Expand Down Expand Up @@ -289,6 +329,14 @@ def _load_pyarrow_in_runner(driver, wheel_name):
await pjs.loadPackage("numpy")
await pjs.loadPackage("pandas")
import pytest
import inspect
if "exc_type" not in inspect.signature(pytest.importorskip).parameters:
_original_importorskip = pytest.importorskip
def _importorskip(modname, minversion=None, reason=None, *,
exc_type=ImportError):
return _original_importorskip(modname, minversion=minversion,
reason=reason)
pytest.importorskip = _importorskip
import pandas # import pandas after pyarrow package load for pandas/pyarrow
# functions to work
import pyarrow
Expand Down
Loading