From a254210c29ec19f4df4510abcacbee1cae6390aa Mon Sep 17 00:00:00 2001 From: Ivana Kellyer Date: Wed, 17 Sep 2025 14:17:27 +0200 Subject: [PATCH 01/10] tests: Move ray under toxgen --- scripts/populate_tox/config.py | 3 +++ scripts/populate_tox/populate_tox.py | 1 - tox.ini | 12 +++++++++++- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/scripts/populate_tox/config.py b/scripts/populate_tox/config.py index 8e35eaba4f..bf97e3433a 100644 --- a/scripts/populate_tox/config.py +++ b/scripts/populate_tox/config.py @@ -250,6 +250,9 @@ "py3.8": ["taskgroup==0.0.0a4"], }, }, + "ray": { + "package": "ray", + }, "redis_py_cluster_legacy": { "package": "redis-py-cluster", }, diff --git a/scripts/populate_tox/populate_tox.py b/scripts/populate_tox/populate_tox.py index e08c2d4b95..d1162eec2b 100644 --- a/scripts/populate_tox/populate_tox.py +++ b/scripts/populate_tox/populate_tox.py @@ -71,7 +71,6 @@ "gcp", "httpx", "pure_eval", - "ray", "redis", "requests", "rq", diff --git a/tox.ini b/tox.ini index 50ac22e886..3125951016 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,7 @@ # The file (and all resulting CI YAMLs) then need to be regenerated via # "scripts/generate-test-files.sh". # -# Last generated: 2025-09-17T07:20:17.058541+00:00 +# Last generated: 2025-09-17T12:17:12.035548+00:00 [tox] requires = @@ -245,6 +245,11 @@ envlist = {py3.6,py3.7}-huey-v2.3.2 {py3.6,py3.11,py3.12}-huey-v2.5.3 + {py3.7,py3.9,py3.10}-ray-v2.7.2 + {py3.8,py3.10,py3.11}-ray-v2.24.0 + {py3.9,py3.11,py3.12}-ray-v2.39.0 + {py3.9,py3.12,py3.13}-ray-v2.49.1 + {py3.8,py3.9}-spark-v3.0.3 {py3.8,py3.10,py3.11}-spark-v3.5.6 {py3.9,py3.12,py3.13}-spark-v4.0.1 @@ -655,6 +660,11 @@ deps = huey-v2.3.2: huey==2.3.2 huey-v2.5.3: huey==2.5.3 + ray-v2.7.2: ray==2.7.2 + ray-v2.24.0: ray==2.24.0 + ray-v2.39.0: ray==2.39.0 + ray-v2.49.1: ray==2.49.1 + spark-v3.0.3: pyspark==3.0.3 spark-v3.5.6: pyspark==3.5.6 spark-v4.0.1: pyspark==4.0.1 From f4a146d206dc9ca916f553df96f92780e01a1253 Mon Sep 17 00:00:00 2001 From: Ivana Kellyer Date: Wed, 17 Sep 2025 14:25:33 +0200 Subject: [PATCH 02/10] . --- .github/workflows/test-integrations-tasks.yml | 2 +- scripts/populate_tox/tox.jinja | 8 -------- tox.ini | 10 +--------- 3 files changed, 2 insertions(+), 18 deletions(-) diff --git a/.github/workflows/test-integrations-tasks.yml b/.github/workflows/test-integrations-tasks.yml index 0038f1d050..d84789b767 100644 --- a/.github/workflows/test-integrations-tasks.yml +++ b/.github/workflows/test-integrations-tasks.yml @@ -29,7 +29,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7","3.10","3.11","3.12","3.13"] + python-version: ["3.7","3.12","3.13"] # python3.6 reached EOL and is no longer being supported on # new versions of hosted runners on Github Actions # ubuntu-20.04 is the last version that supported python3.6 diff --git a/scripts/populate_tox/tox.jinja b/scripts/populate_tox/tox.jinja index 4a4bd96c52..738b15f342 100755 --- a/scripts/populate_tox/tox.jinja +++ b/scripts/populate_tox/tox.jinja @@ -64,10 +64,6 @@ envlist = # pure_eval {py3.6,py3.12,py3.13}-pure_eval - # Ray - {py3.10,py3.11}-ray-v{2.34} - {py3.10,py3.11}-ray-latest - # Redis {py3.6,py3.8}-redis-v{3} {py3.7,py3.8,py3.11}-redis-v{4} @@ -180,10 +176,6 @@ deps = # pure_eval pure_eval: pure_eval - # Ray - ray-v2.34: ray~=2.34.0 - ray-latest: ray - # Redis redis: fakeredis!=1.7.4 redis: pytest<8.0.0 diff --git a/tox.ini b/tox.ini index 3125951016..f1c324cb2d 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,7 @@ # The file (and all resulting CI YAMLs) then need to be regenerated via # "scripts/generate-test-files.sh". # -# Last generated: 2025-09-17T12:17:12.035548+00:00 +# Last generated: 2025-09-17T12:25:23.783060+00:00 [tox] requires = @@ -64,10 +64,6 @@ envlist = # pure_eval {py3.6,py3.12,py3.13}-pure_eval - # Ray - {py3.10,py3.11}-ray-v{2.34} - {py3.10,py3.11}-ray-latest - # Redis {py3.6,py3.8}-redis-v{3} {py3.7,py3.8,py3.11}-redis-v{4} @@ -410,10 +406,6 @@ deps = # pure_eval pure_eval: pure_eval - # Ray - ray-v2.34: ray~=2.34.0 - ray-latest: ray - # Redis redis: fakeredis!=1.7.4 redis: pytest<8.0.0 From e8ebbda58022357375af31418d365cdf13691419 Mon Sep 17 00:00:00 2001 From: Ivana Kellyer Date: Wed, 17 Sep 2025 14:36:49 +0200 Subject: [PATCH 03/10] . --- scripts/populate_tox/config.py | 1 + tox.ini | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/populate_tox/config.py b/scripts/populate_tox/config.py index bf97e3433a..953a23966b 100644 --- a/scripts/populate_tox/config.py +++ b/scripts/populate_tox/config.py @@ -252,6 +252,7 @@ }, "ray": { "package": "ray", + "python": "<=3.11", }, "redis_py_cluster_legacy": { "package": "redis-py-cluster", diff --git a/tox.ini b/tox.ini index f1c324cb2d..039a2a2ee5 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,7 @@ # The file (and all resulting CI YAMLs) then need to be regenerated via # "scripts/generate-test-files.sh". # -# Last generated: 2025-09-17T12:25:23.783060+00:00 +# Last generated: 2025-09-17T12:36:21.578689+00:00 [tox] requires = @@ -243,8 +243,8 @@ envlist = {py3.7,py3.9,py3.10}-ray-v2.7.2 {py3.8,py3.10,py3.11}-ray-v2.24.0 - {py3.9,py3.11,py3.12}-ray-v2.39.0 - {py3.9,py3.12,py3.13}-ray-v2.49.1 + {py3.9,py3.10,py3.11}-ray-v2.39.0 + {py3.9,py3.10,py3.11}-ray-v2.49.1 {py3.8,py3.9}-spark-v3.0.3 {py3.8,py3.10,py3.11}-spark-v3.5.6 From 68575c7164022ce9ddbf19e1c8fc103371980ff3 Mon Sep 17 00:00:00 2001 From: Ivana Kellyer Date: Wed, 17 Sep 2025 14:44:36 +0200 Subject: [PATCH 04/10] . --- scripts/populate_tox/config.py | 2 +- tox.ini | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/populate_tox/config.py b/scripts/populate_tox/config.py index 953a23966b..0efc5f662e 100644 --- a/scripts/populate_tox/config.py +++ b/scripts/populate_tox/config.py @@ -252,7 +252,7 @@ }, "ray": { "package": "ray", - "python": "<=3.11", + "python": ">=3.10,<=3.11", }, "redis_py_cluster_legacy": { "package": "redis-py-cluster", diff --git a/tox.ini b/tox.ini index 039a2a2ee5..ded2d7a986 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,7 @@ # The file (and all resulting CI YAMLs) then need to be regenerated via # "scripts/generate-test-files.sh". # -# Last generated: 2025-09-17T12:36:21.578689+00:00 +# Last generated: 2025-09-17T12:44:17.672600+00:00 [tox] requires = @@ -241,10 +241,10 @@ envlist = {py3.6,py3.7}-huey-v2.3.2 {py3.6,py3.11,py3.12}-huey-v2.5.3 - {py3.7,py3.9,py3.10}-ray-v2.7.2 - {py3.8,py3.10,py3.11}-ray-v2.24.0 - {py3.9,py3.10,py3.11}-ray-v2.39.0 - {py3.9,py3.10,py3.11}-ray-v2.49.1 + {py3.10}-ray-v2.7.2 + {py3.10,py3.11}-ray-v2.24.0 + {py3.10,py3.11}-ray-v2.39.0 + {py3.10,py3.11}-ray-v2.49.1 {py3.8,py3.9}-spark-v3.0.3 {py3.8,py3.10,py3.11}-spark-v3.5.6 From 818dfadf365518d1d746da6ca39845b76ba6467f Mon Sep 17 00:00:00 2001 From: Ivana Kellyer Date: Wed, 17 Sep 2025 15:21:20 +0200 Subject: [PATCH 05/10] remove forks --- tests/integrations/ray/test_ray.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/integrations/ray/test_ray.py b/tests/integrations/ray/test_ray.py index b5bdd473c4..ca531053d4 100644 --- a/tests/integrations/ray/test_ray.py +++ b/tests/integrations/ray/test_ray.py @@ -10,6 +10,12 @@ from tests.conftest import TestTransport +@pytest.fixture(autouse=True) +def shutdown_ray(tmpdir): + yield + ray.shutdown() + + class RayTestTransport(TestTransport): def __init__(self): self.envelopes = [] @@ -58,7 +64,6 @@ def read_error_from_log(job_id): return error -@pytest.mark.forked @pytest.mark.parametrize( "task_options", [{}, {"num_cpus": 0, "memory": 1024 * 1024 * 10}] ) @@ -124,7 +129,6 @@ def example_task(): ) -@pytest.mark.forked def test_errors_in_ray_tasks(): setup_sentry_with_logging_transport() @@ -157,7 +161,6 @@ def example_task(): assert not error["exception"]["values"][0]["mechanism"]["handled"] -@pytest.mark.forked def test_tracing_in_ray_actors(): setup_sentry() @@ -194,7 +197,6 @@ def increment(self): assert worker_envelopes == [] -@pytest.mark.forked def test_errors_in_ray_actors(): setup_sentry_with_logging_transport() From d248658136ed14eb960b1774e52e312c17ee4f85 Mon Sep 17 00:00:00 2001 From: Ivana Kellyer Date: Wed, 17 Sep 2025 16:05:31 +0200 Subject: [PATCH 06/10] . --- tests/integrations/ray/test_ray.py | 155 +++++++++++++++++------------ 1 file changed, 93 insertions(+), 62 deletions(-) diff --git a/tests/integrations/ray/test_ray.py b/tests/integrations/ray/test_ray.py index ca531053d4..3b3bf55e61 100644 --- a/tests/integrations/ray/test_ray.py +++ b/tests/integrations/ray/test_ray.py @@ -1,6 +1,8 @@ import json import os import pytest +import shutil +import uuid import ray @@ -26,9 +28,6 @@ def capture_envelope(self, envelope: Envelope) -> None: class RayLoggingTransport(TestTransport): - def __init__(self): - super().__init__() - def capture_envelope(self, envelope: Envelope) -> None: print(envelope.serialize().decode("utf-8", "replace")) @@ -45,8 +44,18 @@ def setup_sentry(transport=None): ) -def read_error_from_log(job_id): - log_dir = "/tmp/ray/session_latest/logs/" +def read_error_from_log(job_id, ray_temp_dir): + # Find the actual session directory that Ray created + session_dirs = [d for d in os.listdir(ray_temp_dir) if d.startswith("session_")] + if not session_dirs: + raise FileNotFoundError(f"No session directory found in {ray_temp_dir}") + + session_dir = os.path.join(ray_temp_dir, session_dirs[0]) + log_dir = os.path.join(session_dir, "logs") + + if not os.path.exists(log_dir): + raise FileNotFoundError(f"No logs directory found at {log_dir}") + log_file = [ f for f in os.listdir(log_dir) @@ -132,33 +141,44 @@ def example_task(): def test_errors_in_ray_tasks(): setup_sentry_with_logging_transport() - ray.init( - runtime_env={ - "worker_process_setup_hook": setup_sentry_with_logging_transport, - "working_dir": "./", - } - ) - - # Setup ray task - @ray.remote - def example_task(): - 1 / 0 - - with sentry_sdk.start_transaction(op="task", name="ray test transaction"): - with pytest.raises(ZeroDivisionError): - future = example_task.remote() - ray.get(future) - - job_id = future.job_id().hex() - error = read_error_from_log(job_id) - - assert error["level"] == "error" - assert ( - error["transaction"] - == "tests.integrations.ray.test_ray.test_errors_in_ray_tasks..example_task" - ) - assert error["exception"]["values"][0]["mechanism"]["type"] == "ray" - assert not error["exception"]["values"][0]["mechanism"]["handled"] + # Create a short temp directory to avoid Unix socket path length limits + ray_temp_dir = os.path.join("/tmp", f"ray_test_{uuid.uuid4().hex[:8]}") + os.makedirs(ray_temp_dir, exist_ok=True) + + try: + ray.init( + runtime_env={ + "worker_process_setup_hook": setup_sentry_with_logging_transport, + "working_dir": "./", + }, + _temp_dir=ray_temp_dir, + ) + + # Setup ray task + @ray.remote + def example_task(): + 1 / 0 + + with sentry_sdk.start_transaction(op="task", name="ray test transaction"): + with pytest.raises(ZeroDivisionError): + future = example_task.remote() + ray.get(future) + + job_id = future.job_id().hex() + error = read_error_from_log(job_id, ray_temp_dir) + + assert error["level"] == "error" + assert ( + error["transaction"] + == "tests.integrations.ray.test_ray.test_errors_in_ray_tasks..example_task" + ) + assert error["exception"]["values"][0]["mechanism"]["type"] == "ray" + assert not error["exception"]["values"][0]["mechanism"]["handled"] + + finally: + # Clean up the temporary directory + if os.path.exists(ray_temp_dir): + shutil.rmtree(ray_temp_dir, ignore_errors=True) def test_tracing_in_ray_actors(): @@ -200,33 +220,44 @@ def increment(self): def test_errors_in_ray_actors(): setup_sentry_with_logging_transport() - ray.init( - runtime_env={ - "worker_process_setup_hook": setup_sentry_with_logging_transport, - "working_dir": "./", - } - ) - - # Setup ray actor - @ray.remote - class Counter: - def __init__(self): - self.n = 0 - - def increment(self): - with sentry_sdk.start_span(op="task", name="example actor execution"): - 1 / 0 - - return sentry_sdk.get_client().transport.envelopes - - with sentry_sdk.start_transaction(op="task", name="ray test transaction"): - with pytest.raises(ZeroDivisionError): - counter = Counter.remote() - future = counter.increment.remote() - ray.get(future) - - job_id = future.job_id().hex() - error = read_error_from_log(job_id) - - # We do not capture errors in ray actors yet - assert error is None + # Create a short temp directory to avoid Unix socket path length limits + ray_temp_dir = os.path.join("/tmp", f"ray_test_{uuid.uuid4().hex[:8]}") + os.makedirs(ray_temp_dir, exist_ok=True) + + try: + ray.init( + runtime_env={ + "worker_process_setup_hook": setup_sentry_with_logging_transport, + "working_dir": "./", + }, + _temp_dir=ray_temp_dir, + ) + + # Setup ray actor + @ray.remote + class Counter: + def __init__(self): + self.n = 0 + + def increment(self): + with sentry_sdk.start_span(op="task", name="example actor execution"): + 1 / 0 + + return sentry_sdk.get_client().transport.envelopes + + with sentry_sdk.start_transaction(op="task", name="ray test transaction"): + with pytest.raises(ZeroDivisionError): + counter = Counter.remote() + future = counter.increment.remote() + ray.get(future) + + job_id = future.job_id().hex() + error = read_error_from_log(job_id, ray_temp_dir) + + # We do not capture errors in ray actors yet + assert error is None + + finally: + # Clean up the temporary directory + if os.path.exists(ray_temp_dir): + shutil.rmtree(ray_temp_dir, ignore_errors=True) From 50128280c4badd3ebfb3cb94e41e16d43b52f34f Mon Sep 17 00:00:00 2001 From: Ivana Kellyer Date: Wed, 17 Sep 2025 16:12:45 +0200 Subject: [PATCH 07/10] cleanup --- tests/integrations/ray/test_ray.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/integrations/ray/test_ray.py b/tests/integrations/ray/test_ray.py index 3b3bf55e61..2fda33e7ea 100644 --- a/tests/integrations/ray/test_ray.py +++ b/tests/integrations/ray/test_ray.py @@ -61,6 +61,9 @@ def read_error_from_log(job_id, ray_temp_dir): for f in os.listdir(log_dir) if "worker" in f and job_id in f and f.endswith(".out") ][0] + print() + print(">>>>>>", log_file) + print() with open(os.path.join(log_dir, log_file), "r") as file: lines = file.readlines() @@ -141,7 +144,6 @@ def example_task(): def test_errors_in_ray_tasks(): setup_sentry_with_logging_transport() - # Create a short temp directory to avoid Unix socket path length limits ray_temp_dir = os.path.join("/tmp", f"ray_test_{uuid.uuid4().hex[:8]}") os.makedirs(ray_temp_dir, exist_ok=True) @@ -176,7 +178,6 @@ def example_task(): assert not error["exception"]["values"][0]["mechanism"]["handled"] finally: - # Clean up the temporary directory if os.path.exists(ray_temp_dir): shutil.rmtree(ray_temp_dir, ignore_errors=True) @@ -220,7 +221,6 @@ def increment(self): def test_errors_in_ray_actors(): setup_sentry_with_logging_transport() - # Create a short temp directory to avoid Unix socket path length limits ray_temp_dir = os.path.join("/tmp", f"ray_test_{uuid.uuid4().hex[:8]}") os.makedirs(ray_temp_dir, exist_ok=True) @@ -258,6 +258,5 @@ def increment(self): assert error is None finally: - # Clean up the temporary directory if os.path.exists(ray_temp_dir): shutil.rmtree(ray_temp_dir, ignore_errors=True) From 52e74988b66c0ee04c1cc4356c774e0cbcccb870 Mon Sep 17 00:00:00 2001 From: Ivana Kellyer Date: Wed, 17 Sep 2025 16:14:45 +0200 Subject: [PATCH 08/10] lets try now --- scripts/populate_tox/config.py | 1 - tox.ini | 10 +++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/scripts/populate_tox/config.py b/scripts/populate_tox/config.py index 0efc5f662e..bf97e3433a 100644 --- a/scripts/populate_tox/config.py +++ b/scripts/populate_tox/config.py @@ -252,7 +252,6 @@ }, "ray": { "package": "ray", - "python": ">=3.10,<=3.11", }, "redis_py_cluster_legacy": { "package": "redis-py-cluster", diff --git a/tox.ini b/tox.ini index ded2d7a986..7933008f92 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,7 @@ # The file (and all resulting CI YAMLs) then need to be regenerated via # "scripts/generate-test-files.sh". # -# Last generated: 2025-09-17T12:44:17.672600+00:00 +# Last generated: 2025-09-17T14:14:35.090628+00:00 [tox] requires = @@ -241,10 +241,10 @@ envlist = {py3.6,py3.7}-huey-v2.3.2 {py3.6,py3.11,py3.12}-huey-v2.5.3 - {py3.10}-ray-v2.7.2 - {py3.10,py3.11}-ray-v2.24.0 - {py3.10,py3.11}-ray-v2.39.0 - {py3.10,py3.11}-ray-v2.49.1 + {py3.7,py3.9,py3.10}-ray-v2.7.2 + {py3.8,py3.10,py3.11}-ray-v2.24.0 + {py3.9,py3.11,py3.12}-ray-v2.39.0 + {py3.9,py3.12,py3.13}-ray-v2.49.1 {py3.8,py3.9}-spark-v3.0.3 {py3.8,py3.10,py3.11}-spark-v3.5.6 From afed90f676195e0b09624242508c342d680f2b1c Mon Sep 17 00:00:00 2001 From: Ivana Kellyer Date: Wed, 17 Sep 2025 16:24:13 +0200 Subject: [PATCH 09/10] hopefully final bound --- scripts/populate_tox/config.py | 1 + tox.ini | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/populate_tox/config.py b/scripts/populate_tox/config.py index bf97e3433a..288447eda2 100644 --- a/scripts/populate_tox/config.py +++ b/scripts/populate_tox/config.py @@ -252,6 +252,7 @@ }, "ray": { "package": "ray", + "python": ">=3.9", }, "redis_py_cluster_legacy": { "package": "redis-py-cluster", diff --git a/tox.ini b/tox.ini index 7933008f92..b44a58966b 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,7 @@ # The file (and all resulting CI YAMLs) then need to be regenerated via # "scripts/generate-test-files.sh". # -# Last generated: 2025-09-17T14:14:35.090628+00:00 +# Last generated: 2025-09-17T14:24:00.884766+00:00 [tox] requires = @@ -241,8 +241,8 @@ envlist = {py3.6,py3.7}-huey-v2.3.2 {py3.6,py3.11,py3.12}-huey-v2.5.3 - {py3.7,py3.9,py3.10}-ray-v2.7.2 - {py3.8,py3.10,py3.11}-ray-v2.24.0 + {py3.9,py3.10}-ray-v2.7.2 + {py3.9,py3.10,py3.11}-ray-v2.24.0 {py3.9,py3.11,py3.12}-ray-v2.39.0 {py3.9,py3.12,py3.13}-ray-v2.49.1 From 7b9c68f005d0e648651ce26a5cc56cf98a6c4bcf Mon Sep 17 00:00:00 2001 From: Ivana Kellyer Date: Wed, 17 Sep 2025 16:27:00 +0200 Subject: [PATCH 10/10] remove print --- tests/integrations/ray/test_ray.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/integrations/ray/test_ray.py b/tests/integrations/ray/test_ray.py index 2fda33e7ea..f4e67df038 100644 --- a/tests/integrations/ray/test_ray.py +++ b/tests/integrations/ray/test_ray.py @@ -61,9 +61,7 @@ def read_error_from_log(job_id, ray_temp_dir): for f in os.listdir(log_dir) if "worker" in f and job_id in f and f.endswith(".out") ][0] - print() - print(">>>>>>", log_file) - print() + with open(os.path.join(log_dir, log_file), "r") as file: lines = file.readlines()