From bdffa0b50eb2bc1481c394e09be81454933bdaf9 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Wed, 10 Jun 2026 21:34:12 +0200 Subject: [PATCH 01/17] TST: Mark tests as thread-unsafe or limit the number of threads - thread_unsafe: nvml init ref-count, graphMem attr, mock-based tests, OpenGL, peer-access pool state, multiprocessing warning, program-cache race reproduction, and functools.cache mutation tests - parallel_threads_limit: IPC / worker-pool tests that spawn subprocesses or open file descriptors (limit 4), example tests (limit 8), and the event-registration test whose timeouts are slow Signed-off-by: Sebastian Berg --- cuda_core/tests/test_multiprocessing_warning.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cuda_core/tests/test_multiprocessing_warning.py b/cuda_core/tests/test_multiprocessing_warning.py index 0f96e0abfbc..94a671ff2f8 100644 --- a/cuda_core/tests/test_multiprocessing_warning.py +++ b/cuda_core/tests/test_multiprocessing_warning.py @@ -12,12 +12,17 @@ import warnings from unittest.mock import patch +import pytest + from cuda.core import DeviceMemoryResource, DeviceMemoryResourceOptions, EventOptions from cuda.core._event import _reduce_event from cuda.core._memory._device_memory_resource import _deep_reduce_device_memory_resource from cuda.core._memory._ipc import _reduce_allocation_handle from cuda.core._utils.cuda_utils import check_multiprocessing_start_method, reset_fork_warning +# We could move these to a (session) fixtures +pytestmark = pytest.mark.thread_unsafe(reason="all tests use unittest.mock.patch") + def test_warn_on_fork_method_device_memory_resource(ipc_device): """Test that warning is emitted when DeviceMemoryResource is pickled with fork method.""" From 8caf6e67124f8ad1e9ccf218bd1ba7d067ddaa04 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Wed, 10 Jun 2026 21:36:55 +0200 Subject: [PATCH 02/17] TST: use tmp_path fixture in cufile (and mark some as unsafe) Signed-off-by: Sebastian Berg --- cuda_bindings/tests/test_cufile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cuda_bindings/tests/test_cufile.py b/cuda_bindings/tests/test_cufile.py index 46bd8429a62..7de4ceb2e20 100644 --- a/cuda_bindings/tests/test_cufile.py +++ b/cuda_bindings/tests/test_cufile.py @@ -138,7 +138,7 @@ def ctx(): (err,) = cuda.cuCtxSetCurrent(ctx) assert err == cuda.CUresult.CUDA_SUCCESS - yield + yield ctx cuda.cuDevicePrimaryCtxRelease(device) From 2b6fe34b70f94c4f16656bbde0c6d9fd2c516ed6 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Wed, 10 Jun 2026 21:38:28 +0200 Subject: [PATCH 03/17] TST: Move graph definnitions inline and mark "global" ones as thread-unsafe always Signed-off-by: Sebastian Berg --- .../tests/graph/test_graph_definition.py | 107 +++++++++++------- .../tests/graph/test_graph_memory_resource.py | 29 +++-- 2 files changed, 82 insertions(+), 54 deletions(-) diff --git a/cuda_core/tests/graph/test_graph_definition.py b/cuda_core/tests/graph/test_graph_definition.py index 1d63504c7c4..139392e9377 100644 --- a/cuda_core/tests/graph/test_graph_definition.py +++ b/cuda_core/tests/graph/test_graph_definition.py @@ -575,20 +575,6 @@ def node_spec(request, init_cuda): # ============================================================================= -@pytest.fixture -def sample_graphdef(init_cuda): - """A sample GraphDefinition for standalone tests.""" - return GraphDefinition() - - -@pytest.fixture -def dot_file(tmp_path): - """Temporary DOT file path, cleaned up after test.""" - path = tmp_path / "graph.dot" - yield path - path.unlink(missing_ok=True) - - # ============================================================================= # Topology tests (parameterized over graph specs) # ============================================================================= @@ -791,14 +777,16 @@ def registered(node): # ============================================================================= -def test_graphdef_handle_valid(sample_graphdef): +def test_graphdef_handle_valid(init_cuda): """GraphDefinition has a valid non-null handle.""" + sample_graphdef = GraphDefinition() assert sample_graphdef.handle is not None assert int(sample_graphdef.handle) != 0 -def test_graphdef_entry_is_virtual(sample_graphdef): +def test_graphdef_entry_is_virtual(init_cuda): """Internal entry node is virtual (no pred/succ, type is None).""" + sample_graphdef = GraphDefinition() entry = sample_graphdef._entry assert isinstance(entry, GraphNode) assert entry.pred == set() @@ -811,8 +799,9 @@ def test_graphdef_entry_is_virtual(sample_graphdef): # ============================================================================= -def test_alloc_zero_size_fails(sample_graphdef): +def test_alloc_zero_size_fails(init_cuda): """Alloc with zero size raises error (CUDA limitation).""" + sample_graphdef = GraphDefinition() _skip_if_no_mempool() from cuda.core._utils.cuda_utils import CUDAError @@ -820,8 +809,9 @@ def test_alloc_zero_size_fails(sample_graphdef): sample_graphdef.allocate(0) -def test_free_creates_dependency(sample_graphdef): +def test_free_creates_dependency(init_cuda): """Free node depends on its predecessor.""" + sample_graphdef = GraphDefinition() _skip_if_no_mempool() with xfail_on_graph_mempool_oom(): alloc = sample_graphdef.allocate(ALLOC_SIZE) @@ -829,8 +819,9 @@ def test_free_creates_dependency(sample_graphdef): assert alloc in free.pred -def test_alloc_free_chain(sample_graphdef): +def test_alloc_free_chain(init_cuda): """Alloc and free can be chained.""" + sample_graphdef = GraphDefinition() _skip_if_no_mempool() with xfail_on_graph_mempool_oom(): a1 = sample_graphdef.allocate(ALLOC_SIZE) @@ -847,8 +838,9 @@ def test_alloc_free_chain(sample_graphdef): # ============================================================================= -def test_alloc_memory_type_invalid(sample_graphdef): +def test_alloc_memory_type_invalid(init_cuda): """Invalid memory type raises ValueError.""" + sample_graphdef = GraphDefinition() with pytest.raises(ValueError, match="'invalid' is not a valid GraphMemoryType. Must be "): sample_graphdef.allocate(ALLOC_SIZE, memory_type="invalid") @@ -860,8 +852,9 @@ def test_alloc_memory_type_invalid(sample_graphdef): pytest.param(lambda d: d, id="Device_object"), ], ) -def test_alloc_device_option(sample_graphdef, device_spec): +def test_alloc_device_option(init_cuda, device_spec): """Device can be specified as int or Device object.""" + sample_graphdef = GraphDefinition() _skip_if_no_mempool() device = Device() with xfail_on_graph_mempool_oom(device): @@ -884,8 +877,9 @@ def test_alloc_peer_access(mempool_device_x2): @pytest.mark.parametrize("num_branches", [2, 3, 5]) -def test_join_merges_branches(sample_graphdef, num_branches): +def test_join_merges_branches(init_cuda, num_branches): """join() with multiple branches creates correct dependencies.""" + sample_graphdef = GraphDefinition() _skip_if_no_mempool() with xfail_on_graph_mempool_oom(): branches = [sample_graphdef.allocate(ALLOC_SIZE) for _ in range(num_branches)] @@ -899,8 +893,9 @@ def test_join_merges_branches(sample_graphdef, num_branches): # ============================================================================= -def test_launch_creates_node(sample_graphdef): +def test_launch_creates_node(init_cuda): """launch() creates a KernelNode.""" + sample_graphdef = GraphDefinition() mod = compile_common_kernels() kernel = mod.get_kernel("empty_kernel") config = LaunchConfig(grid=1, block=1) @@ -908,8 +903,9 @@ def test_launch_creates_node(sample_graphdef): assert isinstance(node, KernelNode) -def test_launch_chain_dependencies(sample_graphdef): +def test_launch_chain_dependencies(init_cuda): """Chained launches create correct dependencies.""" + sample_graphdef = GraphDefinition() mod = compile_common_kernels() kernel = mod.get_kernel("empty_kernel") config = LaunchConfig(grid=1, block=1) @@ -971,15 +967,17 @@ def _instantiate_and_upload(graph_definition, kwargs, stream): @pytest.mark.parametrize("inst_kwargs", _INSTANTIATE_ONLY_OPTIONS) -def test_instantiate_empty_graph(sample_graphdef, inst_kwargs): +def test_instantiate_empty_graph(init_cuda, inst_kwargs): """Empty graph can be instantiated.""" + sample_graphdef = GraphDefinition() graph = _instantiate(sample_graphdef, inst_kwargs) assert graph is not None @pytest.mark.parametrize("inst_kwargs", _INSTANTIATE_ONLY_OPTIONS) -def test_instantiate_with_nodes(sample_graphdef, inst_kwargs): +def test_instantiate_with_nodes(init_cuda, inst_kwargs): """Graph with nodes can be instantiated.""" + sample_graphdef = GraphDefinition() _skip_if_no_mempool() with xfail_on_graph_mempool_oom(): sample_graphdef.allocate(ALLOC_SIZE) @@ -989,8 +987,9 @@ def test_instantiate_with_nodes(sample_graphdef, inst_kwargs): @pytest.mark.skipif(not Device(0).properties.unified_addressing, reason="requires unified addressing") -def test_instantiate_and_execute_kernel_device_launch(sample_graphdef): +def test_instantiate_and_execute_kernel_device_launch(init_cuda): """Kernel-only graph can be instantiated with device_launch flag.""" + sample_graphdef = GraphDefinition() mod = compile_common_kernels() kernel = mod.get_kernel("empty_kernel") config = LaunchConfig(grid=1, block=1) @@ -1006,8 +1005,9 @@ def test_instantiate_and_execute_kernel_device_launch(sample_graphdef): @pytest.mark.parametrize("inst_kwargs", _EXECUTE_OPTIONS) -def test_instantiate_and_execute_kernel(sample_graphdef, inst_kwargs): +def test_instantiate_and_execute_kernel(init_cuda, inst_kwargs): """Graph with kernel can be instantiated and executed.""" + sample_graphdef = GraphDefinition() mod = compile_common_kernels() kernel = mod.get_kernel("empty_kernel") config = LaunchConfig(grid=1, block=1) @@ -1020,8 +1020,9 @@ def test_instantiate_and_execute_kernel(sample_graphdef, inst_kwargs): @pytest.mark.parametrize("inst_kwargs", _EXECUTE_OPTIONS) -def test_instantiate_and_execute_alloc_free(sample_graphdef, inst_kwargs): +def test_instantiate_and_execute_alloc_free(init_cuda, inst_kwargs): """Graph with alloc/free can be executed.""" + sample_graphdef = GraphDefinition() _skip_if_no_mempool() with xfail_on_graph_mempool_oom(): alloc = sample_graphdef.allocate(ALLOC_SIZE) @@ -1034,8 +1035,9 @@ def test_instantiate_and_execute_alloc_free(sample_graphdef, inst_kwargs): @pytest.mark.parametrize("inst_kwargs", _EXECUTE_OPTIONS) -def test_instantiate_and_execute_memset(sample_graphdef, inst_kwargs): +def test_instantiate_and_execute_memset(init_cuda, inst_kwargs): """Graph with alloc/memset/free can be executed.""" + sample_graphdef = GraphDefinition() _skip_if_no_mempool() with xfail_on_graph_mempool_oom(): alloc = sample_graphdef.allocate(ALLOC_SIZE) @@ -1049,8 +1051,9 @@ def test_instantiate_and_execute_memset(sample_graphdef, inst_kwargs): @pytest.mark.parametrize("inst_kwargs", _EXECUTE_OPTIONS) -def test_instantiate_and_execute_memcpy(sample_graphdef, inst_kwargs): +def test_instantiate_and_execute_memcpy(init_cuda, inst_kwargs): """Graph with alloc/memset/memcpy/free can be executed and data is copied.""" + sample_graphdef = GraphDefinition() _skip_if_no_mempool() import ctypes @@ -1074,8 +1077,9 @@ def test_instantiate_and_execute_memcpy(sample_graphdef, inst_kwargs): assert all(b == 0xAB for b in host_buf) -def test_instantiate_and_execute_child_graph(sample_graphdef): +def test_instantiate_and_execute_child_graph(init_cuda): """Graph with embedded child graph can be executed.""" + sample_graphdef = GraphDefinition() child = GraphDefinition() mod = compile_common_kernels() kernel = mod.get_kernel("empty_kernel") @@ -1091,8 +1095,9 @@ def test_instantiate_and_execute_child_graph(sample_graphdef): stream.sync() -def test_instantiate_and_execute_host_callback(sample_graphdef): +def test_instantiate_and_execute_host_callback(init_cuda): """Graph with host callback can be executed and callback is invoked.""" + sample_graphdef = GraphDefinition() results = [] def my_callback(): @@ -1109,8 +1114,9 @@ def my_callback(): assert results == [42] -def test_instantiate_and_execute_host_callback_cfunc(sample_graphdef): +def test_instantiate_and_execute_host_callback_cfunc(init_cuda): """Graph with ctypes function pointer callback can be executed.""" + sample_graphdef = GraphDefinition() import ctypes CALLBACK = ctypes.CFUNCTYPE(None, ctypes.c_void_p) @@ -1131,8 +1137,9 @@ def raw_fn(data): assert called[0] -def test_host_callback_cfunc_with_user_data(sample_graphdef): +def test_host_callback_cfunc_with_user_data(init_cuda): """Host callback with bytes user_data passes data to C function.""" + sample_graphdef = GraphDefinition() import ctypes CALLBACK = ctypes.CFUNCTYPE(None, ctypes.c_void_p) @@ -1153,14 +1160,16 @@ def read_byte(data): assert result[0] == 0xAB -def test_host_callback_user_data_rejected_for_python_callable(sample_graphdef): +def test_host_callback_user_data_rejected_for_python_callable(init_cuda): """user_data is rejected for Python callables.""" + sample_graphdef = GraphDefinition() with pytest.raises(ValueError, match="user_data is only supported"): sample_graphdef.callback(lambda: None, user_data=b"hello") -def test_instantiate_and_execute_event_record_wait(sample_graphdef): +def test_instantiate_and_execute_event_record_wait(init_cuda): """Graph with event record and wait nodes can be executed.""" + sample_graphdef = GraphDefinition() event = Device().create_event() rec = sample_graphdef.record(event) rec.wait(event) @@ -1182,8 +1191,9 @@ def _skip_unless_cc_90(): pytest.skip("Conditional node execution requires CC >= 9.0 (Hopper)") -def test_instantiate_and_execute_if_then(sample_graphdef): +def test_instantiate_and_execute_if_then(init_cuda): """If-conditional node: body executes only when condition is non-zero.""" + sample_graphdef = GraphDefinition() _skip_unless_cc_90() _skip_if_no_mempool() import ctypes @@ -1215,8 +1225,9 @@ def test_instantiate_and_execute_if_then(sample_graphdef): assert result[0] == 1 -def test_instantiate_and_execute_if_else(sample_graphdef): +def test_instantiate_and_execute_if_else(init_cuda): """If-else node: then or else branch executes based on condition.""" + sample_graphdef = GraphDefinition() _skip_unless_cc_90() _skip_if_no_mempool() import ctypes @@ -1250,8 +1261,9 @@ def test_instantiate_and_execute_if_else(sample_graphdef): assert result[0] == 2 -def test_instantiate_and_execute_switch(sample_graphdef): +def test_instantiate_and_execute_switch(init_cuda): """Switch node: selected branch executes based on condition value.""" + sample_graphdef = GraphDefinition() _skip_unless_cc_90() _skip_if_no_mempool() import ctypes @@ -1284,8 +1296,9 @@ def test_instantiate_and_execute_switch(sample_graphdef): assert result[0] == 1 -def test_conditional_node_type_preserved_by_nodes(sample_graphdef): +def test_conditional_node_type_preserved_by_nodes(init_cuda): """Conditional nodes appear as ConditionalNode base when read back from graph.""" + sample_graphdef = GraphDefinition() condition = try_create_condition(sample_graphdef) if_node = sample_graphdef.if_then(condition) assert isinstance(if_node, IfNode) @@ -1301,8 +1314,10 @@ def test_conditional_node_type_preserved_by_nodes(sample_graphdef): # ============================================================================= -def test_debug_dot_print_creates_file(sample_graphdef, dot_file): +def test_debug_dot_print_creates_file(init_cuda, tmp_path): """debug_dot_print writes a DOT file.""" + sample_graphdef = GraphDefinition() + dot_file = tmp_path / "graph.dot" _skip_if_no_mempool() with xfail_on_graph_mempool_oom(): sample_graphdef.allocate(ALLOC_SIZE) @@ -1312,8 +1327,10 @@ def test_debug_dot_print_creates_file(sample_graphdef, dot_file): assert "digraph" in content -def test_debug_dot_print_with_options(sample_graphdef, dot_file): +def test_debug_dot_print_with_options(init_cuda, tmp_path): """debug_dot_print accepts GraphDebugPrintOptions.""" + sample_graphdef = GraphDefinition() + dot_file = tmp_path / "graph.dot" _skip_if_no_mempool() with xfail_on_graph_mempool_oom(): sample_graphdef.allocate(ALLOC_SIZE) @@ -1322,8 +1339,10 @@ def test_debug_dot_print_with_options(sample_graphdef, dot_file): assert dot_file.exists() -def test_debug_dot_print_invalid_options(sample_graphdef, dot_file): +def test_debug_dot_print_invalid_options(init_cuda, tmp_path): """debug_dot_print rejects invalid options type.""" + sample_graphdef = GraphDefinition() + dot_file = tmp_path / "graph.dot" _skip_if_no_mempool() with xfail_on_graph_mempool_oom(): sample_graphdef.allocate(ALLOC_SIZE) diff --git a/cuda_core/tests/graph/test_graph_memory_resource.py b/cuda_core/tests/graph/test_graph_memory_resource.py index 9fc794f4cca..482e8fe1c57 100644 --- a/cuda_core/tests/graph/test_graph_memory_resource.py +++ b/cuda_core/tests/graph/test_graph_memory_resource.py @@ -21,6 +21,13 @@ from cuda.core._utils.cuda_utils import CUDAError from cuda.core.graph import GraphCompleteOptions +# NOTE(seberg): "global" mode seems thread-unsafe even when working on stream +_GRAPH_MODES = [ + pytest.param("global", marks=pytest.mark.thread_unsafe(reason="gb instances share stream unsafely")), + "thread_local", + "relaxed", +] + def _common_kernels_alloc(): code = """ @@ -80,7 +87,7 @@ def free(self, buffers): self.stream.sync() -@pytest.mark.parametrize("mode", ["no_graph", "global", "thread_local", "relaxed"]) +@pytest.mark.parametrize("mode", ["no_graph"] + _GRAPH_MODES) @pytest.mark.parametrize("action", ["incr", "fill"]) def test_graph_alloc(mempool_device, mode, action): """Test basic graph capture with memory allocated and deallocated by @@ -130,7 +137,7 @@ def apply_kernels(mr, stream, out): assert compare_buffer_to_constant(out, 3) else: # Capture work, then upload and launch. - gb = device.create_graph_builder().begin_building(mode) + gb = stream.create_graph_builder().begin_building(mode) with xfail_on_graph_mempool_oom(device): apply_kernels(mr=gmr, stream=gb, out=out) graph = gb.end_building().complete() @@ -150,7 +157,7 @@ def apply_kernels(mr, stream, out): @pytest.mark.skipif(IS_WINDOWS or IS_WSL, reason="auto_free_on_launch not supported on Windows") -@pytest.mark.parametrize("mode", ["global", "thread_local", "relaxed"]) +@pytest.mark.parametrize("mode", _GRAPH_MODES) def test_graph_alloc_with_output(mempool_device, mode): """Test for memory allocated in a graph being used outside the graph.""" NBYTES = 64 @@ -168,7 +175,7 @@ def test_graph_alloc_with_output(mempool_device, mode): # Construct a graph to copy and increment the input. It returns a new # buffer allocated within the graph. The auto_free_on_launch option # is required to properly use the output buffer. - gb = device.create_graph_builder().begin_building(mode) + gb = stream.create_graph_builder().begin_building(mode) with xfail_on_graph_mempool_oom(device): out = gmr.allocate(NBYTES, stream=gb) out.copy_from(in_, stream=gb) @@ -195,7 +202,8 @@ def test_graph_alloc_with_output(mempool_device, mode): assert compare_buffer_to_constant(out, 6) -@pytest.mark.parametrize("mode", ["global", "thread_local", "relaxed"]) +@pytest.mark.parametrize("mode", _GRAPH_MODES) +@pytest.mark.thread_unsafe(reason="gb instances share default stream") def test_graph_mem_alloc_zero(mempool_device, mode): device = mempool_device gb = device.create_graph_builder().begin_building(mode) @@ -213,7 +221,8 @@ def test_graph_mem_alloc_zero(mempool_device, mode): assert buffer.device_id == int(device) -@pytest.mark.parametrize("mode", ["global", "thread_local", "relaxed"]) +@pytest.mark.parametrize("mode", _GRAPH_MODES) +@pytest.mark.thread_unsafe(reason="GMR is shared, so high mark is global") def test_graph_mem_set_attributes(mempool_device, mode): device = mempool_device stream = device.create_stream() @@ -265,7 +274,7 @@ def test_graph_mem_set_attributes(mempool_device, mode): mman.reset() -@pytest.mark.parametrize("mode", ["global", "thread_local", "relaxed"]) +@pytest.mark.parametrize("mode", _GRAPH_MODES) def test_gmr_check_capture_state(mempool_device, mode): """ Test expected errors (and non-errors) using GraphMemoryResource with graph @@ -284,7 +293,7 @@ def test_gmr_check_capture_state(mempool_device, mode): gmr.allocate(1, stream=stream) # Capturing - gb = device.create_graph_builder().begin_building(mode=mode) + gb = stream.create_graph_builder().begin_building(mode=mode) with xfail_on_graph_mempool_oom(device): gmr.allocate(1, stream=gb) # no error gb.end_building().complete() @@ -320,7 +329,7 @@ def test_graph_memory_resource_attributes_repr(mempool_device): assert "used_mem_high=" in r -@pytest.mark.parametrize("mode", ["global", "thread_local", "relaxed"]) +@pytest.mark.parametrize("mode", _GRAPH_MODES) def test_dmr_check_capture_state(mempool_device, mode): """ Test expected errors (and non-errors) using DeviceMemoryResource with graph @@ -334,7 +343,7 @@ def test_dmr_check_capture_state(mempool_device, mode): dmr.allocate(1, stream=stream).close() # no error # Capturing - gb = device.create_graph_builder().begin_building(mode=mode) + gb = stream.create_graph_builder().begin_building(mode=mode) with pytest.raises( RuntimeError, match=r"cannot perform memory operations on a capturing " From 2232f01aaaf5f9291bec87889133f790eab5218b Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Wed, 10 Jun 2026 21:41:12 +0200 Subject: [PATCH 04/17] TST: Fixup memory tests, mostly work around issue when tearing down mempool Signed-off-by: Sebastian Berg --- cuda_core/tests/test_memory.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cuda_core/tests/test_memory.py b/cuda_core/tests/test_memory.py index 4427b899765..42d9852c60e 100644 --- a/cuda_core/tests/test_memory.py +++ b/cuda_core/tests/test_memory.py @@ -249,9 +249,8 @@ def _pattern_bytes(value) -> bytes: @pytest.fixture(params=["device", "unified", "pinned"]) -def fill_env(request): - device = Device() - device.set_current() +def fill_env(request, init_cuda): + device = init_cuda if request.param == "device": mr = DummyDeviceMemoryResource(device) elif request.param == "unified": From ba564dd71eb0dd26f5b57203e2cd3f714c506925 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Wed, 10 Jun 2026 21:42:38 +0200 Subject: [PATCH 05/17] TST: Thread unsafe markers for test_managed_ops Signed-off-by: Sebastian Berg --- cuda_core/tests/memory/test_managed_ops.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cuda_core/tests/memory/test_managed_ops.py b/cuda_core/tests/memory/test_managed_ops.py index 33def77935f..d9305399de4 100644 --- a/cuda_core/tests/memory/test_managed_ops.py +++ b/cuda_core/tests/memory/test_managed_ops.py @@ -364,6 +364,7 @@ def test_from_handle(self, init_cuda): finally: plain.close() + @pytest.mark.thread_unsafe(reason="external_managed_buffer is shared between threads") def test_read_mostly_roundtrip(self, external_managed_buffer): buf = external_managed_buffer assert buf.read_mostly is False @@ -372,6 +373,7 @@ def test_read_mostly_roundtrip(self, external_managed_buffer): buf.read_mostly = False assert buf.read_mostly is False + @pytest.mark.thread_unsafe(reason="external_managed_buffer is shared between threads") def test_preferred_location_roundtrip(self, location_ops_device, external_managed_buffer): device = location_ops_device buf = external_managed_buffer @@ -386,6 +388,7 @@ def test_preferred_location_roundtrip(self, location_ops_device, external_manage buf.preferred_location = None assert buf.preferred_location is None + @pytest.mark.thread_unsafe(reason="external_managed_buffer is shared between threads") def test_preferred_location_roundtrip_host_numa(self, location_ops_device): """Host(numa_id=N) round-trips correctly on CUDA 13 builds.""" from cuda.core._utils.version import binding_version @@ -406,6 +409,7 @@ def test_preferred_location_roundtrip_host_numa(self, location_ops_device): finally: plain.close() + @pytest.mark.thread_unsafe(reason="external_managed_buffer is shared between threads") def test_accessed_by_add_discard(self, location_ops_device, external_managed_buffer): device = location_ops_device buf = external_managed_buffer @@ -417,6 +421,7 @@ def test_accessed_by_add_discard(self, location_ops_device, external_managed_buf buf.accessed_by.discard(device) assert device not in buf.accessed_by + @pytest.mark.thread_unsafe(reason="external_managed_buffer is shared between threads") def test_accessed_by_mutable_set_interface(self, location_ops_device, external_managed_buffer): """Full MutableSet conformance pass on AccessedBySetProxy. @@ -436,6 +441,7 @@ def test_accessed_by_mutable_set_interface(self, location_ops_device, external_m non_member=Host(numa_id=0), ) + @pytest.mark.thread_unsafe(reason="external_managed_buffer is shared between threads") def test_accessed_by_set_assignment(self, location_ops_device, external_managed_buffer): device = location_ops_device buf = external_managed_buffer From 8c335e817b94a1ac2846e61cf4ecad84139ba3da Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Wed, 10 Jun 2026 21:43:24 +0200 Subject: [PATCH 06/17] Avoid interactive backend when using run_tests.sh locally Signed-off-by: Sebastian Berg --- cuda_bindings/tests/test_examples.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cuda_bindings/tests/test_examples.py b/cuda_bindings/tests/test_examples.py index 63a56c78fb7..6ffce3e7fe7 100644 --- a/cuda_bindings/tests/test_examples.py +++ b/cuda_bindings/tests/test_examples.py @@ -20,6 +20,7 @@ def test_example(example): env = os.environ.copy() env["CUDA_BINDINGS_SKIP_EXAMPLE"] = "100" + env["MPLBACKEND"] = "Agg" # avoid plt.show() from blocking process = subprocess.run([sys.executable, example], capture_output=True, env=env) # noqa: S603 # returncode is a special value used in the examples to indicate that system requirements are not met. From 6757a012fa090d02294f150b5e15422666c86e85 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Wed, 10 Jun 2026 22:13:22 +0200 Subject: [PATCH 07/17] Use indirect fixtures for a nicer pattern and avoid thread issues After my first AI try was a crazy mess, the second run actually found a neat solution... These objects can be created in the main thread, but we can't create them on the fly in many threads as it was... Signed-off-by: Sebastian Berg --- cuda_core/tests/test_object_protocols.py | 160 +++++++++++------------ 1 file changed, 80 insertions(+), 80 deletions(-) diff --git a/cuda_core/tests/test_object_protocols.py b/cuda_core/tests/test_object_protocols.py index e8391c75678..e4e8ee21c9b 100644 --- a/cuda_core/tests/test_object_protocols.py +++ b/cuda_core/tests/test_object_protocols.py @@ -527,6 +527,26 @@ def sample_switch_node_alt(sample_graphdef): return sample_graphdef.switch(condition, 3) +# Indirect-parametrize helpers: request.getfixturevalue() runs here, in the +# fixture (main thread), so the resolved object is already available when the +# test function runs in a worker thread. + + +@pytest.fixture +def sample_object(request): + return request.getfixturevalue(request.param) + + +@pytest.fixture +def sample_object_a(request): + return request.getfixturevalue(request.param) + + +@pytest.fixture +def sample_object_b(request): + return request.getfixturevalue(request.param) + + # ============================================================================= # Type groupings # ============================================================================= @@ -722,12 +742,11 @@ def sample_switch_node_alt(sample_graphdef): # ============================================================================= -@pytest.mark.parametrize("fixture_name", WEAKREF_TYPES) -def test_weakref_supported(fixture_name, request): +@pytest.mark.parametrize("sample_object", WEAKREF_TYPES, indirect=True) +def test_weakref_supported(sample_object): """Object supports weak references.""" - obj = request.getfixturevalue(fixture_name) - ref = weakref.ref(obj) - assert ref() is obj + ref = weakref.ref(sample_object) + assert ref() is sample_object # ============================================================================= @@ -735,27 +754,22 @@ def test_weakref_supported(fixture_name, request): # ============================================================================= -@pytest.mark.parametrize("fixture_name", HASH_TYPES) -def test_hash_consistency(fixture_name, request): +@pytest.mark.parametrize("sample_object", HASH_TYPES, indirect=True) +def test_hash_consistency(sample_object): """Hash is consistent across multiple calls.""" - obj = request.getfixturevalue(fixture_name) - assert hash(obj) == hash(obj) + assert hash(sample_object) == hash(sample_object) -@pytest.mark.parametrize("a_name,b_name", SAME_TYPE_PAIRS) -def test_hash_distinct_same_type(a_name, b_name, request): +@pytest.mark.parametrize("sample_object_a,sample_object_b", SAME_TYPE_PAIRS, indirect=True) +def test_hash_distinct_same_type(sample_object_a, sample_object_b): """Distinct objects of the same type have different hashes.""" - obj_a = request.getfixturevalue(a_name) - obj_b = request.getfixturevalue(b_name) - assert hash(obj_a) != hash(obj_b) # extremely unlikely + assert hash(sample_object_a) != hash(sample_object_b) # extremely unlikely -@pytest.mark.parametrize("a_name,b_name", itertools.combinations(HASH_TYPES, 2)) -def test_hash_distinct_cross_type(a_name, b_name, request): +@pytest.mark.parametrize("sample_object_a,sample_object_b", itertools.combinations(HASH_TYPES, 2), indirect=True) +def test_hash_distinct_cross_type(sample_object_a, sample_object_b): """Distinct objects of different types have different hashes.""" - obj_a = request.getfixturevalue(a_name) - obj_b = request.getfixturevalue(b_name) - assert hash(obj_a) != hash(obj_b) # extremely unlikely + assert hash(sample_object_a) != hash(sample_object_b) # extremely unlikely # ============================================================================= @@ -763,41 +777,35 @@ def test_hash_distinct_cross_type(a_name, b_name, request): # ============================================================================= -@pytest.mark.parametrize("fixture_name", EQ_TYPES) -def test_equality_basic(fixture_name, request): +@pytest.mark.parametrize("sample_object", EQ_TYPES, indirect=True) +def test_equality_basic(sample_object): """Object equality: reflexive, not equal to None or other types.""" - obj = request.getfixturevalue(fixture_name) - assert obj == obj - assert obj is not None - assert obj != "string" - if hasattr(obj, "handle"): - assert obj != obj.handle + assert sample_object == sample_object + assert sample_object is not None + assert sample_object != "string" + if hasattr(sample_object, "handle"): + assert sample_object != sample_object.handle -@pytest.mark.parametrize("a_name,b_name", itertools.combinations(EQ_TYPES, 2)) -def test_no_cross_type_equality(a_name, b_name, request): +@pytest.mark.parametrize("sample_object_a,sample_object_b", itertools.combinations(EQ_TYPES, 2), indirect=True) +def test_no_cross_type_equality(sample_object_a, sample_object_b): """No two distinct objects of different types should compare equal.""" - obj_a = request.getfixturevalue(a_name) - obj_b = request.getfixturevalue(b_name) - assert obj_a != obj_b + assert sample_object_a != sample_object_b -@pytest.mark.parametrize("a_name,b_name", SAME_TYPE_PAIRS) -def test_same_type_inequality(a_name, b_name, request): +@pytest.mark.parametrize("sample_object_a,sample_object_b", SAME_TYPE_PAIRS, indirect=True) +def test_same_type_inequality(sample_object_a, sample_object_b): """Two distinct objects of the same type should not compare equal.""" - obj_a = request.getfixturevalue(a_name) - obj_b = request.getfixturevalue(b_name) - assert obj_a is not obj_b - assert obj_a != obj_b + assert sample_object_a is not sample_object_b + assert sample_object_a != sample_object_b -@pytest.mark.parametrize("fixture_name,copy_fn", FROM_HANDLE_COPIES) -def test_equality_same_handle(fixture_name, copy_fn, request): +@pytest.mark.parametrize("sample_object,copy_fn", FROM_HANDLE_COPIES, indirect=["sample_object"]) +def test_equality_same_handle(sample_object, copy_fn): """Two wrappers around the same handle should compare equal.""" - obj = request.getfixturevalue(fixture_name) - obj2 = copy_fn(obj) - assert obj == obj2 - assert hash(obj) == hash(obj2) + obj2 = copy_fn(sample_object) + assert sample_object == obj2 + assert hash(sample_object) == hash(obj2) # ============================================================================= @@ -805,48 +813,43 @@ def test_equality_same_handle(fixture_name, copy_fn, request): # ============================================================================= -@pytest.mark.parametrize("fixture_name", DICT_KEY_TYPES) -def test_usable_as_dict_key(fixture_name, request): +@pytest.mark.parametrize("sample_object", DICT_KEY_TYPES, indirect=True) +def test_usable_as_dict_key(sample_object): """Object can be used as a dictionary key.""" - obj = request.getfixturevalue(fixture_name) - d = {obj: "value"} - assert d[obj] == "value" - assert obj in d + d = {sample_object: "value"} + assert d[sample_object] == "value" + assert sample_object in d -@pytest.mark.parametrize("fixture_name", DICT_KEY_TYPES) -def test_usable_in_set(fixture_name, request): +@pytest.mark.parametrize("sample_object", DICT_KEY_TYPES, indirect=True) +def test_usable_in_set(sample_object): """Object can be added to a set.""" - obj = request.getfixturevalue(fixture_name) - s = {obj} - assert obj in s + s = {sample_object} + assert sample_object in s -@pytest.mark.parametrize("fixture_name", WEAKREF_TYPES) -def test_usable_in_weak_value_dict(fixture_name, request): +@pytest.mark.parametrize("sample_object", WEAKREF_TYPES, indirect=True) +def test_usable_in_weak_value_dict(sample_object): """Object can be used as a WeakValueDictionary value.""" - obj = request.getfixturevalue(fixture_name) wvd = weakref.WeakValueDictionary() - wvd["key"] = obj - assert wvd["key"] is obj + wvd["key"] = sample_object + assert wvd["key"] is sample_object -@pytest.mark.parametrize("fixture_name", WEAK_KEY_TYPES) -def test_usable_in_weak_key_dict(fixture_name, request): +@pytest.mark.parametrize("sample_object", WEAK_KEY_TYPES, indirect=True) +def test_usable_in_weak_key_dict(sample_object): """Object can be used as a WeakKeyDictionary key.""" - obj = request.getfixturevalue(fixture_name) wkd = weakref.WeakKeyDictionary() - wkd[obj] = "value" - assert wkd[obj] == "value" + wkd[sample_object] = "value" + assert wkd[sample_object] == "value" -@pytest.mark.parametrize("fixture_name", WEAK_KEY_TYPES) -def test_usable_in_weak_set(fixture_name, request): +@pytest.mark.parametrize("sample_object", WEAK_KEY_TYPES, indirect=True) +def test_usable_in_weak_set(sample_object): """Object can be added to a WeakSet.""" - obj = request.getfixturevalue(fixture_name) ws = weakref.WeakSet() - ws.add(obj) - assert obj in ws + ws.add(sample_object) + assert sample_object in ws # ============================================================================= @@ -854,12 +857,10 @@ def test_usable_in_weak_set(fixture_name, request): # ============================================================================= -@pytest.mark.parametrize("fixture_name,pattern", REPR_PATTERNS) -def test_repr_format(fixture_name, pattern, request): +@pytest.mark.parametrize("sample_object,pattern", REPR_PATTERNS, indirect=["sample_object"]) +def test_repr_format(sample_object, pattern): """repr() returns a properly formatted string.""" - obj = request.getfixturevalue(fixture_name) - result = repr(obj) - assert re.fullmatch(pattern, result) + assert re.fullmatch(pattern, repr(sample_object)) # ============================================================================= @@ -868,10 +869,9 @@ def test_repr_format(fixture_name, pattern, request): @pytest.mark.parametrize("pickle_module", PICKLE_MODULES) -@pytest.mark.parametrize("fixture_name", PICKLE_TYPES) -def test_pickle_roundtrip(fixture_name, pickle_module, request): +@pytest.mark.parametrize("sample_object", PICKLE_TYPES, indirect=True) +def test_pickle_roundtrip(sample_object, pickle_module): """Object survives a pickle/cloudpickle roundtrip.""" mod = pytest.importorskip(pickle_module) - obj = request.getfixturevalue(fixture_name) - result = mod.loads(mod.dumps(obj)) - assert type(result) is type(obj) + result = mod.loads(mod.dumps(sample_object)) + assert type(result) is type(sample_object) From a9a211612222b3dd5eae95ef024b45e5edc5880a Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Thu, 11 Jun 2026 14:46:39 +0200 Subject: [PATCH 08/17] Make latch-kernel helper compile only once For some reason the latch kernel helper test started failing now (it did not before my update from CUDA 13.2 to 13.3?). The reason isn't that it is not thread-safe, but that something (presumably module loading/unloading) causes synchronizations which in turn cause threads having to wait on their LatchKernel to finish. And of course the test itself really needs that not to happen. Making sure there is only one LatchKernel compiled and loaded exactly once seems to avoid this problem. Signed-off-by: Sebastian Berg --- cuda_core/tests/helpers/latch.py | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/cuda_core/tests/helpers/latch.py b/cuda_core/tests/helpers/latch.py index c28fb222641..978e2dbdf18 100644 --- a/cuda_core/tests/helpers/latch.py +++ b/cuda_core/tests/helpers/latch.py @@ -1,7 +1,8 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 import ctypes +import threading import pytest @@ -20,9 +21,15 @@ class LatchKernel: Manages a kernel that blocks stream progress until released. """ - def __init__(self, device, timeout_sec=60): - if helpers.CUDA_INCLUDE_PATH is None: - pytest.skip("need CUDA header") + _latch_kernel_lock = threading.Lock() + _latch_kernels = {} + + @classmethod + def _get_kernel(cls, device): + kernel = cls._latch_kernels.get(device.uuid) + if kernel is not None: + return kernel + code = """ #include @@ -41,6 +48,7 @@ def __init__(self, device, timeout_sec=60): // Check for timeout if (clock64() - start >= timeout_cycles) { + signal.store(-1, cuda::memory_order_relaxed); break; // Timeout reached } @@ -56,14 +64,25 @@ def __init__(self, device, timeout_sec=60): ) prog = Program(code, code_type="c++", options=program_options) mod = prog.compile(target_type="cubin") - self.kernel = mod.get_kernel("latch") + kernel = mod.get_kernel("latch") + + return cls._latch_kernels.setdefault(device.uuid, kernel) + + def __init__(self, device, timeout_sec=60): + if helpers.CUDA_INCLUDE_PATH is None: + pytest.skip("need CUDA header") + + with self._latch_kernel_lock: + self.kernel = self._get_kernel(device) mr = LegacyPinnedMemoryResource() self.buffer = mr.allocate(4) - self.busy_wait_flag[0] = 0 + self.busy_wait_flag[0] = 1 clock_rate_hz = device.properties.clock_rate * 1000 self.timeout_cycles = int(timeout_sec * clock_rate_hz) + self.busy_wait_flag[0] = 0 + def launch(self, stream): """Launch the latch kernel, blocking stream progress via busy waiting.""" config = LaunchConfig(grid=1, block=1) From ea7cb70a6dfc86a6c46640b5076000f5e2c2d103 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Wed, 8 Jul 2026 13:40:55 +0200 Subject: [PATCH 09/17] Limit threads for event test that otherwise seems to ptentially fail --- cuda_core/tests/test_event.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cuda_core/tests/test_event.py b/cuda_core/tests/test_event.py index 79f0090ace6..c0eb4654752 100644 --- a/cuda_core/tests/test_event.py +++ b/cuda_core/tests/test_event.py @@ -119,6 +119,7 @@ def test_error_timing_recorded(): @pytest.mark.skipif(Device().compute_capability.major < 7, reason="__nanosleep is only available starting Volta (sm70)") +@pytest.mark.parallel_threads_limit(8) # Very many threads may cause latch to time out def test_error_timing_incomplete(): device = Device() device.set_current() From 2f90d7a62c67fa772b588ce2d6a89a9c66b1ea6c Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Wed, 8 Jul 2026 14:27:43 +0200 Subject: [PATCH 10/17] TST: Mark device-persistence-mode test as thread-unsafe --- cuda_core/tests/system/test_system_device.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cuda_core/tests/system/test_system_device.py b/cuda_core/tests/system/test_system_device.py index b8fe3505674..8e47dce67bd 100644 --- a/cuda_core/tests/system/test_system_device.py +++ b/cuda_core/tests/system/test_system_device.py @@ -349,6 +349,7 @@ def test_c2c_mode_enabled(): @pytest.mark.skipif(helpers.IS_WSL or helpers.IS_WINDOWS, reason="Persistence mode not supported on WSL or Windows") +@pytest.mark.thread_unsafe(reason="device persistence mode is global state") def test_persistence_mode_enabled(): for device in system.Device.get_all_devices(): is_enabled = device.is_persistence_mode_enabled From e90369fac45be3d63dfde445720dce57d022ebc5 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Tue, 14 Jul 2026 16:23:41 +0200 Subject: [PATCH 11/17] TST: Test fails in CI, assume that few enough threads eventually pass... --- cuda_core/tests/test_event.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cuda_core/tests/test_event.py b/cuda_core/tests/test_event.py index c0eb4654752..cc42010ea25 100644 --- a/cuda_core/tests/test_event.py +++ b/cuda_core/tests/test_event.py @@ -119,7 +119,7 @@ def test_error_timing_recorded(): @pytest.mark.skipif(Device().compute_capability.major < 7, reason="__nanosleep is only available starting Volta (sm70)") -@pytest.mark.parallel_threads_limit(8) # Very many threads may cause latch to time out +@pytest.mark.parallel_threads_limit(2) # Many threads seem to cause latch to time out def test_error_timing_incomplete(): device = Device() device.set_current() From 9f142e94ef0d91627b9255b14e6ca7b598dde664 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Tue, 14 Jul 2026 18:01:14 +0200 Subject: [PATCH 12/17] TST: Add another sync to guard against potential deadlocks (seems I missed it) --- cuda_core/tests/test_memory.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cuda_core/tests/test_memory.py b/cuda_core/tests/test_memory.py index 42d9852c60e..957c8804be8 100644 --- a/cuda_core/tests/test_memory.py +++ b/cuda_core/tests/test_memory.py @@ -1195,6 +1195,8 @@ def test_managed_memory_resource_with_options(init_cuda): device.sync() dst_buffer.close() src_buffer.close() + # TODO(seberg): 2026-06: mr close may be unsafe with incomplete `buf.close()` + device.sync() def test_managed_memory_resource_preferred_location_default(init_cuda): From d4192a4e0030b3f18528359603bcf7aeb93fe34b Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Tue, 14 Jul 2026 18:09:13 +0200 Subject: [PATCH 13/17] TST: Limit threads for another LatchKernel test --- cuda_core/tests/test_event.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cuda_core/tests/test_event.py b/cuda_core/tests/test_event.py index cc42010ea25..79e5d68c63d 100644 --- a/cuda_core/tests/test_event.py +++ b/cuda_core/tests/test_event.py @@ -223,6 +223,7 @@ def test_event_ipc_descriptor_non_ipc(init_cuda): _ = event.ipc_descriptor +@pytest.mark.parallel_threads_limit(2) # Many threads seem to cause latch to time out def test_event_is_done_false(init_cuda): """Event.is_done returns False when captured work has not yet completed.""" device = Device() From 443921aed587ff750ab28573d79469a6567640b4 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Tue, 14 Jul 2026 18:42:22 +0200 Subject: [PATCH 14/17] TST: Force test_helpers to single threaded on windows to avoid crash --- cuda_core/tests/test_helpers.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cuda_core/tests/test_helpers.py b/cuda_core/tests/test_helpers.py index 43dbf8887e2..f0937b55a08 100644 --- a/cuda_core/tests/test_helpers.py +++ b/cuda_core/tests/test_helpers.py @@ -16,6 +16,13 @@ NBYTES = 64 +if IS_WINDOWS or IS_WSL: + # see comment below, with threaded test all threads would need + # synchronization (which would require a barrier that we don't have). + # Applies at least to test_latchkernel and test_patterngen_(seeds|values). + pytestmark = pytest.mark.thread_unsafe(reason="windows host-access unsafe while GPU is working") + + @pytest.mark.skipif(Device().compute_capability.major < 7, reason="__nanosleep is only available starting Volta (sm70)") def test_latchkernel(): """Test LatchKernel.""" From a98fc527539abde886bb0a76ec65e78d2bd72cb7 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Wed, 15 Jul 2026 10:42:17 +0200 Subject: [PATCH 15/17] TST: Many buffer related tests cannot run threaded on windows --- .../tests/graph/test_graph_memory_resource.py | 4 +++- cuda_core/tests/helpers/buffers.py | 14 +++++++++++++- cuda_core/tests/test_event.py | 8 ++++++-- cuda_core/tests/test_helpers.py | 12 ++++-------- cuda_core/tests/test_memory.py | 3 ++- 5 files changed, 28 insertions(+), 13 deletions(-) diff --git a/cuda_core/tests/graph/test_graph_memory_resource.py b/cuda_core/tests/graph/test_graph_memory_resource.py index 482e8fe1c57..f6e87543c85 100644 --- a/cuda_core/tests/graph/test_graph_memory_resource.py +++ b/cuda_core/tests/graph/test_graph_memory_resource.py @@ -6,7 +6,7 @@ import pytest from helpers import IS_WINDOWS, IS_WSL -from helpers.buffers import compare_buffer_to_constant, make_scratch_buffer, set_buffer +from helpers.buffers import compare_buffer_to_constant, make_scratch_buffer, set_buffer, thread_unsafe_on_windows from conftest import xfail_on_graph_mempool_oom from cuda.core import ( @@ -89,6 +89,7 @@ def free(self, buffers): @pytest.mark.parametrize("mode", ["no_graph"] + _GRAPH_MODES) @pytest.mark.parametrize("action", ["incr", "fill"]) +@thread_unsafe_on_windows def test_graph_alloc(mempool_device, mode, action): """Test basic graph capture with memory allocated and deallocated by GraphMemoryResource. @@ -158,6 +159,7 @@ def apply_kernels(mr, stream, out): @pytest.mark.skipif(IS_WINDOWS or IS_WSL, reason="auto_free_on_launch not supported on Windows") @pytest.mark.parametrize("mode", _GRAPH_MODES) +@thread_unsafe_on_windows def test_graph_alloc_with_output(mempool_device, mode): """Test for memory allocated in a graph being used outside the graph.""" NBYTES = 64 diff --git a/cuda_core/tests/helpers/buffers.py b/cuda_core/tests/helpers/buffers.py index f4412d57e16..532aadecdcf 100644 --- a/cuda_core/tests/helpers/buffers.py +++ b/cuda_core/tests/helpers/buffers.py @@ -3,10 +3,12 @@ import ctypes +import pytest + from cuda.core import Buffer, Device, MemoryResource from cuda.core._utils.cuda_utils import driver, handle_return -from . import libc +from . import IS_WINDOWS, IS_WSL, libc __all__ = [ "DummyDeviceMemoryResource", @@ -16,9 +18,19 @@ "compare_buffer_to_constant", "compare_equal_buffers", "make_scratch_buffer", + "thread_unsafe_on_windows", ] +def thread_unsafe_on_windows(func): + # Tests that use these buffers and access the memory on the host are + # thread-unsafe on windows. On windows the GPU must be fully quiescent for host + # access to be safe and with threaded tests that would require a barrier. + if IS_WINDOWS or IS_WSL: + return pytest.mark.thread_unsafe(reason="windows host-access unsafe while GPU is working") + return func + + class DummyDeviceMemoryResource(MemoryResource): # cuMemAlloc / cuMemFree are synchronous; stream is accepted for # interface conformance but ignored. diff --git a/cuda_core/tests/test_event.py b/cuda_core/tests/test_event.py index 79e5d68c63d..4ec24f1d9a6 100644 --- a/cuda_core/tests/test_event.py +++ b/cuda_core/tests/test_event.py @@ -5,6 +5,7 @@ import math import pytest +from helpers.buffers import thread_unsafe_on_windows from helpers.latch import LatchKernel from helpers.nanosleep_kernel import NanosleepKernel @@ -119,7 +120,8 @@ def test_error_timing_recorded(): @pytest.mark.skipif(Device().compute_capability.major < 7, reason="__nanosleep is only available starting Volta (sm70)") -@pytest.mark.parallel_threads_limit(2) # Many threads seem to cause latch to time out +@pytest.mark.parallel_threads_limit(2) +@thread_unsafe_on_windows def test_error_timing_incomplete(): device = Device() device.set_current() @@ -223,7 +225,9 @@ def test_event_ipc_descriptor_non_ipc(init_cuda): _ = event.ipc_descriptor -@pytest.mark.parallel_threads_limit(2) # Many threads seem to cause latch to time out +@pytest.mark.skipif(Device().compute_capability.major < 7, reason="__nanosleep is only available starting Volta (sm70)") +@pytest.mark.parallel_threads_limit(2) +@thread_unsafe_on_windows def test_event_is_done_false(init_cuda): """Event.is_done returns False when captured work has not yet completed.""" device = Device() diff --git a/cuda_core/tests/test_helpers.py b/cuda_core/tests/test_helpers.py index f0937b55a08..05d28620501 100644 --- a/cuda_core/tests/test_helpers.py +++ b/cuda_core/tests/test_helpers.py @@ -5,7 +5,7 @@ import time import pytest -from helpers.buffers import PatternGen, compare_equal_buffers, make_scratch_buffer +from helpers.buffers import PatternGen, compare_equal_buffers, make_scratch_buffer, thread_unsafe_on_windows from helpers.latch import LatchKernel from helpers.logging import TimestampedLogger @@ -16,14 +16,8 @@ NBYTES = 64 -if IS_WINDOWS or IS_WSL: - # see comment below, with threaded test all threads would need - # synchronization (which would require a barrier that we don't have). - # Applies at least to test_latchkernel and test_patterngen_(seeds|values). - pytestmark = pytest.mark.thread_unsafe(reason="windows host-access unsafe while GPU is working") - - @pytest.mark.skipif(Device().compute_capability.major < 7, reason="__nanosleep is only available starting Volta (sm70)") +@thread_unsafe_on_windows def test_latchkernel(): """Test LatchKernel.""" log = TimestampedLogger(enabled=ENABLE_LOGGING) @@ -58,6 +52,7 @@ def test_latchkernel(): under_compute_sanitizer(), reason="Too slow under compute-sanitizer (UVM-heavy test).", ) +@thread_unsafe_on_windows def test_patterngen_seeds(): """Test PatternGen with seed argument.""" device = Device() @@ -76,6 +71,7 @@ def test_patterngen_seeds(): pgen.verify_buffer(buffer, seed=j) +@thread_unsafe_on_windows def test_patterngen_values(): """Test PatternGen with value argument, also compare_equal_buffers.""" device = Device() diff --git a/cuda_core/tests/test_memory.py b/cuda_core/tests/test_memory.py index 957c8804be8..f0e1a6ac6bb 100644 --- a/cuda_core/tests/test_memory.py +++ b/cuda_core/tests/test_memory.py @@ -15,7 +15,7 @@ import pytest from helpers import IS_WINDOWS, supports_ipc_mempool -from helpers.buffers import DummyDeviceMemoryResource, DummyUnifiedMemoryResource, TrackingMR +from helpers.buffers import DummyDeviceMemoryResource, DummyUnifiedMemoryResource, TrackingMR, thread_unsafe_on_windows from conftest import ( create_managed_memory_resource_or_skip, @@ -313,6 +313,7 @@ def fill_env(request, init_cuda): ) +@thread_unsafe_on_windows @pytest.mark.parametrize("value,size,exc", _FILL_CASES) def test_buffer_fill(fill_env, value, size, exc): device, mr = fill_env From d920526f185707dc9dfa374e07c4b3f9f131e3b2 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Wed, 29 Jul 2026 20:11:25 +0200 Subject: [PATCH 16/17] These tests seem to test process global cleanup (not sure I follow) --- cuda_core/tests/graph/test_graph_definition_lifetime.py | 2 ++ cuda_core/tests/graph/test_graph_update.py | 1 + 2 files changed, 3 insertions(+) diff --git a/cuda_core/tests/graph/test_graph_definition_lifetime.py b/cuda_core/tests/graph/test_graph_definition_lifetime.py index 804cccc1923..c4e1011da38 100644 --- a/cuda_core/tests/graph/test_graph_definition_lifetime.py +++ b/cuda_core/tests/graph/test_graph_definition_lifetime.py @@ -592,6 +592,7 @@ def test_event_survives_graph_clone_and_execution(init_cuda): # ============================================================================= +@pytest.mark.thread_unsafe(reason="asserts cleanup on main thread") @pytest.mark.agent_authored(model="gpt-5.6") def test_user_object_cleanup_is_coalesced_on_python_thread(init_cuda): """More than 32 CUDA callbacks drain through one main-thread pending call.""" @@ -1319,6 +1320,7 @@ def test_memcpy_buffer_survives_close(init_cuda): assert list(out) == [0xCD] * 4 +@pytest.mark.thread_unsafe(reason="deferred cleanup on main thread which would wait") @pytest.mark.agent_authored(model="claude-opus-4.8") def test_memcpy_buffer_allocations_released_after_graph_destroyed(init_cuda): """Destroying the graph frees both memcpy operand allocations. diff --git a/cuda_core/tests/graph/test_graph_update.py b/cuda_core/tests/graph/test_graph_update.py index 13513830944..add449628b2 100644 --- a/cuda_core/tests/graph/test_graph_update.py +++ b/cuda_core/tests/graph/test_graph_update.py @@ -196,6 +196,7 @@ def new_callback(): assert called == ["new"] +@pytest.mark.thread_unsafe(reason="deferred cleanup on main thread which would wait") @pytest.mark.agent_authored(model="gpt-5.6") def test_failed_graph_update_does_not_adopt_attachments(init_cuda): """A rejected source graph keeps ownership separate from the exec.""" From 0bde899810f14d2cc4d394a95b4f931cae70ab4b Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Wed, 29 Jul 2026 21:38:58 +0200 Subject: [PATCH 17/17] TST: Mark LatchKernel tests as thread-unsafe Concurrent LatchKernel runs can overlap pinned flag alloc/free across workers; mark them thread-unsafe until a barrier_wait is restored. Also drop the compile-once LatchKernel helper changes for now. --- cuda_core/tests/helpers/latch.py | 29 +++++------------------------ cuda_core/tests/test_event.py | 7 ++----- cuda_core/tests/test_helpers.py | 2 +- 3 files changed, 8 insertions(+), 30 deletions(-) diff --git a/cuda_core/tests/helpers/latch.py b/cuda_core/tests/helpers/latch.py index 978e2dbdf18..7702c5f617e 100644 --- a/cuda_core/tests/helpers/latch.py +++ b/cuda_core/tests/helpers/latch.py @@ -2,7 +2,6 @@ # SPDX-License-Identifier: Apache-2.0 import ctypes -import threading import pytest @@ -21,15 +20,9 @@ class LatchKernel: Manages a kernel that blocks stream progress until released. """ - _latch_kernel_lock = threading.Lock() - _latch_kernels = {} - - @classmethod - def _get_kernel(cls, device): - kernel = cls._latch_kernels.get(device.uuid) - if kernel is not None: - return kernel - + def __init__(self, device, timeout_sec=60): + if helpers.CUDA_INCLUDE_PATH is None: + pytest.skip("need CUDA header") code = """ #include @@ -48,7 +41,6 @@ def _get_kernel(cls, device): // Check for timeout if (clock64() - start >= timeout_cycles) { - signal.store(-1, cuda::memory_order_relaxed); break; // Timeout reached } @@ -64,25 +56,14 @@ def _get_kernel(cls, device): ) prog = Program(code, code_type="c++", options=program_options) mod = prog.compile(target_type="cubin") - kernel = mod.get_kernel("latch") - - return cls._latch_kernels.setdefault(device.uuid, kernel) - - def __init__(self, device, timeout_sec=60): - if helpers.CUDA_INCLUDE_PATH is None: - pytest.skip("need CUDA header") - - with self._latch_kernel_lock: - self.kernel = self._get_kernel(device) + self.kernel = mod.get_kernel("latch") mr = LegacyPinnedMemoryResource() self.buffer = mr.allocate(4) - self.busy_wait_flag[0] = 1 + self.busy_wait_flag[0] = 0 clock_rate_hz = device.properties.clock_rate * 1000 self.timeout_cycles = int(timeout_sec * clock_rate_hz) - self.busy_wait_flag[0] = 0 - def launch(self, stream): """Launch the latch kernel, blocking stream progress via busy waiting.""" config = LaunchConfig(grid=1, block=1) diff --git a/cuda_core/tests/test_event.py b/cuda_core/tests/test_event.py index 4ec24f1d9a6..bbe5a16efac 100644 --- a/cuda_core/tests/test_event.py +++ b/cuda_core/tests/test_event.py @@ -5,7 +5,6 @@ import math import pytest -from helpers.buffers import thread_unsafe_on_windows from helpers.latch import LatchKernel from helpers.nanosleep_kernel import NanosleepKernel @@ -120,8 +119,7 @@ def test_error_timing_recorded(): @pytest.mark.skipif(Device().compute_capability.major < 7, reason="__nanosleep is only available starting Volta (sm70)") -@pytest.mark.parallel_threads_limit(2) -@thread_unsafe_on_windows +@pytest.mark.thread_unsafe(reason="requires a barrier wait to avoid overlapping pinned latch allocations") def test_error_timing_incomplete(): device = Device() device.set_current() @@ -226,8 +224,7 @@ def test_event_ipc_descriptor_non_ipc(init_cuda): @pytest.mark.skipif(Device().compute_capability.major < 7, reason="__nanosleep is only available starting Volta (sm70)") -@pytest.mark.parallel_threads_limit(2) -@thread_unsafe_on_windows +@pytest.mark.thread_unsafe(reason="requires a barrier wait to avoid overlapping pinned latch allocations") def test_event_is_done_false(init_cuda): """Event.is_done returns False when captured work has not yet completed.""" device = Device() diff --git a/cuda_core/tests/test_helpers.py b/cuda_core/tests/test_helpers.py index 05d28620501..409060e7228 100644 --- a/cuda_core/tests/test_helpers.py +++ b/cuda_core/tests/test_helpers.py @@ -17,7 +17,7 @@ @pytest.mark.skipif(Device().compute_capability.major < 7, reason="__nanosleep is only available starting Volta (sm70)") -@thread_unsafe_on_windows +@pytest.mark.thread_unsafe(reason="requires a barrier wait to avoid overlapping pinned latch allocations") def test_latchkernel(): """Test LatchKernel.""" log = TimestampedLogger(enabled=ENABLE_LOGGING)