diff --git a/cuda_bindings/tests/test_cufile.py b/cuda_bindings/tests/test_cufile.py index 46bd8429a62..7de4ceb2e20 100644 --- a/cuda_bindings/tests/test_cufile.py +++ b/cuda_bindings/tests/test_cufile.py @@ -138,7 +138,7 @@ def ctx(): (err,) = cuda.cuCtxSetCurrent(ctx) assert err == cuda.CUresult.CUDA_SUCCESS - yield + yield ctx cuda.cuDevicePrimaryCtxRelease(device) diff --git a/cuda_bindings/tests/test_examples.py b/cuda_bindings/tests/test_examples.py index 63a56c78fb7..6ffce3e7fe7 100644 --- a/cuda_bindings/tests/test_examples.py +++ b/cuda_bindings/tests/test_examples.py @@ -20,6 +20,7 @@ def test_example(example): env = os.environ.copy() env["CUDA_BINDINGS_SKIP_EXAMPLE"] = "100" + env["MPLBACKEND"] = "Agg" # avoid plt.show() from blocking process = subprocess.run([sys.executable, example], capture_output=True, env=env) # noqa: S603 # returncode is a special value used in the examples to indicate that system requirements are not met. diff --git a/cuda_core/tests/graph/test_graph_definition.py b/cuda_core/tests/graph/test_graph_definition.py index 1d63504c7c4..139392e9377 100644 --- a/cuda_core/tests/graph/test_graph_definition.py +++ b/cuda_core/tests/graph/test_graph_definition.py @@ -575,20 +575,6 @@ def node_spec(request, init_cuda): # ============================================================================= -@pytest.fixture -def sample_graphdef(init_cuda): - """A sample GraphDefinition for standalone tests.""" - return GraphDefinition() - - -@pytest.fixture -def dot_file(tmp_path): - """Temporary DOT file path, cleaned up after test.""" - path = tmp_path / "graph.dot" - yield path - path.unlink(missing_ok=True) - - # ============================================================================= # Topology tests (parameterized over graph specs) # ============================================================================= @@ -791,14 +777,16 @@ def registered(node): # ============================================================================= -def test_graphdef_handle_valid(sample_graphdef): +def test_graphdef_handle_valid(init_cuda): """GraphDefinition has a valid non-null handle.""" + sample_graphdef = GraphDefinition() assert sample_graphdef.handle is not None assert int(sample_graphdef.handle) != 0 -def test_graphdef_entry_is_virtual(sample_graphdef): +def test_graphdef_entry_is_virtual(init_cuda): """Internal entry node is virtual (no pred/succ, type is None).""" + sample_graphdef = GraphDefinition() entry = sample_graphdef._entry assert isinstance(entry, GraphNode) assert entry.pred == set() @@ -811,8 +799,9 @@ def test_graphdef_entry_is_virtual(sample_graphdef): # ============================================================================= -def test_alloc_zero_size_fails(sample_graphdef): +def test_alloc_zero_size_fails(init_cuda): """Alloc with zero size raises error (CUDA limitation).""" + sample_graphdef = GraphDefinition() _skip_if_no_mempool() from cuda.core._utils.cuda_utils import CUDAError @@ -820,8 +809,9 @@ def test_alloc_zero_size_fails(sample_graphdef): sample_graphdef.allocate(0) -def test_free_creates_dependency(sample_graphdef): +def test_free_creates_dependency(init_cuda): """Free node depends on its predecessor.""" + sample_graphdef = GraphDefinition() _skip_if_no_mempool() with xfail_on_graph_mempool_oom(): alloc = sample_graphdef.allocate(ALLOC_SIZE) @@ -829,8 +819,9 @@ def test_free_creates_dependency(sample_graphdef): assert alloc in free.pred -def test_alloc_free_chain(sample_graphdef): +def test_alloc_free_chain(init_cuda): """Alloc and free can be chained.""" + sample_graphdef = GraphDefinition() _skip_if_no_mempool() with xfail_on_graph_mempool_oom(): a1 = sample_graphdef.allocate(ALLOC_SIZE) @@ -847,8 +838,9 @@ def test_alloc_free_chain(sample_graphdef): # ============================================================================= -def test_alloc_memory_type_invalid(sample_graphdef): +def test_alloc_memory_type_invalid(init_cuda): """Invalid memory type raises ValueError.""" + sample_graphdef = GraphDefinition() with pytest.raises(ValueError, match="'invalid' is not a valid GraphMemoryType. Must be "): sample_graphdef.allocate(ALLOC_SIZE, memory_type="invalid") @@ -860,8 +852,9 @@ def test_alloc_memory_type_invalid(sample_graphdef): pytest.param(lambda d: d, id="Device_object"), ], ) -def test_alloc_device_option(sample_graphdef, device_spec): +def test_alloc_device_option(init_cuda, device_spec): """Device can be specified as int or Device object.""" + sample_graphdef = GraphDefinition() _skip_if_no_mempool() device = Device() with xfail_on_graph_mempool_oom(device): @@ -884,8 +877,9 @@ def test_alloc_peer_access(mempool_device_x2): @pytest.mark.parametrize("num_branches", [2, 3, 5]) -def test_join_merges_branches(sample_graphdef, num_branches): +def test_join_merges_branches(init_cuda, num_branches): """join() with multiple branches creates correct dependencies.""" + sample_graphdef = GraphDefinition() _skip_if_no_mempool() with xfail_on_graph_mempool_oom(): branches = [sample_graphdef.allocate(ALLOC_SIZE) for _ in range(num_branches)] @@ -899,8 +893,9 @@ def test_join_merges_branches(sample_graphdef, num_branches): # ============================================================================= -def test_launch_creates_node(sample_graphdef): +def test_launch_creates_node(init_cuda): """launch() creates a KernelNode.""" + sample_graphdef = GraphDefinition() mod = compile_common_kernels() kernel = mod.get_kernel("empty_kernel") config = LaunchConfig(grid=1, block=1) @@ -908,8 +903,9 @@ def test_launch_creates_node(sample_graphdef): assert isinstance(node, KernelNode) -def test_launch_chain_dependencies(sample_graphdef): +def test_launch_chain_dependencies(init_cuda): """Chained launches create correct dependencies.""" + sample_graphdef = GraphDefinition() mod = compile_common_kernels() kernel = mod.get_kernel("empty_kernel") config = LaunchConfig(grid=1, block=1) @@ -971,15 +967,17 @@ def _instantiate_and_upload(graph_definition, kwargs, stream): @pytest.mark.parametrize("inst_kwargs", _INSTANTIATE_ONLY_OPTIONS) -def test_instantiate_empty_graph(sample_graphdef, inst_kwargs): +def test_instantiate_empty_graph(init_cuda, inst_kwargs): """Empty graph can be instantiated.""" + sample_graphdef = GraphDefinition() graph = _instantiate(sample_graphdef, inst_kwargs) assert graph is not None @pytest.mark.parametrize("inst_kwargs", _INSTANTIATE_ONLY_OPTIONS) -def test_instantiate_with_nodes(sample_graphdef, inst_kwargs): +def test_instantiate_with_nodes(init_cuda, inst_kwargs): """Graph with nodes can be instantiated.""" + sample_graphdef = GraphDefinition() _skip_if_no_mempool() with xfail_on_graph_mempool_oom(): sample_graphdef.allocate(ALLOC_SIZE) @@ -989,8 +987,9 @@ def test_instantiate_with_nodes(sample_graphdef, inst_kwargs): @pytest.mark.skipif(not Device(0).properties.unified_addressing, reason="requires unified addressing") -def test_instantiate_and_execute_kernel_device_launch(sample_graphdef): +def test_instantiate_and_execute_kernel_device_launch(init_cuda): """Kernel-only graph can be instantiated with device_launch flag.""" + sample_graphdef = GraphDefinition() mod = compile_common_kernels() kernel = mod.get_kernel("empty_kernel") config = LaunchConfig(grid=1, block=1) @@ -1006,8 +1005,9 @@ def test_instantiate_and_execute_kernel_device_launch(sample_graphdef): @pytest.mark.parametrize("inst_kwargs", _EXECUTE_OPTIONS) -def test_instantiate_and_execute_kernel(sample_graphdef, inst_kwargs): +def test_instantiate_and_execute_kernel(init_cuda, inst_kwargs): """Graph with kernel can be instantiated and executed.""" + sample_graphdef = GraphDefinition() mod = compile_common_kernels() kernel = mod.get_kernel("empty_kernel") config = LaunchConfig(grid=1, block=1) @@ -1020,8 +1020,9 @@ def test_instantiate_and_execute_kernel(sample_graphdef, inst_kwargs): @pytest.mark.parametrize("inst_kwargs", _EXECUTE_OPTIONS) -def test_instantiate_and_execute_alloc_free(sample_graphdef, inst_kwargs): +def test_instantiate_and_execute_alloc_free(init_cuda, inst_kwargs): """Graph with alloc/free can be executed.""" + sample_graphdef = GraphDefinition() _skip_if_no_mempool() with xfail_on_graph_mempool_oom(): alloc = sample_graphdef.allocate(ALLOC_SIZE) @@ -1034,8 +1035,9 @@ def test_instantiate_and_execute_alloc_free(sample_graphdef, inst_kwargs): @pytest.mark.parametrize("inst_kwargs", _EXECUTE_OPTIONS) -def test_instantiate_and_execute_memset(sample_graphdef, inst_kwargs): +def test_instantiate_and_execute_memset(init_cuda, inst_kwargs): """Graph with alloc/memset/free can be executed.""" + sample_graphdef = GraphDefinition() _skip_if_no_mempool() with xfail_on_graph_mempool_oom(): alloc = sample_graphdef.allocate(ALLOC_SIZE) @@ -1049,8 +1051,9 @@ def test_instantiate_and_execute_memset(sample_graphdef, inst_kwargs): @pytest.mark.parametrize("inst_kwargs", _EXECUTE_OPTIONS) -def test_instantiate_and_execute_memcpy(sample_graphdef, inst_kwargs): +def test_instantiate_and_execute_memcpy(init_cuda, inst_kwargs): """Graph with alloc/memset/memcpy/free can be executed and data is copied.""" + sample_graphdef = GraphDefinition() _skip_if_no_mempool() import ctypes @@ -1074,8 +1077,9 @@ def test_instantiate_and_execute_memcpy(sample_graphdef, inst_kwargs): assert all(b == 0xAB for b in host_buf) -def test_instantiate_and_execute_child_graph(sample_graphdef): +def test_instantiate_and_execute_child_graph(init_cuda): """Graph with embedded child graph can be executed.""" + sample_graphdef = GraphDefinition() child = GraphDefinition() mod = compile_common_kernels() kernel = mod.get_kernel("empty_kernel") @@ -1091,8 +1095,9 @@ def test_instantiate_and_execute_child_graph(sample_graphdef): stream.sync() -def test_instantiate_and_execute_host_callback(sample_graphdef): +def test_instantiate_and_execute_host_callback(init_cuda): """Graph with host callback can be executed and callback is invoked.""" + sample_graphdef = GraphDefinition() results = [] def my_callback(): @@ -1109,8 +1114,9 @@ def my_callback(): assert results == [42] -def test_instantiate_and_execute_host_callback_cfunc(sample_graphdef): +def test_instantiate_and_execute_host_callback_cfunc(init_cuda): """Graph with ctypes function pointer callback can be executed.""" + sample_graphdef = GraphDefinition() import ctypes CALLBACK = ctypes.CFUNCTYPE(None, ctypes.c_void_p) @@ -1131,8 +1137,9 @@ def raw_fn(data): assert called[0] -def test_host_callback_cfunc_with_user_data(sample_graphdef): +def test_host_callback_cfunc_with_user_data(init_cuda): """Host callback with bytes user_data passes data to C function.""" + sample_graphdef = GraphDefinition() import ctypes CALLBACK = ctypes.CFUNCTYPE(None, ctypes.c_void_p) @@ -1153,14 +1160,16 @@ def read_byte(data): assert result[0] == 0xAB -def test_host_callback_user_data_rejected_for_python_callable(sample_graphdef): +def test_host_callback_user_data_rejected_for_python_callable(init_cuda): """user_data is rejected for Python callables.""" + sample_graphdef = GraphDefinition() with pytest.raises(ValueError, match="user_data is only supported"): sample_graphdef.callback(lambda: None, user_data=b"hello") -def test_instantiate_and_execute_event_record_wait(sample_graphdef): +def test_instantiate_and_execute_event_record_wait(init_cuda): """Graph with event record and wait nodes can be executed.""" + sample_graphdef = GraphDefinition() event = Device().create_event() rec = sample_graphdef.record(event) rec.wait(event) @@ -1182,8 +1191,9 @@ def _skip_unless_cc_90(): pytest.skip("Conditional node execution requires CC >= 9.0 (Hopper)") -def test_instantiate_and_execute_if_then(sample_graphdef): +def test_instantiate_and_execute_if_then(init_cuda): """If-conditional node: body executes only when condition is non-zero.""" + sample_graphdef = GraphDefinition() _skip_unless_cc_90() _skip_if_no_mempool() import ctypes @@ -1215,8 +1225,9 @@ def test_instantiate_and_execute_if_then(sample_graphdef): assert result[0] == 1 -def test_instantiate_and_execute_if_else(sample_graphdef): +def test_instantiate_and_execute_if_else(init_cuda): """If-else node: then or else branch executes based on condition.""" + sample_graphdef = GraphDefinition() _skip_unless_cc_90() _skip_if_no_mempool() import ctypes @@ -1250,8 +1261,9 @@ def test_instantiate_and_execute_if_else(sample_graphdef): assert result[0] == 2 -def test_instantiate_and_execute_switch(sample_graphdef): +def test_instantiate_and_execute_switch(init_cuda): """Switch node: selected branch executes based on condition value.""" + sample_graphdef = GraphDefinition() _skip_unless_cc_90() _skip_if_no_mempool() import ctypes @@ -1284,8 +1296,9 @@ def test_instantiate_and_execute_switch(sample_graphdef): assert result[0] == 1 -def test_conditional_node_type_preserved_by_nodes(sample_graphdef): +def test_conditional_node_type_preserved_by_nodes(init_cuda): """Conditional nodes appear as ConditionalNode base when read back from graph.""" + sample_graphdef = GraphDefinition() condition = try_create_condition(sample_graphdef) if_node = sample_graphdef.if_then(condition) assert isinstance(if_node, IfNode) @@ -1301,8 +1314,10 @@ def test_conditional_node_type_preserved_by_nodes(sample_graphdef): # ============================================================================= -def test_debug_dot_print_creates_file(sample_graphdef, dot_file): +def test_debug_dot_print_creates_file(init_cuda, tmp_path): """debug_dot_print writes a DOT file.""" + sample_graphdef = GraphDefinition() + dot_file = tmp_path / "graph.dot" _skip_if_no_mempool() with xfail_on_graph_mempool_oom(): sample_graphdef.allocate(ALLOC_SIZE) @@ -1312,8 +1327,10 @@ def test_debug_dot_print_creates_file(sample_graphdef, dot_file): assert "digraph" in content -def test_debug_dot_print_with_options(sample_graphdef, dot_file): +def test_debug_dot_print_with_options(init_cuda, tmp_path): """debug_dot_print accepts GraphDebugPrintOptions.""" + sample_graphdef = GraphDefinition() + dot_file = tmp_path / "graph.dot" _skip_if_no_mempool() with xfail_on_graph_mempool_oom(): sample_graphdef.allocate(ALLOC_SIZE) @@ -1322,8 +1339,10 @@ def test_debug_dot_print_with_options(sample_graphdef, dot_file): assert dot_file.exists() -def test_debug_dot_print_invalid_options(sample_graphdef, dot_file): +def test_debug_dot_print_invalid_options(init_cuda, tmp_path): """debug_dot_print rejects invalid options type.""" + sample_graphdef = GraphDefinition() + dot_file = tmp_path / "graph.dot" _skip_if_no_mempool() with xfail_on_graph_mempool_oom(): sample_graphdef.allocate(ALLOC_SIZE) diff --git a/cuda_core/tests/graph/test_graph_definition_lifetime.py b/cuda_core/tests/graph/test_graph_definition_lifetime.py index 804cccc1923..c4e1011da38 100644 --- a/cuda_core/tests/graph/test_graph_definition_lifetime.py +++ b/cuda_core/tests/graph/test_graph_definition_lifetime.py @@ -592,6 +592,7 @@ def test_event_survives_graph_clone_and_execution(init_cuda): # ============================================================================= +@pytest.mark.thread_unsafe(reason="asserts cleanup on main thread") @pytest.mark.agent_authored(model="gpt-5.6") def test_user_object_cleanup_is_coalesced_on_python_thread(init_cuda): """More than 32 CUDA callbacks drain through one main-thread pending call.""" @@ -1319,6 +1320,7 @@ def test_memcpy_buffer_survives_close(init_cuda): assert list(out) == [0xCD] * 4 +@pytest.mark.thread_unsafe(reason="deferred cleanup on main thread which would wait") @pytest.mark.agent_authored(model="claude-opus-4.8") def test_memcpy_buffer_allocations_released_after_graph_destroyed(init_cuda): """Destroying the graph frees both memcpy operand allocations. diff --git a/cuda_core/tests/graph/test_graph_memory_resource.py b/cuda_core/tests/graph/test_graph_memory_resource.py index 9fc794f4cca..f6e87543c85 100644 --- a/cuda_core/tests/graph/test_graph_memory_resource.py +++ b/cuda_core/tests/graph/test_graph_memory_resource.py @@ -6,7 +6,7 @@ import pytest from helpers import IS_WINDOWS, IS_WSL -from helpers.buffers import compare_buffer_to_constant, make_scratch_buffer, set_buffer +from helpers.buffers import compare_buffer_to_constant, make_scratch_buffer, set_buffer, thread_unsafe_on_windows from conftest import xfail_on_graph_mempool_oom from cuda.core import ( @@ -21,6 +21,13 @@ from cuda.core._utils.cuda_utils import CUDAError from cuda.core.graph import GraphCompleteOptions +# NOTE(seberg): "global" mode seems thread-unsafe even when working on stream +_GRAPH_MODES = [ + pytest.param("global", marks=pytest.mark.thread_unsafe(reason="gb instances share stream unsafely")), + "thread_local", + "relaxed", +] + def _common_kernels_alloc(): code = """ @@ -80,8 +87,9 @@ def free(self, buffers): self.stream.sync() -@pytest.mark.parametrize("mode", ["no_graph", "global", "thread_local", "relaxed"]) +@pytest.mark.parametrize("mode", ["no_graph"] + _GRAPH_MODES) @pytest.mark.parametrize("action", ["incr", "fill"]) +@thread_unsafe_on_windows def test_graph_alloc(mempool_device, mode, action): """Test basic graph capture with memory allocated and deallocated by GraphMemoryResource. @@ -130,7 +138,7 @@ def apply_kernels(mr, stream, out): assert compare_buffer_to_constant(out, 3) else: # Capture work, then upload and launch. - gb = device.create_graph_builder().begin_building(mode) + gb = stream.create_graph_builder().begin_building(mode) with xfail_on_graph_mempool_oom(device): apply_kernels(mr=gmr, stream=gb, out=out) graph = gb.end_building().complete() @@ -150,7 +158,8 @@ def apply_kernels(mr, stream, out): @pytest.mark.skipif(IS_WINDOWS or IS_WSL, reason="auto_free_on_launch not supported on Windows") -@pytest.mark.parametrize("mode", ["global", "thread_local", "relaxed"]) +@pytest.mark.parametrize("mode", _GRAPH_MODES) +@thread_unsafe_on_windows def test_graph_alloc_with_output(mempool_device, mode): """Test for memory allocated in a graph being used outside the graph.""" NBYTES = 64 @@ -168,7 +177,7 @@ def test_graph_alloc_with_output(mempool_device, mode): # Construct a graph to copy and increment the input. It returns a new # buffer allocated within the graph. The auto_free_on_launch option # is required to properly use the output buffer. - gb = device.create_graph_builder().begin_building(mode) + gb = stream.create_graph_builder().begin_building(mode) with xfail_on_graph_mempool_oom(device): out = gmr.allocate(NBYTES, stream=gb) out.copy_from(in_, stream=gb) @@ -195,7 +204,8 @@ def test_graph_alloc_with_output(mempool_device, mode): assert compare_buffer_to_constant(out, 6) -@pytest.mark.parametrize("mode", ["global", "thread_local", "relaxed"]) +@pytest.mark.parametrize("mode", _GRAPH_MODES) +@pytest.mark.thread_unsafe(reason="gb instances share default stream") def test_graph_mem_alloc_zero(mempool_device, mode): device = mempool_device gb = device.create_graph_builder().begin_building(mode) @@ -213,7 +223,8 @@ def test_graph_mem_alloc_zero(mempool_device, mode): assert buffer.device_id == int(device) -@pytest.mark.parametrize("mode", ["global", "thread_local", "relaxed"]) +@pytest.mark.parametrize("mode", _GRAPH_MODES) +@pytest.mark.thread_unsafe(reason="GMR is shared, so high mark is global") def test_graph_mem_set_attributes(mempool_device, mode): device = mempool_device stream = device.create_stream() @@ -265,7 +276,7 @@ def test_graph_mem_set_attributes(mempool_device, mode): mman.reset() -@pytest.mark.parametrize("mode", ["global", "thread_local", "relaxed"]) +@pytest.mark.parametrize("mode", _GRAPH_MODES) def test_gmr_check_capture_state(mempool_device, mode): """ Test expected errors (and non-errors) using GraphMemoryResource with graph @@ -284,7 +295,7 @@ def test_gmr_check_capture_state(mempool_device, mode): gmr.allocate(1, stream=stream) # Capturing - gb = device.create_graph_builder().begin_building(mode=mode) + gb = stream.create_graph_builder().begin_building(mode=mode) with xfail_on_graph_mempool_oom(device): gmr.allocate(1, stream=gb) # no error gb.end_building().complete() @@ -320,7 +331,7 @@ def test_graph_memory_resource_attributes_repr(mempool_device): assert "used_mem_high=" in r -@pytest.mark.parametrize("mode", ["global", "thread_local", "relaxed"]) +@pytest.mark.parametrize("mode", _GRAPH_MODES) def test_dmr_check_capture_state(mempool_device, mode): """ Test expected errors (and non-errors) using DeviceMemoryResource with graph @@ -334,7 +345,7 @@ def test_dmr_check_capture_state(mempool_device, mode): dmr.allocate(1, stream=stream).close() # no error # Capturing - gb = device.create_graph_builder().begin_building(mode=mode) + gb = stream.create_graph_builder().begin_building(mode=mode) with pytest.raises( RuntimeError, match=r"cannot perform memory operations on a capturing " diff --git a/cuda_core/tests/graph/test_graph_update.py b/cuda_core/tests/graph/test_graph_update.py index 13513830944..add449628b2 100644 --- a/cuda_core/tests/graph/test_graph_update.py +++ b/cuda_core/tests/graph/test_graph_update.py @@ -196,6 +196,7 @@ def new_callback(): assert called == ["new"] +@pytest.mark.thread_unsafe(reason="deferred cleanup on main thread which would wait") @pytest.mark.agent_authored(model="gpt-5.6") def test_failed_graph_update_does_not_adopt_attachments(init_cuda): """A rejected source graph keeps ownership separate from the exec.""" diff --git a/cuda_core/tests/helpers/buffers.py b/cuda_core/tests/helpers/buffers.py index f4412d57e16..532aadecdcf 100644 --- a/cuda_core/tests/helpers/buffers.py +++ b/cuda_core/tests/helpers/buffers.py @@ -3,10 +3,12 @@ import ctypes +import pytest + from cuda.core import Buffer, Device, MemoryResource from cuda.core._utils.cuda_utils import driver, handle_return -from . import libc +from . import IS_WINDOWS, IS_WSL, libc __all__ = [ "DummyDeviceMemoryResource", @@ -16,9 +18,19 @@ "compare_buffer_to_constant", "compare_equal_buffers", "make_scratch_buffer", + "thread_unsafe_on_windows", ] +def thread_unsafe_on_windows(func): + # Tests that use these buffers and access the memory on the host are + # thread-unsafe on windows. On windows the GPU must be fully quiescent for host + # access to be safe and with threaded tests that would require a barrier. + if IS_WINDOWS or IS_WSL: + return pytest.mark.thread_unsafe(reason="windows host-access unsafe while GPU is working") + return func + + class DummyDeviceMemoryResource(MemoryResource): # cuMemAlloc / cuMemFree are synchronous; stream is accepted for # interface conformance but ignored. diff --git a/cuda_core/tests/helpers/latch.py b/cuda_core/tests/helpers/latch.py index c28fb222641..7702c5f617e 100644 --- a/cuda_core/tests/helpers/latch.py +++ b/cuda_core/tests/helpers/latch.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 import ctypes diff --git a/cuda_core/tests/memory/test_managed_ops.py b/cuda_core/tests/memory/test_managed_ops.py index 33def77935f..d9305399de4 100644 --- a/cuda_core/tests/memory/test_managed_ops.py +++ b/cuda_core/tests/memory/test_managed_ops.py @@ -364,6 +364,7 @@ def test_from_handle(self, init_cuda): finally: plain.close() + @pytest.mark.thread_unsafe(reason="external_managed_buffer is shared between threads") def test_read_mostly_roundtrip(self, external_managed_buffer): buf = external_managed_buffer assert buf.read_mostly is False @@ -372,6 +373,7 @@ def test_read_mostly_roundtrip(self, external_managed_buffer): buf.read_mostly = False assert buf.read_mostly is False + @pytest.mark.thread_unsafe(reason="external_managed_buffer is shared between threads") def test_preferred_location_roundtrip(self, location_ops_device, external_managed_buffer): device = location_ops_device buf = external_managed_buffer @@ -386,6 +388,7 @@ def test_preferred_location_roundtrip(self, location_ops_device, external_manage buf.preferred_location = None assert buf.preferred_location is None + @pytest.mark.thread_unsafe(reason="external_managed_buffer is shared between threads") def test_preferred_location_roundtrip_host_numa(self, location_ops_device): """Host(numa_id=N) round-trips correctly on CUDA 13 builds.""" from cuda.core._utils.version import binding_version @@ -406,6 +409,7 @@ def test_preferred_location_roundtrip_host_numa(self, location_ops_device): finally: plain.close() + @pytest.mark.thread_unsafe(reason="external_managed_buffer is shared between threads") def test_accessed_by_add_discard(self, location_ops_device, external_managed_buffer): device = location_ops_device buf = external_managed_buffer @@ -417,6 +421,7 @@ def test_accessed_by_add_discard(self, location_ops_device, external_managed_buf buf.accessed_by.discard(device) assert device not in buf.accessed_by + @pytest.mark.thread_unsafe(reason="external_managed_buffer is shared between threads") def test_accessed_by_mutable_set_interface(self, location_ops_device, external_managed_buffer): """Full MutableSet conformance pass on AccessedBySetProxy. @@ -436,6 +441,7 @@ def test_accessed_by_mutable_set_interface(self, location_ops_device, external_m non_member=Host(numa_id=0), ) + @pytest.mark.thread_unsafe(reason="external_managed_buffer is shared between threads") def test_accessed_by_set_assignment(self, location_ops_device, external_managed_buffer): device = location_ops_device buf = external_managed_buffer diff --git a/cuda_core/tests/system/test_system_device.py b/cuda_core/tests/system/test_system_device.py index b8fe3505674..8e47dce67bd 100644 --- a/cuda_core/tests/system/test_system_device.py +++ b/cuda_core/tests/system/test_system_device.py @@ -349,6 +349,7 @@ def test_c2c_mode_enabled(): @pytest.mark.skipif(helpers.IS_WSL or helpers.IS_WINDOWS, reason="Persistence mode not supported on WSL or Windows") +@pytest.mark.thread_unsafe(reason="device persistence mode is global state") def test_persistence_mode_enabled(): for device in system.Device.get_all_devices(): is_enabled = device.is_persistence_mode_enabled diff --git a/cuda_core/tests/test_event.py b/cuda_core/tests/test_event.py index 79f0090ace6..bbe5a16efac 100644 --- a/cuda_core/tests/test_event.py +++ b/cuda_core/tests/test_event.py @@ -119,6 +119,7 @@ def test_error_timing_recorded(): @pytest.mark.skipif(Device().compute_capability.major < 7, reason="__nanosleep is only available starting Volta (sm70)") +@pytest.mark.thread_unsafe(reason="requires a barrier wait to avoid overlapping pinned latch allocations") def test_error_timing_incomplete(): device = Device() device.set_current() @@ -222,6 +223,8 @@ def test_event_ipc_descriptor_non_ipc(init_cuda): _ = event.ipc_descriptor +@pytest.mark.skipif(Device().compute_capability.major < 7, reason="__nanosleep is only available starting Volta (sm70)") +@pytest.mark.thread_unsafe(reason="requires a barrier wait to avoid overlapping pinned latch allocations") def test_event_is_done_false(init_cuda): """Event.is_done returns False when captured work has not yet completed.""" device = Device() diff --git a/cuda_core/tests/test_helpers.py b/cuda_core/tests/test_helpers.py index 43dbf8887e2..409060e7228 100644 --- a/cuda_core/tests/test_helpers.py +++ b/cuda_core/tests/test_helpers.py @@ -5,7 +5,7 @@ import time import pytest -from helpers.buffers import PatternGen, compare_equal_buffers, make_scratch_buffer +from helpers.buffers import PatternGen, compare_equal_buffers, make_scratch_buffer, thread_unsafe_on_windows from helpers.latch import LatchKernel from helpers.logging import TimestampedLogger @@ -17,6 +17,7 @@ @pytest.mark.skipif(Device().compute_capability.major < 7, reason="__nanosleep is only available starting Volta (sm70)") +@pytest.mark.thread_unsafe(reason="requires a barrier wait to avoid overlapping pinned latch allocations") def test_latchkernel(): """Test LatchKernel.""" log = TimestampedLogger(enabled=ENABLE_LOGGING) @@ -51,6 +52,7 @@ def test_latchkernel(): under_compute_sanitizer(), reason="Too slow under compute-sanitizer (UVM-heavy test).", ) +@thread_unsafe_on_windows def test_patterngen_seeds(): """Test PatternGen with seed argument.""" device = Device() @@ -69,6 +71,7 @@ def test_patterngen_seeds(): pgen.verify_buffer(buffer, seed=j) +@thread_unsafe_on_windows def test_patterngen_values(): """Test PatternGen with value argument, also compare_equal_buffers.""" device = Device() diff --git a/cuda_core/tests/test_memory.py b/cuda_core/tests/test_memory.py index 4427b899765..f0e1a6ac6bb 100644 --- a/cuda_core/tests/test_memory.py +++ b/cuda_core/tests/test_memory.py @@ -15,7 +15,7 @@ import pytest from helpers import IS_WINDOWS, supports_ipc_mempool -from helpers.buffers import DummyDeviceMemoryResource, DummyUnifiedMemoryResource, TrackingMR +from helpers.buffers import DummyDeviceMemoryResource, DummyUnifiedMemoryResource, TrackingMR, thread_unsafe_on_windows from conftest import ( create_managed_memory_resource_or_skip, @@ -249,9 +249,8 @@ def _pattern_bytes(value) -> bytes: @pytest.fixture(params=["device", "unified", "pinned"]) -def fill_env(request): - device = Device() - device.set_current() +def fill_env(request, init_cuda): + device = init_cuda if request.param == "device": mr = DummyDeviceMemoryResource(device) elif request.param == "unified": @@ -314,6 +313,7 @@ def fill_env(request): ) +@thread_unsafe_on_windows @pytest.mark.parametrize("value,size,exc", _FILL_CASES) def test_buffer_fill(fill_env, value, size, exc): device, mr = fill_env @@ -1196,6 +1196,8 @@ def test_managed_memory_resource_with_options(init_cuda): device.sync() dst_buffer.close() src_buffer.close() + # TODO(seberg): 2026-06: mr close may be unsafe with incomplete `buf.close()` + device.sync() def test_managed_memory_resource_preferred_location_default(init_cuda): diff --git a/cuda_core/tests/test_multiprocessing_warning.py b/cuda_core/tests/test_multiprocessing_warning.py index 0f96e0abfbc..94a671ff2f8 100644 --- a/cuda_core/tests/test_multiprocessing_warning.py +++ b/cuda_core/tests/test_multiprocessing_warning.py @@ -12,12 +12,17 @@ import warnings from unittest.mock import patch +import pytest + from cuda.core import DeviceMemoryResource, DeviceMemoryResourceOptions, EventOptions from cuda.core._event import _reduce_event from cuda.core._memory._device_memory_resource import _deep_reduce_device_memory_resource from cuda.core._memory._ipc import _reduce_allocation_handle from cuda.core._utils.cuda_utils import check_multiprocessing_start_method, reset_fork_warning +# We could move these to a (session) fixtures +pytestmark = pytest.mark.thread_unsafe(reason="all tests use unittest.mock.patch") + def test_warn_on_fork_method_device_memory_resource(ipc_device): """Test that warning is emitted when DeviceMemoryResource is pickled with fork method.""" diff --git a/cuda_core/tests/test_object_protocols.py b/cuda_core/tests/test_object_protocols.py index e8391c75678..e4e8ee21c9b 100644 --- a/cuda_core/tests/test_object_protocols.py +++ b/cuda_core/tests/test_object_protocols.py @@ -527,6 +527,26 @@ def sample_switch_node_alt(sample_graphdef): return sample_graphdef.switch(condition, 3) +# Indirect-parametrize helpers: request.getfixturevalue() runs here, in the +# fixture (main thread), so the resolved object is already available when the +# test function runs in a worker thread. + + +@pytest.fixture +def sample_object(request): + return request.getfixturevalue(request.param) + + +@pytest.fixture +def sample_object_a(request): + return request.getfixturevalue(request.param) + + +@pytest.fixture +def sample_object_b(request): + return request.getfixturevalue(request.param) + + # ============================================================================= # Type groupings # ============================================================================= @@ -722,12 +742,11 @@ def sample_switch_node_alt(sample_graphdef): # ============================================================================= -@pytest.mark.parametrize("fixture_name", WEAKREF_TYPES) -def test_weakref_supported(fixture_name, request): +@pytest.mark.parametrize("sample_object", WEAKREF_TYPES, indirect=True) +def test_weakref_supported(sample_object): """Object supports weak references.""" - obj = request.getfixturevalue(fixture_name) - ref = weakref.ref(obj) - assert ref() is obj + ref = weakref.ref(sample_object) + assert ref() is sample_object # ============================================================================= @@ -735,27 +754,22 @@ def test_weakref_supported(fixture_name, request): # ============================================================================= -@pytest.mark.parametrize("fixture_name", HASH_TYPES) -def test_hash_consistency(fixture_name, request): +@pytest.mark.parametrize("sample_object", HASH_TYPES, indirect=True) +def test_hash_consistency(sample_object): """Hash is consistent across multiple calls.""" - obj = request.getfixturevalue(fixture_name) - assert hash(obj) == hash(obj) + assert hash(sample_object) == hash(sample_object) -@pytest.mark.parametrize("a_name,b_name", SAME_TYPE_PAIRS) -def test_hash_distinct_same_type(a_name, b_name, request): +@pytest.mark.parametrize("sample_object_a,sample_object_b", SAME_TYPE_PAIRS, indirect=True) +def test_hash_distinct_same_type(sample_object_a, sample_object_b): """Distinct objects of the same type have different hashes.""" - obj_a = request.getfixturevalue(a_name) - obj_b = request.getfixturevalue(b_name) - assert hash(obj_a) != hash(obj_b) # extremely unlikely + assert hash(sample_object_a) != hash(sample_object_b) # extremely unlikely -@pytest.mark.parametrize("a_name,b_name", itertools.combinations(HASH_TYPES, 2)) -def test_hash_distinct_cross_type(a_name, b_name, request): +@pytest.mark.parametrize("sample_object_a,sample_object_b", itertools.combinations(HASH_TYPES, 2), indirect=True) +def test_hash_distinct_cross_type(sample_object_a, sample_object_b): """Distinct objects of different types have different hashes.""" - obj_a = request.getfixturevalue(a_name) - obj_b = request.getfixturevalue(b_name) - assert hash(obj_a) != hash(obj_b) # extremely unlikely + assert hash(sample_object_a) != hash(sample_object_b) # extremely unlikely # ============================================================================= @@ -763,41 +777,35 @@ def test_hash_distinct_cross_type(a_name, b_name, request): # ============================================================================= -@pytest.mark.parametrize("fixture_name", EQ_TYPES) -def test_equality_basic(fixture_name, request): +@pytest.mark.parametrize("sample_object", EQ_TYPES, indirect=True) +def test_equality_basic(sample_object): """Object equality: reflexive, not equal to None or other types.""" - obj = request.getfixturevalue(fixture_name) - assert obj == obj - assert obj is not None - assert obj != "string" - if hasattr(obj, "handle"): - assert obj != obj.handle + assert sample_object == sample_object + assert sample_object is not None + assert sample_object != "string" + if hasattr(sample_object, "handle"): + assert sample_object != sample_object.handle -@pytest.mark.parametrize("a_name,b_name", itertools.combinations(EQ_TYPES, 2)) -def test_no_cross_type_equality(a_name, b_name, request): +@pytest.mark.parametrize("sample_object_a,sample_object_b", itertools.combinations(EQ_TYPES, 2), indirect=True) +def test_no_cross_type_equality(sample_object_a, sample_object_b): """No two distinct objects of different types should compare equal.""" - obj_a = request.getfixturevalue(a_name) - obj_b = request.getfixturevalue(b_name) - assert obj_a != obj_b + assert sample_object_a != sample_object_b -@pytest.mark.parametrize("a_name,b_name", SAME_TYPE_PAIRS) -def test_same_type_inequality(a_name, b_name, request): +@pytest.mark.parametrize("sample_object_a,sample_object_b", SAME_TYPE_PAIRS, indirect=True) +def test_same_type_inequality(sample_object_a, sample_object_b): """Two distinct objects of the same type should not compare equal.""" - obj_a = request.getfixturevalue(a_name) - obj_b = request.getfixturevalue(b_name) - assert obj_a is not obj_b - assert obj_a != obj_b + assert sample_object_a is not sample_object_b + assert sample_object_a != sample_object_b -@pytest.mark.parametrize("fixture_name,copy_fn", FROM_HANDLE_COPIES) -def test_equality_same_handle(fixture_name, copy_fn, request): +@pytest.mark.parametrize("sample_object,copy_fn", FROM_HANDLE_COPIES, indirect=["sample_object"]) +def test_equality_same_handle(sample_object, copy_fn): """Two wrappers around the same handle should compare equal.""" - obj = request.getfixturevalue(fixture_name) - obj2 = copy_fn(obj) - assert obj == obj2 - assert hash(obj) == hash(obj2) + obj2 = copy_fn(sample_object) + assert sample_object == obj2 + assert hash(sample_object) == hash(obj2) # ============================================================================= @@ -805,48 +813,43 @@ def test_equality_same_handle(fixture_name, copy_fn, request): # ============================================================================= -@pytest.mark.parametrize("fixture_name", DICT_KEY_TYPES) -def test_usable_as_dict_key(fixture_name, request): +@pytest.mark.parametrize("sample_object", DICT_KEY_TYPES, indirect=True) +def test_usable_as_dict_key(sample_object): """Object can be used as a dictionary key.""" - obj = request.getfixturevalue(fixture_name) - d = {obj: "value"} - assert d[obj] == "value" - assert obj in d + d = {sample_object: "value"} + assert d[sample_object] == "value" + assert sample_object in d -@pytest.mark.parametrize("fixture_name", DICT_KEY_TYPES) -def test_usable_in_set(fixture_name, request): +@pytest.mark.parametrize("sample_object", DICT_KEY_TYPES, indirect=True) +def test_usable_in_set(sample_object): """Object can be added to a set.""" - obj = request.getfixturevalue(fixture_name) - s = {obj} - assert obj in s + s = {sample_object} + assert sample_object in s -@pytest.mark.parametrize("fixture_name", WEAKREF_TYPES) -def test_usable_in_weak_value_dict(fixture_name, request): +@pytest.mark.parametrize("sample_object", WEAKREF_TYPES, indirect=True) +def test_usable_in_weak_value_dict(sample_object): """Object can be used as a WeakValueDictionary value.""" - obj = request.getfixturevalue(fixture_name) wvd = weakref.WeakValueDictionary() - wvd["key"] = obj - assert wvd["key"] is obj + wvd["key"] = sample_object + assert wvd["key"] is sample_object -@pytest.mark.parametrize("fixture_name", WEAK_KEY_TYPES) -def test_usable_in_weak_key_dict(fixture_name, request): +@pytest.mark.parametrize("sample_object", WEAK_KEY_TYPES, indirect=True) +def test_usable_in_weak_key_dict(sample_object): """Object can be used as a WeakKeyDictionary key.""" - obj = request.getfixturevalue(fixture_name) wkd = weakref.WeakKeyDictionary() - wkd[obj] = "value" - assert wkd[obj] == "value" + wkd[sample_object] = "value" + assert wkd[sample_object] == "value" -@pytest.mark.parametrize("fixture_name", WEAK_KEY_TYPES) -def test_usable_in_weak_set(fixture_name, request): +@pytest.mark.parametrize("sample_object", WEAK_KEY_TYPES, indirect=True) +def test_usable_in_weak_set(sample_object): """Object can be added to a WeakSet.""" - obj = request.getfixturevalue(fixture_name) ws = weakref.WeakSet() - ws.add(obj) - assert obj in ws + ws.add(sample_object) + assert sample_object in ws # ============================================================================= @@ -854,12 +857,10 @@ def test_usable_in_weak_set(fixture_name, request): # ============================================================================= -@pytest.mark.parametrize("fixture_name,pattern", REPR_PATTERNS) -def test_repr_format(fixture_name, pattern, request): +@pytest.mark.parametrize("sample_object,pattern", REPR_PATTERNS, indirect=["sample_object"]) +def test_repr_format(sample_object, pattern): """repr() returns a properly formatted string.""" - obj = request.getfixturevalue(fixture_name) - result = repr(obj) - assert re.fullmatch(pattern, result) + assert re.fullmatch(pattern, repr(sample_object)) # ============================================================================= @@ -868,10 +869,9 @@ def test_repr_format(fixture_name, pattern, request): @pytest.mark.parametrize("pickle_module", PICKLE_MODULES) -@pytest.mark.parametrize("fixture_name", PICKLE_TYPES) -def test_pickle_roundtrip(fixture_name, pickle_module, request): +@pytest.mark.parametrize("sample_object", PICKLE_TYPES, indirect=True) +def test_pickle_roundtrip(sample_object, pickle_module): """Object survives a pickle/cloudpickle roundtrip.""" mod = pytest.importorskip(pickle_module) - obj = request.getfixturevalue(fixture_name) - result = mod.loads(mod.dumps(obj)) - assert type(result) is type(obj) + result = mod.loads(mod.dumps(sample_object)) + assert type(result) is type(sample_object)