Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
bdffa0b
TST: Mark tests as thread-unsafe or limit the number of threads
seberg Jun 10, 2026
8caf6e6
TST: use tmp_path fixture in cufile (and mark some as unsafe)
seberg Jun 10, 2026
2b6fe34
TST: Move graph definnitions inline and mark "global" ones as thread-…
seberg Jun 10, 2026
2232f01
TST: Fixup memory tests, mostly work around issue when tearing down m…
seberg Jun 10, 2026
ba564dd
TST: Thread unsafe markers for test_managed_ops
seberg Jun 10, 2026
8c335e8
Avoid interactive backend when using run_tests.sh locally
seberg Jun 10, 2026
6757a01
Use indirect fixtures for a nicer pattern and avoid thread issues
seberg Jun 10, 2026
a9a2116
Make latch-kernel helper compile only once
seberg Jun 11, 2026
ea7cb70
Limit threads for event test that otherwise seems to ptentially fail
seberg Jul 8, 2026
2f90d7a
TST: Mark device-persistence-mode test as thread-unsafe
seberg Jul 8, 2026
e90369f
TST: Test fails in CI, assume that few enough threads eventually pass...
seberg Jul 14, 2026
9f142e9
TST: Add another sync to guard against potential deadlocks (seems I m…
seberg Jul 14, 2026
d4192a4
TST: Limit threads for another LatchKernel test
seberg Jul 14, 2026
443921a
TST: Force test_helpers to single threaded on windows to avoid crash
seberg Jul 14, 2026
a98fc52
TST: Many buffer related tests cannot run threaded on windows
seberg Jul 15, 2026
d920526
These tests seem to test process global cleanup (not sure I follow)
seberg Jul 29, 2026
0bde899
TST: Mark LatchKernel tests as thread-unsafe
seberg Jul 29, 2026
ca4c746
Merge branch 'main' into ft-testing-test-fixes
seberg Jul 29, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cuda_bindings/tests/test_cufile.py
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ def ctx():
(err,) = cuda.cuCtxSetCurrent(ctx)
assert err == cuda.CUresult.CUDA_SUCCESS

yield
yield ctx

cuda.cuDevicePrimaryCtxRelease(device)

Expand Down
1 change: 1 addition & 0 deletions cuda_bindings/tests/test_examples.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ def test_example(example):

env = os.environ.copy()
env["CUDA_BINDINGS_SKIP_EXAMPLE"] = "100"
env["MPLBACKEND"] = "Agg" # avoid plt.show() from blocking

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I dunno if others get a pop-up for these tests, I did and with parallel testing, it might be a 100 windows :).


process = subprocess.run([sys.executable, example], capture_output=True, env=env) # noqa: S603
# returncode is a special value used in the examples to indicate that system requirements are not met.
Expand Down
107 changes: 63 additions & 44 deletions cuda_core/tests/graph/test_graph_definition.py
Original file line number Diff line number Diff line change
Expand Up @@ -575,20 +575,6 @@ def node_spec(request, init_cuda):
# =============================================================================


@pytest.fixture
def sample_graphdef(init_cuda):
"""A sample GraphDefinition for standalone tests."""
return GraphDefinition()


@pytest.fixture
def dot_file(tmp_path):
"""Temporary DOT file path, cleaned up after test."""
path = tmp_path / "graph.dot"
yield path
path.unlink(missing_ok=True)


# =============================================================================
# Topology tests (parameterized over graph specs)
# =============================================================================
Expand Down Expand Up @@ -791,14 +777,16 @@ def registered(node):
# =============================================================================


def test_graphdef_handle_valid(sample_graphdef):
def test_graphdef_handle_valid(init_cuda):
"""GraphDefinition has a valid non-null handle."""
sample_graphdef = GraphDefinition()

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Admittedly, we could also do the fixture hack, but somehow this felt just as well for now...

assert sample_graphdef.handle is not None
assert int(sample_graphdef.handle) != 0


def test_graphdef_entry_is_virtual(sample_graphdef):
def test_graphdef_entry_is_virtual(init_cuda):
"""Internal entry node is virtual (no pred/succ, type is None)."""
sample_graphdef = GraphDefinition()
entry = sample_graphdef._entry
assert isinstance(entry, GraphNode)
assert entry.pred == set()
Expand All @@ -811,26 +799,29 @@ def test_graphdef_entry_is_virtual(sample_graphdef):
# =============================================================================


def test_alloc_zero_size_fails(sample_graphdef):
def test_alloc_zero_size_fails(init_cuda):
"""Alloc with zero size raises error (CUDA limitation)."""
sample_graphdef = GraphDefinition()
_skip_if_no_mempool()
from cuda.core._utils.cuda_utils import CUDAError

with pytest.raises(CUDAError):
sample_graphdef.allocate(0)


def test_free_creates_dependency(sample_graphdef):
def test_free_creates_dependency(init_cuda):
"""Free node depends on its predecessor."""
sample_graphdef = GraphDefinition()
_skip_if_no_mempool()
with xfail_on_graph_mempool_oom():
alloc = sample_graphdef.allocate(ALLOC_SIZE)
free = alloc.deallocate(alloc.dptr)
assert alloc in free.pred


def test_alloc_free_chain(sample_graphdef):
def test_alloc_free_chain(init_cuda):
"""Alloc and free can be chained."""
sample_graphdef = GraphDefinition()
_skip_if_no_mempool()
with xfail_on_graph_mempool_oom():
a1 = sample_graphdef.allocate(ALLOC_SIZE)
Expand All @@ -847,8 +838,9 @@ def test_alloc_free_chain(sample_graphdef):
# =============================================================================


def test_alloc_memory_type_invalid(sample_graphdef):
def test_alloc_memory_type_invalid(init_cuda):
"""Invalid memory type raises ValueError."""
sample_graphdef = GraphDefinition()
with pytest.raises(ValueError, match="'invalid' is not a valid GraphMemoryType. Must be "):
sample_graphdef.allocate(ALLOC_SIZE, memory_type="invalid")

Expand All @@ -860,8 +852,9 @@ def test_alloc_memory_type_invalid(sample_graphdef):
pytest.param(lambda d: d, id="Device_object"),
],
)
def test_alloc_device_option(sample_graphdef, device_spec):
def test_alloc_device_option(init_cuda, device_spec):
"""Device can be specified as int or Device object."""
sample_graphdef = GraphDefinition()
_skip_if_no_mempool()
device = Device()
with xfail_on_graph_mempool_oom(device):
Expand All @@ -884,8 +877,9 @@ def test_alloc_peer_access(mempool_device_x2):


@pytest.mark.parametrize("num_branches", [2, 3, 5])
def test_join_merges_branches(sample_graphdef, num_branches):
def test_join_merges_branches(init_cuda, num_branches):
"""join() with multiple branches creates correct dependencies."""
sample_graphdef = GraphDefinition()
_skip_if_no_mempool()
with xfail_on_graph_mempool_oom():
branches = [sample_graphdef.allocate(ALLOC_SIZE) for _ in range(num_branches)]
Expand All @@ -899,17 +893,19 @@ def test_join_merges_branches(sample_graphdef, num_branches):
# =============================================================================


def test_launch_creates_node(sample_graphdef):
def test_launch_creates_node(init_cuda):
"""launch() creates a KernelNode."""
sample_graphdef = GraphDefinition()
mod = compile_common_kernels()
kernel = mod.get_kernel("empty_kernel")
config = LaunchConfig(grid=1, block=1)
node = sample_graphdef.launch(config, kernel)
assert isinstance(node, KernelNode)


def test_launch_chain_dependencies(sample_graphdef):
def test_launch_chain_dependencies(init_cuda):
"""Chained launches create correct dependencies."""
sample_graphdef = GraphDefinition()
mod = compile_common_kernels()
kernel = mod.get_kernel("empty_kernel")
config = LaunchConfig(grid=1, block=1)
Expand Down Expand Up @@ -971,15 +967,17 @@ def _instantiate_and_upload(graph_definition, kwargs, stream):


@pytest.mark.parametrize("inst_kwargs", _INSTANTIATE_ONLY_OPTIONS)
def test_instantiate_empty_graph(sample_graphdef, inst_kwargs):
def test_instantiate_empty_graph(init_cuda, inst_kwargs):
"""Empty graph can be instantiated."""
sample_graphdef = GraphDefinition()
graph = _instantiate(sample_graphdef, inst_kwargs)
assert graph is not None


@pytest.mark.parametrize("inst_kwargs", _INSTANTIATE_ONLY_OPTIONS)
def test_instantiate_with_nodes(sample_graphdef, inst_kwargs):
def test_instantiate_with_nodes(init_cuda, inst_kwargs):
"""Graph with nodes can be instantiated."""
sample_graphdef = GraphDefinition()
_skip_if_no_mempool()
with xfail_on_graph_mempool_oom():
sample_graphdef.allocate(ALLOC_SIZE)
Expand All @@ -989,8 +987,9 @@ def test_instantiate_with_nodes(sample_graphdef, inst_kwargs):


@pytest.mark.skipif(not Device(0).properties.unified_addressing, reason="requires unified addressing")
def test_instantiate_and_execute_kernel_device_launch(sample_graphdef):
def test_instantiate_and_execute_kernel_device_launch(init_cuda):
"""Kernel-only graph can be instantiated with device_launch flag."""
sample_graphdef = GraphDefinition()
mod = compile_common_kernels()
kernel = mod.get_kernel("empty_kernel")
config = LaunchConfig(grid=1, block=1)
Expand All @@ -1006,8 +1005,9 @@ def test_instantiate_and_execute_kernel_device_launch(sample_graphdef):


@pytest.mark.parametrize("inst_kwargs", _EXECUTE_OPTIONS)
def test_instantiate_and_execute_kernel(sample_graphdef, inst_kwargs):
def test_instantiate_and_execute_kernel(init_cuda, inst_kwargs):
"""Graph with kernel can be instantiated and executed."""
sample_graphdef = GraphDefinition()
mod = compile_common_kernels()
kernel = mod.get_kernel("empty_kernel")
config = LaunchConfig(grid=1, block=1)
Expand All @@ -1020,8 +1020,9 @@ def test_instantiate_and_execute_kernel(sample_graphdef, inst_kwargs):


@pytest.mark.parametrize("inst_kwargs", _EXECUTE_OPTIONS)
def test_instantiate_and_execute_alloc_free(sample_graphdef, inst_kwargs):
def test_instantiate_and_execute_alloc_free(init_cuda, inst_kwargs):
"""Graph with alloc/free can be executed."""
sample_graphdef = GraphDefinition()
_skip_if_no_mempool()
with xfail_on_graph_mempool_oom():
alloc = sample_graphdef.allocate(ALLOC_SIZE)
Expand All @@ -1034,8 +1035,9 @@ def test_instantiate_and_execute_alloc_free(sample_graphdef, inst_kwargs):


@pytest.mark.parametrize("inst_kwargs", _EXECUTE_OPTIONS)
def test_instantiate_and_execute_memset(sample_graphdef, inst_kwargs):
def test_instantiate_and_execute_memset(init_cuda, inst_kwargs):
"""Graph with alloc/memset/free can be executed."""
sample_graphdef = GraphDefinition()
_skip_if_no_mempool()
with xfail_on_graph_mempool_oom():
alloc = sample_graphdef.allocate(ALLOC_SIZE)
Expand All @@ -1049,8 +1051,9 @@ def test_instantiate_and_execute_memset(sample_graphdef, inst_kwargs):


@pytest.mark.parametrize("inst_kwargs", _EXECUTE_OPTIONS)
def test_instantiate_and_execute_memcpy(sample_graphdef, inst_kwargs):
def test_instantiate_and_execute_memcpy(init_cuda, inst_kwargs):
"""Graph with alloc/memset/memcpy/free can be executed and data is copied."""
sample_graphdef = GraphDefinition()
_skip_if_no_mempool()
import ctypes

Expand All @@ -1074,8 +1077,9 @@ def test_instantiate_and_execute_memcpy(sample_graphdef, inst_kwargs):
assert all(b == 0xAB for b in host_buf)


def test_instantiate_and_execute_child_graph(sample_graphdef):
def test_instantiate_and_execute_child_graph(init_cuda):
"""Graph with embedded child graph can be executed."""
sample_graphdef = GraphDefinition()
child = GraphDefinition()
mod = compile_common_kernels()
kernel = mod.get_kernel("empty_kernel")
Expand All @@ -1091,8 +1095,9 @@ def test_instantiate_and_execute_child_graph(sample_graphdef):
stream.sync()


def test_instantiate_and_execute_host_callback(sample_graphdef):
def test_instantiate_and_execute_host_callback(init_cuda):
"""Graph with host callback can be executed and callback is invoked."""
sample_graphdef = GraphDefinition()
results = []

def my_callback():
Expand All @@ -1109,8 +1114,9 @@ def my_callback():
assert results == [42]


def test_instantiate_and_execute_host_callback_cfunc(sample_graphdef):
def test_instantiate_and_execute_host_callback_cfunc(init_cuda):
"""Graph with ctypes function pointer callback can be executed."""
sample_graphdef = GraphDefinition()
import ctypes

CALLBACK = ctypes.CFUNCTYPE(None, ctypes.c_void_p)
Expand All @@ -1131,8 +1137,9 @@ def raw_fn(data):
assert called[0]


def test_host_callback_cfunc_with_user_data(sample_graphdef):
def test_host_callback_cfunc_with_user_data(init_cuda):
"""Host callback with bytes user_data passes data to C function."""
sample_graphdef = GraphDefinition()
import ctypes

CALLBACK = ctypes.CFUNCTYPE(None, ctypes.c_void_p)
Expand All @@ -1153,14 +1160,16 @@ def read_byte(data):
assert result[0] == 0xAB


def test_host_callback_user_data_rejected_for_python_callable(sample_graphdef):
def test_host_callback_user_data_rejected_for_python_callable(init_cuda):
"""user_data is rejected for Python callables."""
sample_graphdef = GraphDefinition()
with pytest.raises(ValueError, match="user_data is only supported"):
sample_graphdef.callback(lambda: None, user_data=b"hello")


def test_instantiate_and_execute_event_record_wait(sample_graphdef):
def test_instantiate_and_execute_event_record_wait(init_cuda):
"""Graph with event record and wait nodes can be executed."""
sample_graphdef = GraphDefinition()
event = Device().create_event()
rec = sample_graphdef.record(event)
rec.wait(event)
Expand All @@ -1182,8 +1191,9 @@ def _skip_unless_cc_90():
pytest.skip("Conditional node execution requires CC >= 9.0 (Hopper)")


def test_instantiate_and_execute_if_then(sample_graphdef):
def test_instantiate_and_execute_if_then(init_cuda):
"""If-conditional node: body executes only when condition is non-zero."""
sample_graphdef = GraphDefinition()
_skip_unless_cc_90()
_skip_if_no_mempool()
import ctypes
Expand Down Expand Up @@ -1215,8 +1225,9 @@ def test_instantiate_and_execute_if_then(sample_graphdef):
assert result[0] == 1


def test_instantiate_and_execute_if_else(sample_graphdef):
def test_instantiate_and_execute_if_else(init_cuda):
"""If-else node: then or else branch executes based on condition."""
sample_graphdef = GraphDefinition()
_skip_unless_cc_90()
_skip_if_no_mempool()
import ctypes
Expand Down Expand Up @@ -1250,8 +1261,9 @@ def test_instantiate_and_execute_if_else(sample_graphdef):
assert result[0] == 2


def test_instantiate_and_execute_switch(sample_graphdef):
def test_instantiate_and_execute_switch(init_cuda):
"""Switch node: selected branch executes based on condition value."""
sample_graphdef = GraphDefinition()
_skip_unless_cc_90()
_skip_if_no_mempool()
import ctypes
Expand Down Expand Up @@ -1284,8 +1296,9 @@ def test_instantiate_and_execute_switch(sample_graphdef):
assert result[0] == 1


def test_conditional_node_type_preserved_by_nodes(sample_graphdef):
def test_conditional_node_type_preserved_by_nodes(init_cuda):
"""Conditional nodes appear as ConditionalNode base when read back from graph."""
sample_graphdef = GraphDefinition()
condition = try_create_condition(sample_graphdef)
if_node = sample_graphdef.if_then(condition)
assert isinstance(if_node, IfNode)
Expand All @@ -1301,8 +1314,10 @@ def test_conditional_node_type_preserved_by_nodes(sample_graphdef):
# =============================================================================


def test_debug_dot_print_creates_file(sample_graphdef, dot_file):
def test_debug_dot_print_creates_file(init_cuda, tmp_path):
"""debug_dot_print writes a DOT file."""
sample_graphdef = GraphDefinition()
dot_file = tmp_path / "graph.dot"
_skip_if_no_mempool()
with xfail_on_graph_mempool_oom():
sample_graphdef.allocate(ALLOC_SIZE)
Expand All @@ -1312,8 +1327,10 @@ def test_debug_dot_print_creates_file(sample_graphdef, dot_file):
assert "digraph" in content


def test_debug_dot_print_with_options(sample_graphdef, dot_file):
def test_debug_dot_print_with_options(init_cuda, tmp_path):
"""debug_dot_print accepts GraphDebugPrintOptions."""
sample_graphdef = GraphDefinition()
dot_file = tmp_path / "graph.dot"
_skip_if_no_mempool()
with xfail_on_graph_mempool_oom():
sample_graphdef.allocate(ALLOC_SIZE)
Expand All @@ -1322,8 +1339,10 @@ def test_debug_dot_print_with_options(sample_graphdef, dot_file):
assert dot_file.exists()


def test_debug_dot_print_invalid_options(sample_graphdef, dot_file):
def test_debug_dot_print_invalid_options(init_cuda, tmp_path):
"""debug_dot_print rejects invalid options type."""
sample_graphdef = GraphDefinition()
dot_file = tmp_path / "graph.dot"
_skip_if_no_mempool()
with xfail_on_graph_mempool_oom():
sample_graphdef.allocate(ALLOC_SIZE)
Expand Down
2 changes: 2 additions & 0 deletions cuda_core/tests/graph/test_graph_definition_lifetime.py
Original file line number Diff line number Diff line change
Expand Up @@ -592,6 +592,7 @@ def test_event_survives_graph_clone_and_execution(init_cuda):
# =============================================================================


@pytest.mark.thread_unsafe(reason="asserts cleanup on main thread")
@pytest.mark.agent_authored(model="gpt-5.6")
def test_user_object_cleanup_is_coalesced_on_python_thread(init_cuda):
"""More than 32 CUDA callbacks drain through one main-thread pending call."""
Expand Down Expand Up @@ -1319,6 +1320,7 @@ def test_memcpy_buffer_survives_close(init_cuda):
assert list(out) == [0xCD] * 4


@pytest.mark.thread_unsafe(reason="deferred cleanup on main thread which would wait")

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There are three tests here that fail with _wait_until @Andy-Jost. I just skipped them, the reason seems to be that cleanup can never happen as long as the main thread is waiting in thread.join()?

I suspect that is a potential but very minor issue (you would think eventually this cleanup happens). But wanted to make a comment. I didn't try to understand what is going on here exactly!

@pytest.mark.agent_authored(model="claude-opus-4.8")
def test_memcpy_buffer_allocations_released_after_graph_destroyed(init_cuda):
"""Destroying the graph frees both memcpy operand allocations.
Expand Down
Loading
Loading