-
Notifications
You must be signed in to change notification settings - Fork 420
Description
Bug Description
I am trying to run an example in mujoco_warp. I am getting error related to tile_choleksy function. I tried to build warp from scratch with libmathdx and still keep getting this error.
Is their a way to debug this further?
Error log
libmathdx cuSOLVER error: 3 on /home/user/projects/warp/warp/native/warp.cu:3272
Module mujoco_warp._src.solver._update_gradient 83bb4cc load on device 'cuda:0' took 343.87 ms
Traceback (most recent call last):
File "/home/user/projects/warp/warp/codegen.py", line 1020, in build
adj.eval(adj.tree.body[0])
File "/home/user/projects/warp/warp/codegen.py", line 2777, in eval
return emit_node(adj, node)
^^^^^^^^^^^^^^^^^^^^
File "/home/user/projects/warp/warp/codegen.py", line 1674, in emit_FunctionDef
adj.eval(f)
File "/home/user/projects/warp/warp/codegen.py", line 2777, in eval
return emit_node(adj, node)
^^^^^^^^^^^^^^^^^^^^
File "/home/user/projects/warp/warp/codegen.py", line 2566, in emit_Assign
rhs = adj.eval(node.value)
^^^^^^^^^^^^^^^^^^^^
File "/home/user/projects/warp/warp/codegen.py", line 2777, in eval
return emit_node(adj, node)
^^^^^^^^^^^^^^^^^^^^
File "/home/user/projects/warp/warp/codegen.py", line 2298, in emit_Call
out = adj.add_call(func, args, kwargs, type_args, min_outputs=min_outputs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/user/projects/warp/warp/codegen.py", line 1409, in add_call
func_args, template_args, ltoirs, extra_shared_memory = func.lto_dispatch_func(
^^^^^^^^^^^^^^^^^^^^^^^
File "/home/user/projects/warp/warp/builtins.py", line 6605, in tile_cholesky_generic_lto_dispatch_func
lto_symbol, lto_code_data = warp.build.build_lto_solver(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/user/projects/warp/warp/build.py", line 423, in build_lto_solver
raise RuntimeError("Failed to compile tile_cholesky")
RuntimeError: Failed to compile tile_cholesky
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/user/projects/mujoco_warp/.venv/bin/mjwarp-testspeed", line 8, in
sys.exit(main())
^^^^^^
File "/home/user/projects/mujoco_warp/mujoco_warp/testspeed.py", line 207, in main
app.run(_main)
File "/home/user/projects/mujoco_warp/.venv/lib/python3.11/site-packages/absl/app.py", line 316, in run
_run_main(main, args)
File "/home/user/projects/mujoco_warp/.venv/lib/python3.11/site-packages/absl/app.py", line 261, in _run_main
sys.exit(main(argv))
^^^^^^^^^^
File "/home/user/projects/mujoco_warp/mujoco_warp/testspeed.py", line 131, in _main
jit_time, run_time, trace, ncon, nefc = mjwarp.benchmark(
^^^^^^^^^^^^^^^^^
File "/home/user/projects/mujoco_warp/mujoco_warp/_src/test_util.py", line 69, in benchmark
fn(m, d)
File "/home/user/projects/mujoco_warp/mujoco_warp/_src/warp_util.py", line 101, in wrapper
return fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^
File "/home/user/projects/mujoco_warp/mujoco_warp/_src/forward.py", line 684, in step
forward(m, d)
File "/home/user/projects/mujoco_warp/mujoco_warp/_src/warp_util.py", line 101, in wrapper
return fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^
File "/home/user/projects/mujoco_warp/mujoco_warp/_src/forward.py", line 678, in forward
solver.solve(m, d)
File "/home/user/projects/mujoco_warp/mujoco_warp/_src/warp_util.py", line 101, in wrapper
return fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^
File "/home/user/projects/mujoco_warp/mujoco_warp/_src/solver.py", line 1002, in solve
_create_context(m, d, grad=True)
File "/home/user/projects/mujoco_warp/mujoco_warp/_src/solver.py", line 69, in _create_context
_update_gradient(m, d)
File "/home/user/projects/mujoco_warp/mujoco_warp/_src/solver.py", line 296, in _update_gradient
wp.launch(_zero_grad_dot, dim=(d.nworld), inputs=[d])
File "/home/user/projects/warp/warp/context.py", line 5723, in launch
module_exec = kernel.module.load(device, block_dim)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/user/projects/warp/warp/context.py", line 2186, in load
builder = ModuleBuilder(self, builder_options, hasher=self.hashers[active_block_dim])
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/user/projects/warp/warp/context.py", line 1692, in init
self.build_kernel(kernel)
File "/home/user/projects/warp/warp/context.py", line 1721, in build_kernel
kernel.adj.build(self)
File "/home/user/projects/warp/warp/codegen.py", line 1031, in build
raise e
File "/home/user/projects/warp/warp/codegen.py", line 1020, in build
adj.eval(adj.tree.body[0])
File "/home/user/projects/warp/warp/codegen.py", line 2777, in eval
return emit_node(adj, node)
^^^^^^^^^^^^^^^^^^^^
File "/home/user/projects/warp/warp/codegen.py", line 1674, in emit_FunctionDef
adj.eval(f)
File "/home/user/projects/warp/warp/codegen.py", line 2777, in eval
return emit_node(adj, node)
^^^^^^^^^^^^^^^^^^^^
File "/home/user/projects/warp/warp/codegen.py", line 2566, in emit_Assign
rhs = adj.eval(node.value)
^^^^^^^^^^^^^^^^^^^^
File "/home/user/projects/warp/warp/codegen.py", line 2777, in eval
return emit_node(adj, node)
^^^^^^^^^^^^^^^^^^^^
File "/home/user/projects/warp/warp/codegen.py", line 2298, in emit_Call
out = adj.add_call(func, args, kwargs, type_args, min_outputs=min_outputs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/user/projects/warp/warp/codegen.py", line 1409, in add_call
func_args, template_args, ltoirs, extra_shared_memory = func.lto_dispatch_func(
^^^^^^^^^^^^^^^^^^^^^^^
File "/home/user/projects/warp/warp/builtins.py", line 6605, in tile_cholesky_generic_lto_dispatch_func
lto_symbol, lto_code_data = warp.build.build_lto_solver(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/user/projects/warp/warp/build.py", line 423, in build_lto_solver
raise RuntimeError("Failed to compile tile_cholesky")
RuntimeError: Error while parsing function "_cholesky" at /home/user/projects/mujoco_warp/mujoco_warp/_src/solver.py:290:
fact_tile = wp.tile_cholesky(mat_tile)
;Failed to compile tile_cholesky
System Information
OS: Ubuntu 22.04
CUDA: 12.8
Nvidia Driver: 560.35.05
python: 3.11.9
g++ - 11.4.0
Warp - 1.7.0-rc.4 (d7e9b9e641f7b2ad4c4056e9dfb30ad9f4c40c0a)
GPU: RTX 4500 Ada