Help me!!!Thanks you very much!!!About "THCBlas.cu:334"

[lurenjia384]

🐛 Describe the bug

Traceback (most recent call last):
File "/home/ubuntu/miniconda3/envs/ycc_peract/lib/python3.8/site-packages/torch/multiprocessing/spawn.py", line 19, in _wrap
fn(i, *args)
File "/home/ubuntu/data0/ycc/peract_4_30/peract/run_seed_fn.py", line 158, in run_seed
train_runner.start()
File "/home/ubuntu/data0/ycc/peract/YARR/yarr/runners/offline_train_runner.py", line 161, in start
loss = self._step(i, batches.pop(0), batches) #传入数据2 batches未来四帧
File "/home/ubuntu/data0/ycc/peract/YARR/yarr/runners/offline_train_runner.py", line 93, in _step
update_dict = self._agent.update(i, sampled_batch, batches) #无处理直接将数据传入该函数3 batches 为未来四帧，共五帧
File "/home/ubuntu/data0/ycc/peract_4_30/peract/helpers/preprocess_agent.py", line 42, in update
return self._pose_agent.update(step, replay_sample, batches_n) #进一步传入数据4 batches_n 为未来四帧，共五帧
File "/home/ubuntu/data0/ycc/peract_4_30/peract/agents/peract_bc/qattention_stack_agent.py", line 39, in update
update_dict = qa.update(step, replay_sample, batches) #数据5 batches 为未来四帧，共五帧
File "/home/ubuntu/data0/ycc/peract_4_30/peract/agents/peract_bc/qattention_peract_bc_agent.py", line 520, in update
voxel_grid ,numbb = self._q(obs,
File "/home/ubuntu/miniconda3/envs/ycc_peract/lib/python3.8/site-packages/torch/nn/modules/module.py", line 727, in _call_impl
result = self.forward(*input, **kwargs)
File "/home/ubuntu/data0/ycc/peract_4_30/peract/agents/peract_bc/qattention_peract_bc_agent.py", line 144, in forward
q_ignore_collisions, numb = self._qnet(voxel_grid,
File "/home/ubuntu/miniconda3/envs/ycc_peract/lib/python3.8/site-packages/torch/nn/modules/module.py", line 727, in _call_impl
result = self.forward(*input, **kwargs)
File "/home/ubuntu/miniconda3/envs/ycc_peract/lib/python3.8/site-packages/torch/nn/parallel/distributed.py", line 619, in forward
output = self.module(*inputs[0], **kwargs[0])
File "/home/ubuntu/miniconda3/envs/ycc_peract/lib/python3.8/site-packages/torch/nn/modules/module.py", line 727, in _call_impl
result = self.forward(*input, **kwargs)
File "/home/ubuntu/data0/ycc/peract_4_30/peract/agents/peract_bc/perceiver_lang_io.py", line 890, in forward
tens2=self.mamba_model1(tens1)
File "/home/ubuntu/miniconda3/envs/ycc_peract/lib/python3.8/site-packages/torch/nn/modules/module.py", line 727, in _call_impl
result = self.forward(*input, **kwargs)
File "/home/ubuntu/data0/ycc/peract_4_30/peract/agents/peract_bc/perceiver_lang_io.py", line 208, in forward
x = layer(x)
File "/home/ubuntu/miniconda3/envs/ycc_peract/lib/python3.8/site-packages/torch/nn/modules/module.py", line 727, in _call_impl
result = self.forward(*input, **kwargs)
File "/home/ubuntu/data0/ycc/peract_4_30/peract/agents/peract_bc/perceiver_lang_io.py", line 399, in forward
output = self.mixer(self.norm(x)) + x
File "/home/ubuntu/miniconda3/envs/ycc_peract/lib/python3.8/site-packages/torch/nn/modules/module.py", line 727, in _call_impl
result = self.forward(*input, **kwargs)
File "/home/ubuntu/data0/ycc/peract_4_30/peract/agents/peract_bc/perceiver_lang_io.py", line 459, in forward
y = self.ssm(x)
File "/home/ubuntu/data0/ycc/peract_4_30/peract/agents/peract_bc/perceiver_lang_io.py", line 498, in ssm
y = self.selective_scan(x, delta, A, B, C, D) # This is similar to run_SSM(A, B, C, u) in The Annotated S4 [2]
File "/home/ubuntu/data0/ycc/peract_4_30/peract/agents/peract_bc/perceiver_lang_io.py", line 547, in selective_scan
y = einsum(x, C[:, i, :], 'b d_in n, b n -> b d_in')
File "/home/ubuntu/miniconda3/envs/ycc_peract/lib/python3.8/site-packages/einops/einops.py", line 901, in einsum
return get_backend(tensors[0]).einsum(pattern, *tensors)
File "/home/ubuntu/miniconda3/envs/ycc_peract/lib/python3.8/site-packages/einops/_backends.py", line 287, in einsum
return self.torch.einsum(pattern, *x)
File "/home/ubuntu/miniconda3/envs/ycc_peract/lib/python3.8/site-packages/torch/functional.py", line 344, in einsum
return _VF.einsum(equation, operands) # type: ignore
RuntimeError: cublas runtime error : the GPU program failed to execute at /pytorch/aten/src/THC/THCBlas.cu:334

Versions

Traceback (most recent call last):
File "/home/ubuntu/miniconda3/envs/ycc_peract/lib/python3.8/site-packages/torch/multiprocessing/spawn.py", line 19, in _wrap
fn(i, *args)
File "/home/ubuntu/data0/ycc/peract_4_30/peract/run_seed_fn.py", line 158, in run_seed
train_runner.start()
File "/home/ubuntu/data0/ycc/peract/YARR/yarr/runners/offline_train_runner.py", line 161, in start
loss = self._step(i, batches.pop(0), batches) #传入数据2 batches未来四帧
File "/home/ubuntu/data0/ycc/peract/YARR/yarr/runners/offline_train_runner.py", line 93, in _step
update_dict = self._agent.update(i, sampled_batch, batches) #无处理直接将数据传入该函数3 batches 为未来四帧，共五帧
File "/home/ubuntu/data0/ycc/peract_4_30/peract/helpers/preprocess_agent.py", line 42, in update
return self._pose_agent.update(step, replay_sample, batches_n) #进一步传入数据4 batches_n 为未来四帧，共五帧
File "/home/ubuntu/data0/ycc/peract_4_30/peract/agents/peract_bc/qattention_stack_agent.py", line 39, in update
update_dict = qa.update(step, replay_sample, batches) #数据5 batches 为未来四帧，共五帧
File "/home/ubuntu/data0/ycc/peract_4_30/peract/agents/peract_bc/qattention_peract_bc_agent.py", line 520, in update
voxel_grid ,numbb = self._q(obs,
File "/home/ubuntu/miniconda3/envs/ycc_peract/lib/python3.8/site-packages/torch/nn/modules/module.py", line 727, in _call_impl
result = self.forward(*input, **kwargs)
File "/home/ubuntu/data0/ycc/peract_4_30/peract/agents/peract_bc/qattention_peract_bc_agent.py", line 144, in forward
q_ignore_collisions, numb = self._qnet(voxel_grid,
File "/home/ubuntu/miniconda3/envs/ycc_peract/lib/python3.8/site-packages/torch/nn/modules/module.py", line 727, in _call_impl
result = self.forward(*input, **kwargs)
File "/home/ubuntu/miniconda3/envs/ycc_peract/lib/python3.8/site-packages/torch/nn/parallel/distributed.py", line 619, in forward
output = self.module(*inputs[0], **kwargs[0])
File "/home/ubuntu/miniconda3/envs/ycc_peract/lib/python3.8/site-packages/torch/nn/modules/module.py", line 727, in _call_impl
result = self.forward(*input, **kwargs)
File "/home/ubuntu/data0/ycc/peract_4_30/peract/agents/peract_bc/perceiver_lang_io.py", line 890, in forward
tens2=self.mamba_model1(tens1)
File "/home/ubuntu/miniconda3/envs/ycc_peract/lib/python3.8/site-packages/torch/nn/modules/module.py", line 727, in _call_impl
result = self.forward(*input, **kwargs)
File "/home/ubuntu/data0/ycc/peract_4_30/peract/agents/peract_bc/perceiver_lang_io.py", line 208, in forward
x = layer(x)
File "/home/ubuntu/miniconda3/envs/ycc_peract/lib/python3.8/site-packages/torch/nn/modules/module.py", line 727, in _call_impl
result = self.forward(*input, **kwargs)
File "/home/ubuntu/data0/ycc/peract_4_30/peract/agents/peract_bc/perceiver_lang_io.py", line 399, in forward
output = self.mixer(self.norm(x)) + x
File "/home/ubuntu/miniconda3/envs/ycc_peract/lib/python3.8/site-packages/torch/nn/modules/module.py", line 727, in _call_impl
result = self.forward(*input, **kwargs)
File "/home/ubuntu/data0/ycc/peract_4_30/peract/agents/peract_bc/perceiver_lang_io.py", line 459, in forward
y = self.ssm(x)
File "/home/ubuntu/data0/ycc/peract_4_30/peract/agents/peract_bc/perceiver_lang_io.py", line 498, in ssm
y = self.selective_scan(x, delta, A, B, C, D) # This is similar to run_SSM(A, B, C, u) in The Annotated S4 [2]
File "/home/ubuntu/data0/ycc/peract_4_30/peract/agents/peract_bc/perceiver_lang_io.py", line 547, in selective_scan
y = einsum(x, C[:, i, :], 'b d_in n, b n -> b d_in')
File "/home/ubuntu/miniconda3/envs/ycc_peract/lib/python3.8/site-packages/einops/einops.py", line 901, in einsum
return get_backend(tensors[0]).einsum(pattern, *tensors)
File "/home/ubuntu/miniconda3/envs/ycc_peract/lib/python3.8/site-packages/einops/_backends.py", line 287, in einsum
return self.torch.einsum(pattern, *x)
File "/home/ubuntu/miniconda3/envs/ycc_peract/lib/python3.8/site-packages/torch/functional.py", line 344, in einsum
return _VF.einsum(equation, operands) # type: ignore
RuntimeError: cublas runtime error : the GPU program failed to execute at /pytorch/aten/src/THC/THCBlas.cu:334

Tasks

Beta Give feedback

1. NVIDIA RTX A6000 CUDA Version: 12.0 print(torch.version) 1.7.1+cu110

[albanD]

Hey!
Can you please open a new issue and follow the template to provide all the right details about your environment so we can help you effectively.
Thanks!