RuntimeError: tinycudann/bindings.cpp:81 check failed params.device().is_cuda() #358
Description
The program worked fine before, but an error was reported when I opened it today. After debugging, I felt that it was not a program error. I don't know whether the program could not be carried out because of my wrong operation about CUDA。
Epoch 0: : 0it [00:00, ?it/s]TCNN WARNING: input must be a CUDA tensor, but isn't. This indicates suboptimal performance.
Traceback (most recent call last):
File "launch.py", line 213, in
main(args, extras)
File "launch.py", line 163, in main
trainer.fit(system, datamodule=dm, ckpt_path=cfg.resume)
File "/media/sobey/work/chenys/yes/envs/tstudio/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 603, in fit
call._call_and_handle_interrupt(
File "/media/sobey/work/chenys/yes/envs/tstudio/lib/python3.8/site-packages/pytorch_lightning/trainer/call.py", line 38, in _call_and_handle_interrupt
return trainer_fn(*args, **kwargs)
File "/media/sobey/work/chenys/yes/envs/tstudio/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 645, in _fit_impl
self._run(model, ckpt_path=self.ckpt_path)
File "/media/sobey/work/chenys/yes/envs/tstudio/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1098, in _run
results = self._run_stage()
File "/media/sobey/work/chenys/yes/envs/tstudio/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1177, in _run_stage
self._run_train()
File "/media/sobey/work/chenys/yes/envs/tstudio/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1200, in _run_train
self.fit_loop.run()
File "/media/sobey/work/chenys/yes/envs/tstudio/lib/python3.8/site-packages/pytorch_lightning/loops/loop.py", line 199, in run
self.advance(*args, **kwargs)
File "/media/sobey/work/chenys/yes/envs/tstudio/lib/python3.8/site-packages/pytorch_lightning/loops/fit_loop.py", line 267, in advance
self._outputs = self.epoch_loop.run(self._data_fetcher)
File "/media/sobey/work/chenys/yes/envs/tstudio/lib/python3.8/site-packages/pytorch_lightning/loops/loop.py", line 199, in run
self.advance(*args, **kwargs)
File "/media/sobey/work/chenys/yes/envs/tstudio/lib/python3.8/site-packages/pytorch_lightning/loops/epoch/training_epoch_loop.py", line 205, in advance
response = self.trainer._call_lightning_module_hook("on_train_batch_start", batch, batch_idx)
File "/media/sobey/work/chenys/yes/envs/tstudio/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1342, in _call_lightning_module_hook
output = fn(*args, **kwargs)
File "/media/sobey/work/chenys/CV3D/threestudio1/threestudio/systems/base.py", line 152, in on_train_batch_start
self.do_update_step(self.true_current_epoch, self.true_global_step)
File "/media/sobey/work/chenys/CV3D/threestudio1/threestudio/utils/base.py", line 33, in do_update_step
module.do_update_step(
File "/media/sobey/work/chenys/CV3D/threestudio1/threestudio/utils/base.py", line 33, in do_update_step
module.do_update_step(
File "/media/sobey/work/chenys/CV3D/threestudio1/threestudio/utils/base.py", line 36, in do_update_step
self.update_step(epoch, global_step, on_load_weights=on_load_weights)
File "/media/sobey/work/chenys/CV3D/threestudio1/threestudio/models/renderers/nerf_volume_renderer.py", line 267, in update_step
self.estimator.update_every_n_steps(
File "/media/sobey/work/chenys/yes/envs/tstudio/lib/python3.8/site-packages/torch/autograd/grad_mode.py", line 27, in decorate_context
return func(*args, **kwargs)
File "/media/sobey/work/chenys/yes/envs/tstudio/lib/python3.8/site-packages/nerfacc/estimators/occ_grid.py", line 253, in update_every_n_steps
self._update(
File "/media/sobey/work/chenys/yes/envs/tstudio/lib/python3.8/site-packages/torch/autograd/grad_mode.py", line 27, in decorate_context
return func(*args, **kwargs)
File "/media/sobey/work/chenys/yes/envs/tstudio/lib/python3.8/site-packages/nerfacc/estimators/occ_grid.py", line 393, in _update
occ = occ_eval_fn(x).squeeze(-1)
File "/media/sobey/work/chenys/CV3D/threestudio1/threestudio/models/renderers/nerf_volume_renderer.py", line 262, in occ_eval_fn
density = self.geometry.forward_density(x)
File "/media/sobey/work/chenys/CV3D/threestudio1/threestudio/models/geometry/implicit_volume.py", line 200, in forward_density
self.encoding(points.reshape(-1, self.cfg.n_input_dims))
File "/media/sobey/work/chenys/yes/envs/tstudio/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1130, in _call_impl
return forward_call(*input, **kwargs)
File "/media/sobey/work/chenys/CV3D/threestudio1/threestudio/models/networks.py", line 124, in forward
self.encoding(x, *args)
File "/media/sobey/work/chenys/yes/envs/tstudio/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1130, in _call_impl
return forward_call(*input, **kwargs)
File "/media/sobey/work/chenys/CV3D/threestudio1/threestudio/models/networks.py", line 64, in forward
return self.encoding(x)
File "/media/sobey/work/chenys/yes/envs/tstudio/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1130, in _call_impl
return forward_call(*input, **kwargs)
File "/media/sobey/work/chenys/yes/envs/tstudio/lib/python3.8/site-packages/tinycudann-1.6-py3.8-linux-x86_64.egg/tinycudann/modules.py", line 118, in forward
output = _module_function.apply(
File "/media/sobey/work/chenys/yes/envs/tstudio/lib/python3.8/site-packages/tinycudann-1.6-py3.8-linux-x86_64.egg/tinycudann/modules.py", line 30, in forward
native_ctx, output = native_tcnn_module.fwd(input, params)
RuntimeError: test/tiny-cuda-nn/bindings/torch/tinycudann/bindings.cpp:81 check failed params.device().is_cuda()