cupy_backends.cuda.api.driver.CUDADriverError #3541
Description
Mark the Blossom error for future reference:
2021-12-22T14:40:04.5597947Z Traceback (most recent call last):
2021-12-22T14:40:04.5599020Z File "/opt/conda/lib/python3.8/site-packages/parameterized/parameterized.py", line 533, in standalone_func
2021-12-22T14:40:04.5599815Z return func(*(a + p.args), **p.kwargs)
2021-12-22T14:40:04.5600883Z File "/home/jenkins/agent/workspace/MONAI-premerge/monai/tests/test_cucim_transform.py", line 79, in test_tramsforms_numpy_single
2021-12-22T14:40:04.5601701Z output = CuCIM(**params)(input)
2021-12-22T14:40:04.5602662Z File "/home/jenkins/agent/workspace/MONAI-premerge/monai/monai/transforms/utility/array.py", line 1216, in __call__
2021-12-22T14:40:04.5603525Z return self.transform(data, *self.args, **self.kwargs)
2021-12-22T14:40:04.5604556Z File "/opt/conda/lib/python3.8/site-packages/cucim/core/operations/intensity/zoom.py", line 153, in zoom
2021-12-22T14:40:04.5605353Z kernel = CUDA_KERNELS.get_function("zoom_out_kernel")
2021-12-22T14:40:04.5606092Z File "cupy/_core/raw.pyx", line 485, in cupy._core.raw.RawModule.get_function
2021-12-22T14:40:04.5606984Z File "cupy/_core/raw.pyx", line 96, in cupy._core.raw.RawKernel.kernel.__get__
2021-12-22T14:40:04.5607845Z File "cupy/_core/raw.pyx", line 113, in cupy._core.raw.RawKernel._kernel
2021-12-22T14:40:04.5608663Z File "cupy/_util.pyx", line 59, in cupy._util.memoize.decorator.ret
2021-12-22T14:40:04.5609428Z File "cupy/_core/raw.pyx", line 547, in cupy._core.raw._get_raw_module
2021-12-22T14:40:04.5610153Z File "cupy/_core/core.pyx", line 1961, in cupy._core.core.compile_with_cache
2021-12-22T14:40:04.5610923Z File "cupy/_core/core.pyx", line 2030, in cupy._core.core.compile_with_cache
2021-12-22T14:40:04.5612189Z File "/opt/conda/lib/python3.8/site-packages/cupy/cuda/compiler.py", line 434, in compile_with_cache
2021-12-22T14:40:04.5612955Z return _compile_with_cache_cuda(
2021-12-22T14:40:04.5613953Z File "/opt/conda/lib/python3.8/site-packages/cupy/cuda/compiler.py", line 502, in _compile_with_cache_cuda
2021-12-22T14:40:04.5614666Z mod.load(cubin)
2021-12-22T14:40:04.5615438Z File "cupy/cuda/function.pyx", line 241, in cupy.cuda.function.Module.load
2021-12-22T14:40:04.5616490Z File "cupy/cuda/function.pyx", line 243, in cupy.cuda.function.Module.load
2021-12-22T14:40:04.5617757Z File "cupy_backends/cuda/api/driver.pyx", line 253, in cupy_backends.cuda.api.driver.moduleLoadData
2021-12-22T14:40:04.5619107Z File "cupy_backends/cuda/api/driver.pyx", line 125, in cupy_backends.cuda.api.driver.check_status
2021-12-22T14:40:04.5620761Z cupy_backends.cuda.api.driver.CUDADriverError: CUDA_ERROR_UNSUPPORTED_PTX_VERSION: the provided PTX was compiled with an unsupported toolchain.
Thanks.
Originally posted by @Nic-Ma in #3532 (comment)
I had another look at this, it seems 21.12 docker container is not compatible with the NVTX utilities.
nightly test error logs:
======================================================================
ERROR: test_wrapper_tranforms_0 (tests.test_nvtx_decorator.TestNVTXRangeDecorator)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/opt/monai/monai/transforms/transform.py", line 82, in apply_transform
return _apply_transform(transform, data, unpack_items)
File "/opt/monai/monai/transforms/transform.py", line 53, in _apply_transform
return transform(parameters)
File "/opt/monai/monai/utils/nvtx.py", line 114, in range_wrapper
output = _temp_func(*args, **kwargs)
File "/opt/monai/monai/transforms/utility/array.py", line 1258, in __call__
return super().__call__(data)
File "/opt/monai/monai/transforms/utility/array.py", line 1216, in __call__
return self.transform(data, *self.args, **self.kwargs)
File "/opt/conda/lib/python3.8/site-packages/cucim/core/operations/spatial/rotate_and_flip.py", line 212, in rand_image_rotate_90
return image_rotate_90(img, _rand_k, spatial_axis)
File "/opt/conda/lib/python3.8/site-packages/cucim/core/operations/spatial/rotate_and_flip.py", line 118, in image_rotate_90
cupy_img = cupy.ascontiguousarray(img)
File "/opt/conda/lib/python3.8/site-packages/cupy/_creation/from_data.py", line 97, in ascontiguousarray
return _core.ascontiguousarray(a, dtype)
File "cupy/_core/core.pyx", line 2442, in cupy._core.core.ascontiguousarray
File "cupy/_core/core.pyx", line 2459, in cupy._core.core.ascontiguousarray
File "cupy/_core/_kernel.pyx", line 1185, in cupy._core._kernel.ufunc.__call__
File "cupy/_core/_kernel.pyx", line 1212, in cupy._core._kernel.ufunc._get_ufunc_kernel
File "cupy/_core/_kernel.pyx", line 952, in cupy._core._kernel._get_ufunc_kernel
File "cupy/_core/_kernel.pyx", line 62, in cupy._core._kernel._get_simple_elementwise_kernel
File "cupy/_core/core.pyx", line 2030, in cupy._core.core.compile_with_cache
File "/opt/conda/lib/python3.8/site-packages/cupy/cuda/compiler.py", line 434, in compile_with_cache
return _compile_with_cache_cuda(
File "/opt/conda/lib/python3.8/site-packages/cupy/cuda/compiler.py", line 555, in _compile_with_cache_cuda
mod.load(cubin)
File "cupy/cuda/function.pyx", line 241, in cupy.cuda.function.Module.load
File "cupy/cuda/function.pyx", line 243, in cupy.cuda.function.Module.load
File "cupy_backends/cuda/api/driver.pyx", line 253, in cupy_backends.cuda.api.driver.moduleLoadData
File "cupy_backends/cuda/api/driver.pyx", line 125, in cupy_backends.cuda.api.driver.check_status
cupy_backends.cuda.api.driver.CUDADriverError: CUDA_ERROR_UNSUPPORTED_PTX_VERSION: the provided PTX was compiled with an unsupported toolchain.
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/conda/lib/python3.8/site-packages/parameterized/parameterized.py", line 533, in standalone_func
return func(*(a + p.args), **p.kwargs)
File "/opt/monai/tests/test_nvtx_decorator.py", line 122, in test_wrapper_tranforms
output = transforms(input)
File "/opt/monai/monai/transforms/compose.py", line 160, in __call__
input_ = apply_transform(_transform, input_, self.map_items, self.unpack_items)
File "/opt/monai/monai/transforms/transform.py", line 105, in apply_transform
_log_stats(data=data)
File "/opt/monai/monai/transforms/transform.py", line 99, in _log_stats
datastats(img=data, data_value=True, prefix=prefix)
File "/opt/monai/monai/transforms/utility/array.py", line 626, in __call__
lines.append(f"Value: {img}")
File "cupy/_core/core.pyx", line 1544, in cupy._core.core.ndarray.__format__
File "cupy/_core/core.pyx", line 1623, in cupy._core.core.ndarray.get
File "cupy/_core/core.pyx", line 1625, in cupy._core.core.ndarray.get
File "cupy/_core/core.pyx", line 2404, in cupy._core.core._internal_ascontiguousarray
File "cupy/_core/_kernel.pyx", line 1185, in cupy._core._kernel.ufunc.__call__
File "cupy/_core/_kernel.pyx", line 1212, in cupy._core._kernel.ufunc._get_ufunc_kernel
File "cupy/_core/_kernel.pyx", line 952, in cupy._core._kernel._get_ufunc_kernel
File "cupy/_core/_kernel.pyx", line 62, in cupy._core._kernel._get_simple_elementwise_kernel
File "cupy/_core/core.pyx", line 2030, in cupy._core.core.compile_with_cache
File "/opt/conda/lib/python3.8/site-packages/cupy/cuda/compiler.py", line 434, in compile_with_cache
return _compile_with_cache_cuda(
File "/opt/conda/lib/python3.8/site-packages/cupy/cuda/compiler.py", line 502, in _compile_with_cache_cuda
mod.load(cubin)
File "cupy/cuda/function.pyx", line 241, in cupy.cuda.function.Module.load
File "cupy/cuda/function.pyx", line 243, in cupy.cuda.function.Module.load
File "cupy_backends/cuda/api/driver.pyx", line 253, in cupy_backends.cuda.api.driver.moduleLoadData
File "cupy_backends/cuda/api/driver.pyx", line 125, in cupy_backends.cuda.api.driver.check_status
cupy_backends.cuda.api.driver.CUDADriverError: CUDA_ERROR_UNSUPPORTED_PTX_VERSION: the provided PTX was compiled with an unsupported toolchain.
======================================================================
ERROR: test_tramsforms_cupy_batch_1 (tests.test_rand_cucim_dict_transform.TestRandCuCIMDict)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/opt/conda/lib/python3.8/site-packages/parameterized/parameterized.py", line 533, in standalone_func
return func(*(a + p.args), **p.kwargs)
File "/opt/monai/tests/test_rand_cucim_dict_transform.py", line 176, in test_tramsforms_cupy_batch
cp.testing.assert_allclose(output, expected)
File "/opt/conda/lib/python3.8/site-packages/cupy/testing/_array.py", line 25, in assert_allclose
cupy.asnumpy(actual), cupy.asnumpy(desired),
File "/opt/conda/lib/python3.8/site-packages/cupy/__init__.py", line 773, in asnumpy
return a.get(stream=stream, order=order)
File "cupy/_core/core.pyx", line 1567, in cupy._core.core.ndarray.get
File "cupy/_core/core.pyx", line 1623, in cupy._core.core.ndarray.get
File "cupy/_core/core.pyx", line 1625, in cupy._core.core.ndarray.get
File "cupy/_core/core.pyx", line 2404, in cupy._core.core._internal_ascontiguousarray
File "cupy/_core/_kernel.pyx", line 1185, in cupy._core._kernel.ufunc.__call__
File "cupy/_core/_kernel.pyx", line 1212, in cupy._core._kernel.ufunc._get_ufunc_kernel
File "cupy/_core/_kernel.pyx", line 952, in cupy._core._kernel._get_ufunc_kernel
File "cupy/_core/_kernel.pyx", line 62, in cupy._core._kernel._get_simple_elementwise_kernel
File "cupy/_core/core.pyx", line 2030, in cupy._core.core.compile_with_cache
File "/opt/conda/lib/python3.8/site-packages/cupy/cuda/compiler.py", line 434, in compile_with_cache
return _compile_with_cache_cuda(
File "/opt/conda/lib/python3.8/site-packages/cupy/cuda/compiler.py", line 555, in _compile_with_cache_cuda
mod.load(cubin)
File "cupy/cuda/function.pyx", line 241, in cupy.cuda.function.Module.load
File "cupy/cuda/function.pyx", line 243, in cupy.cuda.function.Module.load
File "cupy_backends/cuda/api/driver.pyx", line 253, in cupy_backends.cuda.api.driver.moduleLoadData
File "cupy_backends/cuda/api/driver.pyx", line 125, in cupy_backends.cuda.api.driver.check_status
cupy_backends.cuda.api.driver.CUDADriverError: CUDA_ERROR_UNSUPPORTED_PTX_VERSION: the provided PTX was compiled with an unsupported toolchain.
======================================================================
ERROR: test_tramsforms_cupy_batch_2 (tests.test_rand_cucim_dict_transform.TestRandCuCIMDict)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/opt/conda/lib/python3.8/site-packages/parameterized/parameterized.py", line 533, in standalone_func
return func(*(a + p.args), **p.kwargs)
File "/opt/monai/tests/test_rand_cucim_dict_transform.py", line 176, in test_tramsforms_cupy_batch
cp.testing.assert_allclose(output, expected)
File "/opt/conda/lib/python3.8/site-packages/cupy/testing/_array.py", line 25, in assert_allclose
cupy.asnumpy(actual), cupy.asnumpy(desired),
File "/opt/conda/lib/python3.8/site-packages/cupy/__init__.py", line 773, in asnumpy
return a.get(stream=stream, order=order)
File "cupy/_core/core.pyx", line 1567, in cupy._core.core.ndarray.get
File "cupy/_core/core.pyx", line 1623, in cupy._core.core.ndarray.get
File "cupy/_core/core.pyx", line 1625, in cupy._core.core.ndarray.get
File "cupy/_core/core.pyx", line 2404, in cupy._core.core._internal_ascontiguousarray
File "cupy/_core/_kernel.pyx", line 1185, in cupy._core._kernel.ufunc.__call__
File "cupy/_core/_kernel.pyx", line 1212, in cupy._core._kernel.ufunc._get_ufunc_kernel
File "cupy/_core/_kernel.pyx", line 952, in cupy._core._kernel._get_ufunc_kernel
File "cupy/_core/_kernel.pyx", line 62, in cupy._core._kernel._get_simple_elementwise_kernel
File "cupy/_core/core.pyx", line 2030, in cupy._core.core.compile_with_cache
File "/opt/conda/lib/python3.8/site-packages/cupy/cuda/compiler.py", line 434, in compile_with_cache
return _compile_with_cache_cuda(
File "/opt/conda/lib/python3.8/site-packages/cupy/cuda/compiler.py", line 555, in _compile_with_cache_cuda
mod.load(cubin)
File "cupy/cuda/function.pyx", line 241, in cupy.cuda.function.Module.load
File "cupy/cuda/function.pyx", line 243, in cupy.cuda.function.Module.load
File "cupy_backends/cuda/api/driver.pyx", line 253, in cupy_backends.cuda.api.driver.moduleLoadData
File "cupy_backends/cuda/api/driver.pyx", line 125, in cupy_backends.cuda.api.driver.check_status
cupy_backends.cuda.api.driver.CUDADriverError: CUDA_ERROR_UNSUPPORTED_PTX_VERSION: the provided PTX was compiled with an unsupported toolchain.
======================================================================
ERROR: test_tramsforms_cupy_batch_4 (tests.test_rand_cucim_dict_transform.TestRandCuCIMDict)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/opt/conda/lib/python3.8/site-packages/parameterized/parameterized.py", line 533, in standalone_func
return func(*(a + p.args), **p.kwargs)
File "/opt/monai/tests/test_rand_cucim_dict_transform.py", line 173, in test_tramsforms_cupy_batch
output = RandCuCIMd(keys="image", **params)(input)["image"]
File "/opt/monai/monai/transforms/utility/dictionary.py", line 1593, in __call__
return super().__call__(data)
File "/opt/monai/monai/transforms/utility/dictionary.py", line 1548, in __call__
d[key] = self.trans(d[key])
File "/opt/monai/monai/transforms/utility/array.py", line 1216, in __call__
return self.transform(data, *self.args, **self.kwargs)
File "/opt/conda/lib/python3.8/site-packages/cucim/core/operations/intensity/scaling.py", line 98, in scale_intensity_range
scale = CUDA_KERNELS.get_function("scaleVolume")
File "cupy/_core/raw.pyx", line 485, in cupy._core.raw.RawModule.get_function
File "cupy/_core/raw.pyx", line 96, in cupy._core.raw.RawKernel.kernel.__get__
File "cupy/_core/raw.pyx", line 113, in cupy._core.raw.RawKernel._kernel
File "cupy/_util.pyx", line 59, in cupy._util.memoize.decorator.ret
File "cupy/_core/raw.pyx", line 547, in cupy._core.raw._get_raw_module
File "cupy/_core/core.pyx", line 1961, in cupy._core.core.compile_with_cache
File "cupy/_core/core.pyx", line 2030, in cupy._core.core.compile_with_cache
File "/opt/conda/lib/python3.8/site-packages/cupy/cuda/compiler.py", line 434, in compile_with_cache
return _compile_with_cache_cuda(
File "/opt/conda/lib/python3.8/site-packages/cupy/cuda/compiler.py", line 555, in _compile_with_cache_cuda
mod.load(cubin)
File "cupy/cuda/function.pyx", line 241, in cupy.cuda.function.Module.load
File "cupy/cuda/function.pyx", line 243, in cupy.cuda.function.Module.load
File "cupy_backends/cuda/api/driver.pyx", line 253, in cupy_backends.cuda.api.driver.moduleLoadData
File "cupy_backends/cuda/api/driver.pyx", line 125, in cupy_backends.cuda.api.driver.check_status
cupy_backends.cuda.api.driver.CUDADriverError: CUDA_ERROR_UNSUPPORTED_PTX_VERSION: the provided PTX was compiled with an unsupported toolchain.
======================================================================
ERROR: test_tramsforms_cupy_batch_5 (tests.test_rand_cucim_dict_transform.TestRandCuCIMDict)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/opt/conda/lib/python3.8/site-packages/parameterized/parameterized.py", line 533, in standalone_func
return func(*(a + p.args), **p.kwargs)
File "/opt/monai/tests/test_rand_cucim_dict_transform.py", line 173, in test_tramsforms_cupy_batch
output = RandCuCIMd(keys="image", **params)(input)["image"]
File "/opt/monai/monai/transforms/utility/dictionary.py", line 1593, in __call__
return super().__call__(data)
File "/opt/monai/monai/transforms/utility/dictionary.py", line 1548, in __call__
d[key] = self.trans(d[key])
File "/opt/monai/monai/transforms/utility/array.py", line 1216, in __call__
return self.transform(data, *self.args, **self.kwargs)
File "/opt/conda/lib/python3.8/site-packages/cucim/core/operations/intensity/zoom.py", line 153, in zoom
kernel = CUDA_KERNELS.get_function("zoom_out_kernel")
File "cupy/_core/raw.pyx", line 485, in cupy._core.raw.RawModule.get_function
File "cupy/_core/raw.pyx", line 96, in cupy._core.raw.RawKernel.kernel.__get__
File "cupy/_core/raw.pyx", line 113, in cupy._core.raw.RawKernel._kernel
File "cupy/_util.pyx", line 59, in cupy._util.memoize.decorator.ret
File "cupy/_core/raw.pyx", line 547, in cupy._core.raw._get_raw_module
File "cupy/_core/core.pyx", line 1961, in cupy._core.core.compile_with_cache
File "cupy/_core/core.pyx", line 2030, in cupy._core.core.compile_with_cache
File "/opt/conda/lib/python3.8/site-packages/cupy/cuda/compiler.py", line 434, in compile_with_cache
return _compile_with_cache_cuda(
File "/opt/conda/lib/python3.8/site-packages/cupy/cuda/compiler.py", line 502, in _compile_with_cache_cuda
mod.load(cubin)
File "cupy/cuda/function.pyx", line 241, in cupy.cuda.function.Module.load
File "cupy/cuda/function.pyx", line 243, in cupy.cuda.function.Module.load
File "cupy_backends/cuda/api/driver.pyx", line 253, in cupy_backends.cuda.api.driver.moduleLoadData
File "cupy_backends/cuda/api/driver.pyx", line 125, in cupy_backends.cuda.api.driver.check_status
cupy_backends.cuda.api.driver.CUDADriverError: CUDA_ERROR_UNSUPPORTED_PTX_VERSION: the provided PTX was compiled with an unsupported toolchain.
======================================================================
ERROR: test_tramsforms_cupy_batch_6 (tests.test_rand_cucim_dict_transform.TestRandCuCIMDict)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/opt/conda/lib/python3.8/site-packages/parameterized/parameterized.py", line 533, in standalone_func
return func(*(a + p.args), **p.kwargs)
File "/opt/monai/tests/test_rand_cucim_dict_transform.py", line 173, in test_tramsforms_cupy_batch
output = RandCuCIMd(keys="image", **params)(input)["image"]
File "/opt/monai/monai/transforms/utility/dictionary.py", line 1593, in __call__
return super().__call__(data)
File "/opt/monai/monai/transforms/utility/dictionary.py", line 1548, in __call__
d[key] = self.trans(d[key])
File "/opt/monai/monai/transforms/utility/array.py", line 1216, in __call__
return self.transform(data, *self.args, **self.kwargs)
File "/opt/conda/lib/python3.8/site-packages/cucim/core/operations/intensity/zoom.py", line 253, in rand_zoom
return zoom(img, zoom_factor)
File "/opt/conda/lib/python3.8/site-packages/cucim/core/operations/intensity/zoom.py", line 153, in zoom
kernel = CUDA_KERNELS.get_function("zoom_out_kernel")
File "cupy/_core/raw.pyx", line 485, in cupy._core.raw.RawModule.get_function
File "cupy/_core/raw.pyx", line 96, in cupy._core.raw.RawKernel.kernel.__get__
File "cupy/_core/raw.pyx", line 113, in cupy._core.raw.RawKernel._kernel
File "cupy/_util.pyx", line 59, in cupy._util.memoize.decorator.ret
File "cupy/_core/raw.pyx", line 547, in cupy._core.raw._get_raw_module
File "cupy/_core/core.pyx", line 1961, in cupy._core.core.compile_with_cache
File "cupy/_core/core.pyx", line 2030, in cupy._core.core.compile_with_cache
File "/opt/conda/lib/python3.8/site-packages/cupy/cuda/compiler.py", line 434, in compile_with_cache
return _compile_with_cache_cuda(
File "/opt/conda/lib/python3.8/site-packages/cupy/cuda/compiler.py", line 502, in _compile_with_cache_cuda
mod.load(cubin)
File "cupy/cuda/function.pyx", line 241, in cupy.cuda.function.Module.load
File "cupy/cuda/function.pyx", line 243, in cupy.cuda.function.Module.load
File "cupy_backends/cuda/api/driver.pyx", line 253, in cupy_backends.cuda.api.driver.moduleLoadData
File "cupy_backends/cuda/api/driver.pyx", line 125, in cupy_backends.cuda.api.driver.check_status
cupy_backends.cuda.api.driver.CUDADriverError: CUDA_ERROR_UNSUPPORTED_PTX_VERSION: the provided PTX was compiled with an unsupported toolchain.
======================================================================
ERROR: test_tramsforms_cupy_single_1 (tests.test_rand_cucim_dict_transform.TestRandCuCIMDict)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/opt/conda/lib/python3.8/site-packages/parameterized/parameterized.py", line 533, in standalone_func
return func(*(a + p.args), **p.kwargs)
File "/opt/monai/tests/test_rand_cucim_dict_transform.py", line 150, in test_tramsforms_cupy_single
cp.testing.assert_allclose(output, expected)
File "/opt/conda/lib/python3.8/site-packages/cupy/testing/_array.py", line 25, in assert_allclose
cupy.asnumpy(actual), cupy.asnumpy(desired),
File "/opt/conda/lib/python3.8/site-packages/cupy/__init__.py", line 773, in asnumpy
return a.get(stream=stream, order=order)
File "cupy/_core/core.pyx", line 1567, in cupy._core.core.ndarray.get
File "cupy/_core/core.pyx", line 1623, in cupy._core.core.ndarray.get
File "cupy/_core/core.pyx", line 1625, in cupy._core.core.ndarray.get
File "cupy/_core/core.pyx", line 2404, in cupy._core.core._internal_ascontiguousarray
File "cupy/_core/_kernel.pyx", line 1185, in cupy._core._kernel.ufunc.__call__
File "cupy/_core/_kernel.pyx", line 1212, in cupy._core._kernel.ufunc._get_ufunc_kernel
File "cupy/_core/_kernel.pyx", line 952, in cupy._core._kernel._get_ufunc_kernel
File "cupy/_core/_kernel.pyx", line 62, in cupy._core._kernel._get_simple_elementwise_kernel
File "cupy/_core/core.pyx", line 2030, in cupy._core.core.compile_with_cache
File "/opt/conda/lib/python3.8/site-packages/cupy/cuda/compiler.py", line 434, in compile_with_cache
return _compile_with_cache_cuda(
File "/opt/conda/lib/python3.8/site-packages/cupy/cuda/compiler.py", line 502, in _compile_with_cache_cuda
mod.load(cubin)
File "cupy/cuda/function.pyx", line 241, in cupy.cuda.function.Module.load
File "cupy/cuda/function.pyx", line 243, in cupy.cuda.function.Module.load
File "cupy_backends/cuda/api/driver.pyx", line 253, in cupy_backends.cuda.api.driver.moduleLoadData
File "cupy_backends/cuda/api/driver.pyx", line 125, in cupy_backends.cuda.api.driver.check_status
cupy_backends.cuda.api.driver.CUDADriverError: CUDA_ERROR_UNSUPPORTED_PTX_VERSION: the provided PTX was compiled with an unsupported toolchain.
======================================================================
ERROR: test_tramsforms_cupy_single_2 (tests.test_rand_cucim_dict_transform.TestRandCuCIMDict)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/opt/conda/lib/python3.8/site-packages/parameterized/parameterized.py", line 533, in standalone_func
return func(*(a + p.args), **p.kwargs)
File "/opt/monai/tests/test_rand_cucim_dict_transform.py", line 150, in test_tramsforms_cupy_single
cp.testing.assert_allclose(output, expected)
File "/opt/conda/lib/python3.8/site-packages/cupy/testing/_array.py", line 25, in assert_allclose
cupy.asnumpy(actual), cupy.asnumpy(desired),
File "/opt/conda/lib/python3.8/site-packages/cupy/__init__.py", line 773, in asnumpy
return a.get(stream=stream, order=order)
File "cupy/_core/core.pyx", line 1567, in cupy._core.core.ndarray.get
File "cupy/_core/core.pyx", line 1623, in cupy._core.core.ndarray.get
File "cupy/_core/core.pyx", line 1625, in cupy._core.core.ndarray.get
File "cupy/_core/core.pyx", line 2404, in cupy._core.core._internal_ascontiguousarray
File "cupy/_core/_kernel.pyx", line 1185, in cupy._core._kernel.ufunc.__call__
File "cupy/_core/_kernel.pyx", line 1212, in cupy._core._kernel.ufunc._get_ufunc_kernel
File "cupy/_core/_kernel.pyx", line 952, in cupy._core._kernel._get_ufunc_kernel
File "cupy/_core/_kernel.pyx", line 62, in cupy._core._kernel._get_simple_elementwise_kernel
File "cupy/_core/core.pyx", line 2030, in cupy._core.core.compile_with_cache
File "/opt/conda/lib/python3.8/site-packages/cupy/cuda/compiler.py", line 434, in compile_with_cache
return _compile_with_cache_cuda(
File "/opt/conda/lib/python3.8/site-packages/cupy/cuda/compiler.py", line 502, in _compile_with_cache_cuda
mod.load(cubin)
File "cupy/cuda/function.pyx", line 241, in cupy.cuda.function.Module.load
File "cupy/cuda/function.pyx", line 243, in cupy.cuda.function.Module.load
File "cupy_backends/cuda/api/driver.pyx", line 253, in cupy_backends.cuda.api.driver.moduleLoadData
File "cupy_backends/cuda/api/driver.pyx", line 125, in cupy_backends.cuda.api.driver.check_status
cupy_backends.cuda.api.driver.CUDADriverError: CUDA_ERROR_UNSUPPORTED_PTX_VERSION: the provided PTX was compiled with an unsupported toolchain.
======================================================================
ERROR: test_tramsforms_cupy_single_4 (tests.test_rand_cucim_dict_transform.TestRandCuCIMDict)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/opt/conda/lib/python3.8/site-packages/parameterized/parameterized.py", line 533, in standalone_func
return func(*(a + p.args), **p.kwargs)
File "/opt/monai/tests/test_rand_cucim_dict_transform.py", line 147, in test_tramsforms_cupy_single
output = RandCuCIMd(keys="image", apply_prob=1.0, **params)(input)["image"]
File "/opt/monai/monai/transforms/utility/dictionary.py", line 1593, in __call__
return super().__call__(data)
File "/opt/monai/monai/transforms/utility/dictionary.py", line 1548, in __call__
d[key] = self.trans(d[key])
File "/opt/monai/monai/transforms/utility/array.py", line 1216, in __call__
return self.transform(data, *self.args, **self.kwargs)
File "/opt/conda/lib/python3.8/site-packages/cucim/core/operations/intensity/scaling.py", line 98, in scale_intensity_range
scale = CUDA_KERNELS.get_function("scaleVolume")
File "cupy/_core/raw.pyx", line 485, in cupy._core.raw.RawModule.get_function
File "cupy/_core/raw.pyx", line 96, in cupy._core.raw.RawKernel.kernel.__get__
File "cupy/_core/raw.pyx", line 113, in cupy._core.raw.RawKernel._kernel
File "cupy/_util.pyx", line 59, in cupy._util.memoize.decorator.ret
File "cupy/_core/raw.pyx", line 547, in cupy._core.raw._get_raw_module
File "cupy/_core/core.pyx", line 1961, in cupy._core.core.compile_with_cache
File "cupy/_core/core.pyx", line 2030, in cupy._core.core.compile_with_cache
File "/opt/conda/lib/python3.8/site-packages/cupy/cuda/compiler.py", line 434, in compile_with_cache
return _compile_with_cache_cuda(
File "/opt/conda/lib/python3.8/site-packages/cupy/cuda/compiler.py", line 502, in _compile_with_cache_cuda
mod.load(cubin)
File "cupy/cuda/function.pyx", line 241, in cupy.cuda.function.Module.load
File "cupy/cuda/function.pyx", line 243, in cupy.cuda.function.Module.load
File "cupy_backends/cuda/api/driver.pyx", line 253, in cupy_backends.cuda.api.driver.moduleLoadData
File "cupy_backends/cuda/api/driver.pyx", line 125, in cupy_backends.cuda.api.driver.check_status
cupy_backends.cuda.api.driver.CUDADriverError: CUDA_ERROR_UNSUPPORTED_PTX_VERSION: the provided PTX was compiled with an unsupported toolchain.
======================================================================
ERROR: test_tramsforms_cupy_single_5 (tests.test_rand_cucim_dict_transform.TestRandCuCIMDict)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/opt/conda/lib/python3.8/site-packages/parameterized/parameterized.py", line 533, in standalone_func
return func(*(a + p.args), **p.kwargs)
File "/opt/monai/tests/test_rand_cucim_dict_transform.py", line 147, in test_tramsforms_cupy_single
output = RandCuCIMd(keys="image", apply_prob=1.0, **params)(input)["image"]
File "/opt/monai/monai/transforms/utility/dictionary.py", line 1593, in __call__
return super().__call__(data)
File "/opt/monai/monai/transforms/utility/dictionary.py", line 1548, in __call__
d[key] = self.trans(d[key])
File "/opt/monai/monai/transforms/utility/array.py", line 1216, in __call__
return self.transform(data, *self.args, **self.kwargs)
File "/opt/conda/lib/python3.8/site-packages/cucim/core/operations/intensity/zoom.py", line 153, in zoom
kernel = CUDA_KERNELS.get_function("zoom_out_kernel")
File "cupy/_core/raw.pyx", line 485, in cupy._core.raw.RawModule.get_function
File "cupy/_core/raw.pyx", line 96, in cupy._core.raw.RawKernel.kernel.__get__
File "cupy/_core/raw.pyx", line 113, in cupy._core.raw.RawKernel._kernel
File "cupy/_util.pyx", line 59, in cupy._util.memoize.decorator.ret
File "cupy/_core/raw.pyx", line 547, in cupy._core.raw._get_raw_module
File "cupy/_core/core.pyx", line 1961, in cupy._core.core.compile_with_cache
File "cupy/_core/core.pyx", line 2030, in cupy._core.core.compile_with_cache
File "/opt/conda/lib/python3.8/site-packages/cupy/cuda/compiler.py", line 434, in compile_with_cache
return _compile_with_cache_cuda(
File "/opt/conda/lib/python3.8/site-packages/cupy/cuda/compiler.py", line 502, in _compile_with_cache_cuda
mod.load(cubin)
File "cupy/cuda/function.pyx", line 241, in cupy.cuda.function.Module.load
File "cupy/cuda/function.pyx", line 243, in cupy.cuda.function.Module.load
File "cupy_backends/cuda/api/driver.pyx", line 253, in cupy_backends.cuda.api.driver.moduleLoadData
File "cupy_backends/cuda/api/driver.pyx", line 125, in cupy_backends.cuda.api.driver.check_status
cupy_backends.cuda.api.driver.CUDADriverError: CUDA_ERROR_UNSUPPORTED_PTX_VERSION: the provided PTX was compiled with an unsupported toolchain.
======================================================================
ERROR: test_tramsforms_cupy_single_6 (tests.test_rand_cucim_dict_transform.TestRandCuCIMDict)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/opt/conda/lib/python3.8/site-packages/parameterized/parameterized.py", line 533, in standalone_func
return func(*(a + p.args), **p.kwargs)
File "/opt/monai/tests/test_rand_cucim_dict_transform.py", line 147, in test_tramsforms_cupy_single
output = RandCuCIMd(keys="image", apply_prob=1.0, **params)(input)["image"]
File "/opt/monai/monai/transforms/utility/dictionary.py", line 1593, in __call__
return super().__call__(data)
File "/opt/monai/monai/transforms/utility/dictionary.py", line 1548, in __call__
d[key] = self.trans(d[key])
File "/opt/monai/monai/transforms/utility/array.py", line 1216, in __call__
return self.transform(data, *self.args, **self.kwargs)
File "/opt/conda/lib/python3.8/site-packages/cucim/core/operations/intensity/zoom.py", line 253, in rand_zoom
return zoom(img, zoom_factor)
File "/opt/conda/lib/python3.8/site-packages/cucim/core/operations/intensity/zoom.py", line 153, in zoom
kernel = CUDA_KERNELS.get_function("zoom_out_kernel")
File "cupy/_core/raw.pyx", line 485, in cupy._core.raw.RawModule.get_function
File "cupy/_core/raw.pyx", line 96, in cupy._core.raw.RawKernel.kernel.__get__
File "cupy/_core/raw.pyx", line 113, in cupy._core.raw.RawKernel._kernel
File "cupy/_util.pyx", line 59, in cupy._util.memoize.decorator.ret
File "cupy/_core/raw.pyx", line 547, in cupy._core.raw._get_raw_module
File "cupy/_core/core.pyx", line 1961, in cupy._core.core.compile_with_cache
File "cupy/_core/core.pyx", line 2030, in cupy._core.core.compile_with_cache
File "/opt/conda/lib/python3.8/site-packages/cupy/cuda/compiler.py", line 434, in compile_with_cache
return _compile_with_cache_cuda(
File "/opt/conda/lib/python3.8/site-packages/cupy/cuda/compiler.py", line 502, in _compile_with_cache_cuda
mod.load(cubin)
File "cupy/cuda/function.pyx", line 241, in cupy.cuda.function.Module.load
File "cupy/cuda/function.pyx", line 243, in cupy.cuda.function.Module.load
File "cupy_backends/cuda/api/driver.pyx", line 253, in cupy_backends.cuda.api.driver.moduleLoadData
File "cupy_backends/cuda/api/driver.pyx", line 125, in cupy_backends.cuda.api.driver.check_status
cupy_backends.cuda.api.driver.CUDADriverError: CUDA_ERROR_UNSUPPORTED_PTX_VERSION: the provided PTX was compiled with an unsupported toolchain.
======================================================================
ERROR: test_tramsforms_numpy_batch_1 (tests.test_rand_cucim_dict_transform.TestRandCuCIMDict)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/opt/conda/lib/python3.8/site-packages/parameterized/parameterized.py", line 533, in standalone_func
return func(*(a + p.args), **p.kwargs)
File "/opt/monai/tests/test_rand_cucim_dict_transform.py", line 121, in test_tramsforms_numpy_batch
output = RandCuCIMd(keys="image", apply_prob=1.0, **params)(input)["image"]
File "/opt/monai/monai/transforms/utility/dictionary.py", line 1593, in __call__
return super().__call__(data)
File "/opt/monai/monai/transforms/utility/dictionary.py", line 1548, in __call__
d[key] = self.trans(d[key])
File "/opt/monai/monai/transforms/utility/array.py", line 1216, in __call__
return self.transform(data, *self.args, **self.kwargs)
File "/opt/conda/lib/python3.8/site-packages/cucim/core/operations/spatial/rotate_and_flip.py", line 68, in image_flip
result = cupy.asnumpy(result)
File "/opt/conda/lib/python3.8/site-packages/cupy/__init__.py", line 773, in asnumpy
return a.get(stream=stream, order=order)
File "cupy/_core/core.pyx", line 1567, in cupy._core.core.ndarray.get
File "cupy/_core/core.pyx", line 1623, in cupy._core.core.ndarray.get
File "cupy/_core/core.pyx", line 1625, in cupy._core.core.ndarray.get
File "cupy/_core/core.pyx", line 2404, in cupy._core.core._internal_ascontiguousarray
File "cupy/_core/_kernel.pyx", line 1185, in cupy._core._kernel.ufunc.__call__
File "cupy/_core/_kernel.pyx", line 1212, in cupy._core._kernel.ufunc._get_ufunc_kernel
File "cupy/_core/_kernel.pyx", line 952, in cupy._core._kernel._get_ufunc_kernel
File "cupy/_core/_kernel.pyx", line 62, in cupy._core._kernel._get_simple_elementwise_kernel
File "cupy/_core/core.pyx", line 2030, in cupy._core.core.compile_with_cache
File "/opt/conda/lib/python3.8/site-packages/cupy/cuda/compiler.py", line 434, in compile_with_cache
return _compile_with_cache_cuda(
File "/opt/conda/lib/python3.8/site-packages/cupy/cuda/compiler.py", line 502, in _compile_with_cache_cuda
mod.load(cubin)
File "cupy/cuda/function.pyx", line 241, in cupy.cuda.function.Module.load
File "cupy/cuda/function.pyx", line 243, in cupy.cuda.function.Module.load
File "cupy_backends/cuda/api/driver.pyx", line 253, in cupy_backends.cuda.api.driver.moduleLoadData
File "cupy_backends/cuda/api/driver.pyx", line 125, in cupy_backends.cuda.api.driver.check_status
cupy_backends.cuda.api.driver.CUDADriverError: CUDA_ERROR_UNSUPPORTED_PTX_VERSION: the provided PTX was compiled with an unsupported toolchain.