Merge pull request #2174 from asi1024/fix-deterministic-mode

Avoid using Tensor Core with cuDNN deterministic mode in convolution backward
cupy · May 10, 2019 · 666e0ae · 666e0ae
2 parents 380d919 + e2d0f82
commit 666e0ae
Showing 1 changed file with 10 additions and 2 deletions.
diff --git a/cupy/cudnn.pyx b/cupy/cudnn.pyx
@@ -1652,7 +1652,10 @@ def convolution_backward_filter(
         zero = <size_t>&float_zero
         one = <size_t>&float_one
 
-    cdef bint use_tensor_core = _should_use_tensor_core(tensor_core, x.dtype)
+    # Disable use_tensor_core in deterministic mode because
+    # CUDNN_CONVOLUTION_BWD_FILTER_ALGO_1 does not use Tensor Core.
+    cdef bint use_tensor_core = (
+        not deterministic and _should_use_tensor_core(tensor_core, x.dtype))
     cdef tuple conv_param = (pad, stride, x.dtype, use_tensor_core)
 
     handle = get_handle()
@@ -1678,6 +1681,7 @@ def convolution_backward_filter(
             cudnn.CUDNN_CROSS_CORRELATION, use_tensor_core)
 
         if deterministic:
+            # TODO(imanishi): Support Tensor Core in deterministic mode.
             algo = cudnn.CUDNN_CONVOLUTION_BWD_FILTER_ALGO_1
             workspace_size = cudnn.getConvolutionBackwardFilterWorkspaceSize(
                 handle, x_desc, gy_desc, conv_desc, filter_desc, algo)
@@ -1732,7 +1736,10 @@ def convolution_backward_data(
         zero = <size_t>&float_zero
         one = <size_t>&float_one
 
-    cdef bint use_tensor_core = _should_use_tensor_core(tensor_core, x.dtype)
+    # Disable use_tensor_core in deterministic mode because
+    # CUDNN_CONVOLUTION_BWD_DATA_ALGO_1 does not use Tensor Core.
+    cdef bint use_tensor_core = (
+        not deterministic and _should_use_tensor_core(tensor_core, x.dtype))
     cdef tuple conv_param = (pad, stride, x.dtype, use_tensor_core)
 
     # cuDNN 7 supports dilation only in *_FWD_ALGO_IMPLICIT_GEMM, but
@@ -1768,6 +1775,7 @@ def convolution_backward_data(
             cudnn.CUDNN_CROSS_CORRELATION, use_tensor_core)
 
         if deterministic:
+            # TODO(imanishi): Support Tensor Core in deterministic mode.
             algo = cudnn.CUDNN_CONVOLUTION_BWD_DATA_ALGO_1
             workspace_size = cudnn.getConvolutionBackwardDataWorkspaceSize(
                 handle, filter_desc, x_desc, conv_desc, y_desc, algo)