CMake CUDA fixes + NCCL (#9672)

* CMake CUDA fixes + NCCL

* 9.1

* default nccl to off

* Change default cuda version selection logic

* revert "${__condition}" change

* no operator tuning for msvc

* ..

* NO LAPACK FOR MSVC

* set default project languages at top

* Add cmake toolset for Unix Makefiles

CMakeLists.txt

@@ -1,43 +1,25 @@
 cmake_minimum_required(VERSION 3.0.2)
-message(STATUS "CMake version '${CMAKE_VERSION}' using generator '${CMAKE_GENERATOR}'")
-if(((${CMAKE_GENERATOR} MATCHES "Visual Studio.*") OR (${CMAKE_GENERATOR} MATCHES "Xcode.*"))
-        AND ((${CMAKE_VERSION} VERSION_GREATER "3.9.0") OR (${CMAKE_VERSION} VERSION_EQUAL "3.9.0")))
-  # Toolsets are only supported for Visual Studio and Xcode
-  set(FIRST_CUDA TRUE)
-else()
-  set(FIRST_CUDA FALSE)
-endif()
-include(cmake/Utils.cmake)
 
-#Some things have order. This must be put in front alone
-mxnet_option(USE_CUDA             "Build with CUDA support"   ON)
-mxnet_option(USE_OLDCMAKECUDA           "Build with old cmake cuda" OFF)
-if(USE_CUDA)
-  add_definitions(-DMSHADOW_USE_CUDA=1)
-  IF(FIRST_CUDA AND (NOT USE_OLDCMAKECUDA))
-    set(__cuda_toolset "7.5" "8.0" "9.0")
-    set(CUDA_TOOLSET "8.0" CACHE STRING "Select CUDA Version.")
-    set_property( CACHE CUDA_TOOLSET PROPERTY STRINGS "" ${__cuda_toolset} )
-    set(CMAKE_GENERATOR_TOOLSET "cuda=${CUDA_TOOLSET},host=x64")
-    project(mxnet C CXX CUDA)
-  else()
-    project(mxnet C CXX)
-    set(FIRST_CUDA FALSE)
-  endif()
-else()
-  project(mxnet C CXX)
-  add_definitions(-DMSHADOW_USE_CUDA=0)
+project(mxnet C CXX)
+
+if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/build/private/local_config.cmake)
+  include(${CMAKE_CURRENT_SOURCE_DIR}/build/private/local_config.cmake)
 endif()
 
+include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/Utils.cmake)
 
+#Some things have order. This must be put in front alone
+mxnet_option(USE_CUDA             "Build with CUDA support"   ON)
+mxnet_option(USE_OLDCMAKECUDA     "Build with old cmake cuda" OFF)
+mxnet_option(USE_NCCL             "Use NVidia NCCL with CUDA" OFF)
 mxnet_option(USE_OPENCV           "Build with OpenCV support" ON)
 mxnet_option(USE_OPENMP           "Build with Openmp support" ON)
 mxnet_option(USE_CUDNN            "Build with cudnn support"  ON) # one could set CUDNN_ROOT for search path
 mxnet_option(USE_LAPACK           "Build with lapack support" ON IF NOT MSVC)
 mxnet_option(USE_MKL_IF_AVAILABLE "Use MKL if found" ON)
 mxnet_option(USE_MKLML_MKL        "Use MKLML variant of MKL (if MKL found)" ON IF USE_MKL_IF_AVAILABLE AND UNIX AND (NOT APPLE))
 mxnet_option(USE_MKL_EXPERIMENTAL "Use experimental MKL (if MKL enabled and found)" OFF)
-mxnet_option(USE_OPERATOR_TUNING  "Enable auto-tuning of operators" ON AND NOT MSVC)
+mxnet_option(USE_OPERATOR_TUNING  "Enable auto-tuning of operators" ON IF NOT MSVC)
 mxnet_option(USE_GPERFTOOLS       "Build with GPerfTools support (if found)" ON)
 mxnet_option(USE_JEMALLOC         "Build with Jemalloc support"   ON)
 mxnet_option(USE_PROFILER         "Build with Profiler support"   ON)
@@ -47,30 +29,49 @@ mxnet_option(USE_PLUGIN_CAFFE     "Use Caffe Plugin" OFF)
 mxnet_option(USE_CPP_PACKAGE      "Build C++ Package" OFF)
 mxnet_option(USE_MXNET_LIB_NAMING "Use MXNet library naming conventions." ON)
 mxnet_option(USE_GPROF            "Compile with gprof (profiling) flag" OFF)
+mxnet_option(USE_CXX14_IF_AVAILABLE "Build with C++14 if the compiler supports it" OFF)
 mxnet_option(USE_VTUNE            "Enable use of Intel Amplifier XE (VTune)" OFF) # one could set VTUNE_ROOT for search path
 mxnet_option(ENABLE_CUDA_RTC      "Build with CUDA runtime compilation support" ON)
 mxnet_option(INSTALL_EXAMPLES     "Install the example source files." OFF)
 mxnet_option(USE_SIGNAL_HANDLER   "Print stack traces on segfaults." OFF)
 
-
-
-if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/build/private/local_config.cmake)
-  include(${CMAKE_CURRENT_SOURCE_DIR}/build/private/local_config.cmake)
+if(USE_CUDA AND NOT USE_OLDCMAKECUDA)
+  message(STATUS "CMake version '${CMAKE_VERSION}' using generator '${CMAKE_GENERATOR}'")
+  if(
+      (
+        (${CMAKE_GENERATOR} MATCHES "Visual Studio.*")
+        OR (${CMAKE_GENERATOR} MATCHES "Xcode.*")
+        OR (${CMAKE_GENERATOR} STREQUAL "Unix Makefiles")
+      ) AND (
+        (${CMAKE_VERSION} VERSION_GREATER "3.9.0") OR (${CMAKE_VERSION} VERSION_EQUAL "3.9.0")
+      )
+    )
+    set(FIRST_CUDA TRUE)
+    project(mxnet C CXX CUDA)
+    message(ERROR " foo")
+  else()
+    set(FIRST_CUDA FALSE)
+    set(USE_OLDCMAKECUDA TRUE)
+    project(mxnet C CXX)
+  endif()
+else()
+  project(mxnet C CXX)
 endif()
 
+
 if(MSVC)
   set(SYSTEM_ARCHITECTURE x86_64)
 else()
-  EXECUTE_PROCESS( COMMAND uname -m COMMAND tr -d '\n' OUTPUT_VARIABLE SYSTEM_ARCHITECTURE)
+  execute_process(COMMAND uname -m COMMAND tr -d '\n' OUTPUT_VARIABLE SYSTEM_ARCHITECTURE)
 endif()
 
-set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake/Modules;${CMAKE_MODULE_PATH}")
+set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules;${CMAKE_MODULE_PATH}")
 
 SET(EXTRA_OPERATORS "" CACHE PATH "EXTRA OPERATORS PATH")
 
 if("$ENV{VERBOSE}" STREQUAL "1")
   message(STATUS " Verbose Makefile ACTIVATED")
-  set(CMAKE_VERBOISE_MAKEFILE ON)
+  set(CMAKE_VERBOSE_MAKEFILE ON)
 endif()
 
 
@@ -83,10 +84,14 @@ if(MSVC)
   add_definitions(-DMXNET_EXPORTS)
   add_definitions(-DNNVM_EXPORTS)
   add_definitions(-DDMLC_STRICT_CXX11)
+  add_definitions(-DNOMINMAX)
   set(CMAKE_C_FLAGS "/MP")
   set(CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} /bigobj")
 else(MSVC)
   include(CheckCXXCompilerFlag)
+  if(USE_CXX14_IF_AVAILABLE)
+    check_cxx_compiler_flag("-std=c++14"   SUPPORT_CXX14)
+  endif()
   check_cxx_compiler_flag("-std=c++11"   SUPPORT_CXX11)
   check_cxx_compiler_flag("-std=c++0x"   SUPPORT_CXX0X)
   check_cxx_compiler_flag("-msse2"       SUPPORT_MSSE2)
@@ -107,7 +112,11 @@ else(MSVC)
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -msse2")
   endif()
   set(CMAKE_CXX_FLAGS ${CMAKE_C_FLAGS})
-  if(SUPPORT_CXX11)
+  if(SUPPORT_CXX14)
+    add_definitions(-DDMLC_USE_CXX11=1)
+    add_definitions(-DDMLC_USE_CXX14=1)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14")
+  elseif(SUPPORT_CXX11)
     add_definitions(-DDMLC_USE_CXX11=1)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
   elseif(SUPPORT_CXX0X)
@@ -120,8 +129,8 @@ set(mxnet_LINKER_LIBS "")
 
 if(USE_GPROF)
   message(STATUS "Using GPROF")
-  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -pg")
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -pg")
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-omit-frame-pointer -g -pg")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer -g -pg")
   set(CMAKE_LINK_LIBRARY_FILE_FLAG "${CMAKE_LINK_LIBRARY_FILE_FLAG} -g -pg")
 endif()
 
@@ -170,12 +179,43 @@ endif()
 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include)
 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/src)
 
-if(FIRST_CUDA)
+if(USE_CUDA)
+  find_package(CUDA REQUIRED)
+  add_definitions(-DMSHADOW_USE_CUDA=1)
+  if(FIRST_CUDA AND (NOT USE_OLDCMAKECUDA))
+    if(CUDA_TOOLSET STREQUAL "")
+      set(CUDA_TOOLSET "${CUDA_VERSION_STRING}")
+    endif()
+    set(CMAKE_GENERATOR_TOOLSET "cuda=${CUDA_TOOLSET},host=x64")
+  else()
+    set(FIRST_CUDA FALSE)
+  endif()
+  if(USE_NCCL)
+    find_package(NCCL)
+    if(NCCL_FOUND)
+      include_directories(${NCCL_INCLUDE_DIRS})
+      list(APPEND mxnet_LINKER_LIBS ${NCCL_LIBRARIES})
+    else()
+      message(WARNING "Could not find NCCL libraries")
+    endif()
+  endif()
+else()
+  add_definitions(-DMSHADOW_USE_CUDA=0)
+endif()
+
+if(NCCL_FOUND)
+  add_definitions(-DMXNET_USE_NCCL=1)
+else()
+  add_definitions(-DMXNET_USE_NCCL=0)
+endif()
+
+if(USE_CUDA AND FIRST_CUDA)
   include(cmake/ChooseBlas.cmake)
   include(mshadow/cmake/Utils.cmake)
   include(cmake/FirstClassLangCuda.cmake)
+  include_directories(${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
 else()
-  if(EXISTS ${PROJECT_SOURCE_DIR}/mshadow/cmake)
+  if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/mshadow/cmake)
     include(mshadow/cmake/mshadow.cmake)
     include(mshadow/cmake/Utils.cmake)
     include(mshadow/cmake/Cuda.cmake)
@@ -238,8 +278,8 @@ endif()
 
 # ---[ jemalloc
 if(USE_JEMALLOC)
-  if(USE_GPERFTOOLS)
-    message(ERROR "Only one of USE_JEMALLOC and USE_GPERFTOOLS can be defined at once")
+  if(GPERFTOOLS_FOUND)
+    message(ERROR " Only one of USE_JEMALLOC and USE_GPERFTOOLS can be defined at once")
   endif()
   find_package(JeMalloc)
   if(JEMALLOC_FOUND)
@@ -295,12 +335,12 @@ elseif(UNIX)
 endif()
 
 # ---[ LAPack
-if(USE_LAPACK)
+if(USE_LAPACK AND NOT MSVC)
   add_definitions(-DMXNET_USE_LAPACK=1)
   list(APPEND mxnet_LINKER_LIBS lapack)
 else(USE_LAPACK)
   # Workaround for Windows until using new Jenkinsfile.
-  if(BLAS STREQUAL "Open" OR BLAS STREQUAL "open")
+  if(BLAS STREQUAL "Open" OR BLAS STREQUAL "open" OR USE_BLAS STREQUAL "Open" OR USE_BLAS STREQUAL "open")
     add_definitions(-DMXNET_USE_LAPACK=1)
   endif()
 endif()
@@ -367,7 +407,8 @@ list(APPEND CUDA ${MSHADOW_CUDASOURCE})
 # add source group
 FILE(GLOB_RECURSE GROUP_SOURCE "src/*.cc" "nnvm/*.cc" "plugin/*.cc")
 FILE(GLOB_RECURSE GROUP_Include "src/*.h" "nnvm/*.h" "mshadow/mshadow/*.h" "plugin/*.h")
-FILE(GLOB_RECURSE GROUP_CUDA "src/*.cu" "src/*.cuh" "mshadow/mshadow/*.cuh" "plugin/*.cu" "plugin/*.cuh")
+FILE(GLOB_RECURSE GROUP_CUDA "src/*.cu" "src/*.cuh" "mshadow/mshadow/*.cuh" "plugin/*.cu"
+  "plugin/*.cuh" "3rdparty/cub/cub/*.cuh")
 assign_source_group("Source" ${GROUP_SOURCE})
 assign_source_group("Include" ${GROUP_Include})
 assign_source_group("CUDA" ${GROUP_CUDA})
@@ -376,8 +417,6 @@ if(USE_PLUGINS_WARPCTC)
     set(WARPCTC_INCLUDE  "" CACHE PATH "WARPCTC include")
     set(WARPCTC_LIB_DEBUG  "" CACHE FILEPATH "WARPCTC lib")
     set(WARPCTC_LIB_RELEASE  "" CACHE FILEPATH "WARPCTC lib")
-
-
     include_directories(SYSTEM ${WARPCTC_INCLUDE})
     list(APPEND mxnet_LINKER_LIBS ${WARPCTC_LIB})
     FILE(GLOB_RECURSE PLUGINS_SOURCE "plugin/warpctc/*.cc" "plugin/warpctc/*.h")
@@ -399,7 +438,7 @@ if(USE_PLUGIN_CAFFE)
     if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/caffe)
       # Need newer FindCUDA.cmake that correctly handles -std=c++11
       cmake_minimum_required(VERSION 3.3)
-      set(CAFFE_PATH ${PROJECT_SOURCE_DIR}/caffe)
+      set(CAFFE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/caffe)
     else()
       set(CAFFE_PATH $ENV{CAFFE_PATH})
     endif()
@@ -456,6 +495,7 @@ if(USE_CUDA)
     endif()
     list(APPEND SOURCE ${CUDA})
     add_definitions(-DMXNET_USE_CUDA=1)
+    link_directories(${CUDA_TOOLKIT_ROOT_DIR}/lib64)
   else()
     list(APPEND CUDA_INCLUDE_DIRS ${INCLUDE_DIRECTORIES})
     # define preprocessor macro so that we will not include the generated forcelink header
@@ -472,6 +512,8 @@ if(USE_CUDA)
         list(APPEND mxnet_LINKER_LIBS "${CUDA_cufft_LIBRARY}/../cufft.lib") # For fft operator
         FIND_LIBRARY(CUDA_cusolver_LIBRARY nvrtc "${CUDA_TOOLKIT_ROOT_DIR}/lib/x64"  "${CUDA_TOOLKIT_ROOT_DIR}/lib/win32")
         list(APPEND mxnet_LINKER_LIBS "${CUDA_cusolver_LIBRARY}/../cusolver.lib") # For cusolver
+        link_directories(${CUDA_TOOLKIT_ROOT_DIR}/lib/win32)
+        link_directories(${CUDA_TOOLKIT_ROOT_DIR}/lib/x64)
     else(MSVC)
         list(APPEND mxnet_LINKER_LIBS cufft cusolver)
         if(ENABLE_CUDA_RTC)
@@ -605,13 +647,13 @@ if(AUTO_INSTALL_DIR)
   # ---[ Install Includes
   add_custom_command(TARGET mxnet POST_BUILD
     COMMAND ${CMAKE_COMMAND} -E copy_directory
-    ${PROJECT_SOURCE_DIR}/include ${AUTO_INSTALL_DIR}/include
+    ${CMAKE_CURRENT_SOURCE_DIR}/include ${AUTO_INSTALL_DIR}/include
     )
 
   # ---[ Install Examples
   add_custom_command(TARGET mxnet POST_BUILD
     COMMAND ${CMAKE_COMMAND} -E copy_directory
-    ${PROJECT_SOURCE_DIR}/example ${AUTO_INSTALL_DIR}/example
+    ${CMAKE_CURRENT_SOURCE_DIR}/example ${AUTO_INSTALL_DIR}/example
     )
 endif()
 

cmake/Modules/FindNCCL.cmake

@@ -0,0 +1,65 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Find the nccl libraries
+#
+# The following variables are optionally searched for defaults
+#  NCCL_ROOT_DIR: Base directory where all NCCL components are found
+#  NCCL_INCLUDE_DIR: Directory where NCCL header is found
+#  NCCL_LIB_DIR: Directory where NCCL library is found
+#
+# The following are set after configuration is done:
+#  NCCL_FOUND
+#  NCCL_INCLUDE_DIRS
+#  NCCL_LIBRARIES
+#
+# The path hints include CUDA_TOOLKIT_ROOT_DIR seeing as some folks
+# install NCCL in the same location as the CUDA toolkit.
+# See https://github.com/caffe2/caffe2/issues/1601
+
+set(NCCL_ROOT_DIR "" CACHE PATH "Folder contains NVIDIA NCCL")
+
+find_path(NCCL_INCLUDE_DIRS
+  NAMES nccl.h
+  HINTS
+  ${NCCL_INCLUDE_DIR}
+  ${NCCL_ROOT_DIR}
+  ${NCCL_ROOT_DIR}/include
+  ${CUDA_TOOLKIT_ROOT_DIR}/include
+  $ENV{NCCL_DIR}/include
+  )
+
+find_library(NCCL_LIBRARIES
+  NAMES nccl
+  HINTS
+  ${NCCL_LIB_DIR}
+  ${NCCL_ROOT_DIR}
+  ${NCCL_ROOT_DIR}/lib
+  ${NCCL_ROOT_DIR}/lib/x86_64-linux-gnu
+  ${NCCL_ROOT_DIR}/lib64
+  ${CUDA_TOOLKIT_ROOT_DIR}/lib64
+  $ENV{NCCL_DIR}/lib
+  )
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(NCCL DEFAULT_MSG NCCL_INCLUDE_DIRS NCCL_LIBRARIES)
+
+if(NCCL_FOUND)
+  message(STATUS "Found NCCL (include: ${NCCL_INCLUDE_DIRS}, library: ${NCCL_LIBRARIES})")
+  mark_as_advanced(NCCL_ROOT_DIR NCCL_INCLUDE_DIRS NCCL_LIBRARIES)
+endif()
+

tests/CMakeLists.txt

@@ -26,6 +26,7 @@ if(GTEST_FOUND AND NOT MSVC)
 
   file(GLOB_RECURSE UNIT_TEST_SOURCE "cpp/*.cc" "cpp/*.h")
 
+  include_directories(${GTEST_INCLUDE_DIR})
   include_directories(cpp/include)
 
   if (NOT PRIVATE_RUNTIME_DIR)