Asadchev/feature/gitlab cuda build

.gitlab-ci.yml

@@ -5,6 +5,38 @@ default:
   # timeout: 3h # NB doesnt work as of 1/2021
   interruptible: true
 
+variables:
+  MAD_NUM_THREADS : 2
+  TA_CI_TARGETS : "tiledarray examples ta_test check"
+  TA_CI_CONFIG : >
+                 TA_BUILD_UNITTEST=TRUE
+                 CMAKE_BUILD_TYPE=${BUILD_TYPE}
+                 ${TA_PYTHON}
+                 ${ENABLE_CUDA}
+                 ${BLA_VENDOR}
+                 ${ENABLE_SCALAPACK}
+
+before_script:
+  - echo 'localhost slots=2' > /etc/openmpi/openmpi-default-hostfile
+  - echo "CI_RUNNER_TAGS=$CI_RUNNER_TAGS"
+  # NB: tag parsing below is not robust
+  - CMAKE_BUILD_PARALLEL_LEVEL=$(echo $CI_RUNNER_TAGS | sed -n 's/CMAKE_BUILD_PARALLEL_LEVEL=\([0-9]\+\).*/\1/p')
+  - export CMAKE_BUILD_PARALLEL_LEVEL=${CMAKE_BUILD_PARALLEL_LEVEL:=1}
+  - echo "CMAKE_BUILD_PARALLEL_LEVEL=$CMAKE_BUILD_PARALLEL_LEVEL"
+  - |-
+    if [[ "$BLA_VENDOR" == "BLA_VENDOR=Intel10"* ]]; then
+      # apt-get install -yq intel-mkl-core-c-2020.4-304
+      # source /opt/intel/compilers_and_libraries_2020.4.304/linux/mkl/bin/mklvars.sh intel64
+      make -C /home/ValeevGroup install/intel-mkl
+      source /opt/intel/mkl/bin/mklvars.sh intel64
+      echo "MKLROOT=$MKLROOT"
+    fi
+  - |-
+    if [[ "$ENABLE_CUDA" == "ENABLE_CUDA=ON" ]]; then
+      make -C /home/ValeevGroup install/cuda
+      export CUDACXX=/usr/local/cuda/bin/nvcc
+    fi
+
 ubuntu:
   stage: build
   tags: [ docker ]
@@ -13,23 +45,15 @@ ubuntu:
   variables:
     TA_PYTHON : "TA_PYTHON=ON"
     ENABLE_SCALAPACK : "ENABLE_SCALAPACK=OFF"
-  before_script:
-    - echo 'localhost slots=8' > /etc/openmpi/openmpi-default-hostfile
-    - |-
-      if [[ "$BLA_VENDOR" == "BLA_VENDOR=Intel10"* ]]; then
-        apt-get install -yq intel-mkl-core-c-2020.4-304
-        source /opt/intel/compilers_and_libraries_2020.4.304/linux/mkl/bin/mklvars.sh intel64
-        echo "MKLROOT=$MKLROOT"
-      fi
   script:
     - mkdir build
     - cd build
-    - TA_CI_CONFIG="CMAKE_BUILD_TYPE=${BUILD_TYPE} ${TA_PYTHON} ${BLA_VENDOR} ${ENABLE_SCALAPACK}"
-    - unset BUILD_TYPE TA_PYTHON BLA_VENDOR ENABLE_SCALAPACK
     # !!! Unset env vars that may conflict with build, eg FindBLAS uses $ENV{BLA_VENDOR}
+    - unset BUILD_TYPE TA_PYTHON BLA_VENDOR ENABLE_SCALAPACK ENABLE_CUDA
     - ../bin/gitlab-ci.sh ..
+        ${TA_CI_TARGETS}
         ${TA_CI_CONFIG}
-        MPIEXEC_PREFLAGS="--allow-run-as-root"
+        MPIEXEC_PREFLAGS='--bind-to;none;--allow-run-as-root'
         blacs_LIBRARIES=scalapack-openmpi
         scalapack_LIBRARIES=scalapack-openmpi
         #lapack_LIBRARIES=lapack
@@ -39,9 +63,14 @@ ubuntu:
         CXX: [ g++ ]
         BUILD_TYPE : [ "Release" ]
         BLA_VENDOR : [ "BLA_VENDOR=Intel10_64lp_seq", "BLA_VENDOR=Intel10_64lp" ]
-        ENABLE_SCALAPACK : [ "ENABLE_SCALAPACK=ON", "ENABLE_SCALAPACK=OFF" ]
+        # ENABLE_SCALAPACK : [ "ENABLE_SCALAPACK=ON", "ENABLE_SCALAPACK=OFF" ]
         TA_PYTHON : [ "TA_PYTHON=OFF" ] # needs to be fixed for MKL
       - IMAGE : [ "ubuntu:18.04", "ubuntu:20.04" ]
         CXX: [ g++, clang++-9 ]
         BUILD_TYPE : [ "Release", "Debug" ]
         ENABLE_SCALAPACK : [ "ENABLE_SCALAPACK=ON", "ENABLE_SCALAPACK=OFF" ]
+      - IMAGE : [ "ubuntu:18.04", "ubuntu:20.04" ]
+        CXX: [ g++ ]
+        BUILD_TYPE : [ "Release", "Debug" ]
+        ENABLE_CUDA : [ "ENABLE_CUDA=ON" ]
+        TA_CI_TARGETS : [ "tiledarray examples" ]

bin/gitlab-ci.sh

@@ -10,13 +10,20 @@ fi
 project_dir=$1
 shift
 
-cmake_args="-DTA_BUILD_UNITTEST=TRUE"
-
-for arg in $@; do
-    cmake_args+=" -D$arg"
+targets=""
+cmake_args=""
+
+for arg in "$@"; do
+    #echo $arg
+    case $arg in
+         *=*) cmake_args+=" \"-D$arg\"" ;;
+         *)   targets+=" $arg" ;;
+    esac
 done
 
-cmake_build_target="cmake --build . --target "
+echo "CMake args: $cmake_args"
+echo "Build targets: $targets"
+echo ""
 
 set -e
 set -x
@@ -26,10 +33,9 @@ export OMPI_ALLOW_RUN_AS_ROOT=1
 export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
 export OMPI_MCA_btl_vader_single_copy_mechanism="none"
 
-cmake $project_dir $cmake_args
-
-$cmake_build_target tiledarray
-$cmake_build_target examples
-$cmake_build_target ta_test
-$cmake_build_target check
+eval "cmake $project_dir $cmake_args"
 
+for target in $targets; do
+    echo "Building target $target"
+    eval "cmake --build . --target $target"
+done

external/cuda.cmake

@@ -21,13 +21,11 @@ set(CMAKE_CUDA_SEPARABLE_COMPILATION ON)
 # NB CUDAToolkit does NOT have COMPONENTS
 find_package(CUDAToolkit REQUIRED)
 
-if (NOT TARGET CUDA::cublas)
-  message(FATAL_ERROR "CUDA::cublas not found")
-endif()
-
-if (NOT TARGET CUDA::nvToolsExt)
-  message(FATAL_ERROR "CUDA::nvToolsExt not found")
-endif()
+foreach (library cublas;nvToolsExt)
+  if (NOT TARGET CUDA::${library})
+    message(FATAL_ERROR "CUDA::${library} not found")
+  endif()
+endforeach()
 
 if (NOT DEFINED CUDAToolkit_ROOT)
   get_filename_component(CUDAToolkit_ROOT "${CUDAToolkit_INCLUDE_DIR}/../" ABSOLUTE CACHE)

external/cutt.cmake

@@ -19,6 +19,9 @@ else()
 
     include(ExternalProject)
 
+    # to pass CMAKE_C_* vars to external project
+    enable_language(C)
+
     # set source and build path for cuTT in the TiledArray project
     set(EXTERNAL_SOURCE_DIR   ${PROJECT_BINARY_DIR}/external/source/cutt)
     # cutt only supports in source build

external/umpire.cmake

@@ -23,6 +23,9 @@ else()
 
     include(ExternalProject)
 
+    # to pass CMAKE_C_* vars to external project
+    enable_language(C)
+
     # set source and build path for Umpire in the TiledArray project
     set(EXTERNAL_SOURCE_DIR   ${PROJECT_BINARY_DIR}/external/source/Umpire)
     set(EXTERNAL_BUILD_DIR  ${PROJECT_BINARY_DIR}/external/build/Umpire)