ValeevGroup · evaleev · Dec 31, 2020 · Dec 22, 2020 · Dec 22, 2020 · Dec 22, 2020
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -7,7 +7,6 @@ default:
 before_script:
   - echo 'localhost slots=8' > /etc/openmpi/openmpi-default-hostfile
   - apt-get install -yq intel-mkl-core-c-2020.4-304
-  - ln -s /opt/intel/compilers_and_libraries_2020.4.304/linux/mkl/ /opt/intel/
 
 build:
   stage: build
@@ -16,16 +15,21 @@ build:
   image: valeevgroup/${IMAGE}
   variables:
     CXX: ${CXX}
-    ENABLE_MKL : "ENABLE_MKL=OFF"
+    TA_PYTHON : "TA_PYTHON=OFF"
+    BLAS : ""
     ENABLE_SCALAPACK : "ENABLE_SCALAPACK=OFF"
   script:
     - mkdir build
     - cd build
+    - |-
+      if [[ "X$BLAS" != "X" ]]; then
+        source /opt/intel/compilers_and_libraries_2020.4.304/linux/mkl/bin/mklvars.sh intel64
+      fi
     - ../bin/gitlab-ci.sh ..
         CMAKE_BUILD_TYPE=${BUILD_TYPE}
         ${TA_PYTHON}
-        ${ENABLE_MKL}
         ${ENABLE_SCALAPACK}
+        ${BLAS}
         MPIEXEC_PREFLAGS="--allow-run-as-root"
         blacs_LIBRARIES=scalapack-openmpi
         scalapack_LIBRARIES=scalapack-openmpi
@@ -35,7 +39,7 @@ build:
       - IMAGE : [ "ubuntu:20.04" ]
         CXX: [ g++, clang++-9 ]
         BUILD_TYPE : [ "Release", "Debug" ]
-        ENABLE_MKL : [ "ENABLE_MKL=ON", "ENABLE_MKL=OFF" ]
+        BLAS : [ "BLA_VENDOR=Intel10_64lp_seq", "BLA_VENDOR=Intel10_64lp", "" ]
         ENABLE_SCALAPACK : [ "ENABLE_SCALAPACK=ON", "ENABLE_SCALAPACK=OFF" ]
       - IMAGE : [ "ubuntu:18.04", "ubuntu:20.04" ]
         CXX: [ g++ ]

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -105,9 +105,6 @@ add_feature_info(MPI ENABLE_MPI "Message-Passing Interface supports distributed-
 option(ENABLE_SCALAPACK "Enable ScaLAPACK Bindings in TiledArray" OFF)
 add_feature_info(ScaLAPACK ENABLE_SCALAPACK "ScaLAPACK provides distributed linear algebra")
 
-redefaultable_option(ENABLE_MKL "Enable use of MKL (info passed to MADNESS)" ON)
-add_feature_info(MKL ENABLE_MKL "Intel Math Kernel Library provides linear algebra and other math functionality")
-
 redefaultable_option(ENABLE_TBB "Enable use of TBB with MADNESS" OFF)
 add_feature_info(TBB ENABLE_TBB "Intel Thread-Building Blocks (TBB) supports programming shared-memory systems")
 
@@ -293,15 +290,19 @@ include_directories(${PROJECT_SOURCE_DIR}/src ${PROJECT_BINARY_DIR}/src)
 add_custom_target(External-tiledarray)
 
 # required deps:
+# 1. CUDA first since others may depend on it
+if(ENABLE_CUDA)
+  include(external/cuda.cmake)
+endif()
 include(external/madness.cmake)
 detect_MADNESS_configuration()
 include(external/eigen.cmake)
-# find BTAS first, so that it finds Boost more carefully and also has a chance to memorize where it found it
+# BTAS does a better job of building and checking Boost since it uses Boost::serialization
+# it also memorized the location of its config for use from install tree
 include(FindOrFetchBTAS)
-include(external/boost.cmake)
-if(ENABLE_CUDA)
-  include(external/cuda.cmake)
-endif()
+# this generates blaspp_headers needed to include header files associated with the BLAS++ libraries
+include(FindBLASPPHeaders)
+include(FindOrFetchBoost)
 if(ENABLE_SCALAPACK)
   include(external/scalapack.cmake)
 endif()

diff --git a/INSTALL.md b/INSTALL.md
diff --git a/bin/admin/dependency-versions-update-hook.py b/bin/admin/dependency-versions-update-hook.py
@@ -131,10 +131,6 @@ def replace_dep_id(topsrc, file_ext, dep_name, old_id, new_id, search_prefix = '
 
 # Boost version in INSTALL.md
 any_files_changed |= replace_dep_id(topsrc, 'md', 'Boost', boost_old_version, boost_new_version, 'boost.org/), version ', ' or higher')
-# Boost install version in boost.cmake
-any_files_changed |= replace_dep_id(topsrc, 'cmake', 'Boost', boost_old_install_version, boost_new_install_version, 'boostorg/release/', '/source/boost')
-any_files_changed |= replace_dep_id(topsrc, 'cmake', 'Boost', dots_to_undescores(boost_old_install_version), dots_to_undescores(boost_new_install_version), 'source/boost_', '.tar.gz')
-any_files_changed |= replace_dep_id(topsrc, 'cmake', 'Boost', boost_old_install_url_hash, boost_new_install_url_hash, 'SHA256=', '')
 
 # Eigen version in INSTALL.md
 any_files_changed |= replace_dep_id(topsrc, 'md', 'Eigen', eigen_old_version, eigen_new_version, 'eigen.tuxfamily.org), version ', ' or higher')

diff --git a/bin/build-linux.sh b/bin/build-linux.sh
@@ -134,6 +134,8 @@ fi
 # Build all libraries, examples, and applications
 make -j2 all VERBOSE=1
 make install
+# remove install dir to avoid broken artifacts like BTAS polluting the next build via cached copy
+rm -rf $INSTALL_DIR
 
 # Validate
 make -j1 ta_test VERBOSE=1

diff --git a/bin/build-madness-linux.sh b/bin/build-madness-linux.sh
@@ -62,19 +62,19 @@ if [ "$BUILD_TYPE" = "Debug" ]; then
       -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \
       -DCMAKE_BUILD_TYPE=$BUILD_TYPE \
       -DCMAKE_CXX_FLAGS="-ftemplate-depth=1024 -Wno-unused-command-line-argument ${EXTRACXXFLAGS}" \
+      -DMADNESS_BUILD_MADWORLD_ONLY=ON \
       -DENABLE_MPI=ON \
       -DMPI_THREAD=multiple \
       -DENABLE_TBB=OFF \
       -DTBB_ROOT_DIR=/usr \
-      -DENABLE_MKL=OFF \
       -DFORTRAN_INTEGER_SIZE=4 \
       -DENABLE_LIBXC=OFF \
       -DENABLE_GPERFTOOLS=OFF \
       -DASSERTION_TYPE=throw \
       -DDISABLE_WORLD_GET_DEFAULT=ON
 
     # Build MADworld + LAPACK/BLAS interface
-    make -j2 install-madness-world install-madness-clapack install-madness-common install-madness-config VERBOSE=1
+    make -j2 install-madness-world install-madness-common install-madness-config VERBOSE=1
   fi
 
 fi
diff --git a/cmake/modules/DetectMADNESSConfig.cmake b/cmake/modules/DetectMADNESSConfig.cmake
@@ -23,24 +23,6 @@ macro (detect_MADNESS_configuration)
           CACHE STRING "Sanitized list of MADNESS include directories usable in build tree")
 
   list(APPEND CMAKE_REQUIRED_INCLUDES ${MADNESS_INTERNAL_INCLUDE_DIRS})
-  if (NOT DEFINED MADNESS_HAS_MKL)
-    CHECK_CXX_SOURCE_COMPILES(
-            "
-    #include <madness/config.h>
-    #ifndef HAVE_INTEL_MKL
-    # error \"MADNESS does not have MKL\"
-    #endif
-    int main(int argc, char** argv) {
-      return 0;
-    }
-    "  MADNESS_HAS_MKL)
-  endif()
-
-  if (MADNESS_HAS_MKL)
-    unset(MADNESS_HAS_MKL)
-    set(MADNESS_HAS_MKL ON CACHE BOOL "MADNESS detected usable Intel MKL" FORCE)
-  endif()
-
   if (NOT DEFINED MADNESS_HAS_TBB)
     CHECK_CXX_SOURCE_COMPILES(
         "
@@ -59,24 +41,6 @@ macro (detect_MADNESS_configuration)
     set(MADNESS_HAS_TBB ON CACHE BOOL "MADNESS detected usable Intel TBB" FORCE)
   endif()
 
-  if (NOT DEFINED MADNESS_FORTRAN_DEFAULT_INTEGER4)
-    CHECK_CXX_SOURCE_COMPILES(
-        "
-    #include <madness/config.h>
-    #if MADNESS_FORTRAN_DEFAULT_INTEGER_SIZE != 4
-    # error \"MADNESS does not assume integer*4 for Fortran BLAS/LAPACK interfaces\"
-    #endif
-    int main(int argc, char** argv) {
-      return 0;
-    }
-    "  MADNESS_FORTRAN_DEFAULT_INTEGER4)
-  endif()
-
-  if (MADNESS_FORTRAN_DEFAULT_INTEGER4)
-    unset(MADNESS_FORTRAN_DEFAULT_INTEGER4)
-    set(MADNESS_FORTRAN_DEFAULT_INTEGER4 ON CACHE BOOL "MADNESS assumes integer*4 for Fortran BLAS/LAPACK integers" FORCE)
-  endif()
-
   unset(CMAKE_REQUIRED_QUIET)
   cmake_pop_check_state()
 

diff --git a/cmake/modules/FindBLASPPHeaders.cmake b/cmake/modules/FindBLASPPHeaders.cmake
@@ -0,0 +1,54 @@
+#
+#  This file is a part of TiledArray.
+#  Copyright (C) 2019  Virginia Tech
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+#  Edward Valeev
+#  Department of Chemistry, Virginia Tech
+#
+#  FindBLASPPHeaders.cmake
+#  Dec 28, 2020
+#
+
+# check for MKL
+include( CheckFunctionExists )
+include(CMakePushCheckState)
+cmake_push_check_state( RESET )
+find_package( OpenMP QUIET ) #XXX Open LAPACKPP issue for this...
+set( CMAKE_REQUIRED_LIBRARIES "${blaspp_libraries}" m )
+check_function_exists( mkl_dimatcopy  BLAS_IS_MKL )
+cmake_pop_check_state()
+
+add_library(blaspp_headers INTERFACE)
+
+if (BLAS_IS_MKL)
+  foreach(_lib ${blaspp_libraries})
+    if (EXISTS ${_lib} AND _lib MATCHES libmkl_)
+      string(REGEX REPLACE "/lib/(|intel64/|intel64_lin/)libmkl_.*" "" _mklroot "${_lib}")
+    elseif (_lib MATCHES "^-L")
+      string(REGEX REPLACE "^-L" "" _mklroot "${_lib}")
+      string(REGEX REPLACE "/lib(|/intel64|/intel64_lin)(|/)" "" _mklroot "${_mklroot}")
+    endif()
+    if (_mklroot)
+      break()
+    endif(_mklroot)
+  endforeach()
+
+  target_include_directories(blaspp_headers INTERFACE "${_mklroot}/include")
+endif()
+
+install(TARGETS blaspp_headers EXPORT tiledarray COMPONENT blaspp_headers
+    LIBRARY DESTINATION "${TILEDARRAY_INSTALL_LIBDIR}"
+    ARCHIVE DESTINATION "${TILEDARRAY_INSTALL_LIBDIR}")
diff --git a/cmake/modules/FindOrFetchBTAS.cmake b/cmake/modules/FindOrFetchBTAS.cmake
@@ -4,15 +4,9 @@ if (TARGET BTAS::BTAS)
   message(STATUS "Found BTAS CONFIG at ${BTAS_CONFIG}")
 else (TARGET BTAS::BTAS)
 
-  set(BTAS_ENABLE_MKL ${ENABLE_MKL} CACHE BOOL "Whether BTAS should seek MKL")
-  if (MADNESS_FORTRAN_DEFAULT_INTEGER4)
-    set(mkl_pref_ilp64 OFF)
-  else(MADNESS_FORTRAN_DEFAULT_INTEGER4)
-    set(mkl_pref_ilp64 ON)
-  endif(MADNESS_FORTRAN_DEFAULT_INTEGER4)
-  set(MKL_PREFER_ILP64 ${mkl_pref_ilp64} CACHE BOOL "MKL preference: ILP64 (yes) or {LP64,LP32} (no)")
-  if (MADNESS_HAS_TBB)
-    set(MKL_THREADING "TBB" CACHE STRING "MKL flavor: SEQ, TBB or OMP (default)")
+  if (NOT TILEDARRAY_HAS_CUDA)
+    # tell BLAS++/LAPACK++ to ignore CUDA
+    set(use_cuda FALSE CACHE BOOL "Whether to look for CUDA-enabled libs in BLAS++/LAPACK++")
   endif()
 
   FetchContent_Declare(

diff --git a/cmake/modules/FindOrFetchBoost.cmake b/cmake/modules/FindOrFetchBoost.cmake
@@ -0,0 +1,41 @@
+# Limit scope of the search if BOOST_ROOT or BOOST_INCLUDEDIR is provided.
+if (BOOST_ROOT OR BOOST_INCLUDEDIR)
+  set(Boost_NO_SYSTEM_PATHS TRUE)
+endif()
+
+# Check for Boost
+# - require version 1.68 where Boost.Test preconditions work correctly (see https://svn.boost.org/trac10/ticket/12095)
+find_package(Boost ${TA_TRACKED_BOOST_VERSION} QUIET CONFIG)
+
+if (TARGET Boost::boost)
+  message(STATUS "Found Boost CONFIG at ${Boost_CONFIG}")
+else (TARGET Boost::boost)
+  include (FetchContent)
+  cmake_minimum_required (VERSION 3.14.0)  # for FetchContent_MakeAvailable
+
+  FetchContent_Declare(
+          CMAKEBOOST
+          GIT_REPOSITORY      https://github.com/Orphis/boost-cmake
+  )
+  FetchContent_MakeAvailable(CMAKEBOOST)
+  FetchContent_GetProperties(CMAKEBOOST
+          SOURCE_DIR CMAKEBOOST_SOURCE_DIR
+          BINARY_DIR CMAKEBOOST_BINARY_DIR
+          )
+
+  # current boost-cmake/master does not install boost correctly, so warn that installed TiledArray will not be usable
+  # boost-cmake/install_rules https://github.com/Orphis/boost-cmake/pull/45 is supposed to fix it but is inactive
+  message(WARNING "Building Boost from source makes TiledArray unusable from the install location! Install TA using package manager or manually and reconfigure/reinstall TiledArray to fix this")
+  export(EXPORT tiledarray
+      FILE "${PROJECT_BINARY_DIR}/boost-targets.cmake")
+  install(EXPORT tiledarray
+      FILE "boost-targets.cmake"
+      DESTINATION "${TILEDARRAY_INSTALL_CMAKEDIR}"
+      COMPONENT boost-libs)
+
+endif(TARGET Boost::boost)
+
+# postcond check
+if (NOT TARGET Boost::boost)
+  message(FATAL_ERROR "FindOrFetchBoost could not make Boost::boost target available")
+endif(NOT TARGET Boost::boost)
diff --git a/examples/dgemm/ta_blas.cpp b/examples/dgemm/ta_blas.cpp
@@ -60,7 +60,8 @@ int main(int argc, char** argv) {
   std::fill_n(c, matrix_size * matrix_size, 0.0);
 
   // BLAS dgemm arguments
-  char opa = 'n', opb = 'n';
+  using namespace TiledArray::math::blas;
+  auto opa = Op::NoTrans, opb = Op::NoTrans;
   const double alpha = 1l, beta = 0l;
   const integer m = matrix_size, n = matrix_size, k = matrix_size;
   const integer lda = matrix_size, ldb = matrix_size, ldc = matrix_size;
@@ -72,7 +73,7 @@ int main(int argc, char** argv) {
   // Note: If TiledArray has not been configured with blas, this will be an
   // eigen call.
   for (int i = 0; i < repeat; ++i) {
-    F77_DGEMM(&opb, &opa, &n, &m, &k, &alpha, b, &ldb, a, &lda, &beta, c, &ldc);
+    gemm(opa, opb, m, n, k, alpha, a, lda, b, ldb, beta, c, ldc);
   }
 
   // Stop clock

diff --git a/examples/dgemm/ta_cc_abcd.cpp b/examples/dgemm/ta_cc_abcd.cpp
@@ -328,8 +328,8 @@ make_contract(const unsigned int result_rank, const unsigned int left_rank,
               const TA::Permutation& perm = TA::Permutation()) {
   return TA::detail::ContractReduce<Tile, Tile, Tile,
                                     typename Tile::value_type>(
-      madness::cblas::NoTrans, madness::cblas::NoTrans, 1, result_rank,
-      left_rank, right_rank, perm);
+      TiledArray::math::blas::Op::NoTrans, TiledArray::math::blas::Op::NoTrans,
+      1, result_rank, left_rank, right_rank, perm);
 }
 
 template <typename Tile>

diff --git a/examples/vector_tests/vector.cpp b/examples/vector_tests/vector.cpp
@@ -391,7 +391,7 @@ int main(int argc, char** argv) {
 
   start = madness::wall_time();
   for (std::size_t r = 0ul; r < repeat; ++r) {
-    TiledArray::blas::scale(n, 3.0, c);
+    TiledArray::math::blas::scale(n, 3.0, c);
   }
   stop = madness::wall_time();