discover/fetch TTG + use for cholesky_linv

CMakeLists.txt

@@ -26,12 +26,30 @@
 
 cmake_minimum_required (VERSION 3.15.0) # need list(PREPEND for toolchains
 
+# Set TiledArray version =======================================================
+
+# see https://semver.org/
+set(TILEDARRAY_MAJOR_VERSION 1)
+set(TILEDARRAY_MINOR_VERSION 0)
+set(TILEDARRAY_MICRO_VERSION 0)
+set(TILEDARRAY_PRERELEASE_ID )
+
+set(TILEDARRAY_VERSION "${TILEDARRAY_MAJOR_VERSION}.${TILEDARRAY_MINOR_VERSION}.${TILEDARRAY_MICRO_VERSION}")
+if (TILEDARRAY_PRERELEASE_ID)
+  set(TILEDARRAY_EXT_VERSION "${TILEDARRAY_VERSION}-${TILEDARRAY_PRERELEASE_ID}")
+else(TILEDARRAY_PRERELEASE_ID)
+  set(TILEDARRAY_EXT_VERSION "${TILEDARRAY_VERSION}")
+endif(TILEDARRAY_PRERELEASE_ID)
+
 # Preload versions/tags of all dependencies ====================================
 include(external/versions.cmake)
 
-###############################################################################
-# Bring ValeevGroup cmake toolkit
-###############################################################################
+# Safety net for dev workflow: accidental install will not affect FindOrFetch*
+if (NOT DEFINED CACHE{CMAKE_FIND_NO_INSTALL_PREFIX})
+  set(CMAKE_FIND_NO_INSTALL_PREFIX ON CACHE BOOL "Whether find_* commands will search CMAKE_INSTALL_PREFIX and CMAKE_STAGING_PREFIX; see https://cmake.org/cmake/help/latest/variable/CMAKE_FIND_NO_INSTALL_PREFIX.html#variable:CMAKE_FIND_NO_INSTALL_PREFIX")
+endif()
+
+# Bring ValeevGroup cmake toolkit ==============================================
 include(FetchContent)
 if (DEFINED PROJECT_BINARY_DIR)
   set(VG_CMAKE_KIT_PREFIX_DIR PROJECT_BINARY_DIR)
@@ -50,23 +68,13 @@ FetchContent_Declare(
 FetchContent_MakeAvailable(vg_cmake_kit)
 list(APPEND CMAKE_MODULE_PATH "${vg_cmake_kit_SOURCE_DIR}/modules")
 
-project (TiledArray LANGUAGES)
-enable_language(CXX)
-
-# Set TiledArray version =======================================================
-
-# see https://semver.org/
-set(TILEDARRAY_MAJOR_VERSION 1)
-set(TILEDARRAY_MINOR_VERSION 0)
-set(TILEDARRAY_MICRO_VERSION 0)
-set(TILEDARRAY_PRERELEASE_ID )
-
-set(TILEDARRAY_VERSION "${TILEDARRAY_MAJOR_VERSION}.${TILEDARRAY_MINOR_VERSION}.${TILEDARRAY_MICRO_VERSION}")
-if (TILEDARRAY_PRERELEASE_ID)
-  set(TILEDARRAY_EXT_VERSION "${TILEDARRAY_VERSION}-${TILEDARRAY_PRERELEASE_ID}")
-else(TILEDARRAY_PRERELEASE_ID)
-  set(TILEDARRAY_EXT_VERSION "${TILEDARRAY_VERSION}")
-endif(TILEDARRAY_PRERELEASE_ID)
+# Declare ourselves ============================================================
+project(TiledArray
+        VERSION ${TILEDARRAY_VERSION}
+        DESCRIPTION "TiledArray: block-sparse tensor framework for modern (distributed-memory and heterogeneous) computing"
+        LANGUAGES CXX
+        HOMEPAGE_URL "https://valeevgroup.github.io/tiledarray/")
+enable_language(C)  # C needed even for basic platform introspection
 
 # Set install paths ============================================================
 
@@ -157,8 +165,11 @@ if(TA_ENABLE_TILE_OPS_LOGGING AND NOT DEFINED TA_TILE_OPS_LOG_LEVEL)
   set(TA_TILE_OPS_LOG_LEVEL 1)
 endif(TA_ENABLE_TILE_OPS_LOGGING AND NOT DEFINED TA_TILE_OPS_LOG_LEVEL)
 
-option(TA_ENABLE_RANGEV3 "Enable Range-V3 library" OFF)
-add_feature_info(ENABLE_RANGEV3 TA_ENABLE_RANGEV3 "Range-V3 ranges library")
+option(TA_RANGEV3 "Enable Range-V3 library" OFF)
+add_feature_info(TA_RANGEV3 TA_RANGEV3 "Range-V3 ranges library")
+
+option(TA_TTG "Enable search/build of TTG library" OFF)
+add_feature_info(TA_TTG TA_TTG "TTG library")
 
 # Enable shared library support options
 redefaultable_option(TA_ASSUMES_ASLR_DISABLED "TiledArray assumes the Address Space Layout Randomization (ASLR) to be disabled" OFF)
@@ -303,6 +314,9 @@ add_custom_target(External-tiledarray)
 if(ENABLE_CUDA)
   include(external/cuda.cmake)
 endif()
+if (TA_TTG)
+  include(FindOrFetchTTG)
+endif(TA_TTG)
 include(FindOrFetchMADWorld)
 detect_MADNESS_configuration()
 include(external/eigen.cmake)
@@ -328,9 +342,6 @@ include(FindOrFetchBoost)
 if(ENABLE_SCALAPACK)
   include(external/scalapackpp.cmake)
 endif()
-if (TA_ENABLE_RANGEV3)
-  include(FindOrFetchRangeV3)
-endif(TA_ENABLE_RANGEV3)
 
 # optional deps:
 # 1. ccache
@@ -340,6 +351,15 @@ if(CCACHE)
     set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE})
     set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE})
 endif(CCACHE)
+# 2. range-v3
+if (TA_RANGEV3)
+  include(FindOrFetchRangeV3)
+endif(TA_RANGEV3)
+# 3. TTG
+# N.B. make sure TA configures MADNESS correctly
+#if (TA_TTG)
+#  include(FindOrFetchTTG)
+#endif(TA_TTG)
 
 ##########################
 # sources

INSTALL.md

@@ -40,9 +40,9 @@ Both methods are supported. However, for most users we _strongly_ recommend to b
   - Boost.Container: header-only
   - Boost.Test: header-only or (optionally) as a compiled library, *only used for unit testing*
   - Boost.Range: header-only, *only used for unit testing*
-- [BTAS](http://github.com/ValeevGroup/BTAS), tag 242871710dabd5ef337e5253000d3e38c1d977ba . If usable BTAS installation is not found, TiledArray will download and compile
+- [BTAS](http://github.com/ValeevGroup/BTAS), tag 240b49b033864b34d74f2b8d6dd55f2ab524eae3 . If usable BTAS installation is not found, TiledArray will download and compile
   BTAS from source. *This is the recommended way to compile BTAS for all users*.
-- [MADNESS](https://github.com/m-a-d-n-e-s-s/madness), tag 997e8b458c4234fb6c8c2781a5df59cb14b7e700 .
+- [MADNESS](https://github.com/m-a-d-n-e-s-s/madness), tag 20d4d7cffbbc98dd7b7d6a5feb7d68f1df067cd6 .
   Only the MADworld runtime and BLAS/LAPACK C API component of MADNESS is used by TiledArray.
   If usable MADNESS installation is not found, TiledArray will download and compile
   MADNESS from source. *This is the recommended way to compile MADNESS for all users*.
@@ -72,12 +72,13 @@ Optional prerequisites:
     - [blacspp](https://github.com/wavefunction91/blacspp.git) -- a modern C++ (C++17) wrapper for BLACS
 - Python3 interpreter -- to test (optionally-built) Python bindings
 - [Range-V3](https://github.com/ericniebler/range-v3.git) -- a Ranges library that served as the basis for Ranges component of C++20; only used for some unit testing of the functionality anticipated to be supported by future C++ standards.
+- [TTG](https://github.com/TESSEorg/ttg.git) -- C++ implementation of the Template Task Graph programming model for fine-grained flow-graph composition of distributed memory programs.
 
-Most of the dependencies (except for MADNESS and BTAS) can be installed with a package manager,
+Many of these dependencies can be installed with a package manager,
 such as Homebrew on OS X or apt-get on Debian Linux distributions;
-this is the preferred method. Since on some systems configuring
-and building MADNESS can be difficult even for experts, we recommend letting the
-TiledArray download and build MADNESS for you.
+this is the preferred method. Since configuring and building other
+dependencies (such as MADNESS) can be difficult even for experts,
+we recommend letting the TiledArray download and build them for you.
 
 ## Obtain TiledArray source code
 
@@ -411,7 +412,8 @@ support may be added.
 * `TA_ASSERT_POLICY` -- Set to `TA_ASSERT_IGNORE` to disable `TA_ASSERT` assertions, `TA_ASSERT_THROW` to cause `TA_ASSERT` assertions to throw, `TA_ASSERT_ABORT` to cause `TA_ASSERT` assertions to abort. The default is `TA_ASSERT_IGNORE` if CMake uses a single-configuration generator and`CMAKE_BUILD_TYPE` is set to `Release` or `MinSizeRel`, else the default is `TA_ASSERT_THROW`.
 * `BUILD_TESTING` -- Set of `OFF` to disable building unit tests. The default is `ON`.
 * `TA_TRACE_TASKS` -- Set to `ON` to enable tracing of MADNESS tasks using custom task tracer. Note that standard profilers/tracers are generally useless (except in the trivial cases) with MADWorld-based programs since the submission context of tasks is not captured by standard tracing tools; this makes it impossible in a nontrivial program to attribute tasks to source code. WARNING: task tracing his will greatly increase the memory requirements. [Default=OFF].
-* `TA_ENABLE_RANGEV3` -- Set to `ON` to find or fetch the Range-V3 library and enable additional tests of TA components with constructs anticipated to be supported in the future. [Default=OFF].
+* `TA_RANGEV3` -- Set to `ON` to find or fetch the Range-V3 library and enable additional tests of TA components with constructs anticipated to be supported in the future. [Default=OFF].
+* `TA_TTG` -- Set to `ON` to find or fetch the TTG library. [Default=OFF]. 
 * `TA_SIGNED_1INDEX_TYPE` -- Set to `OFF` to use unsigned 1-index coordinate type (default for TiledArray 1.0.0-alpha.2 and older). The default is `ON`, which enables the use of negative indices in coordinates.
 * `TA_MAX_SOO_RANK_METADATA` -- Specifies the maximum rank for which to use Small Object Optimization (hence, avoid the use of the heap) for metadata. The default is `8`.
 * `TA_TENSOR_MEM_PROFILE` -- Set to `ON` to profile memory allocations in TA::Tensor.

cmake/modules/FindOrFetchBoost.cmake

@@ -11,11 +11,15 @@ if (NOT TARGET Boost::boost)
     message(STATUS "Found Boost ${Boost_VERSION}: ${Boost_INCLUDE_DIRS}")
   endif(TARGET Boost::boost)
 
-  # Boost::* targets by default are not GLOBAL, so to allow users of LINALG_LIBRARIES to safely use them we need to make them global
+  # Boost::* targets by default are not GLOBAL, so to allow users of TA to safely use them we need to make them global
   # more discussion here: https://gitlab.kitware.com/cmake/cmake/-/issues/17256
-  foreach(tgt boost;headers)
+  foreach(tgt boost;headers;${Boost_BTAS_DEPS_LIBRARIES})
     if (TARGET Boost::${tgt})
-      set_target_properties(Boost::${tgt} PROPERTIES IMPORTED_GLOBAL TRUE)
+      get_target_property(_boost_tgt_${tgt}_is_imported_global Boost::${tgt} IMPORTED_GLOBAL)
+      if (NOT _boost_tgt_${tgt}_is_imported_global)
+        set_target_properties(Boost::${tgt} PROPERTIES IMPORTED_GLOBAL TRUE)
+      endif()
+      unset(_boost_tgt_${tgt}_is_imported_global)
     endif()
   endforeach()
 

cmake/modules/FindOrFetchTTG.cmake

@@ -0,0 +1,28 @@
+if (NOT TARGET ttg-parsec)
+  find_package(ttg CONFIG)
+endif(NOT TARGET ttg-parsec)
+
+if (TARGET ttg-parsec)
+    message(STATUS "Found ttg CONFIG at ${ttg_CONFIG}")
+else (TARGET ttg-parsec)
+
+  include(FetchContent)
+  FetchContent_Declare(
+      ttg
+      GIT_REPOSITORY      ${TA_TRACKED_TTG_URL}
+      GIT_TAG             ${TA_TRACKED_TTG_TAG}
+  )
+  FetchContent_MakeAvailable(ttg)
+  FetchContent_GetProperties(ttg
+      SOURCE_DIR TTG_SOURCE_DIR
+      BINARY_DIR TTG_BINARY_DIR
+      )
+
+endif(TARGET ttg-parsec)
+
+# postcond check
+if (NOT TARGET ttg-parsec)
+  message(FATAL_ERROR "FindOrFetchTTG could not make ttg-parsec target available")
+else()
+  set(TILEDARRAY_HAS_TTG 1)
+endif()

external/versions.cmake

@@ -19,13 +19,13 @@ set(TA_INSTALL_EIGEN_PREVIOUS_VERSION 3.3.7)
 set(TA_INSTALL_EIGEN_URL_HASH SHA256=b4c198460eba6f28d34894e3a5710998818515104d6e74e5cc331ce31e46e626)
 set(TA_INSTALL_EIGEN_PREVIOUS_URL_HASH MD5=b9e98a200d2455f06db9c661c5610496)
 
-set(TA_TRACKED_MADNESS_TAG 997e8b458c4234fb6c8c2781a5df59cb14b7e700)
-set(TA_TRACKED_MADNESS_PREVIOUS_TAG fae8081179b9d074968b08e064a32e3ca07ab0f1)
+set(TA_TRACKED_MADNESS_TAG 20d4d7cffbbc98dd7b7d6a5feb7d68f1df067cd6)
+set(TA_TRACKED_MADNESS_PREVIOUS_TAG 851ef271858129c47a912a088cc833b085c382aa)
 set(TA_TRACKED_MADNESS_VERSION 0.10.1)
 set(TA_TRACKED_MADNESS_PREVIOUS_VERSION 0.10.1)
 
-set(TA_TRACKED_BTAS_TAG 242871710dabd5ef337e5253000d3e38c1d977ba)
-set(TA_TRACKED_BTAS_PREVIOUS_TAG db884b020b5c13c312c07df9d5c03cea2d65afb2)
+set(TA_TRACKED_BTAS_TAG 240b49b033864b34d74f2b8d6dd55f2ab524eae3)
+set(TA_TRACKED_BTAS_PREVIOUS_TAG ab866a760b72ff23053266bf46b87f19d3df44b7)
 
 set(TA_TRACKED_CUTT_TAG 0e8685bf82910bc7435835f846e88f1b39f47f09)
 set(TA_TRACKED_CUTT_PREVIOUS_TAG 592198b93c93b7ca79e7900b9a9f2e79f9dafec3)
@@ -38,3 +38,7 @@ set(TA_TRACKED_SCALAPACKPP_PREVIOUS_TAG 043f85d7f31ec6009740ab466bcb5008af7b0814
 
 set(TA_TRACKED_RANGEV3_TAG 2e0591c57fce2aca6073ad6e4fdc50d841827864)
 set(TA_TRACKED_RANGEV3_PREVIOUS_TAG dbdaa247a25a0daa24c68f1286a5693c72ea0006)
+
+set(TA_TRACKED_TTG_URL https://github.com/therault/ttg.git)
+set(TA_TRACKED_TTG_TAG bb5309a5224e2546a5316daf7fc5c143f450f17b)
+set(TA_TRACKED_TTG_PREVIOUS_TAG 5107143b418384c44587c2776a9e87065d33d670)

src/CMakeLists.txt

@@ -40,6 +40,7 @@ TiledArray/perm_index.h
 TiledArray/permutation.h
 TiledArray/proc_grid.h
 TiledArray/range.h
+TiledArray/range1.h
 TiledArray/range_iterator.h
 TiledArray/reduce_task.h
 TiledArray/replicator.h
@@ -58,6 +59,7 @@ TiledArray/val_array.h
 ${PROJECT_BINARY_DIR}/src/TiledArray/version.h
 TiledArray/zero_tensor.h
 TiledArray/math/linalg/forward.h
+TiledArray/math/linalg/basic.h
 TiledArray/math/linalg/conjgrad.h
 TiledArray/math/linalg/diis.h
 TiledArray/math/linalg/util.h
@@ -73,6 +75,8 @@ TiledArray/math/linalg/scalapack/heig.h
 TiledArray/math/linalg/scalapack/lu.h
 TiledArray/math/linalg/scalapack/qr.h
 TiledArray/math/linalg/scalapack/svd.h
+TiledArray/math/linalg/ttg/cholesky.h
+TiledArray/math/linalg/ttg/util.h
 TiledArray/conversions/btas.h
 TiledArray/conversions/clone.h
 TiledArray/conversions/dense_to_sparse.h
@@ -223,6 +227,7 @@ TiledArray/array_impl.cpp
 TiledArray/dist_array.cpp
 TiledArray/util/backtrace.cpp
 TiledArray/util/bug.cpp
+TiledArray/math/linalg/basic.cpp
 TiledArray/math/linalg/rank-local.cpp
 )
 
@@ -254,6 +259,10 @@ if( TARGET TiledArray_SCALAPACK )
 endif()
 list(APPEND _TILEDARRAY_DEPENDENCIES "${LAPACK_LIBRARIES}")
 
+if( TARGET ttg-parsec )
+  list(APPEND _TILEDARRAY_DEPENDENCIES ttg-parsec)
+endif()
+
 # cache deps as TILEDARRAY_PRIVATE_LINK_LIBRARIES
 set(TILEDARRAY_PRIVATE_LINK_LIBRARIES ${_TILEDARRAY_DEPENDENCIES} CACHE STRING "List of libraries on which TiledArray depends on")
 
@@ -295,6 +304,10 @@ add_library(tiledarray ${TILEDARRAY_SOURCE_FILES} ${TILEDARRAY_HEADER_FILES})
     target_compile_definitions(${targetname} PUBLIC ${LAPACK_COMPILE_DEFINITIONS})
   endif (LAPACK_COMPILE_DEFINITIONS)
 
+if( TARGET ttg-parsec )
+  target_compile_definitions(${targetname} PUBLIC TTG_USE_PARSEC=1)
+endif()
+
 # Add library to the list of installed components
 install(TARGETS tiledarray EXPORT tiledarray COMPONENT tiledarray
     LIBRARY DESTINATION "${TILEDARRAY_INSTALL_LIBDIR}"

src/TiledArray/array_impl.h

@@ -451,7 +451,7 @@ class ArrayImpl : public TensorImpl<Policy> {
   /// \throw TiledArray::Exception When the size of shape is not equal to
   /// zero
   ArrayImpl(World& world, const trange_type& trange, const shape_type& shape,
-            const std::shared_ptr<pmap_interface>& pmap)
+            const std::shared_ptr<const pmap_interface>& pmap)
       : TensorImpl_(world, trange, shape, pmap),
         data_(world, trange.tiles_range().volume(), pmap) {}
 
@@ -636,9 +636,7 @@ class ArrayImpl : public TensorImpl<Policy> {
   /// DistributedStorage
 
   /// @return const reference to the atomic counter of live DelayedSet requests
-  const madness::AtomicInt& num_live_ds() const {
-    return data_.num_live_ds();
-  }
+  const madness::AtomicInt& num_live_ds() const { return data_.num_live_ds(); }
 
 };  // class ArrayImpl
 

src/TiledArray/config.h.in

@@ -81,6 +81,9 @@
 /* Is TA::Tensor memory profiling enabled? */
 #cmakedefine TA_TENSOR_MEM_PROFILE 1
 
+/* Is TTG available? */
+#cmakedefine TILEDARRAY_HAS_TTG 1
+
 /* Use preprocessor to check if BTAS is available */
 #ifndef TILEDARRAY_HAS_BTAS
 #ifdef __has_include

src/TiledArray/conversions/make_array.h

@@ -70,9 +70,9 @@ namespace TiledArray {
 /// \return An array object of type `Array`
 template <typename Array, typename Op,
           typename std::enable_if<is_dense<Array>::value>::type* = nullptr>
-inline Array make_array(World& world, const detail::trange_t<Array>& trange,
-                        const std::shared_ptr<detail::pmap_t<Array> >& pmap,
-                        Op&& op) {
+inline Array make_array(
+    World& world, const detail::trange_t<Array>& trange,
+    const std::shared_ptr<const detail::pmap_t<Array> >& pmap, Op&& op) {
   typedef typename Array::value_type value_type;
   typedef typename value_type::range_type range_type;
 
@@ -134,9 +134,9 @@ inline Array make_array(World& world, const detail::trange_t<Array>& trange,
 /// \return An array object of type `Array`
 template <typename Array, typename Op,
           typename std::enable_if<!is_dense<Array>::value>::type* = nullptr>
-inline Array make_array(World& world, const detail::trange_t<Array>& trange,
-                        const std::shared_ptr<detail::pmap_t<Array> >& pmap,
-                        Op&& op) {
+inline Array make_array(
+    World& world, const detail::trange_t<Array>& trange,
+    const std::shared_ptr<const detail::pmap_t<Array> >& pmap, Op&& op) {
   typedef typename Array::value_type value_type;
   typedef typename Array::ordinal_type ordinal_type;
   typedef std::pair<ordinal_type, Future<value_type> > datum_type;
@@ -146,8 +146,8 @@ inline Array make_array(World& world, const detail::trange_t<Array>& trange,
   tiles.reserve(pmap->size());
 
   // Construct a tensor to hold updated tile norms for the result shape.
-  TiledArray::Tensor<typename detail::shape_t<Array>::value_type>
-      tile_norms(trange.tiles_range(), 0);
+  TiledArray::Tensor<typename detail::shape_t<Array>::value_type> tile_norms(
+      trange.tiles_range(), 0);
 
   // Construct the task function used to construct the result tiles.
   madness::AtomicInt counter;

src/TiledArray/conversions/sparse_to_dense.h

@@ -39,7 +39,7 @@ to_dense(DistArray<Tile, ArgPolicy> const& sparse_array) {
   ArrayType dense_array(sparse_array.world(), sparse_array.trange());
 
   typedef typename ArrayType::pmap_interface pmap_interface;
-  std::shared_ptr<pmap_interface> const& pmap = dense_array.pmap();
+  auto& pmap = dense_array.pmap();
 
   typename pmap_interface::const_iterator end = pmap->end();
 

src/TiledArray/dense_shape.h

@@ -387,6 +387,16 @@ constexpr inline bool is_replicated(World& world, const DenseShape& t) {
   return true;
 }
 
+/// Add the shape to an output stream
+
+/// \param os The output stream
+/// \param shape the DenseShape object
+/// \return A reference to the output stream
+inline std::ostream& operator<<(std::ostream& os, const DenseShape& shape) {
+  os << "DenseShape:" << std::endl;
+  return os;
+}
+
 }  // namespace TiledArray
 
 #endif  // TILEDARRAY_DENSE_SHAPE_H__INCLUDED