kmp5VT [Feature] Round Robin pmap

INSTALL.md

@@ -40,9 +40,9 @@ Both methods are supported. However, for most users we _strongly_ recommend to b
   - Boost.Container: header-only
   - Boost.Test: header-only or (optionally) as a compiled library, *only used for unit testing*
   - Boost.Range: header-only, *only used for unit testing*
-- [BTAS](http://github.com/ValeevGroup/BTAS), tag 1c6099ed2d709896430a892b05bcb94b306f76c9 . If usable BTAS installation is not found, TiledArray will download and compile
+  - [BTAS](http://github.com/ValeevGroup/BTAS), tag 4f160466504d0405cefcc844b5db20b6bf70c732 . If usable BTAS installation is not found, TiledArray will download and compile 
   BTAS from source. *This is the recommended way to compile BTAS for all users*.
-- [MADNESS](https://github.com/m-a-d-n-e-s-s/madness), tag b22ee85059e6ccc9a6e803ba0550652ece8d9df1 .
+  - [MADNESS](https://github.com/m-a-d-n-e-s-s/madness), tag b22ee85059e6ccc9a6e803ba0550652ece8d9df1 .
   Only the MADworld runtime and BLAS/LAPACK C API component of MADNESS is used by TiledArray.
   If usable MADNESS installation is not found, TiledArray will download and compile
   MADNESS from source. *This is the recommended way to compile MADNESS for all users*.

external/versions.cmake

@@ -1,9 +1,8 @@
 # for each dependency track both current and previous id (the variable for the latter must contain PREVIOUS)
 # to be able to auto-update them
 
-# Boost explicitly downgraded to 1.59 from 1.68
-set(TA_TRACKED_BOOST_VERSION 1.59)
-set(TA_TRACKED_BOOST_PREVIOUS_VERSION 1.68)
+set(TA_TRACKED_BOOST_VERSION 1.33)
+set(TA_TRACKED_BOOST_PREVIOUS_VERSION 1.33)
 set(TA_INSTALL_BOOST_VERSION 1.70.0)
 set(TA_INSTALL_BOOST_PREVIOUS_VERSION 1.70.0)
 set(TA_INSTALL_BOOST_URL_HASH 882b48708d211a5f48e60b0124cf5863c1534cd544ecd0664bb534a4b5d506e9)
@@ -14,16 +13,16 @@ set(TA_TRACKED_EIGEN_VERSION 3.3)
 set(TA_TRACKED_EIGEN_PREVIOUS_VERSION 3.3)
 set(TA_INSTALL_EIGEN_VERSION 3.3.7)
 set(TA_INSTALL_EIGEN_PREVIOUS_VERSION 3.3.7)
-set(TA_INSTALL_EIGEN_URL_HASH b9e98a200d2455f06db9c661c5610496)
-set(TA_INSTALL_EIGEN_PREVIOUS_URL_HASH b9e98a200d2455f06db9c661c5610496)
+set(TA_INSTALL_EIGEN_URL_HASH 05b1f7511c93980c385ebe11bd3c93fa)
+set(TA_INSTALL_EIGEN_PREVIOUS_URL_HASH 05b1f7511c93980c385ebe11bd3c93fa)
 
 set(TA_TRACKED_MADNESS_TAG b22ee85059e6ccc9a6e803ba0550652ece8d9df1)
 set(TA_TRACKED_MADNESS_PREVIOUS_TAG 925552feaf326cca8e83de7bd042074ad3cfd3f1)
 set(TA_TRACKED_MADNESS_VERSION 0.10.1)
 set(TA_TRACKED_MADNESS_PREVIOUS_VERSION 0.10.1)
 
-set(TA_TRACKED_BTAS_TAG 1c6099ed2d709896430a892b05bcb94b306f76c9)
-set(TA_TRACKED_BTAS_PREVIOUS_TAG ef198e3fa53911eac308c8100c8651b5952c4f26)
+set(TA_TRACKED_BTAS_TAG 74553a6c7d7f0657951646a96c45c3170c71e40f)
+set(TA_TRACKED_BTAS_PREVIOUS_TAG 08f22d26b9f42a5a52e39088f157c87ac26822c8)
 
 set(TA_TRACKED_CUTT_TAG 0e8685bf82910bc7435835f846e88f1b39f47f09)
 set(TA_TRACKED_CUTT_PREVIOUS_TAG 592198b93c93b7ca79e7900b9a9f2e79f9dafec3)

src/CMakeLists.txt

@@ -129,6 +129,7 @@ TiledArray/pmap/cyclic_pmap.h
 TiledArray/pmap/hash_pmap.h
 TiledArray/pmap/pmap.h
 TiledArray/pmap/replicated_pmap.h
+TiledArray/pmap/round_robin_pmap.h
 TiledArray/policies/dense_policy.h
 TiledArray/policies/sparse_policy.h
 TiledArray/special/diagonal_array.h

src/TiledArray/conversions/vector_of_arrays.h

@@ -109,11 +109,11 @@ TA::SparseShape<float> fuse_vector_of_shapes_tiles(
     bool have_rank = (rank == fused_vidx % size);
     // how many arrays actually constribute to this fused tile ... last fused
     // tile may have fewer than block_size
-    const auto vblk_size =
-        (narrays - vidx) >= block_size ? block_size : narrays - vidx;
-    for (size_t tile_ord = 0; tile_ord != ntiles_per_array;
-         ++tile_ord, ++fused_tile_ord) {
-      if (have_rank) {
+    if (have_rank) {
+      const auto vblk_size =
+          (narrays - vidx) >= block_size ? block_size : narrays - vidx;
+      for (size_t tile_ord = 0; tile_ord != ntiles_per_array;
+           ++tile_ord, ++fused_tile_ord) {
         auto array_ptr = arrays.begin() + element_offset_in_owner * vblk_size;
         float unscaled_fused_tile_norm2 = 0;
         const auto tile_volume = tile_volumes[tile_ord];
@@ -129,9 +129,10 @@ TA::SparseShape<float> fuse_vector_of_shapes_tiles(
 
         *(fused_tile_norms.data() + fused_tile_ord) = fused_tile_norm;
       }
+      element_offset_in_owner += 1;
+    } else {
+      fused_tile_ord += ntiles_per_array;
     }
-    element_offset_in_owner =
-        have_rank ? element_offset_in_owner + 1 : element_offset_in_owner;
   }
 
   auto fused_shapes = TA::SparseShape<float>(global_world, fused_tile_norms,

src/TiledArray/pmap/round_robin_pmap.h

@@ -0,0 +1,67 @@
+//
+// Created by Karl Pierce on 7/26/20.
+//
+
+#ifndef TILEDARRAY_PMAP_ROUND_ROBIN_PMAP_H__INCLUDED
+#define TILEDARRAY_PMAP_ROUND_ROBIN_PMAP_H__INCLUDED
+
+#include <TiledArray/pmap/pmap.h>
+
+namespace TiledArray {
+namespace detail {
+
+/// A blocked process map
+
+/// Map N elements among P processes into blocks that are approximately N/P
+/// elements in size. A minimum block size may also be specified.
+class RoundRobinPmap : public Pmap {
+ protected:
+  // Import Pmap protected variables
+  using Pmap::procs_;  ///< The number of processes
+  using Pmap::rank_;   ///< The rank of this process
+  using Pmap::size_;   ///< The number of tiles mapped among all processes
+
+ private:
+  const size_type remainder_;  ///< tile remainder (= size_ % procs_)
+
+ public:
+  typedef Pmap::size_type size_type;  ///< Key type
+
+  /// Construct Round Robin map
+
+  /// \param world The world where the tiles will be mapped
+  /// \param size The number of tiles to be mapped
+  RoundRobinPmap(World& world, size_type size)
+      : Pmap(world, size), remainder_(size_ % procs_) {
+    auto num_tiles_per_proc = size / procs_;
+    if (remainder_ == 0 || rank_ >= remainder_)
+      this->local_size_ = num_tiles_per_proc;
+    else
+      this->local_size_ = num_tiles_per_proc + 1;
+  }
+
+  virtual ~RoundRobinPmap() {}
+
+  /// Maps \c tile to the processor that owns it
+
+  /// \param tile The tile to be queried
+  /// \return Processor that logically owns \c tile
+  virtual size_type owner(const size_type tile) const {
+    TA_ASSERT(tile < size_);
+    return (tile % procs_);
+  }
+
+  /// Check that the tile is owned by this process
+
+  /// \param tile The tile to be checked
+  /// \return \c true if \c tile is owned by this process, otherwise \c false .
+  virtual bool is_local(const size_type tile) const {
+    return (tile % procs_ == rank_);
+  }
+
+};  // class RoundRobinPmap
+
+}  // namespace detail
+}  // namespace TiledArray
+
+#endif  // TILEDARRAY_PMAP_ROUND_ROBIN_PMAP_H__INCLUDED

tests/CMakeLists.txt

@@ -56,6 +56,7 @@ set(ta_test_src_files  ta_test.cpp
     tiled_range1.cpp
     tiled_range.cpp
     blocked_pmap.cpp
+    round_robin_pmap.cpp
     hash_pmap.cpp
     cyclic_pmap.cpp
     replicated_pmap.cpp

tests/round_robin_pmap.cpp

@@ -0,0 +1,112 @@
+/*
+ *  This file is a part of TiledArray.
+ *  Copyright (C) 2013  Virginia Tech
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "TiledArray/pmap/round_robin_pmap.h"
+#include "global_fixture.h"
+#include "tiledarray.h"
+#include "unit_test_config.h"
+
+using namespace TiledArray;
+
+struct RoundRobinPmapFixture {
+  RoundRobinPmapFixture() {}
+};
+
+// =============================================================================
+// RoundRobinPmap Test Suite
+
+BOOST_FIXTURE_TEST_SUITE(round_robin_pmap_suite, RoundRobinPmapFixture)
+
+BOOST_AUTO_TEST_CASE(constructor) {
+  for (std::size_t tiles = 1ul; tiles < 100ul; ++tiles) {
+    BOOST_REQUIRE_NO_THROW(
+        TiledArray::detail::RoundRobinPmap pmap(*GlobalFixture::world, tiles));
+    TiledArray::detail::RoundRobinPmap pmap(*GlobalFixture::world, tiles);
+    BOOST_CHECK_EQUAL(pmap.rank(), GlobalFixture::world->rank());
+    BOOST_CHECK_EQUAL(pmap.procs(), GlobalFixture::world->size());
+    BOOST_CHECK_EQUAL(pmap.size(), tiles);
+  }
+}
+
+BOOST_AUTO_TEST_CASE(owner) {
+  const std::size_t rank = GlobalFixture::world->rank();
+  const std::size_t size = GlobalFixture::world->size();
+
+  ProcessID *p_owner = new ProcessID[size];
+
+  // Check various pmap sizes
+  for (std::size_t tiles = 1ul; tiles < 100ul; ++tiles) {
+    TiledArray::detail::RoundRobinPmap pmap(*GlobalFixture::world, tiles);
+
+    for (std::size_t tile = 0; tile < tiles; ++tile) {
+      std::fill_n(p_owner, size, 0);
+      p_owner[rank] = pmap.owner(tile);
+      // check that the value is in range
+      BOOST_CHECK_LT(p_owner[rank], size);
+      GlobalFixture::world->gop.sum(p_owner, size);
+
+      // Make sure everyone agrees on who owns what.
+      for (std::size_t p = 0ul; p < size; ++p)
+        BOOST_CHECK_EQUAL(p_owner[p], p_owner[rank]);
+    }
+  }
+
+  delete[] p_owner;
+}
+
+BOOST_AUTO_TEST_CASE(local_size) {
+  for (std::size_t tiles = 1ul; tiles < 100ul; ++tiles) {
+    TiledArray::detail::RoundRobinPmap pmap(*GlobalFixture::world, tiles);
+
+    std::size_t total_size = pmap.local_size();
+    GlobalFixture::world->gop.sum(total_size);
+
+    // Check that the total number of elements in all local groups is equal to
+    // the number of tiles in the map.
+    BOOST_CHECK_EQUAL(total_size, tiles);
+    BOOST_CHECK(pmap.empty() == (pmap.local_size() == 0ul));
+  }
+}
+
+BOOST_AUTO_TEST_CASE(local_group) {
+  ProcessID tile_owners[100];
+
+  for (std::size_t tiles = 1ul; tiles < 100ul; ++tiles) {
+    TiledArray::detail::RoundRobinPmap pmap(*GlobalFixture::world, tiles);
+
+    // Check that all local elements map to this rank
+    for (detail::RoundRobinPmap::const_iterator it = pmap.begin();
+         it != pmap.end(); ++it) {
+      BOOST_CHECK_EQUAL(pmap.owner(*it), GlobalFixture::world->rank());
+    }
+
+    std::fill_n(tile_owners, tiles, 0);
+    for (detail::RoundRobinPmap::const_iterator it = pmap.begin();
+         it != pmap.end(); ++it) {
+      tile_owners[*it] += GlobalFixture::world->rank();
+    }
+
+    GlobalFixture::world->gop.sum(tile_owners, tiles);
+    for (std::size_t tile = 0; tile < tiles; ++tile) {
+      BOOST_CHECK_EQUAL(tile_owners[tile], pmap.owner(tile));
+    }
+  }
+}
+
+BOOST_AUTO_TEST_SUITE_END()