refactor: VecMem-ification of the Sycl Seedfinder Plugin (#998)

The current implementation of the Sycl Seedfinder uses pointers to manage memory allocations on the host and on the device. This PR changes this approach by implementing VecMem for memory management.

CMakeLists.txt

@@ -36,6 +36,7 @@ option(ACTS_BUILD_PLUGIN_JSON "Build json plugin" OFF)
 option(ACTS_USE_SYSTEM_NLOHMANN_JSON "Use nlohmann::json provided by the system instead of the bundled version" OFF)
 option(ACTS_BUILD_PLUGIN_LEGACY "Build legacy plugin" OFF)
 option(ACTS_BUILD_PLUGIN_ONNX "Build ONNX plugin" OFF)
+option(ACTS_USE_SYSTEM_VECMEM "Use a system-provided vecmem installation" OFF)
 option(ACTS_BUILD_PLUGIN_SYCL "Build SYCL plugin" OFF)
 option(ACTS_BUILD_PLUGIN_TGEO "Build TGeo plugin" OFF)
 # fatras related options
@@ -213,6 +214,11 @@ if(ACTS_BUILD_PLUGIN_ONNX)
 endif()
 if(ACTS_BUILD_PLUGIN_SYCL)
   find_package(SYCL REQUIRED)
+  if (ACTS_USE_SYSTEM_VECMEM)
+    find_package(vecmem REQUIRED)
+  else()
+    add_subdirectory(thirdparty/vecmem)
+  endif()
 endif()
 if(ACTS_BUILD_PLUGIN_TGEO)
   find_package(ROOT ${_acts_root_version} REQUIRED CONFIG COMPONENTS Geom)

Plugins/Sycl/CMakeLists.txt

@@ -1,3 +1,11 @@
+# This file is part of the Acts project.
+#
+# Copyright (C) 2020-2021 CERN for the benefit of the Acts project
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
 add_library(
   ActsPluginSycl SHARED
   # header files
@@ -29,7 +37,8 @@ target_include_directories(
 
 target_link_libraries(
   ActsPluginSycl
-  PUBLIC ActsCore)
+  PUBLIC ActsCore vecmem::core
+  PRIVATE vecmem::sycl)
 
 acts_target_setup_sycl(ActsPluginSycl DEPENDENCY PRIVATE)
 

Plugins/Sycl/include/Acts/Plugins/Sycl/Seeding/CreateSeedsForGroupSycl.hpp

@@ -1,6 +1,6 @@
 // This file is part of the Acts project.
 //
-// Copyright (C) 2020 CERN for the benefit of the Acts project
+// Copyright (C) 2020-2021 CERN for the benefit of the Acts project
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -11,6 +11,11 @@
 // System include(s)
 #include <vector>
 
+// VecMem include(s).
+#include "vecmem/containers/jagged_vector.hpp"
+#include "vecmem/containers/vector.hpp"
+#include "vecmem/memory/memory_resource.hpp"
+
 // SYCL plugin include(s)
 #include "Acts/Plugins/Sycl/Seeding/DeviceExperimentCuts.hpp"
 #include "Acts/Plugins/Sycl/Seeding/detail/Types.hpp"
@@ -21,6 +26,10 @@ namespace Acts::Sycl {
 /// @brief Seedfinding algorithm implemented in SYCL.
 ///
 /// @param[in] wrappedQueue is a wrapper object of the SYCL queue
+/// @param[in] resource is the host-accessible memory resource to use
+/// @param[in] device_resource is the optional device-accessible memory
+///                            resource, necessary if @c resource is not
+///                            device-accessible
 /// @param[in] seedfinderConfig includes the required configuration
 /// parameters for the algorithm
 /// @param[in] deviceCuts is an experiment specific object with customizable
@@ -33,11 +42,12 @@ namespace Acts::Sycl {
 ///                      point structures of top space points
 /// @param[out] seeds holds of the generated seed indices and weight
 void createSeedsForGroupSycl(
-    const QueueWrapper& wrappedQueue,
+    QueueWrapper wrappedQueue, vecmem::memory_resource& resource,
+    vecmem::memory_resource* device_resource,
     const detail::DeviceSeedfinderConfig& seedfinderConfig,
     const DeviceExperimentCuts& deviceCuts,
-    const std::vector<detail::DeviceSpacePoint>& bottomSPs,
-    const std::vector<detail::DeviceSpacePoint>& middleSPs,
-    const std::vector<detail::DeviceSpacePoint>& topSPs,
+    vecmem::vector<detail::DeviceSpacePoint>& bottomSPs,
+    vecmem::vector<detail::DeviceSpacePoint>& middleSPs,
+    vecmem::vector<detail::DeviceSpacePoint>& topSPs,
     std::vector<std::vector<detail::SeedData>>& seeds);
 }  // namespace Acts::Sycl

Plugins/Sycl/include/Acts/Plugins/Sycl/Seeding/Seedfinder.hpp

@@ -1,6 +1,6 @@
 // This file is part of the Acts project.
 //
-// Copyright (C) 2020 CERN for the benefit of the Acts project
+// Copyright (C) 2020-2021 CERN for the benefit of the Acts project
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -17,15 +17,19 @@
 #include "Acts/Plugins/Sycl/Seeding/detail/Types.hpp"
 #include "Acts/Plugins/Sycl/Utilities/QueueWrapper.hpp"
 
+// VecMem include(s).
+#include "vecmem/memory/memory_resource.hpp"
+
 namespace Acts::Sycl {
 
 template <typename external_spacepoint_t>
 class Seedfinder {
  public:
-  Seedfinder(
-      Acts::SeedfinderConfig<external_spacepoint_t> config,
-      const Acts::Sycl::DeviceExperimentCuts& cuts,
-      Acts::Sycl::QueueWrapper wrappedQueue = Acts::Sycl::QueueWrapper());
+  Seedfinder(Acts::SeedfinderConfig<external_spacepoint_t> config,
+             const Acts::Sycl::DeviceExperimentCuts& cuts,
+             Acts::Sycl::QueueWrapper wrappedQueue,
+             vecmem::memory_resource& resource,
+             vecmem::memory_resource* device_resource = nullptr);
 
   ~Seedfinder() = default;
   Seedfinder() = delete;
@@ -56,6 +60,12 @@ class Seedfinder {
 
   /// Wrapper around a SYCL queue object.
   QueueWrapper m_wrappedQueue;
+
+  /// host/shared memory resource to use in the seed-finder
+  vecmem::memory_resource* m_resource;
+
+  /// Device memory resource for use
+  vecmem::memory_resource* m_device_resource;
 };
 
 }  // namespace Acts::Sycl

Plugins/Sycl/include/Acts/Plugins/Sycl/Seeding/Seedfinder.ipp

@@ -1,6 +1,6 @@
 // This file is part of the Acts project.
 //
-// Copyright (C) 2020 CERN for the benefit of the Acts project
+// Copyright (C) 2020-2021 CERN for the benefit of the Acts project
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -11,6 +11,9 @@
 #include <cmath>
 #include <utility>
 
+// VecMem include(s).
+#include "vecmem/containers/vector.hpp"
+
 // Acts include(s).
 #include "Acts/Seeding/InternalSeed.hpp"
 #include "Acts/Seeding/InternalSpacePoint.hpp"
@@ -24,10 +27,13 @@ template <typename external_spacepoint_t>
 Seedfinder<external_spacepoint_t>::Seedfinder(
     Acts::SeedfinderConfig<external_spacepoint_t> config,
     const Acts::Sycl::DeviceExperimentCuts& cuts,
-    Acts::Sycl::QueueWrapper wrappedQueue)
+    Acts::Sycl::QueueWrapper wrappedQueue, vecmem::memory_resource& resource,
+    vecmem::memory_resource* device_resource)
     : m_config(config.toInternalUnits()),
       m_deviceCuts(cuts),
-      m_wrappedQueue(std::move(wrappedQueue)) {
+      m_wrappedQueue(std::move(wrappedQueue)),
+      m_resource(&resource),
+      m_device_resource(device_resource) {
   // init m_config
   m_config.highland = 13.6f * std::sqrt(m_config.radLengthPerSeed) *
                       (1 + 0.038f * std::log(m_config.radLengthPerSeed));
@@ -72,9 +78,11 @@ Seedfinder<external_spacepoint_t>::createSeedsForGroup(
   // that are easily comprehensible by the GPU. This allows us
   // less memory access operations than with simple (float) arrays.
 
-  std::vector<detail::DeviceSpacePoint> deviceBottomSPs;
-  std::vector<detail::DeviceSpacePoint> deviceMiddleSPs;
-  std::vector<detail::DeviceSpacePoint> deviceTopSPs;
+  // Creating VecMem vectors of the space points, linked to the host/shared
+  // memory resource They will be filled and passed to CreateSeedsForGroup().
+  vecmem::vector<detail::DeviceSpacePoint> deviceBottomSPs(m_resource);
+  vecmem::vector<detail::DeviceSpacePoint> deviceMiddleSPs(m_resource);
+  vecmem::vector<detail::DeviceSpacePoint> deviceTopSPs(m_resource);
 
   std::vector<const Acts::InternalSpacePoint<external_spacepoint_t>*>
       bottomSPvec;
@@ -84,31 +92,40 @@ Seedfinder<external_spacepoint_t>::createSeedsForGroup(
 
   for (const Acts::InternalSpacePoint<external_spacepoint_t>* SP : bottomSPs) {
     bottomSPvec.push_back(SP);
-    deviceBottomSPs.push_back(
-        detail::DeviceSpacePoint{SP->x(), SP->y(), SP->z(), SP->radius(),
-                                 SP->varianceR(), SP->varianceZ()});
+  }
+  deviceBottomSPs.reserve(bottomSPvec.size());
+  for (const Acts::InternalSpacePoint<external_spacepoint_t>* SP :
+       bottomSPvec) {
+    deviceBottomSPs.push_back({SP->x(), SP->y(), SP->z(), SP->radius(),
+                               SP->varianceR(), SP->varianceZ()});
   }
 
   for (const Acts::InternalSpacePoint<external_spacepoint_t>* SP : middleSPs) {
     middleSPvec.push_back(SP);
-    deviceMiddleSPs.push_back(
-        detail::DeviceSpacePoint{SP->x(), SP->y(), SP->z(), SP->radius(),
-                                 SP->varianceR(), SP->varianceZ()});
+  }
+  deviceMiddleSPs.reserve(middleSPvec.size());
+  for (const Acts::InternalSpacePoint<external_spacepoint_t>* SP :
+       middleSPvec) {
+    deviceMiddleSPs.push_back({SP->x(), SP->y(), SP->z(), SP->radius(),
+                               SP->varianceR(), SP->varianceZ()});
   }
 
   for (const Acts::InternalSpacePoint<external_spacepoint_t>* SP : topSPs) {
     topSPvec.push_back(SP);
-    deviceTopSPs.push_back(
-        detail::DeviceSpacePoint{SP->x(), SP->y(), SP->z(), SP->radius(),
-                                 SP->varianceR(), SP->varianceZ()});
+  }
+  deviceTopSPs.reserve(topSPvec.size());
+  for (const Acts::InternalSpacePoint<external_spacepoint_t>* SP : topSPvec) {
+    deviceTopSPs.push_back({SP->x(), SP->y(), SP->z(), SP->radius(),
+                            SP->varianceR(), SP->varianceZ()});
   }
 
+  // std::vector<std::vector<detail::SeedData>> seeds;
   std::vector<std::vector<detail::SeedData>> seeds;
 
   // Call the SYCL seeding algorithm
-  createSeedsForGroupSycl(m_wrappedQueue, m_deviceConfig, m_deviceCuts,
-                          deviceBottomSPs, deviceMiddleSPs, deviceTopSPs,
-                          seeds);
+  createSeedsForGroupSycl(m_wrappedQueue, *m_resource, m_device_resource,
+                          m_deviceConfig, m_deviceCuts, deviceBottomSPs,
+                          deviceMiddleSPs, deviceTopSPs, seeds);
 
   // Iterate through seeds returned by the SYCL algorithm and perform the last
   // step of filtering for fixed middle SP.

Plugins/Sycl/include/Acts/Plugins/Sycl/Utilities/QueueWrapper.hpp

@@ -1,6 +1,6 @@
 // This file is part of the Acts project.
 //
-// Copyright (C) 2020 CERN for the benefit of the Acts project
+// Copyright (C) 2020-2021 CERN for the benefit of the Acts project
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -31,6 +31,10 @@ class QueueWrapper {
   QueueWrapper(const std::string& = "",
                std::unique_ptr<const Logger> logger =
                    getDefaultLogger("Sycl::QueueWrapper", Logging::INFO));
+  /// Constructor around an existing queue object
+  QueueWrapper(cl::sycl::queue& queue,
+               std::unique_ptr<const Logger> logger =
+                   getDefaultLogger("Sycl::QueueWrapper", Logging::INFO));
   /// Move constructor
   /// It takes ownership (if it is given).
   QueueWrapper(QueueWrapper&& parent) noexcept;
@@ -46,8 +50,25 @@ class QueueWrapper {
   /// Copy assignment operator
   QueueWrapper& operator=(const QueueWrapper& other);
 
-  /// Get stored pointer
-  cl::sycl::queue* getQueue() const;
+  /// @name Accessor functions/operators
+  /// @{
+
+  /// Get stored pointer (const)
+  const cl::sycl::queue* getQueue() const;
+  /// Get stored pointer (non-const)
+  cl::sycl::queue* getQueue();
+
+  /// Operator implementing smart pointer behaviour (const)
+  const cl::sycl::queue* operator->() const;
+  /// Operator implementing smart pointer behaviour (non-const)
+  cl::sycl::queue* operator->();
+
+  /// De-referencing operator (const)
+  const cl::sycl::queue& operator*() const;
+  /// De-referencing operator (non-const)
+  cl::sycl::queue& operator*();
+
+  /// @}
 
  private:
   /// Raw pointer to SYCL queue object