Tiogia TPL Update / Build Fixes

host-configs/LLNL/tioga-cce@15.0.0.cmake

@@ -1,22 +1,22 @@
 
-set(CONFIG_NAME "tioga-cce@15.0.0" CACHE PATH "")
+set(CONFIG_NAME "tioga-cce@15.0.0c" CACHE PATH "")
 include( ${CMAKE_CURRENT_LIST_DIR}/tioga-base.cmake )
 
 # Set up the tpls
-set(TPL_INSTALL_DATE 2023-01-19)
+set(TPL_INSTALL_DATE 2023-05-17)
 set(GEOSX_TPL_DIR "/usr/WS1/GEOS/GEOSX/TPLs_${TPL_INSTALL_DATE}/install-${CONFIG_NAME}" CACHE PATH "")
 
-set(CAMP_DIR "${GEOSX_TPL_DIR}/camp" CACHE PATH "" )
-set(RAJA_DIR "${GEOSX_TPL_DIR}/raja" CACHE PATH "" )
+set(CAMP_DIR "${GEOSX_TPL_DIR}/camp-main" CACHE PATH "" )
+set(RAJA_DIR "${GEOSX_TPL_DIR}/raja-develop" CACHE PATH "" )
 
 set(ENABLE_UMPIRE TRUE CACHE BOOL "" )
-set(UMPIRE_DIR "${GEOSX_TPL_DIR}/umpire" CACHE PATH "" )
+set(UMPIRE_DIR "${GEOSX_TPL_DIR}/umpire-develop" CACHE PATH "" )
 
 set(ENABLE_CHAI TRUE CACHE BOOL "" )
-set(CHAI_DIR "${GEOSX_TPL_DIR}/chai" CACHE PATH "" )
+set(CHAI_DIR "${GEOSX_TPL_DIR}/chai-develop" CACHE PATH "" )
 
-set(METIS_DIR "${GEOSX_TPL_DIR}/metis" CACHE PATH "" )
-set(PARMETIS_DIR "${GEOSX_TPL_DIR}/parmetis" CACHE PATH "" )
+set(METIS_DIR "${GEOSX_TPL_DIR}/metis-5.1.0" CACHE PATH "" )
+set(PARMETIS_DIR "${GEOSX_TPL_DIR}/parmetis-4.0.3" CACHE PATH "" )
 
 # C++ options
 set(CRAYPE_VERSION "2.7.19")
@@ -25,13 +25,13 @@ set(CMAKE_CXX_COMPILER "/opt/cray/pe/craype/${CRAYPE_VERSION}/bin/CC" CACHE PATH
 set(CMAKE_Fortran_COMPILER "/opt/cray/pe/craype/${CRAYPE_VERSION}/bin/ftn" CACHE PATH "")
 
 if( ENABLE_HIP )
-  set( ENABLE_CLANG_HIP ON CACHE BOOL "" FORCE ) # don't invoke hipcc, rely on cce link-time compilation
+  set( ENABLE_CLANG_HIP ON CACHE BOOL "" FORCE )
 
   set( HIP_VERSION_STRING "5.4.0" CACHE STRING "" )
   set( HIP_ROOT "/opt/rocm-${HIP_VERSION_STRING}" CACHE PATH "" )
   set( ROCM_PATH ${HIP_ROOT} CACHE PATH "" )
 
   set( CMAKE_HIP_ARCHITECTURES "gfx90a" CACHE STRING "" FORCE )
-  set( CMAKE_CXX_FLAGS "-mno-unsafe-fp-atomics -fgpu-rdc" CACHE STRING "" FORCE )
+  set( CMAKE_CXX_FLAGS "-munsafe-fp-atomics -fgpu-rdc" CACHE STRING "" FORCE )
   set( CMAKE_CXX_LINK_FLAGS "-fgpu-rdc --hip-link" CACHE STRING "" FORCE )
 endif()

src/ChaiBuffer.hpp

@@ -138,33 +138,52 @@ class ChaiBuffer
    * @brief Constructor for creating an empty Buffer.
    * @details An empty buffer may hold resources and needs to be free'd.
    * @note The unused boolean parameter is to distinguish this from default constructor.
+   * @note Although it is marked as a host-device method, this is only valid to call from the host.
    */
+  LVARRAY_HOST_DEVICE
   ChaiBuffer( bool ):
     m_pointer( nullptr ),
-    m_capacity( 0 ),
-    m_pointerRecord( new chai::PointerRecord{} )
+    m_capacity( 0 )
+  #if !defined(LVARRAY_DEVICE_COMPILE)
+    , m_pointerRecord( new chai::PointerRecord{} )
+  #else
+    , m_pointerRecord( nullptr )
+  #endif
   {
+  #if defined(LVARRAY_DEVICE_COMPILE)
+    LVARRAY_ERROR( "Creating a new ChaiBuffer on device is not supported. This is often the result of capturing an array on device instead of a view." );
+  #else
     m_pointerRecord->m_size = 0;
     setName( "" );
 
     for( int space = chai::CPU; space < chai::NUM_EXECUTION_SPACES; ++space )
     {
       m_pointerRecord->m_allocators[ space ] = internal::getArrayManager().getAllocatorId( chai::ExecutionSpace( space ) );
     }
+  #endif
   }
 
   /**
    * @brief Construct a ChaiBuffer which uses the specific allocator for each space.
    * @param spaces The list of spaces.
    * @param allocators The allocators, must be the same length as @p spaces.
    * @details @code allocator[ i ] @endcode is used for the memory space @code spaces[ i ] @endcode.
+   * @note Although it is marked as a host-device method, this is only valid to call from the host.
    */
+  LVARRAY_HOST_DEVICE
   ChaiBuffer( std::initializer_list< MemorySpace > const & spaces,
               std::initializer_list< umpire::Allocator > const & allocators ):
     m_pointer( nullptr ),
-    m_capacity( 0 ),
-    m_pointerRecord( new chai::PointerRecord{} )
+    m_capacity( 0 )
+  #if !defined(LVARRAY_DEVICE_COMPILE)
+    , m_pointerRecord( new chai::PointerRecord{} )
+  #else
+    , m_pointerRecord( nullptr )
+  #endif
   {
+  #if defined(LVARRAY_DEVICE_COMPILE)
+    LVARRAY_ERROR( "Creating a new ChaiBuffer on device is not supported." );
+  #else
     m_pointerRecord->m_size = 0;
     setName( "" );
 
@@ -179,6 +198,7 @@ class ChaiBuffer
     {
       m_pointerRecord->m_allocators[ internal::toChaiExecutionSpace( spaces.begin()[ i ] ) ] = allocators.begin()[ i ].getId();
     }
+  #endif
   }
 
   /**
@@ -285,9 +305,14 @@ class ChaiBuffer
    * @param space The space to perform the reallocation in. If space is the CPU then the buffer is reallocated
    *   only on the CPU and it is free'd in the other spaces. If the space is the GPU the the current size must be zero.
    * @param newCapacity the new capacity of the buffer.
+   * @note Although it is marked as a host-device method, this is only valid to call from the host.
    */
+  LVARRAY_HOST_DEVICE
   void reallocate( std::ptrdiff_t const size, MemorySpace const space, std::ptrdiff_t const newCapacity )
   {
+  #if defined(LVARRAY_DEVICE_COMPILE)
+    LVARRAY_ERROR( "Allocation from device is not supported." );
+  #else
     move( space, true );
     chai::PointerRecord * const newRecord = new chai::PointerRecord{};
     newRecord->m_size = newCapacity * sizeof( T );
@@ -319,20 +344,26 @@ class ChaiBuffer
     m_pointer = newPointer;
     m_pointerRecord = newRecord;
     registerTouch( space );
+  #endif
   }
 
   /**
    * @brief Free the data in the buffer but does not destroy any values.
    * @note To destroy the values and free the data call bufferManipulation::free.
+   * @note Although it is marked as a host-device method, this is only valid to call from the host.
    */
-  inline
+  LVARRAY_HOST_DEVICE inline
   void free()
   {
+  #if defined(LVARRAY_DEVICE_COMPILE)
+    LVARRAY_ERROR( "Deallocation from device is not supported." );
+  #else
     std::lock_guard< std::mutex > lock( internal::chaiLock );
     internal::getArrayManager().free( m_pointerRecord );
     m_capacity = 0;
     m_pointer = nullptr;
     m_pointerRecord = nullptr;
+  #endif
   }
 
   /**

src/MallocBuffer.hpp

@@ -16,6 +16,7 @@
 #include "LvArrayConfig.hpp"
 #include "Macros.hpp"
 #include "bufferManipulation.hpp"
+#include "math.hpp"
 
 // System includes
 #include <stddef.h>
@@ -125,14 +126,15 @@ class MallocBuffer : public bufferManipulation::VoidBuffer
    * @param space The space to perform the reallocation in, not used.
    * @param newCapacity The new capacity of the buffer.
    */
+  LVARRAY_HOST_DEVICE inline
   void reallocate( std::ptrdiff_t const size, MemorySpace const space, std::ptrdiff_t const newCapacity )
   {
     LVARRAY_ERROR_IF_NE( space, MemorySpace::host );
 
     // TODO: If std::is_trivially_copyable_v< T > then we could use std::realloc.
     T * const newPtr = reinterpret_cast< T * >( std::malloc( newCapacity * sizeof( T ) ) );
 
-    std::ptrdiff_t const overlapAmount = std::min( newCapacity, size );
+    std::ptrdiff_t const overlapAmount = math::min( newCapacity, size );
     arrayManipulation::uninitializedMove( newPtr, overlapAmount, m_data );
     arrayManipulation::destroy( m_data, size );