Memkind persistent

.travis.yml

@@ -14,6 +14,39 @@ addons:
       - hwloc
 matrix:
   allow_failures:
+  exclude:
+     - os: osx
+       compiler: gcc
+       env: MPI_IMPL=mpich
+            ARMCI_STRIDED_METHOD=DIRECT
+            ARMCI_IOV_METHOD=DIRECT
+            ARMCI_USE_WIN_ALLOCATE=-100
+            ARMCI_SHM_LIMIT=$((32*1024*1024))
+            ARMCI_MEMKIND_POOL_PATH=/tmp
+     - os: osx
+       compiler: clang
+       env: MPI_IMPL=mpich
+            ARMCI_STRIDED_METHOD=DIRECT
+            ARMCI_IOV_METHOD=DIRECT
+            ARMCI_USE_WIN_ALLOCATE=-100
+            ARMCI_SHM_LIMIT=$((32*1024*1024))
+            ARMCI_MEMKIND_POOL_PATH=/tmp
+     - os: osx
+       compiler: gcc
+       env: MPI_IMPL=openmpi
+            ARMCI_STRIDED_METHOD=IOV
+            ARMCI_IOV_METHOD=BATCHED
+            ARMCI_USE_WIN_ALLOCATE=-100
+            ARMCI_SHM_LIMIT=$((32*1024*1024))
+            ARMCI_MEMKIND_POOL_PATH=/tmp
+     - os: osx
+       compiler: clang
+       env: MPI_IMPL=openmpi
+            ARMCI_STRIDED_METHOD=IOV
+            ARMCI_IOV_METHOD=BATCHED
+            ARMCI_USE_WIN_ALLOCATE=-100
+            ARMCI_SHM_LIMIT=$((32*1024*1024))
+            ARMCI_MEMKIND_POOL_PATH=/tmp
 env:
   #
   # Test the default config since that is what Debian users will get.
@@ -73,6 +106,21 @@ env:
   #  ARMCI_IOV_METHOD=DIRECT
   #  ARMCI_NO_FLUSH_LOCAL=1
   #  ARMCI_USE_WIN_ALLOCATE=1
+  #
+  #  MEMKIND testing
+  #
+  #- MPI_IMPL=mpich
+  #  ARMCI_STRIDED_METHOD=DIRECT
+  #  ARMCI_IOV_METHOD=DIRECT
+  #  ARMCI_USE_WIN_ALLOCATE=-100
+  #  ARMCI_SHM_LIMIT=$((32*1024*1024))
+  #  ARMCI_MEMKIND_POOL_PATH=/tmp
+  #- MPI_IMPL=openmpi
+  #  ARMCI_STRIDED_METHOD=IOV
+  #  ARMCI_IOV_METHOD=BATCHED
+  #  ARMCI_USE_WIN_ALLOCATE=-100
+  #  ARMCI_SHM_LIMIT=$((32*1024*1024))
+  #  ARMCI_MEMKIND_POOL_PATH=/tmp
 before_install:
   - export TRAVIS_ROOT=$HOME/travis
   - mkdir $TRAVIS_ROOT

configure.ac

@@ -126,14 +126,37 @@ if test "$ac_cv_prog_cc_c99" = "no" ; then
    AC_ERROR([C99 not supported by the compiler])
 fi
 
-AC_CHECK_HEADERS([execinfo.h stdint.h inttypes.h unistd.h time.h])
+AC_CHECK_HEADERS([execinfo.h string.h strings.h stdint.h inttypes.h unistd.h errno.h time.h])
 AC_TYPE_UINT8_T
 AX_PTHREAD([AC_DEFINE(HAVE_PTHREADS,1,[Defined when Pthread library is detected])])
 CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
 LIBS="$PTHREAD_LIBS $LIBS"
 AC_CHECK_FUNC([usleep], [AC_DEFINE([HAVE_USLEEP], [1], [Define if usleep exists.])])
 AC_CHECK_FUNC([nanosleep], [AC_DEFINE([HAVE_NANOSLEEP], [1], [Define if nanosleep exists.])])
 
+## MEMKIND support
+AC_ARG_WITH(memkind,
+            AC_HELP_STRING([--with-memkind],[Enable MEMKIND library]),
+            [use_memkind=$withval],
+            [use_memkind=no])
+AC_MSG_CHECKING(MEMKIND support)
+AC_MSG_RESULT($use_memkind)
+if test "x$use_memkind" != "xno"; then
+    if test "x$use_memkind" = "x"; then
+        AC_CHECK_HEADERS([memkind.h])
+        LIBS="$LIBS -lmemkind"
+        AC_CHECK_FUNCS([memkind_destroy_kind], [have_memkind_destroy_kind=yes], [have_memkind_destroy_kind=no])
+    else
+        CPPFLAGS="$CPPFLAGS -I$use_memkind/include"
+        AC_CHECK_HEADERS([memkind.h])
+        LIBS="-L$use_memkind/lib -lmemkind $LIBS"
+        AC_CHECK_FUNCS([memkind_destroy_kind], [have_memkind_destroy_kind=yes], [have_memkind_destroy_kind=no])
+    fi
+    if test "x$have_memkind_destroy_kind" = "xno"; then
+        AC_MSG_ERROR([Your MEMKIND library version is too old and lacks memkind_destroy_kind.  Please install a newer version.])
+    fi
+fi
+
 ## Debugging support
 AC_ARG_ENABLE(g, AC_HELP_STRING([--enable-g],[Enable Debugging]),
                  [ debug=$enableval ],

src/armci_internals.h

@@ -8,16 +8,37 @@
 #include <armci.h>
 #include <armciconf.h>
 
-#if   HAVE_STDINT_H
+#include <stdio.h>
+#include <stdlib.h>
+#ifdef HAVE_STRING_H
+#include <string.h>
+#endif
+#if HAVE_STDINT_H
 #  include <stdint.h>
 #elif HAVE_INTTYPES_H
 #  include <inttypes.h>
 #endif
+#ifdef HAVE_ERRNO_H
+#include <errno.h>
+#endif
+
+#include <mpi.h>
 
 #ifdef HAVE_PTHREADS
 #  include <pthread.h>
 #endif
 
+
+#ifdef HAVE_MEMKIND_H
+#include <memkind.h>
+#define ARMCII_MEMKIND_WINDOW_TYPE -100
+
+#ifndef MEMKIND_PMEM_MIN_SIZE
+#define MEMKIND_PMEM_MIN_SIZE (1024 * 1024 * 16)
+#endif
+
+#endif
+
 /* Likely/Unlikely macros borrowed from MPICH:
  */
 
@@ -83,14 +104,18 @@ typedef struct {
   int           progress_thread;        /* Create progress thread                                               */
   int           progress_usleep;        /* Argument to usleep() to throttling polling                           */
 #endif
-  int           use_win_allocate;       /* Use win_allocate or win_create                                       */
+  int           use_win_allocate;       /* Use win_allocate or win_create (or special memory...)                */
   int           explicit_nb_progress;   /* Poke the MPI progress engine at the end of nonblocking (NB) calls    */
   int           use_alloc_shm;          /* Pass alloc_shm info to win_allocate / alloc_mem                      */
   int           rma_atomicity;          /* Use Accumulate and Get_accumulate for Put and Get                    */
   int           end_to_end_flush;       /* All flush_local calls become flush                                   */
   int           rma_nocheck;            /* Use MPI_MODE_NOCHECK on synchronization calls that take assertion    */
 
   size_t        memory_limit;           /* upper bound on how much memory ARMCI can allocate                    */
+#ifdef HAVE_MEMKIND_H
+  struct
+  memkind* memkind_handle;              /* memkind volatile memory pool opaque object                           */
+#endif
 
   enum ARMCII_Strided_methods_e strided_method; /* Strided transfer method              */
   enum ARMCII_Iov_methods_e     iov_method;     /* IOV transfer method                  */

src/gmr.c

@@ -2,15 +2,6 @@
  * Copyright (C) 2010. See COPYRIGHT in top-level directory.
  */
 
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <mpi.h>
-
-#ifdef HAVE_PTHREADS
-#  include <pthread.h>
-#endif
-
 #include <armci.h>
 #include <armcix.h>
 #include <armci_internals.h>
@@ -92,7 +83,17 @@ gmr_t *gmr_create(gmr_size_t local_size, void **base_ptrs, ARMCI_Group *group) {
       alloc_shm_info = MPI_INFO_NULL;
   }
 
-  if (ARMCII_GLOBAL_STATE.use_win_allocate) {
+  if (ARMCII_GLOBAL_STATE.use_win_allocate == 0) {
+
+      if (local_size == 0) {
+        alloc_slices[alloc_me].base = NULL;
+      } else {
+        MPI_Alloc_mem(local_size, alloc_shm_info, &(alloc_slices[alloc_me].base));
+        ARMCII_Assert(alloc_slices[alloc_me].base != NULL);
+      }
+      MPI_Win_create(alloc_slices[alloc_me].base, (MPI_Aint) local_size, 1, MPI_INFO_NULL, group->comm, &mreg->window);
+  }
+  else if (ARMCII_GLOBAL_STATE.use_win_allocate == 1) {
 
       /* give hint to CASPER to avoid extra work for lock permission */
       if (alloc_shm_info == MPI_INFO_NULL)
@@ -107,16 +108,25 @@ gmr_t *gmr_create(gmr_size_t local_size, void **base_ptrs, ARMCI_Group *group) {
       } else {
         ARMCII_Assert(alloc_slices[alloc_me].base != NULL);
       }
-  } else /* use win create */ {
+  }
+#ifdef HAVE_MEMKIND_H
+  else if (ARMCII_GLOBAL_STATE.use_win_allocate == ARMCII_MEMKIND_WINDOW_TYPE) {
+
       if (local_size == 0) {
         alloc_slices[alloc_me].base = NULL;
       } else {
-        MPI_Alloc_mem(local_size, alloc_shm_info, &(alloc_slices[alloc_me].base));
-        ARMCII_Assert(alloc_slices[alloc_me].base != NULL);
+        ARMCII_Assert(ARMCII_GLOBAL_STATE.memkind_handle != NULL);
+        alloc_slices[alloc_me].base = memkind_malloc(ARMCII_GLOBAL_STATE.memkind_handle, local_size);
+        if (alloc_slices[alloc_me].base == NULL) {
+            ARMCII_Error("MEMKIND failed to allocate memory! (errno=%d)\n", errno);
+        }
       }
       MPI_Win_create(alloc_slices[alloc_me].base, (MPI_Aint) local_size, 1, MPI_INFO_NULL, group->comm, &mreg->window);
-
-  } /* win allocate/create */
+  }
+#endif
+  else {
+      ARMCII_Error("invalid window type!\n");
+  }
 
   if (alloc_shm_info != MPI_INFO_NULL) {
       MPI_Info_free(&alloc_shm_info);
@@ -308,11 +318,19 @@ void gmr_destroy(gmr_t *mreg, ARMCI_Group *group) {
   /* Destroy the window and free all buffers */
   MPI_Win_free(&mreg->window);
 
-  if (!ARMCII_GLOBAL_STATE.use_win_allocate) {
+  if (ARMCII_GLOBAL_STATE.use_win_allocate == 0) {
     if (mreg->slices[world_me].base != NULL) {
       MPI_Free_mem(mreg->slices[world_me].base);
     }
   }
+#ifdef HAVE_MEMKIND_H
+  else if (ARMCII_GLOBAL_STATE.use_win_allocate == ARMCII_MEMKIND_WINDOW_TYPE) {
+    if (mreg->slices[world_me].base != NULL) {
+      ARMCII_Assert(ARMCII_GLOBAL_STATE.memkind_handle != NULL);
+      memkind_free(ARMCII_GLOBAL_STATE.memkind_handle, mreg->slices[world_me].base);
+    }
+  }
+#endif
 
   free(mreg->slices);
   free(mreg);