From 54998e5745424a683fec8653831c85a5dc8ac9b5 Mon Sep 17 00:00:00 2001 From: Nathan Hjelm Date: Mon, 24 Aug 2015 13:26:47 -0600 Subject: [PATCH 1/2] opal: add recursive mutex This new class is the same as the opal_mutex_t class but has a different constructor. This constructor adds the recursive flag to the mutex attributes for the lock. This class can be used where there may be re-enty into the lock from within the same thread. Signed-off-by: Nathan Hjelm --- opal/threads/mutex.c | 26 ++++++++++++++++++++++++++ opal/threads/mutex.h | 2 +- opal/threads/mutex_unix.h | 1 + 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/opal/threads/mutex.c b/opal/threads/mutex.c index d7b78053080..23153ea843e 100644 --- a/opal/threads/mutex.c +++ b/opal/threads/mutex.c @@ -72,3 +72,29 @@ OBJ_CLASS_INSTANCE(opal_mutex_t, opal_object_t, opal_mutex_construct, opal_mutex_destruct); + +static void opal_recursive_mutex_construct(opal_recursive_mutex_t *m) +{ + pthread_mutexattr_t attr; + pthread_mutexattr_init(&attr); + +#if OPAL_ENABLE_DEBUG + m->m_lock_debug = 0; + m->m_lock_file = NULL; + m->m_lock_line = 0; +#endif + + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); + + pthread_mutex_init(&m->m_lock_pthread, &attr); + pthread_mutexattr_destroy(&attr); + +#if OPAL_HAVE_ATOMIC_SPINLOCKS + opal_atomic_init( &m->m_lock_atomic, OPAL_ATOMIC_UNLOCKED ); +#endif +} + +OBJ_CLASS_INSTANCE(opal_recursive_mutex_t, + opal_object_t, + opal_recursive_mutex_construct, + opal_mutex_destruct); diff --git a/opal/threads/mutex.h b/opal/threads/mutex.h index 542cd2a9521..1e8682ec2bc 100644 --- a/opal/threads/mutex.h +++ b/opal/threads/mutex.h @@ -50,7 +50,7 @@ OPAL_DECLSPEC extern bool opal_uses_threads; * Opaque mutex object */ typedef struct opal_mutex_t opal_mutex_t; - +typedef struct opal_mutex_t opal_recursive_mutex_t; /** * Try to acquire a mutex. diff --git a/opal/threads/mutex_unix.h b/opal/threads/mutex_unix.h index c8cad4975f6..d159285d8e5 100644 --- a/opal/threads/mutex_unix.h +++ b/opal/threads/mutex_unix.h @@ -61,6 +61,7 @@ struct opal_mutex_t { opal_atomic_lock_t m_lock_atomic; }; OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_mutex_t); +OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_recursive_mutex_t); /************************************************************************ * From 1d56007ab107d5dfe646a08d1cd18bceef2f6275 Mon Sep 17 00:00:00 2001 From: Nathan Hjelm Date: Mon, 24 Aug 2015 13:29:35 -0600 Subject: [PATCH 2/2] rcache/vma: make rcache lock recursive There is currently a path through the grdma mpool and vma rcache that leads to deadlock. It happens during the rcache insert. Before the insert the rcache mutex is locked. During the call a new vma item is allocated and then inserted into the rcache tree. The allocation currently goes through the malloc hooks which may (and does) call back into the mpool if the ptmalloc heap needs to be reallocated. This callback tries to lock the rcache mutex which leads to the deadlock. This has been observed with multi-threaded tests and the openib btl. This change may lead to some minor slowdown in the rcache vma when threading is enabled. This will only affect larger message paths in some of the btls. Signed-off-by: Nathan Hjelm --- opal/mca/rcache/vma/rcache_vma.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opal/mca/rcache/vma/rcache_vma.c b/opal/mca/rcache/vma/rcache_vma.c index 7910e671f05..8c9bd5e6f0b 100644 --- a/opal/mca/rcache/vma/rcache_vma.c +++ b/opal/mca/rcache/vma/rcache_vma.c @@ -42,7 +42,7 @@ void mca_rcache_vma_module_init( mca_rcache_vma_module_t* rcache ) { rcache->base.rcache_clean = mca_rcache_vma_clean; rcache->base.rcache_finalize = mca_rcache_vma_finalize; rcache->base.rcache_dump_range = mca_rcache_vma_dump_range; - OBJ_CONSTRUCT(&rcache->base.lock, opal_mutex_t); + OBJ_CONSTRUCT(&rcache->base.lock, opal_recursive_mutex_t); mca_rcache_vma_tree_init(rcache); }