Skip to content
This repository was archived by the owner on Sep 30, 2022. It is now read-only.

mpool/rgpusm: update for rcache threading fixes #1218

Merged
merged 1 commit into from
Jun 7, 2016
Merged

mpool/rgpusm: update for rcache threading fixes #1218

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
108 changes: 39 additions & 69 deletions opal/mca/mpool/rgpusm/mpool_rgpusm_module.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
* Copyright (c) 2007 Mellanox Technologies. All rights reserved.
* Copyright (c) 2010 IBM Corporation. All rights reserved.
* Copyright (c) 2012-2015 NVIDIA Corporation. All rights reserved.
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights
* Copyright (c) 2015-2016 Los Alamos National Security, LLC. All rights
* reserved.
*
* $COPYRIGHT$
Expand Down Expand Up @@ -113,11 +113,11 @@ static inline bool mca_mpool_rgpusm_deregister_lru (mca_mpool_base_module_t *mpo
mpool->rcache->rcache_delete(mpool->rcache, old_reg);

/* Drop the rcache lock while we deregister the memory */
OPAL_THREAD_UNLOCK(&mpool->rcache->lock);
opal_mutex_unlock (&mpool->rcache->lock);
assert(old_reg->ref_count == 0);
rc = mpool_rgpusm->resources.deregister_mem(mpool_rgpusm->resources.reg_data,
old_reg);
OPAL_THREAD_LOCK(&mpool->rcache->lock);
opal_mutex_lock (&mpool->rcache->lock);

/* This introduces a potential leak of registrations if
the deregistration fails to occur as we no longer have
Expand Down Expand Up @@ -242,7 +242,7 @@ int mca_mpool_rgpusm_register (mca_mpool_base_module_t *mpool, void *addr,
}

/* Check to see if memory is registered and stored in the cache. */
OPAL_THREAD_LOCK(&mpool->rcache->lock);
opal_mutex_lock (&mpool->rcache->lock);
mpool->rcache->rcache_find(mpool->rcache, addr, size, reg);

/* If *reg is not NULL, we have a registration. Let us see if the
Expand Down Expand Up @@ -306,7 +306,7 @@ int mca_mpool_rgpusm_register (mca_mpool_base_module_t *mpool, void *addr,
(opal_list_item_t*)(*reg));
}
(*reg)->ref_count++;
OPAL_THREAD_UNLOCK(&mpool->rcache->lock);
opal_mutex_unlock (&mpool->rcache->lock);
opal_output(-1, "reg->ref_count=%d", (int)(*reg)->ref_count);
opal_output_verbose(80, mca_mpool_rgpusm_component.output,
"RGPUSM: Found entry in cache addr=%p, size=%d", addr, (int)size);
Expand All @@ -322,7 +322,7 @@ int mca_mpool_rgpusm_register (mca_mpool_base_module_t *mpool, void *addr,

item = opal_free_list_get (&mpool_rgpusm->reg_list);
if(NULL == item) {
OPAL_THREAD_UNLOCK(&mpool->rcache->lock);
opal_mutex_unlock (&mpool->rcache->lock);
return OPAL_ERR_OUT_OF_RESOURCE;
}
rgpusm_reg = (mca_mpool_common_cuda_reg_t*)item;
Expand Down Expand Up @@ -399,7 +399,7 @@ int mca_mpool_rgpusm_register (mca_mpool_base_module_t *mpool, void *addr,
}

if(rc != OPAL_SUCCESS) {
OPAL_THREAD_UNLOCK(&mpool->rcache->lock);
opal_mutex_unlock (&mpool->rcache->lock);
opal_free_list_return (&mpool_rgpusm->reg_list, item);
return rc;
}
Expand Down Expand Up @@ -439,7 +439,7 @@ int mca_mpool_rgpusm_register (mca_mpool_base_module_t *mpool, void *addr,
}

if(rc != OPAL_SUCCESS) {
OPAL_THREAD_UNLOCK(&mpool->rcache->lock);
opal_mutex_unlock (&mpool->rcache->lock);
opal_free_list_return (&mpool_rgpusm->reg_list, item);
/* We cannot recover from this. We can be here if the size of
* the cache is smaller than the amount of memory we are
Expand All @@ -454,10 +454,8 @@ int mca_mpool_rgpusm_register (mca_mpool_base_module_t *mpool, void *addr,

rgpusm_reg->base.ref_count++;
*reg = (mca_mpool_base_registration_t *)rgpusm_reg;
OPAL_THREAD_UNLOCK(&mpool->rcache->lock);
opal_mutex_unlock (&mpool->rcache->lock);

/* Cleanup any vmas that we have deferred deletion on */
mpool->rcache->rcache_clean(mpool->rcache);
return OPAL_SUCCESS;
}

Expand All @@ -483,7 +481,7 @@ int mca_mpool_rgpusm_find(struct mca_mpool_base_module_t *mpool, void *addr,
base = addr;
bound = base + size - 1; /* To keep cache hits working correctly */

OPAL_THREAD_LOCK(&mpool->rcache->lock);
opal_mutex_lock (&mpool->rcache->lock);
opal_output(-1, "Looking for addr=%p, size=%d", addr, (int)size);
rc = mpool->rcache->rcache_find(mpool->rcache, addr, size, reg);
if(*reg != NULL && mca_mpool_rgpusm_component.leave_pinned) {
Expand All @@ -495,12 +493,12 @@ int mca_mpool_rgpusm_find(struct mca_mpool_base_module_t *mpool, void *addr,
} else {
mpool_rgpusm->stat_cache_notfound++;
}
OPAL_THREAD_UNLOCK(&mpool->rcache->lock);
opal_mutex_unlock (&mpool->rcache->lock);

return rc;
}

static inline bool registration_is_cachebale(mca_mpool_base_registration_t *reg)
static inline bool registration_is_cacheable(mca_mpool_base_registration_t *reg)
{
return !(reg->flags &
(MCA_MPOOL_FLAGS_CACHE_BYPASS |
Expand All @@ -514,14 +512,14 @@ int mca_mpool_rgpusm_deregister(struct mca_mpool_base_module_t *mpool,
int rc = OPAL_SUCCESS;
assert(reg->ref_count > 0);

OPAL_THREAD_LOCK(&mpool->rcache->lock);
opal_mutex_lock (&mpool->rcache->lock);
reg->ref_count--;
opal_output(-1, "Deregister: reg->ref_count=%d", (int)reg->ref_count);
if(reg->ref_count > 0) {
OPAL_THREAD_UNLOCK(&mpool->rcache->lock);
opal_mutex_unlock (&mpool->rcache->lock);
return OPAL_SUCCESS;
}
if(mca_mpool_rgpusm_component.leave_pinned && registration_is_cachebale(reg))
if(mca_mpool_rgpusm_component.leave_pinned && registration_is_cacheable(reg))
{
/* if leave_pinned is set don't deregister memory, but put it
* on LRU list for future use */
Expand All @@ -535,7 +533,7 @@ int mca_mpool_rgpusm_deregister(struct mca_mpool_base_module_t *mpool,
mpool->rcache->rcache_delete(mpool->rcache, reg);

/* Drop the rcache lock before deregistring the memory */
OPAL_THREAD_UNLOCK(&mpool->rcache->lock);
opal_mutex_unlock (&mpool->rcache->lock);

{
mca_mpool_rgpusm_module_t *mpool_rgpusm = (mca_mpool_rgpusm_module_t *)mpool;
Expand All @@ -545,17 +543,14 @@ int mca_mpool_rgpusm_deregister(struct mca_mpool_base_module_t *mpool,
reg);
}

OPAL_THREAD_LOCK(&mpool->rcache->lock);
opal_mutex_lock (&mpool->rcache->lock);

if(OPAL_SUCCESS == rc) {
opal_free_list_return (&mpool_rgpusm->reg_list,
(opal_free_list_item_t*)reg);
}
}
OPAL_THREAD_UNLOCK(&mpool->rcache->lock);

/* Cleanup any vmas that we have deferred deletion on */
mpool->rcache->rcache_clean(mpool->rcache);
opal_mutex_unlock (&mpool->rcache->lock);

return rc;
}
Expand All @@ -572,7 +567,7 @@ int mca_mpool_rgpusm_deregister_no_lock(struct mca_mpool_base_module_t *mpool,
if(reg->ref_count > 0) {
return OPAL_SUCCESS;
}
if(mca_mpool_rgpusm_component.leave_pinned && registration_is_cachebale(reg))
if(mca_mpool_rgpusm_component.leave_pinned && registration_is_cacheable(reg))
{
/* if leave_pinned is set don't deregister memory, but put it
* on LRU list for future use */
Expand All @@ -599,15 +594,28 @@ int mca_mpool_rgpusm_deregister_no_lock(struct mca_mpool_base_module_t *mpool,
return rc;
}

#define RGPUSM_MPOOL_NREGS 100
static int iterate_dereg_finalize (mca_mpool_base_registration_t *rgpusm_reg, void *ctx)
{
mca_mpool_rgpusm_module_t *mpool_rgpusm = (mca_mpool_rgpusm_module_t *) ctx;

if ((mca_mpool_base_module_t *) mpool_rgpusm != rgpusm_reg->mpool) {
return 0;
}

if (registration_is_cacheable (rgpusm_reg)) {
opal_list_remove_item (&mpool_rgpusm->lru_list, (opal_list_item_t *) rgpusm_reg);
}

/* set the reference count to 0 otherwise dereg will fail on assert */
rgpusm_reg->ref_count = 0;
(void) mpool_rgpusm->resources.deregister_mem (mpool_rgpusm->resources.reg_data, rgpusm_reg);

return 0;
}

void mca_mpool_rgpusm_finalize(struct mca_mpool_base_module_t *mpool)
{
mca_mpool_rgpusm_module_t *mpool_rgpusm = (mca_mpool_rgpusm_module_t*)mpool;
mca_mpool_base_registration_t *reg;
mca_mpool_base_registration_t *regs[RGPUSM_MPOOL_NREGS];
int reg_cnt, i;
int rc;

/* Statistic */
if(true == mca_mpool_rgpusm_component.print_stats) {
Expand All @@ -619,49 +627,11 @@ void mca_mpool_rgpusm_finalize(struct mca_mpool_base_module_t *mpool)
mpool_rgpusm->stat_evicted);
}

OPAL_THREAD_LOCK(&mpool->rcache->lock);
do {
reg_cnt = mpool->rcache->rcache_find_all(mpool->rcache, 0, (size_t)-1,
regs, RGPUSM_MPOOL_NREGS);
opal_output(-1, "Registration size at finalize = %d", reg_cnt);

for(i = 0; i < reg_cnt; i++) {
reg = regs[i];

if(reg->ref_count) {
reg->ref_count = 0; /* otherway dereg will fail on assert */
} else if (mca_mpool_rgpusm_component.leave_pinned) {
opal_list_remove_item(&mpool_rgpusm->lru_list,
(opal_list_item_t*)reg);
}

/* Remove from rcache first */
mpool->rcache->rcache_delete(mpool->rcache, reg);

/* Drop lock before deregistering memory */
OPAL_THREAD_UNLOCK(&mpool->rcache->lock);
assert(reg->ref_count == 0);
rc = mpool_rgpusm->resources.deregister_mem(mpool_rgpusm->resources.reg_data,
reg);
OPAL_THREAD_LOCK(&mpool->rcache->lock);

if(rc != OPAL_SUCCESS) {
/* Potentially lose track of registrations
do we have to put it back? */
continue;
}

opal_free_list_return (&mpool_rgpusm->reg_list,
(opal_free_list_item_t *) reg);
}
} while(reg_cnt == RGPUSM_MPOOL_NREGS);
(void) mpool->rcache->rcache_iterate (mpool->rcache, NULL, (size_t) -1,
iterate_dereg_finalize, (void *) mpool);

OBJ_DESTRUCT(&mpool_rgpusm->lru_list);
OBJ_DESTRUCT(&mpool_rgpusm->reg_list);
OPAL_THREAD_UNLOCK(&mpool->rcache->lock);

/* Cleanup any vmas that we have deferred deletion on */
mpool->rcache->rcache_clean(mpool->rcache);

}