Skip to content

osc/sm: fix pscw synchronization #925

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Sep 23, 2015
Merged

osc/sm: fix pscw synchronization #925

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
903762e
osc/sm: fix pscw synchronization
hjelmn Sep 22, 2015
f6920aa
osc/rdma: check for usable btls during query
hjelmn Sep 22, 2015
ee58108
osc/pt2pt: fix regression in pscw sync on 0 size groups
hjelmn Sep 22, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 20 additions & 26 deletions ompi/mca/osc/pt2pt/osc_pt2pt_active_target.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,11 +114,13 @@ static ompi_osc_pt2pt_peer_t **ompi_osc_pt2pt_get_peers (ompi_osc_pt2pt_module_t

static void ompi_osc_pt2pt_release_peers (ompi_osc_pt2pt_peer_t **peers, int npeers)
{
for (int i = 0 ; i < npeers ; ++i) {
OBJ_RELEASE(peers[i]);
}
if (peers) {
for (int i = 0 ; i < npeers ; ++i) {
OBJ_RELEASE(peers[i]);
}

free (peers);
free (peers);
}
}

int ompi_osc_pt2pt_fence(int assert, ompi_win_t *win)
Expand Down Expand Up @@ -228,31 +230,31 @@ int ompi_osc_pt2pt_start (ompi_group_t *group, int assert, ompi_win_t *win)
"ompi_osc_pt2pt_start entering with group size %d...",
sync->num_peers));

sync->type = OMPI_OSC_PT2PT_SYNC_TYPE_PSCW;

/* prevent us from entering a passive-target, fence, or another pscw access epoch until
* the matching complete is called */
sync->epoch_active = true;

/* save the group */
OBJ_RETAIN(group);

if (0 == ompi_group_size (group)) {
/* nothing more to do. this is an empty start epoch */
sync->eager_send_active = true;
OPAL_THREAD_UNLOCK(&module->lock);
return OMPI_SUCCESS;
}

opal_atomic_wmb ();

sync->type = OMPI_OSC_PT2PT_SYNC_TYPE_PSCW;

/* prevent us from entering a passive-target, fence, or another pscw access epoch until
* the matching complete is called */
sync->epoch_active = true;

/* translate the group ranks into the communicator */
sync->peer_list.peers = ompi_osc_pt2pt_get_peers (module, group);
if (NULL == sync->peer_list.peers) {
OPAL_THREAD_UNLOCK(&module->lock);
return OMPI_ERR_OUT_OF_RESOURCE;
}

/* save the group */
OBJ_RETAIN(group);
ompi_group_increment_proc_count(group);

if (!(assert & MPI_MODE_NOCHECK)) {
OPAL_THREAD_LOCK(&sync->lock);
for (int i = 0 ; i < sync->num_peers ; ++i) {
Expand Down Expand Up @@ -318,12 +320,6 @@ int ompi_osc_pt2pt_complete (ompi_win_t *win)
group_size = sync->num_peers;

peers = sync->peer_list.peers;
if (NULL == peers) {
/* empty peer list */
OPAL_THREAD_UNLOCK(&(module->lock));
OBJ_RELEASE(group);
return OMPI_SUCCESS;
}

OPAL_THREAD_UNLOCK(&module->lock);

Expand Down Expand Up @@ -383,8 +379,10 @@ int ompi_osc_pt2pt_complete (ompi_win_t *win)
module->epoch_outgoing_frag_count[rank] = 0;
}

/* release our reference to peers in this group */
ompi_osc_pt2pt_release_peers (peers, group_size);
if (peers) {
/* release our reference to peers in this group */
ompi_osc_pt2pt_release_peers (peers, group_size);
}

if (OMPI_SUCCESS != ret) {
return ret;
Expand All @@ -403,7 +401,6 @@ int ompi_osc_pt2pt_complete (ompi_win_t *win)
OPAL_THREAD_UNLOCK(&module->lock);

/* phase 2 cleanup group */
ompi_group_decrement_proc_count(group);
OBJ_RELEASE(group);

OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output,
Expand Down Expand Up @@ -439,7 +436,6 @@ int ompi_osc_pt2pt_post (ompi_group_t *group, int assert, ompi_win_t *win)

/* save the group */
OBJ_RETAIN(group);
ompi_group_increment_proc_count(group);

module->pw_group = group;

Expand Down Expand Up @@ -523,7 +519,6 @@ int ompi_osc_pt2pt_wait (ompi_win_t *win)
module->pw_group = NULL;
OPAL_THREAD_UNLOCK(&module->lock);

ompi_group_decrement_proc_count(group);
OBJ_RELEASE(group);

OPAL_OUTPUT_VERBOSE((25, ompi_osc_base_framework.framework_output,
Expand Down Expand Up @@ -561,7 +556,6 @@ int ompi_osc_pt2pt_test (ompi_win_t *win, int *flag)

OPAL_THREAD_UNLOCK(&(module->lock));

ompi_group_decrement_proc_count(group);
OBJ_RELEASE(group);

return OMPI_SUCCESS;
Expand Down
2 changes: 2 additions & 0 deletions ompi/mca/osc/pt2pt/osc_pt2pt_sync.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,8 @@ static inline void ompi_osc_pt2pt_sync_reset (ompi_osc_pt2pt_sync_t *sync)
sync->type = OMPI_OSC_PT2PT_SYNC_TYPE_NONE;
sync->eager_send_active = 0;
sync->epoch_active = 0;
sync->peer_list.peers = NULL;
sync->sync.pscw.group = NULL;
}

#endif /* OMPI_OSC_PT2PT_SYNC_H */
51 changes: 33 additions & 18 deletions ompi/mca/osc/rdma/osc_rdma_component.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,8 @@ static int ompi_osc_rdma_component_select (struct ompi_win_t *win, void **base,
static int ompi_osc_rdma_set_info (struct ompi_win_t *win, struct ompi_info_t *info);
static int ompi_osc_rdma_get_info (struct ompi_win_t *win, struct ompi_info_t **info_used);

static int ompi_osc_rdma_query_btls (ompi_communicator_t *comm, struct mca_btl_base_module_t **btl);

static char *ompi_osc_rdma_btl_names;

ompi_osc_rdma_component_t mca_osc_rdma_component = {
Expand Down Expand Up @@ -296,10 +298,16 @@ static int ompi_osc_rdma_component_query (struct ompi_win_t *win, void **base, s
struct ompi_communicator_t *comm, struct ompi_info_t *info,
int flavor)
{

if (MPI_WIN_FLAVOR_SHARED == flavor) {
return -1;
}

if (OMPI_SUCCESS != ompi_osc_rdma_query_btls (comm, NULL)) {
return -1;
}


return mca_osc_rdma_component.priority;
}

Expand Down Expand Up @@ -619,32 +627,31 @@ static int allocate_state_shared (ompi_osc_rdma_module_t *module, void **base, s
return ret;
}

static int ompi_osc_rdma_find_rdma_endpoints (ompi_osc_rdma_module_t *module)
static int ompi_osc_rdma_query_btls (ompi_communicator_t *comm, struct mca_btl_base_module_t **btl)
{
struct mca_btl_base_module_t **possible_btls = NULL;
int comm_size = ompi_comm_size (module->comm);
int comm_size = ompi_comm_size (comm);
int rc = OMPI_SUCCESS, max_btls = 0;
unsigned int selected_latency = INT_MAX;
mca_btl_base_selected_module_t *selected_btl;
struct mca_btl_base_module_t *selected_btl = NULL;
mca_btl_base_selected_module_t *item;
int *btl_counts = NULL;
char **btls_to_use;
void *tmp;

module->selected_btl = NULL;

btls_to_use = opal_argv_split (ompi_osc_rdma_btl_names, ',');
if (btls_to_use) {
/* rdma and atomics are only supported with BTLs at the moment */
OPAL_LIST_FOREACH(selected_btl, &mca_btl_base_modules_initialized, mca_btl_base_selected_module_t) {
OPAL_LIST_FOREACH(item, &mca_btl_base_modules_initialized, mca_btl_base_selected_module_t) {
for (int i = 0 ; btls_to_use[i] ; ++i) {
if (0 != strcmp (btls_to_use[i], selected_btl->btl_module->btl_component->btl_version.mca_component_name)) {
if (0 != strcmp (btls_to_use[i], item->btl_module->btl_component->btl_version.mca_component_name)) {
continue;
}

if ((selected_btl->btl_module->btl_flags & (MCA_BTL_FLAGS_RDMA)) == MCA_BTL_FLAGS_RDMA &&
(selected_btl->btl_module->btl_flags & (MCA_BTL_FLAGS_ATOMIC_FOPS | MCA_BTL_FLAGS_ATOMIC_OPS))) {
if (!module->selected_btl || selected_btl->btl_module->btl_latency < module->selected_btl->btl_latency) {
module->selected_btl = selected_btl->btl_module;
if ((item->btl_module->btl_flags & (MCA_BTL_FLAGS_RDMA)) == MCA_BTL_FLAGS_RDMA &&
(item->btl_module->btl_flags & (MCA_BTL_FLAGS_ATOMIC_FOPS | MCA_BTL_FLAGS_ATOMIC_OPS))) {
if (!selected_btl || item->btl_module->btl_latency < selected_btl->btl_latency) {
selected_btl = item->btl_module;
}
}
}
Expand All @@ -653,14 +660,18 @@ static int ompi_osc_rdma_find_rdma_endpoints (ompi_osc_rdma_module_t *module)
opal_argv_free (btls_to_use);
}

if (NULL != module->selected_btl) {
if (btl) {
*btl = selected_btl;
}

if (NULL != selected_btl) {
OPAL_OUTPUT_VERBOSE((11, ompi_osc_base_framework.framework_output, "Selected BTL: %s",
module->selected_btl->btl_component->btl_version.mca_component_name));
selected_btl->btl_component->btl_version.mca_component_name));
return OMPI_SUCCESS;
}

for (int i = 0 ; i < comm_size ; ++i) {
ompi_proc_t *proc = ompi_comm_peer_lookup (module->comm, i);
ompi_proc_t *proc = ompi_comm_peer_lookup (comm, i);
mca_bml_base_endpoint_t *endpoint;
int num_btls, prev_max;

Expand Down Expand Up @@ -733,21 +744,25 @@ static int ompi_osc_rdma_find_rdma_endpoints (ompi_osc_rdma_module_t *module)
}

if (btl_counts[i] == comm_size && possible_btls[i]->btl_latency < selected_latency) {
module->selected_btl = possible_btls[i];
selected_btl = possible_btls[i];
selected_latency = possible_btls[i]->btl_latency;
}
}

free (possible_btls);
free (btl_counts);

if (NULL == module->selected_btl) {
if (btl) {
*btl = selected_btl;
}

if (NULL == selected_btl) {
/* no btl = no rdma/atomics */
return OMPI_ERR_NOT_AVAILABLE;
}

OPAL_OUTPUT_VERBOSE((11, ompi_osc_base_framework.framework_output, "Selected BTL: %s",
module->selected_btl->btl_component->btl_version.mca_component_name));
selected_btl->btl_component->btl_version.mca_component_name));

return OMPI_SUCCESS;
}
Expand Down Expand Up @@ -1017,7 +1032,7 @@ static int ompi_osc_rdma_component_select (struct ompi_win_t *win, void **base,
}

/* find rdma capable endpoints */
ret = ompi_osc_rdma_find_rdma_endpoints (module);
ret = ompi_osc_rdma_query_btls (module->comm, &module->selected_btl);
if (OMPI_SUCCESS != ret) {
OPAL_OUTPUT_VERBOSE((11, ompi_osc_base_framework.framework_output, "Failed finding RDMA endpoints"));
ompi_osc_rdma_free (win);
Expand Down
4 changes: 3 additions & 1 deletion ompi/mca/osc/sm/osc_sm.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@ struct ompi_osc_sm_lock_t {
typedef struct ompi_osc_sm_lock_t ompi_osc_sm_lock_t;

struct ompi_osc_sm_node_state_t {
int32_t post_count;
int32_t complete_count;
ompi_osc_sm_lock_t lock;
opal_atomic_lock_t accumulate_lock;
Expand Down Expand Up @@ -81,6 +80,9 @@ struct ompi_osc_sm_module_t {
ompi_osc_sm_global_state_t *global_state;
ompi_osc_sm_node_state_t *my_node_state;
ompi_osc_sm_node_state_t *node_states;
uint64_t **posts;

opal_mutex_t lock;
};
typedef struct ompi_osc_sm_module_t ompi_osc_sm_module_t;

Expand Down
Loading