Skip to content

v4.0.x: neighbor alltoall fix #7243

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Dec 23, 2019
Merged

v4.0.x: neighbor alltoall fix #7243

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
21221eb
coll/basic: fix neighbor alltoall message ordering
hjelmn Oct 9, 2019
be58cf7
Fix the communication ordering for all cartesian neighbor collectives.
bosilca Dec 11, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions ompi/mca/coll/base/coll_tags.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@
#define MCA_COLL_BASE_TAG_SCATTERV -26
#define MCA_COLL_BASE_TAG_NONBLOCKING_BASE -27
#define MCA_COLL_BASE_TAG_NONBLOCKING_END ((-1 * INT_MAX/2) + 1)
#define MCA_COLL_BASE_TAG_NEIGHBOR_BASE (MCA_COLL_BASE_TAG_NONBLOCKING_END - 1)
#define MCA_COLL_BASE_TAG_NEIGHBOR_END (MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 1024)
#define MCA_COLL_BASE_TAG_HCOLL_BASE (-1 * INT_MAX/2)
#define MCA_COLL_BASE_TAG_HCOLL_END (-1 * INT_MAX)
#endif /* MCA_COLL_BASE_TAGS_H */
10 changes: 5 additions & 5 deletions ompi/mca/coll/basic/coll_basic_neighbor_allgather.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2016 The University of Tennessee and The University
* Copyright (c) 2004-2019 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
Expand Down Expand Up @@ -70,15 +70,15 @@ mca_coll_basic_neighbor_allgather_cart(const void *sbuf, int scount,
if (MPI_PROC_NULL != srank) {
nreqs++;
rc = MCA_PML_CALL(irecv(rbuf, rcount, rdtype, srank,
MCA_COLL_BASE_TAG_ALLGATHER,
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim,
comm, preqs++));
if (OMPI_SUCCESS != rc) break;

nreqs++;
/* remove cast from const when the pml layer is updated to take
* a const for the send buffer. */
rc = MCA_PML_CALL(isend((void *) sbuf, scount, sdtype, srank,
MCA_COLL_BASE_TAG_ALLGATHER,
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1,
MCA_PML_BASE_SEND_STANDARD,
comm, preqs++));
if (OMPI_SUCCESS != rc) break;
Expand All @@ -89,13 +89,13 @@ mca_coll_basic_neighbor_allgather_cart(const void *sbuf, int scount,
if (MPI_PROC_NULL != drank) {
nreqs++;
rc = MCA_PML_CALL(irecv(rbuf, rcount, rdtype, drank,
MCA_COLL_BASE_TAG_ALLGATHER,
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1,
comm, preqs++));
if (OMPI_SUCCESS != rc) break;

nreqs++;
rc = MCA_PML_CALL(isend((void *) sbuf, scount, sdtype, drank,
MCA_COLL_BASE_TAG_ALLGATHER,
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim,
MCA_PML_BASE_SEND_STANDARD,
comm, preqs++));
if (OMPI_SUCCESS != rc) break;
Expand Down
12 changes: 7 additions & 5 deletions ompi/mca/coll/basic/coll_basic_neighbor_allgatherv.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2016 The University of Tennessee and The University
* Copyright (c) 2004-2019 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
Expand Down Expand Up @@ -69,25 +69,27 @@ mca_coll_basic_neighbor_allgatherv_cart(const void *sbuf, int scount, struct omp
if (MPI_PROC_NULL != srank) {
nreqs++;
rc = MCA_PML_CALL(irecv((char *) rbuf + disps[i] * extent, rcounts[i], rdtype, srank,
MCA_COLL_BASE_TAG_ALLGATHER, comm, preqs++));
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim, comm, preqs++));
if (OMPI_SUCCESS != rc) break;

/* remove cast from const when the pml layer is updated to take
* a const for the send buffer. */
nreqs++;
rc = MCA_PML_CALL(isend((void *) sbuf, scount, sdtype, srank, MCA_COLL_BASE_TAG_ALLGATHER,
rc = MCA_PML_CALL(isend((void *) sbuf, scount, sdtype, srank,
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1,
MCA_PML_BASE_SEND_STANDARD, comm, preqs++));
if (OMPI_SUCCESS != rc) break;
}

if (MPI_PROC_NULL != drank) {
nreqs++;
rc = MCA_PML_CALL(irecv((char *) rbuf + disps[i+1] * extent, rcounts[i+1], rdtype, drank,
MCA_COLL_BASE_TAG_ALLGATHER, comm, preqs++));
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1, comm, preqs++));
if (OMPI_SUCCESS != rc) break;

nreqs++;
rc = MCA_PML_CALL(isend((void *) sbuf, scount, sdtype, drank, MCA_COLL_BASE_TAG_ALLGATHER,
rc = MCA_PML_CALL(isend((void *) sbuf, scount, sdtype, drank,
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim,
MCA_PML_BASE_SEND_STANDARD, comm, preqs++));
if (OMPI_SUCCESS != rc) break;
}
Expand Down
15 changes: 10 additions & 5 deletions ompi/mca/coll/basic/coll_basic_neighbor_alltoall.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2016 The University of Tennessee and The University
* Copyright (c) 2004-2019 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
Expand All @@ -15,6 +15,7 @@
* Copyright (c) 2014-2015 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* Copyright (c) 2017 IBM Corporation. All rights reserved.
* Copyright (c) 2019 Google, LLC. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
Expand All @@ -36,6 +37,10 @@
#include "coll_basic.h"
#include "ompi/mca/topo/base/base.h"

/**
* We only have 1024 tags for the neighbor collective, so for now we only support
* 512 dimensions.
*/
static int
mca_coll_basic_neighbor_alltoall_cart(const void *sbuf, int scount, struct ompi_datatype_t *sdtype, void *rbuf,
int rcount, struct ompi_datatype_t *rdtype, struct ompi_communicator_t *comm,
Expand Down Expand Up @@ -67,7 +72,7 @@ mca_coll_basic_neighbor_alltoall_cart(const void *sbuf, int scount, struct ompi_
if (MPI_PROC_NULL != srank) {
nreqs++;
rc = MCA_PML_CALL(irecv(rbuf, rcount, rdtype, srank,
MCA_COLL_BASE_TAG_ALLTOALL,
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim,
comm, preqs++));
if (OMPI_SUCCESS != rc) break;
}
Expand All @@ -77,7 +82,7 @@ mca_coll_basic_neighbor_alltoall_cart(const void *sbuf, int scount, struct ompi_
if (MPI_PROC_NULL != drank) {
nreqs++;
rc = MCA_PML_CALL(irecv(rbuf, rcount, rdtype, drank,
MCA_COLL_BASE_TAG_ALLTOALL,
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1,
comm, preqs++));
if (OMPI_SUCCESS != rc) break;
}
Expand All @@ -104,7 +109,7 @@ mca_coll_basic_neighbor_alltoall_cart(const void *sbuf, int scount, struct ompi_
* a const for the send buffer. */
nreqs++;
rc = MCA_PML_CALL(isend((void *) sbuf, scount, sdtype, srank,
MCA_COLL_BASE_TAG_ALLTOALL,
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1,
MCA_PML_BASE_SEND_STANDARD,
comm, preqs++));
if (OMPI_SUCCESS != rc) break;
Expand All @@ -115,7 +120,7 @@ mca_coll_basic_neighbor_alltoall_cart(const void *sbuf, int scount, struct ompi_
if (MPI_PROC_NULL != drank) {
nreqs++;
rc = MCA_PML_CALL(isend((void *) sbuf, scount, sdtype, drank,
MCA_COLL_BASE_TAG_ALLTOALL,
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim,
MCA_PML_BASE_SEND_STANDARD,
comm, preqs++));
if (OMPI_SUCCESS != rc) break;
Expand Down
10 changes: 5 additions & 5 deletions ompi/mca/coll/basic/coll_basic_neighbor_alltoallv.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2016 The University of Tennessee and The University
* Copyright (c) 2004-2019 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
Expand Down Expand Up @@ -68,14 +68,14 @@ mca_coll_basic_neighbor_alltoallv_cart(const void *sbuf, const int scounts[], co
if (MPI_PROC_NULL != srank) {
nreqs++;
rc = MCA_PML_CALL(irecv((char *) rbuf + rdisps[i] * rdextent, rcounts[i], rdtype, srank,
MCA_COLL_BASE_TAG_ALLTOALL, comm, preqs++));
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim, comm, preqs++));
if (OMPI_SUCCESS != rc) break;
}

if (MPI_PROC_NULL != drank) {
nreqs++;
rc = MCA_PML_CALL(irecv((char *) rbuf + rdisps[i+1] * rdextent, rcounts[i+1], rdtype, drank,
MCA_COLL_BASE_TAG_ALLTOALL, comm, preqs++));
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1, comm, preqs++));
if (OMPI_SUCCESS != rc) break;
}
}
Expand All @@ -98,14 +98,14 @@ mca_coll_basic_neighbor_alltoallv_cart(const void *sbuf, const int scounts[], co
nreqs++;
/* remove cast from const when the pml layer is updated to take a const for the send buffer */
rc = MCA_PML_CALL(isend((char *) sbuf + sdisps[i] * sdextent, scounts[i], sdtype, srank,
MCA_COLL_BASE_TAG_ALLTOALL, MCA_PML_BASE_SEND_STANDARD, comm, preqs++));
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1, MCA_PML_BASE_SEND_STANDARD, comm, preqs++));
if (OMPI_SUCCESS != rc) break;
}

if (MPI_PROC_NULL != drank) {
nreqs++;
rc = MCA_PML_CALL(isend((char *) sbuf + sdisps[i+1] * sdextent, scounts[i+1], sdtype, drank,
MCA_COLL_BASE_TAG_ALLTOALL, MCA_PML_BASE_SEND_STANDARD, comm, preqs++));
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim, MCA_PML_BASE_SEND_STANDARD, comm, preqs++));
if (OMPI_SUCCESS != rc) break;
}
}
Expand Down
10 changes: 5 additions & 5 deletions ompi/mca/coll/basic/coll_basic_neighbor_alltoallw.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2016 The University of Tennessee and The University
* Copyright (c) 2004-2019 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
Expand Down Expand Up @@ -65,14 +65,14 @@ mca_coll_basic_neighbor_alltoallw_cart(const void *sbuf, const int scounts[], co
if (MPI_PROC_NULL != srank) {
nreqs++;
rc = MCA_PML_CALL(irecv((char *) rbuf + rdisps[i], rcounts[i], rdtypes[i], srank,
MCA_COLL_BASE_TAG_ALLTOALL, comm, preqs++));
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim, comm, preqs++));
if (OMPI_SUCCESS != rc) break;
}

if (MPI_PROC_NULL != drank) {
nreqs++;
rc = MCA_PML_CALL(irecv((char *) rbuf + rdisps[i+1], rcounts[i+1], rdtypes[i+1], drank,
MCA_COLL_BASE_TAG_ALLTOALL, comm, preqs++));
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1, comm, preqs++));
if (OMPI_SUCCESS != rc) break;
}
}
Expand All @@ -95,14 +95,14 @@ mca_coll_basic_neighbor_alltoallw_cart(const void *sbuf, const int scounts[], co
nreqs++;
/* remove cast from const when the pml layer is updated to take a const for the send buffer */
rc = MCA_PML_CALL(isend((char *) sbuf + sdisps[i], scounts[i], sdtypes[i], srank,
MCA_COLL_BASE_TAG_ALLTOALL, MCA_PML_BASE_SEND_STANDARD, comm, preqs++));
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1, MCA_PML_BASE_SEND_STANDARD, comm, preqs++));
if (OMPI_SUCCESS != rc) break;
}

if (MPI_PROC_NULL != drank) {
nreqs++;
rc = MCA_PML_CALL(isend((char *) sbuf + sdisps[i+1], scounts[i+1], sdtypes[i+1], drank,
MCA_COLL_BASE_TAG_ALLTOALL, MCA_PML_BASE_SEND_STANDARD, comm, preqs++));
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim, MCA_PML_BASE_SEND_STANDARD, comm, preqs++));
if (OMPI_SUCCESS != rc) break;
}
}
Expand Down