Skip to content

Commit

Permalink
coll/HAN: Add support for XHC on the intra-comm
Browse files Browse the repository at this point in the history 
Signed-off-by: George Katevenis <gkatev@ics.forth.gr>
  • Loading branch information
@gkatev
gkatev committed Mar 27, 2023
1 parent eae34b6 commit 17c4dba
Show file tree
Hide file tree
Showing 4 changed files with 24 additions and 10 deletions.
8 changes: 5 additions & 3 deletions ompi/mca/coll/han/coll_han.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
* reserved.
* Copyright (c) 2022 IBM Corporation. All rights reserved
* Copyright (c) 2020-2022 Bull S.A.S. All rights reserved.
* Copyright (c) 2023 Computer Architecture and VLSI Systems (CARV)
* Laboratory, ICS Forth. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
Expand Down Expand Up @@ -44,11 +46,11 @@

/*
* Today;
* . only 2 modules available for intranode (low) level
* . only 2 modules available for internode (up) level
* . 3 modules available for intranode (low) level (tuned, sm, xhc)
* . only 2 modules available for internode (up) level (libnbc, adapt)
*/

#define COLL_HAN_LOW_MODULES 2
#define COLL_HAN_LOW_MODULES 3
#define COLL_HAN_UP_MODULES 2

struct mca_coll_han_bcast_args_s {
Expand Down
17 changes: 10 additions & 7 deletions ompi/mca/coll/han/coll_han_component.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
* reserved.
* Copyright (c) 2022 IBM Corporation. All rights reserved
* Copyright (c) 2020-2022 Bull S.A.S. All rights reserved.
* Copyright (c) 2023 Computer Architecture and VLSI Systems (CARV)
* Laboratory, ICS Forth. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
Expand Down Expand Up @@ -43,7 +45,8 @@ ompi_coll_han_components ompi_coll_han_available_components[COMPONENTS_COUNT] =
{ TUNED, "tuned", NULL },
{ SM, "sm", NULL },
{ ADAPT, "adapt", NULL },
{ HAN, "han", NULL }
{ HAN, "han", NULL },
{ XHC, "xhc", NULL }
};

/*
Expand Down Expand Up @@ -277,7 +280,7 @@ static int han_register(void)

cs->han_bcast_low_module = 0;
(void) mca_coll_han_query_module_from_mca(c, "bcast_low_module",
"low level module for bcast, 0 tuned, 1 sm",
"low level module for bcast, 0 tuned, 1 sm, 2 xhc",
OPAL_INFO_LVL_9,
&cs->han_bcast_low_module,
&cs->han_op_module_name.bcast.han_op_low_module_name);
Expand All @@ -297,7 +300,7 @@ static int han_register(void)

cs->han_reduce_low_module = 0;
(void) mca_coll_han_query_module_from_mca(c, "reduce_low_module",
"low level module for allreduce, 0 tuned, 1 sm",
"low level module for allreduce, 0 tuned, 1 sm, 2 xhc",
OPAL_INFO_LVL_9, &cs->han_reduce_low_module,
&cs->han_op_module_name.reduce.han_op_low_module_name);

Expand All @@ -316,7 +319,7 @@ static int han_register(void)

cs->han_allreduce_low_module = 0;
(void) mca_coll_han_query_module_from_mca(c, "allreduce_low_module",
"low level module for allreduce, 0 tuned, 1 sm",
"low level module for allreduce, 0 tuned, 1 sm, 2 xhc",
OPAL_INFO_LVL_9, &cs->han_allreduce_low_module,
&cs->han_op_module_name.allreduce.han_op_low_module_name);

Expand All @@ -328,7 +331,7 @@ static int han_register(void)

cs->han_allgather_low_module = 0;
(void) mca_coll_han_query_module_from_mca(c, "allgather_low_module",
"low level module for allgather, 0 tuned, 1 sm",
"low level module for allgather, 0 tuned, 1 sm, 2 xhc",
OPAL_INFO_LVL_9, &cs->han_allgather_low_module,
&cs->han_op_module_name.allgather.han_op_low_module_name);

Expand All @@ -340,7 +343,7 @@ static int han_register(void)

cs->han_gather_low_module = 0;
(void) mca_coll_han_query_module_from_mca(c, "gather_low_module",
"low level module for gather, 0 tuned, 1 sm",
"low level module for gather, 0 tuned, 1 sm, 2 xhc",
OPAL_INFO_LVL_9, &cs->han_gather_low_module,
&cs->han_op_module_name.gather.han_op_low_module_name);

Expand All @@ -352,7 +355,7 @@ static int han_register(void)

cs->han_scatter_low_module = 0;
(void) mca_coll_han_query_module_from_mca(c, "scatter_low_module",
"low level module for scatter, 0 tuned, 1 sm",
"low level module for scatter, 0 tuned, 1 sm, 2 xhc",
OPAL_INFO_LVL_9, &cs->han_scatter_low_module,
&cs->han_op_module_name.scatter.han_op_low_module_name);

Expand Down
3 changes: 3 additions & 0 deletions ompi/mca/coll/han/coll_han_dynamic.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
* reserved.
* Copyright (c) 2020 Bull S.A.S. All rights reserved.
* Copyright (c) 2022 IBM Corporation. All rights reserved
* Copyright (c) 2023 Computer Architecture and VLSI Systems (CARV)
* Laboratory, ICS Forth. All rights reserved.
*
* $COPYRIGHT$
*
Expand Down Expand Up @@ -105,6 +107,7 @@ typedef enum COMPONENTS {
SM,
ADAPT,
HAN,
XHC,
COMPONENTS_COUNT
} COMPONENT_T;

Expand Down
6 changes: 6 additions & 0 deletions ompi/mca/coll/han/coll_han_subcomms.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2020 Bull S.A.S. All rights reserved.
* Copyright (c) 2023 Computer Architecture and VLSI Systems (CARV)
* Laboratory, ICS Forth. All rights reserved.
*
* $COPYRIGHT$
*
Expand Down Expand Up @@ -297,6 +299,10 @@ int mca_coll_han_comm_create(struct ompi_communicator_t *comm,
ompi_comm_split_type(comm, MPI_COMM_TYPE_SHARED, 0,
&comm_info, &(low_comms[1]));

opal_info_set(&comm_info, "ompi_comm_coll_preference", "xhc,^han");
ompi_comm_split_type(comm, MPI_COMM_TYPE_SHARED, 0,
&comm_info, &(low_comms[2]));

/*
* Upgrade libnbc module priority to set up up_comms[0] with libnbc module
* This sub-communicator contains one process per node: processes with the
Expand Down

0 comments on commit 17c4dba

Please sign in to comment.