From e29e7fadb3c1d396f7e2ed19266cd7921c3dd082 Mon Sep 17 00:00:00 2001 From: Joseph Schuchart Date: Tue, 31 Jan 2023 09:49:08 -0500 Subject: [PATCH 1/2] coll/han: increase segment sizes to 512k Increase segment sizes for bcast, reduce, and allreduce to 512k. On modern machines, higher segment sizes seem to be more efficient as they reduce the overhead of segmenting. Signed-off-by: Joseph Schuchart --- ompi/mca/coll/han/coll_han_component.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ompi/mca/coll/han/coll_han_component.c b/ompi/mca/coll/han/coll_han_component.c index 0f5c1a85d2c..642a3967982 100644 --- a/ompi/mca/coll/han/coll_han_component.c +++ b/ompi/mca/coll/han/coll_han_component.c @@ -261,7 +261,7 @@ static int han_register(void) OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &cs->han_output_verbose); - cs->han_bcast_segsize = 65536; + cs->han_bcast_segsize = 524288; (void) mca_base_component_var_register(c, "bcast_segsize", "segment size for bcast", MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, @@ -281,7 +281,7 @@ static int han_register(void) &cs->han_bcast_low_module, &cs->han_op_module_name.bcast.han_op_low_module_name); - cs->han_reduce_segsize = 65536; + cs->han_reduce_segsize = 524288; (void) mca_base_component_var_register(c, "reduce_segsize", "segment size for reduce", MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, @@ -300,7 +300,7 @@ static int han_register(void) OPAL_INFO_LVL_9, &cs->han_reduce_low_module, &cs->han_op_module_name.reduce.han_op_low_module_name); - cs->han_allreduce_segsize = 65536; + cs->han_allreduce_segsize = 524288; (void) mca_base_component_var_register(c, "allreduce_segsize", "segment size for allreduce", MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, From 867c3dfc4de4ecdc2f5fcb3add718e1d0274b3f7 Mon Sep 17 00:00:00 2001 From: Joseph Schuchart Date: Tue, 31 Jan 2023 09:51:14 -0500 Subject: [PATCH 2/2] coll/adapt: Increase ireduce segment size A larger segment size helps reduce the overhead of segmenting. The 512k size matches the size of coll/han. Signed-off-by: Joseph Schuchart --- ompi/mca/coll/adapt/coll_adapt_ireduce.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ompi/mca/coll/adapt/coll_adapt_ireduce.c b/ompi/mca/coll/adapt/coll_adapt_ireduce.c index da6486401f9..b3f7877bbe7 100644 --- a/ompi/mca/coll/adapt/coll_adapt_ireduce.c +++ b/ompi/mca/coll/adapt/coll_adapt_ireduce.c @@ -45,7 +45,7 @@ int ompi_coll_adapt_ireduce_register(void) mca_coll_adapt_component.adapt_ireduce_algorithm = 1; } - mca_coll_adapt_component.adapt_ireduce_segment_size = 163740; + mca_coll_adapt_component.adapt_ireduce_segment_size = 524288; mca_base_component_var_register(c, "reduce_segment_size", "Segment size in bytes used by default for reduce algorithms. Only has meaning if algorithm is forced and supports segmenting. 0 bytes means no segmentation.", MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0,