From 76f3d859ac0c17c7a956df520d583499a4286862 Mon Sep 17 00:00:00 2001 From: Eric Bohm Date: Tue, 12 Nov 2024 22:22:15 +0000 Subject: [PATCH 1/2] bugfix: improve CXI support for ALCF Aurora configuration --- src/arch/ofi/machine.C | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/arch/ofi/machine.C b/src/arch/ofi/machine.C index baefce5736..daf21d18de 100644 --- a/src/arch/ofi/machine.C +++ b/src/arch/ofi/machine.C @@ -696,6 +696,7 @@ void LrtsInit(int *argc, char ***argv, int *numNodes, int *myNodeID) * should not be considered predictive of proximity. That * relationship has to be detected by other means. + * 2. HWLOC doesn't have a hwloc_get_closest_nic because... NIC * doesn't even rate an object type in their ontology, let * alone get first class treatment. Given that PCI devices @@ -714,7 +715,7 @@ void LrtsInit(int *argc, char ***argv, int *numNodes, int *myNodeID) * do *not* have such convenient labeling as something special * needs to happen to get their linuxfs utilities to inject * that derived information into your topology object. As an - * interim solution we allow the user to map their cxi[0..3] + * interim solution we allow the user to map their cxi[0..7] * selection using command line arguments. * 2b. Likewise the 1:1 relationship we assume here between @@ -741,6 +742,8 @@ void LrtsInit(int *argc, char ***argv, int *numNodes, int *myNodeID) * CPU nodes. The user could easily be confused, so we can't * rely on them telling us. This has to be determined at * run time. + + * 6. Aurora can apparently go up to cxi7. */ char *cximap=NULL; @@ -814,7 +817,18 @@ void LrtsInit(int *argc, char ***argv, int *numNodes, int *myNodeID) /// short hsnOrder[numcxi]={2,1,3,0}; if(numcxi==4) { - short hsnOrder[4]= {1,3,0,2}; + short hsnOrder[8]= {1,1,3,3,0,0,2,2}; + if(myRank%quad>numcxi) + { + CmiPrintf("Error: myrank %d quad %d myrank/quad %n",myRank,quad, myRank/quad); + CmiAbort("cxi mapping failure"); + } + myNet=hsnOrder[myRank%quad]; + } + else if(numcxi==8) + { + // no idea if this is a good ordering + short hsnOrder[8]= {0,1,2,3,4,5,6,7}; if(myRank%quad>numcxi) { CmiPrintf("Error: myrank %d quad %d myrank/quad %n",myRank,quad, myRank/quad); From 589e6814c6da1183536f8a1261c52f72988b08a2 Mon Sep 17 00:00:00 2001 From: Eric Bohm Date: Thu, 16 Jan 2025 16:26:51 -0500 Subject: [PATCH 2/2] !fixup: clean up logic for cxi defaults and clarifying whitespace --- src/arch/ofi/machine.C | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/arch/ofi/machine.C b/src/arch/ofi/machine.C index daf21d18de..031491b94d 100644 --- a/src/arch/ofi/machine.C +++ b/src/arch/ofi/machine.C @@ -815,34 +815,34 @@ void LrtsInit(int *argc, char ***argv, int *numNodes, int *myNodeID) /// short hsnOrder[numcxi]={2,1,3,0}; - if(numcxi==4) + if(numcxi == 4) { - short hsnOrder[8]= {1,1,3,3,0,0,2,2}; - if(myRank%quad>numcxi) + short hsnOrder[4]= {1,3,0,2}; + if(myRank % quad > numcxi) { CmiPrintf("Error: myrank %d quad %d myrank/quad %n",myRank,quad, myRank/quad); CmiAbort("cxi mapping failure"); } - myNet=hsnOrder[myRank%quad]; + myNet = hsnOrder[myRank % quad]; } - else if(numcxi==8) + else if(numcxi == 8) { - // no idea if this is a good ordering + // this appears to be a good ordering on aurora short hsnOrder[8]= {0,1,2,3,4,5,6,7}; - if(myRank%quad>numcxi) + if(myRank % quad > numcxi) { CmiPrintf("Error: myrank %d quad %d myrank/quad %n",myRank,quad, myRank/quad); CmiAbort("cxi mapping failure"); } - myNet=hsnOrder[myRank%quad]; + myNet = hsnOrder[myRank % quad]; } else { - CmiAssert(numcxi==1); - //theoretically there are cases other than 4 and 1, but + CmiAssert(numcxi == 1); + //theoretically there are cases other than 8, 4 and 1, but //until someone sights such an incrayptid on a machine floor, //we're just going to assume they don't exist. - myNet=0; + myNet = 0; } } snprintf(myDomainName,5, "cxi%d", myNet);