From 4d6e3fa104d19c8acd5fd81fd4eb3a49abf303f1 Mon Sep 17 00:00:00 2001 From: Ron Lieberman Date: Wed, 19 Nov 2025 07:27:03 -0600 Subject: [PATCH] [gpurun] add numactl check and fallback for -nm and -nr --- offload/utils/gpurun/gpurun | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/offload/utils/gpurun/gpurun b/offload/utils/gpurun/gpurun index ab86f491fa7f6..b8a0daf64b913 100755 --- a/offload/utils/gpurun/gpurun +++ b/offload/utils/gpurun/gpurun @@ -203,13 +203,26 @@ if [ -z "$_num_local_ranks" ] && [ ! -z $SLURM_CPUS_ON_NODE ] ; then _num_local_ranks=$SLURM_CPUS_ON_NODE _local_rank_num=$SLURM_LOCALID fi + if [ "$_use_numactl_rocr" == "1" ] ; then - numactl --cpunodebind $ROCR_VISIBLE_DEVICES --membind $ROCR_VISIBLE_DEVICES $* - exit $? + _cmd_binary=`which numactl` + if [ $? == 0 ] ; then + numactl --cpunodebind $ROCR_VISIBLE_DEVICES --membind $ROCR_VISIBLE_DEVICES $* + exit $? + else + $* + exit $? + fi fi if [ "$_use_numactl_ompi" == "1" ] ; then - numactl --cpunodebind $OMPI_COMM_WORLD_LOCAL_RANK --membind $OMPI_COMM_WORLD_LOCAL_RANK $* - exit $? + _cmd_binary=`which numactl` + if [ $? == 0 ] ; then + numactl --cpunodebind $OMPI_COMM_WORLD_LOCAL_RANK --membind $OMPI_COMM_WORLD_LOCAL_RANK $* + exit $? + else + $* + exit $? + fi fi # If none of the above MPIs, assume gpurun is wrapper for single process on single GPU if [ -z "$_num_local_ranks" ] ; then