Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Intel Benchmark P2P support #141

Merged
merged 3 commits into from Apr 23, 2019
Merged

Intel Benchmark P2P support #141

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
81170fc
Intel Benchmark P2P support
juhlee-microsoft Apr 19, 2019
4cfcfb4
Added P2P verification string in output
juhlee-microsoft Apr 20, 2019
67eadb5
Fxied variable name for intel MPI
juhlee-microsoft Apr 22, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
15 changes: 12 additions & 3 deletions Testscripts/Linux/SetupRDMA.sh
View file
Open in desktop
Expand Up @@ -224,7 +224,7 @@ function Main() {
elif [ $mpi_type == "intel" ]; then
# if HPC images comes with MPI binary pre-installed, (CentOS HPC)
# there is no action required except binary verification
mpirun_path=$(find / -name mpirun | grep intel64) # $mpirun_path is not empty or null and file path should exists
mpirun_path=$(find / -name mpirun | grep intel64) # $mpirun_path is not empty or null and file path should exists
if [[ -f $mpirun_path && ! -z "$mpirun_path" ]]; then
LogMsg "Found pre-installed mpirun binary"

Expand Down Expand Up @@ -390,9 +390,18 @@ function Main() {
cd mpi-benchmarks/src_c
LogMsg "Building Intel MPI Benchmarks tests"
make -j $(nproc)
LogMsg "Completed Intel MPI Benchmarks"
Verify_Result
LogMsg "Intel Benchmark test installation completed"

# install P2P test
LogMsg "Change directory to P2P"
cd P2P
LogMsg "Renaming from mpiicc to mpicc in Makefile"
sed -i -e 's/CC=mpiicc/CC=mpicc/g' Makefile
LogMsg "Building P2P binary"
make -j $(nproc)
LogMsg "Completed P2P2 binary compilation"
Verify_Result
LogMsg "Intel MPI Benchmark test installation completed"

# set string to verify Intel Benchmark binary
benchmark_bin=$HOMEDIR/mpi-benchmarks/src_c/IMB-MPI1
Expand Down
68 changes: 68 additions & 0 deletions Testscripts/Linux/TestRDMA_MultiVM.sh
View file
Open in desktop
Expand Up @@ -25,6 +25,7 @@ UtilsInit
imb_mpi1_final_status=0
imb_rma_final_status=0
imb_nbc_final_status=0
imb_p2p_final_status=0

# Get all the Kernel-Logs from all VMs.
function Collect_Logs() {
Expand Down Expand Up @@ -270,6 +271,69 @@ function Run_IMB_NBC() {
fi
}

function Run_IMB_P2P() {
total_attempts=$(seq 1 1 $imb_p2p_tests_iterations)
imb_p2p_final_status=0
if [[ $imb_p2p_tests == "all" ]]; then
extra_params=""
else
extra_params=$imb_p2p_tests
fi
for attempt in $total_attempts; do
retries=0
while [ $retries -lt 3 ]; do
case "$mpi_type" in
ibm)
LogMsg "$mpi_run_path -hostlist $modified_slaves:$VM_Size -np $(($VM_Size * $total_virtual_machines)) -e MPI_IB_PKEY=$MPI_IB_PKEY $imb_p2p_path $extra_params"
$mpi_run_path -hostlist $modified_slaves:$VM_Size -np $(($VM_Size * $total_virtual_machines)) -e MPI_IB_PKEY=$MPI_IB_PKEY $imb_p2p_path $extra_params > IMB-P2P-AllNodes-output-Attempt-${attempt}.txt
;;
open)
LogMsg "$mpi_run_path --allow-run-as-root --host $master,$slaves -n $(($p2p_ppn * $total_virtual_machines)) $mpi_settings $imb_p2p_path $extra_params"
$mpi_run_path --allow-run-as-root --host $master,$slaves -n $(($p2p_ppn * $total_virtual_machines)) $mpi_settings $imb_p2p_path $extra_params > IMB-P2P-AllNodes-output-Attempt-${attempt}.txt
;;
intel)
LogMsg "$mpi_run_path -hosts $master,$slaves -ppn $p2p_ppn -n $(($VM_Size * $total_virtual_machines)) $mpi_settings $imb_p2p_path $extra_params"
$mpi_run_path -hosts $master,$slaves -ppn $p2p_ppn -n $(($VM_Size * $total_virtual_machines)) $mpi_settings $imb_p2p_path $extra_params > IMB-P2P-AllNodes-output-Attempt-${attempt}.txt
;;
mvapich)
LogMsg "$mpi_run_path -n $(($p2p_ppn * $total_virtual_machines)) $master $slaves_array $mpi_settings $imb_p2p_path $extra_params"
$mpi_run_path -n $(($p2p_ppn * $total_virtual_machines)) $master $slaves_array $mpi_settings $imb_p2p_path $extra_params > IMB-P2P-AllNodes-output-Attempt-${attempt}.txt
;;
esac
p2p_status=$?
if [ $p2p_status -eq 0 ]; then
LogMsg "IMB-P2P test iteration $attempt - Succeeded."
sleep 1
retries=4
else
sleep 10
let retries=retries+1
failed_p2p=$(cat IMB-P2P-AllNodes-output-Attempt-${attempt}.txt | grep Benchmarking | tail -1| awk '{print $NF}')
p2p_benchmarks=$(echo $p2p_benchmarks | sed "s/^.*${failed_p2p}//")
extra_params=$p2p_benchmarks
fi
done
if [ $p2p_status -eq 0 ]; then
LogMsg "IMB-P2P test iteration $attempt - Succeeded."
sleep 1
else
LogErr "IMB-P2P test iteration $attempt - Failed."
imb_p2p_final_status=$(($imb_p2p_final_status + $p2p_status))
sleep 1
fi
done

if [ $imb_p2p_final_status -ne 0 ]; then
LogErr "IMB-P2P tests returned non-zero exit code. Aborting further tests."
SetTestStateFailed
Collect_Logs
LogErr "INFINIBAND_VERIFICATION_FAILED_P2P_ALLNODES"
exit 0
else
LogMsg "INFINIBAND_VERIFICATION_SUCCESS_P2P_ALLNODES"
fi
}

function Main() {
LogMsg "Starting $mpi_type MPI tests..."

Expand Down Expand Up @@ -505,6 +569,7 @@ function Main() {
imb_mpi1_path=$(find / -name IMB-MPI1 | head -n 1)
imb_rma_path=$(find / -name IMB-RMA | head -n 1)
imb_nbc_path=$(find / -name IMB-NBC | head -n 1)
imb_p2p_path=$(find / -name IMB-P2P | head -n 1)
case "$mpi_type" in
ibm)
modified_slaves=${slaves//,/:$VM_Size,}
Expand All @@ -531,12 +596,14 @@ function Main() {
LogMsg "IMB-MPI1 Path: $imb_mpi1_path"
LogMsg "IMB-RMA Path: $imb_rma_path"
LogMsg "IMB-NBC Path: $imb_nbc_path"
LogMsg "IMB-P2P Path: $imb_p2p_path"

# Run all benchmarks
Run_IMB_Intranode
Run_IMB_MPI1
Run_IMB_RMA
Run_IMB_NBC
Run_IMB_P2P

# Get all results and finish the test
Collect_Logs
Expand All @@ -549,6 +616,7 @@ function Main() {
LogMsg "imb_mpi1_final_status: $imb_mpi1_final_status"
LogMsg "imb_rma_final_status: $imb_rma_final_status"
LogMsg "imb_nbc_final_status: $imb_nbc_final_status"
LogMsg "imb_p2p_final_status: $imb_p2p_final_status"
LogErr "INFINIBAND_VERIFICATION_FAILED"
SetTestStateFailed
else
Expand Down
22 changes: 22 additions & 0 deletions Testscripts/Windows/VERIFY-INFINIBAND-MultiVM.ps1
View file
Open in desktop
Expand Up @@ -112,6 +112,9 @@ function Main {
if ($TestParam -imatch "imb_nbc_tests_iterations") {
$ImbNbcTestIterations = [int]($TestParam.Replace("imb_nbc_tests_iterations=", "").Trim('"'))
}
if ($TestParam -imatch "imb_p2p_tests_iterations") {
$ImbP2pTestIterations = [int]($TestParam.Replace("imb_p2p_tests_iterations=", "").Trim('"'))
}
if ($TestParam -imatch "ib_nic") {
$InfinibandNic = [string]($TestParam.Replace("ib_nic=", "").Trim('"'))
}
Expand Down Expand Up @@ -333,6 +336,25 @@ function Main {
}
#endregion

#region Check P2P all nodes tests
if ($ImbP2pTestIterations -ge 1) {
$logFileName = "$LogDir\InfiniBand-Verification-$Iteration-$TempName\TestExecution.log"
$pattern = "INFINIBAND_VERIFICATION_SUCCESS_P2P_ALLNODES"
Write-LogInfo "Analyzing $logFileName"
$metaData = "InfiniBand-Verification-$Iteration-$TempName : IMB-P2P"
$SucessLogs = Select-String -Path $logFileName -Pattern $pattern
if ($SucessLogs.Count -eq 1) {
$currentResult = "PASS"
} else {
$currentResult = "FAIL"
}
Write-LogInfo "$pattern : $currentResult"
$resultArr += $currentResult
$CurrentTestResult.TestSummary += New-ResultSummary -testResult $currentResult -metaData $metaData `
-checkValues "PASS,FAIL,ABORTED" -testName $CurrentTestData.testName
}
#endregion

#region Check RMA all nodes tests
if ($ImbRmaTestIterations -ge 1) {
$logFileName = "$LogDir\InfiniBand-Verification-$Iteration-$TempName\TestExecution.log"
Expand Down
16 changes: 16 additions & 0 deletions XML/Other/ReplaceableTestParameters.xml
View file
Open in desktop
Expand Up @@ -228,6 +228,22 @@ Scenario 2 : You need to run all the performance tests quickly.
<ReplaceThis>INFINIBAND_MVAPICH_NBC_TEST_COUNT</ReplaceThis>
<ReplaceWith>1</ReplaceWith>
</Parameter>
<Parameter>
<ReplaceThis>INFINIBAND_INTEL_P2P_TEST_COUNT</ReplaceThis>
<ReplaceWith>1</ReplaceWith>
</Parameter>
<Parameter>
<ReplaceThis>INFINIBAND_IBM_P2P_TEST_COUNT</ReplaceThis>
<ReplaceWith>1</ReplaceWith>
</Parameter>
<Parameter>
<ReplaceThis>INFINIBAND_OPEN_P2P_TEST_COUNT</ReplaceThis>
<ReplaceWith>1</ReplaceWith>
</Parameter>
<Parameter>
<ReplaceThis>INFINIBAND_MVAPICH_P2P_TEST_COUNT</ReplaceThis>
<ReplaceWith>1</ReplaceWith>
</Parameter>
<Parameter>
<ReplaceThis>NESTED-PERF-STORAGE-IO-MODES</ReplaceThis>
<ReplaceWith>(randread randwrite read write)</ReplaceWith>
Expand Down
16 changes: 16 additions & 0 deletions XML/TestCases/FunctionalTests-InfiniBand.xml
View file
Open in desktop
Expand Up @@ -21,6 +21,8 @@
<param>imb_nbc_tests="INFINIBAND_INTEL_NBC_TESTNAMES"</param>
<param>nbc_ppn=INFINIBAND_INTEL_PPN</param>
<param>imb_nbc_tests_iterations=INFINIBAND_INTEL_NBC_TEST_COUNT</param>
<param>p2p_ppn=INFINIBAND_INTEL_PPN</param>
<param>imb_p2p_tests_iterations=INFINIBAND_INTEL_P2P_TEST_COUNT</param>
<param>mpi_type="intel"</param>
<param>mlx_ofed_partial_link=MLX_OFED_PARTIAL_LINK</param>
<param>ibm_platform_mpi=IBM_PLATFORM_MPI_DOWNLOAD</param>
Expand Down Expand Up @@ -56,6 +58,8 @@
<param>imb_nbc_tests="INFINIBAND_INTEL_NBC_TESTNAMES"</param>
<param>nbc_ppn=INFINIBAND_INTEL_PPN</param>
<param>imb_nbc_tests_iterations=INFINIBAND_INTEL_NBC_TEST_COUNT</param>
<param>p2p_ppn=INFINIBAND_INTEL_PPN</param>
<param>imb_p2p_tests_iterations=INFINIBAND_INTEL_P2P_TEST_COUNT</param>
<param>mpi_type="intel"</param>
<param>mlx_ofed_partial_link=MLX_OFED_PARTIAL_LINK</param>
<param>ibm_platform_mpi=IBM_PLATFORM_MPI_DOWNLOAD</param>
Expand Down Expand Up @@ -92,6 +96,8 @@
<param>imb_nbc_tests="INFINIBAND_OPEN_NBC_TESTNAMES"</param>
<param>nbc_ppn=INFINIBAND_OPEN_PPN</param>
<param>imb_nbc_tests_iterations=INFINIBAND_OPEN_NBC_TEST_COUNT</param>
<param>p2p_ppn=INFINIBAND_OPEN_PPN</param>
<param>imb_p2p_tests_iterations=INFINIBAND_OPEN_P2P_TEST_COUNT</param>
<param>mpi_type="open"</param>
<param>mlx_ofed_partial_link=MLX_OFED_PARTIAL_LINK</param>
<param>ibm_platform_mpi=IBM_PLATFORM_MPI_DOWNLOAD</param>
Expand Down Expand Up @@ -127,6 +133,8 @@
<param>imb_nbc_tests="INFINIBAND_OPEN_NBC_TESTNAMES"</param>
<param>nbc_ppn=INFINIBAND_OPEN_PPN</param>
<param>imb_nbc_tests_iterations=INFINIBAND_OPEN_NBC_TEST_COUNT</param>
<param>p2p_ppn=INFINIBAND_OPEN_PPN</param>
<param>imb_p2p_tests_iterations=INFINIBAND_OPEN_P2P_TEST_COUNT</param>
<param>mpi_type="open"</param>
<param>mlx_ofed_partial_link=MLX_OFED_PARTIAL_LINK</param>
<param>ibm_platform_mpi=IBM_PLATFORM_MPI_DOWNLOAD</param>
Expand Down Expand Up @@ -163,6 +171,8 @@
<param>imb_nbc_tests="INFINIBAND_IBM_NBC_TESTNAMES"</param>
<param>nbc_ppn=INFINIBAND_IBM_PPN</param>
<param>imb_nbc_tests_iterations=INFINIBAND_IBM_NBC_TEST_COUNT</param>
<param>p2p_ppn=INFINIBAND_IBM_PPN</param>
<param>imb_p2p_tests_iterations=INFINIBAND_IBM_P2P_TEST_COUNT</param>
<param>mpi_type="ibm"</param>
<param>mlx_ofed_partial_link=MLX_OFED_PARTIAL_LINK</param>
<param>ibm_platform_mpi=IBM_PLATFORM_MPI_DOWNLOAD</param>
Expand Down Expand Up @@ -198,6 +208,8 @@
<param>imb_nbc_tests="INFINIBAND_IBM_NBC_TESTNAMES"</param>
<param>nbc_ppn=INFINIBAND_IBM_PPN</param>
<param>imb_nbc_tests_iterations=INFINIBAND_IBM_NBC_TEST_COUNT</param>
<param>p2p_ppn=INFINIBAND_IBM_PPN</param>
<param>imb_p2p_tests_iterations=INFINIBAND_IBM_P2P_TEST_COUNT</param>
<param>mpi_type="ibm"</param>
<param>mlx_ofed_partial_link=MLX_OFED_PARTIAL_LINK</param>
<param>ibm_platform_mpi=IBM_PLATFORM_MPI_DOWNLOAD</param>
Expand Down Expand Up @@ -234,6 +246,8 @@
<param>imb_nbc_tests="INFINIBAND_MVAPICH_NBC_TESTNAMES"</param>
<param>nbc_ppn=INFINIBAND_MVAPICH_PPN</param>
<param>imb_nbc_tests_iterations=INFINIBAND_MVAPICH_NBC_TEST_COUNT</param>
<param>p2p_ppn=INFINIBAND_MVAPICH_PPN</param>
<param>imb_p2p_tests_iterations=INFINIBAND_MVAPICH_P2P_TEST_COUNT</param>
<param>mpi_type="mvapich"</param>
<param>mlx_ofed_partial_link=MLX_OFED_PARTIAL_LINK</param>
<param>ibm_platform_mpi=IBM_PLATFORM_MPI_DOWNLOAD</param>
Expand Down Expand Up @@ -269,6 +283,8 @@
<param>imb_nbc_tests="INFINIBAND_MVAPICH_NBC_TESTNAMES"</param>
<param>nbc_ppn=INFINIBAND_MVAPICH_PPN</param>
<param>imb_nbc_tests_iterations=INFINIBAND_MVAPICH_NBC_TEST_COUNT</param>
<param>p2p_ppn=INFINIBAND_MVAPICH_PPN</param>
<param>imb_p2p_tests_iterations=INFINIBAND_MVAPICH_P2P_TEST_COUNT</param>
<param>mpi_type="mvapich"</param>
<param>mlx_ofed_partial_link=MLX_OFED_PARTIAL_LINK</param>
<param>ibm_platform_mpi=IBM_PLATFORM_MPI_DOWNLOAD</param>
Expand Down