Set Up a CI Pipeline for H100

.azure-pipelines/integration-test.yml

@@ -10,8 +10,8 @@ pr:
   drafts: false
 
 jobs:
-- job: IntegrationTest
-  displayName: Integration test
+- job: IntegrationTestA100
+  displayName: Integration test A100
   strategy:
     matrix:
       cuda11:
@@ -22,210 +22,31 @@ jobs:
   pool:
     name: msccl-ci
   container:
-    image: $[ variables['containerImage'] ]
+    image: $(containerImage)
 
   steps:
-  - task: Bash@3
-    name: Build
-    displayName: Build
-    inputs:
-      targetType: 'inline'
-      script: |
-        mkdir build && cd build
-        cmake -DCMAKE_BUILD_TYPE=Release -DMSCCLPP_BYPASS_GPU_CHECK=ON -DMSCCLPP_USE_CUDA=ON ..
-        make -j
-      workingDirectory: '$(System.DefaultWorkingDirectory)'
-
-  - task: Bash@3
-    name: InstallPackages
-    displayName: Install Packages
-    inputs:
-      targetType: 'inline'
-      script: |
-        sudo apt-get update -y
-        sudo apt-get install pssh -y
-        curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
-
-  - task: DownloadSecureFile@1
-    name: SshKeyFile
-    displayName: Download key file
-    inputs:
-      secureFile: mscclpp.pem
-
-  - task: AzureCLI@2
-    name: StartVMSS
-    displayName: Start VMSS
-    inputs:
-      azureSubscription: mscclpp-ci
-      scriptType: bash
-      scriptLocation: inlineScript
-      inlineScript: |
-        az vmss start --name mscclpp-ci --resource-group mscclpp
-
-  - task: Bash@3
-    name: DeployTestEnv
-    displayName: Deploy Test Env
-    inputs:
-      targetType: filePath
-      filePath: test/deploy/deploy.sh
-      arguments: "single-node-test"
-      workingDirectory: '$(System.DefaultWorkingDirectory)'
-
-  - task: Bash@3
-    name: AllGatherTest
-    displayName: Run mscclpp AllGather test
-    inputs:
-      targetType: 'inline'
-      script: |
-        set -e
-        HOSTFILE=$(System.DefaultWorkingDirectory)/test/deploy/hostfile_ci
-        SSH_OPTION="StrictHostKeyChecking=no"
-        KeyFilePath=${SSHKEYFILE_SECUREFILEPATH}
-        : > azureuser@10.0.0.4
-        tail -f azureuser@10.0.0.4 &
-        CHILD_PID=$!
-        parallel-ssh -o . -t 0 -h ${HOSTFILE} -x "-i ${KeyFilePath}" \
-          -O $SSH_OPTION 'sudo docker exec -t mscclpp-test bash -c " \
-          export PATH=/usr/local/mpi/bin:\$PATH;                     \
-          export LD_LIBRARY_PATH=/root/mscclpp/build:\$LD_LIBRARY_PATH;  \
-          cd /root/mscclpp;                                         \
-          set -e;                                                   \
-          mpirun --allow-run-as-root -np 8 --bind-to numa -x MSCCLPP_DEBUG=WARN ./build/test/mscclpp-test/allgather_test_perf -b 1K -e 1G -f 2 -o output.jsonl;       \
-          mpirun --allow-run-as-root -np 8 --bind-to numa -x MSCCLPP_DEBUG=WARN ./build/test/mscclpp-test/allgather_test_perf -b 1K -e 1G -f 2 -k 1 -o output.jsonl;  \
-          mpirun --allow-run-as-root -np 8 --bind-to numa -x MSCCLPP_DEBUG=WARN ./build/test/mscclpp-test/allgather_test_perf -b 1K -e 1G -f 2 -k 2 -o output.jsonl;  \
-          mpirun --allow-run-as-root -np 8 --bind-to numa -x MSCCLPP_DEBUG=WARN ./build/test/mscclpp-test/allgather_test_perf -b 1K -e 1G -f 2 -k 3 -o output.jsonl"'
-        kill $CHILD_PID
-      workingDirectory: '$(System.DefaultWorkingDirectory)'
-
-  - task: Bash@3
-    name: SendRecvTest
-    displayName: Run mscclpp SendRecv test
-    inputs:
-      targetType: 'inline'
-      script: |
-        set -e
-        HOSTFILE=$(System.DefaultWorkingDirectory)/test/deploy/hostfile_ci
-        SSH_OPTION="StrictHostKeyChecking=no"
-        KeyFilePath=${SSHKEYFILE_SECUREFILEPATH}
-        : > azureuser@10.0.0.4
-        tail -f azureuser@10.0.0.4 &
-        CHILD_PID=$!
-        parallel-ssh -o . -t 0 -h ${HOSTFILE} -x "-i ${KeyFilePath}"\
-          -O $SSH_OPTION 'sudo docker exec -t mscclpp-test bash -c "\
-          set -e;                                                   \
-          export PATH=/usr/local/mpi/bin:\$PATH;                    \
-          export LD_LIBRARY_PATH=/root/mscclpp/build:\$LD_LIBRARY_PATH;  \
-          cd /root/mscclpp;                                         \
-          mpirun --allow-run-as-root -np 8 --bind-to numa -x MSCCLPP_DEBUG=WARN ./build/test/mscclpp-test/sendrecv_test_perf -b 1K -e 1G -f 2 -o output.jsonl"'
-        kill $CHILD_PID
-      workingDirectory: '$(System.DefaultWorkingDirectory)'
-
-  - task: Bash@3
-    name: AllReduceTest
-    displayName: Run mscclpp AllReduce test
-    inputs:
-      targetType: 'inline'
-      script: |
-        set -e
-        HOSTFILE=$(System.DefaultWorkingDirectory)/test/deploy/hostfile_ci
-        SSH_OPTION="StrictHostKeyChecking=no"
-        KeyFilePath=${SSHKEYFILE_SECUREFILEPATH}
-        : > azureuser@10.0.0.4
-        tail -f azureuser@10.0.0.4 &
-        CHILD_PID=$!
-        parallel-ssh -o . -t 0 -h ${HOSTFILE} -x "-i ${KeyFilePath}"\
-          -O $SSH_OPTION 'sudo docker exec -t mscclpp-test bash -c "\
-          set -e;                                                   \
-          export PATH=/usr/local/mpi/bin:\$PATH;                     \
-          export LD_LIBRARY_PATH=/root/mscclpp/build:\$LD_LIBRARY_PATH;  \
-          cd /root/mscclpp;                                         \
-          mpirun --allow-run-as-root -np 8 --bind-to numa -x MSCCLPP_DEBUG=WARN ./build/test/mscclpp-test/allreduce_test_perf -b 1K -e 1G -f 2 -o output.jsonl;                 \
-          mpirun --allow-run-as-root -np 8 --bind-to numa -x MSCCLPP_DEBUG=WARN ./build/test/mscclpp-test/allreduce_test_perf -b 1K -e 1G -f 2 -k 1 -o output.jsonl;            \
-          mpirun --allow-run-as-root -np 8 --bind-to numa -x MSCCLPP_DEBUG=WARN ./build/test/mscclpp-test/allreduce_test_perf -b 1K -e 1G -f 2 -k 2 -o output.jsonl;            \
-          mpirun --allow-run-as-root -np 8 --bind-to numa -x MSCCLPP_DEBUG=WARN ./build/test/mscclpp-test/allreduce_test_perf -b 1K -e 1G -f 2 -k 3 -o output.jsonl;            \
-          mpirun --allow-run-as-root -np 8 --bind-to numa -x MSCCLPP_DEBUG=WARN ./build/test/mscclpp-test/allreduce_test_perf -b 1K -e 1G -f 2 -k 4 -o output.jsonl;            \
-          mpirun --allow-run-as-root -np 8 --bind-to numa -x MSCCLPP_DEBUG=WARN ./build/test/mscclpp-test/allreduce_test_perf -b 12M -e 48M -i 3145728 2 -k 5 -o output.jsonl;  \
-          mpirun --allow-run-as-root -np 8 --bind-to numa -x MSCCLPP_DEBUG=WARN ./build/test/mscclpp-test/allreduce_test_perf -b 24K -e 768K -i 24576 -k 6 -w 100 -n 100 -o output.jsonl"'
-        kill $CHILD_PID
-      workingDirectory: '$(System.DefaultWorkingDirectory)'
-
-  - task: Bash@3
-    name: AllToAll
-    displayName: Run mscclpp AllToAll test
-    inputs:
-      targetType: 'inline'
-      script: |
-        HOSTFILE=$(System.DefaultWorkingDirectory)/test/deploy/hostfile_ci
-        SSH_OPTION="StrictHostKeyChecking=no"
-        KeyFilePath=${SSHKEYFILE_SECUREFILEPATH}
-        : > azureuser@10.0.0.4
-        tail -f azureuser@10.0.0.4 &
-        CHILD_PID=$!
-        parallel-ssh -o . -t 0 -h ${HOSTFILE} -x "-i ${KeyFilePath}"\
-          -O $SSH_OPTION 'sudo docker exec -t mscclpp-test bash -c "\
-          set -e;                                                   \
-          export PATH=/usr/local/mpi/bin:\$PATH;                    \
-          export LD_LIBRARY_PATH=/root/mscclpp/build:\$LD_LIBRARY_PATH; \
-          cd /root/mscclpp;                                         \
-          mpirun --allow-run-as-root -np 8 --bind-to numa -x MSCCLPP_DEBUG=WARN ./build/test/mscclpp-test/alltoall_test_perf -b 1K -e 1G -f 2 -o output.jsonl; \
-          mpirun --allow-run-as-root -np 8 --bind-to numa -x MSCCLPP_DEBUG=WARN ./build/test/mscclpp-test/alltoall_test_perf -b 1K -e 1G -f 2 -k 1 -o output.jsonl"'
-        kill $CHILD_PID
-      workingDirectory: '$(System.DefaultWorkingDirectory)'
-
-  - task: Bash@3
-    name: CheckPerfNumber
-    displayName: Check collective primitives performance
-    inputs:
-      targetType: 'inline'
-      script: |
-        set -e
-        HOSTFILE=$(System.DefaultWorkingDirectory)/test/deploy/hostfile_ci
-        SSH_OPTION="StrictHostKeyChecking=no"
-        KeyFilePath=${SSHKEYFILE_SECUREFILEPATH}
-        : > azureuser@10.0.0.4
-        tail -f azureuser@10.0.0.4 &
-        CHILD_PID=$!
-        parallel-ssh -o . -t 0 -h ${HOSTFILE} -x "-i ${KeyFilePath}"\
-          -O $SSH_OPTION 'sudo docker exec -t mscclpp-test bash -c "\
-          set -e;                                                   \
-          cd /root/mscclpp;                                         \
-          export PATH=/usr/local/mpi/bin:\$PATH;                    \
-          export LD_LIBRARY_PATH=/root/mscclpp/build:\$LD_LIBRARY_PATH; \
-          python3 test/mscclpp-test/check_perf_result.py --perf-file output.jsonl --baseline-file test/deploy/perf_ndmv4.jsonl"'
-        kill $CHILD_PID
-      workingDirectory: '$(System.DefaultWorkingDirectory)'
+  - template: templates/integration-test.yaml
+    parameters:
+      subscription:     mscclpp-ci
+      vmssName:         mscclpp-ci
+      sshKeySecureFile: mscclpp.pem
+
+- job: IntegrationTestH100
+  displayName: Integration test H100
+  strategy:
+    matrix:
+      cuda12:
+        containerImage: ghcr.io/microsoft/mscclpp/mscclpp:base-dev-cuda12.4
 
-  - task: Bash@3
-    name: PythonAllReduceBenchmark
-    displayName: Python Allreduce Benchmark
-    inputs:
-      targetType: 'inline'
-      script: |
-        set -e
-        HOSTFILE=$(System.DefaultWorkingDirectory)/test/deploy/hostfile_ci
-        SSH_OPTION="StrictHostKeyChecking=no"
-        KeyFilePath=${SSHKEYFILE_SECUREFILEPATH}
-        : > azureuser@10.0.0.4
-        tail -f azureuser@10.0.0.4 &
-        CHILD_PID=$!
-        parallel-ssh -o . -t 0 -h ${HOSTFILE} -x "-i ${KeyFilePath}" \
-          -O $SSH_OPTION 'sudo docker exec -t mscclpp-test bash -c " \
-          set -e;                                                    \
-          cd /root/mscclpp;                                          \
-          export PATH=/usr/local/mpi/bin:\$PATH;                     \
-          export LD_LIBRARY_PATH=/root/mscclpp/build:\$LD_LIBRARY_PATH; \
-          python3 -m pip install .;                                     \
-          mpirun --allow-run-as-root -tag-output -x MSCCLPP_HOME=/root/mscclpp -np 8 python3 ./python/mscclpp_benchmark/allreduce_bench.py"'
-        kill $CHILD_PID
-      workingDirectory: '$(System.DefaultWorkingDirectory)'
+  pool:
+    name: msccl-ci-h100
+  container:
+    image: $(containerImage)
 
-  - task: AzureCLI@2
-    name: StopVMSS
-    displayName: Deallocate VMSS
-    condition: always()
-    inputs:
-      azureSubscription: mscclpp-ci
-      scriptType: bash
-      scriptLocation: inlineScript
-      inlineScript: |
-        az vmss deallocate --name mscclpp-ci --resource-group mscclpp
+  steps:
+  - template: templates/integration-test.yaml
+    parameters:
+      subscription:     mscclpp-ci-h100
+      vmssName:         mscclpp-h100-ci
+      sshKeySecureFile: mscclpp.pem
+      perfBaselineFile: test/deploy/perf_ndmv5.jsonl