Merge branch 'release/0.3' into guzhao/fix_build

.github/workflows/build-image.yml

@@ -8,11 +8,21 @@ on:
     branches:
     - main
     - release/*
+  workflow_dispatch:
 
 jobs:
   docker:
-    name: Docker build
+    name: Docker build ${{ matrix.name }}
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+        - name: cuda11.1.1
+          tags: superbench/main:cuda11.1.1,superbench/superbench:latest
+        - name: rocm4.2-pytorch1.7.0
+          tags: superbench/main:rocm4.2-pytorch1.7.0
+        # - name: rocm4.0-pytorch1.7.0
+        #   tags: superbench/main:rocm4.0-pytorch1.7.0
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -30,14 +40,12 @@ jobs:
       - name: Prepare metadata
         id: metadata
         run: |
-          DOCKER_IMAGE=superbench/superbench
-          IMAGE_TAG=latest
+          TAGS=${{ matrix.tags }}
+          DOCKERFILE=dockerfile/${{ matrix.name }}.dockerfile
 
-          DOCKERFILE=dockerfile/cuda11.1.1.dockerfile
-          TAGS="${DOCKER_IMAGE}:${IMAGE_TAG}"
-          CACHE_FROM="type=registry,ref=${DOCKER_IMAGE}:${IMAGE_TAG}"
+          CACHE_FROM="type=registry,ref=$(cut -d, -f1 <<< ${TAGS})"
           CACHE_TO=""
-          if [ "${{ github.event_name }}" = "push" ]; then
+          if [ "${{ github.event_name }}" != "pull_request" ]; then
             CACHE_TO="type=inline,mode=max"
           fi
 
@@ -51,7 +59,7 @@ jobs:
         uses: docker/setup-buildx-action@v1
       - name: Login to Docker Hub
         uses: docker/login-action@v1
-        if: ${{ github.event_name == 'push' }}
+        if: ${{ github.event_name != 'pull_request' }}
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -62,7 +70,7 @@ jobs:
           platforms: linux/amd64
           context: .
           file: ${{ steps.metadata.outputs.dockerfile }}
-          push: ${{ github.event_name == 'push' }}
+          push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.metadata.outputs.tags }}
           cache-from: ${{ steps.metadata.outputs.cache_from }}
           cache-to: ${{ steps.metadata.outputs.cache_to }}

superbench/benchmarks/model_benchmarks/pytorch_base.py

@@ -174,6 +174,7 @@ def _postprocess(self):
 
         try:
             if self._args.distributed_impl == DistributedImpl.DDP:
+                torch.distributed.barrier()
                 torch.distributed.destroy_process_group()
         except BaseException as e:
             self._result.set_return_code(ReturnCode.DISTRIBUTED_SETTING_DESTROY_FAILURE)

superbench/runner/playbooks/deploy.yaml

@@ -101,7 +101,7 @@
           {{ '--security-opt seccomp=unconfined --group-add video' if amd_gpu_exist else '' }} \
           -w /root -v {{ workspace }}:/root -v /mnt:/mnt \
           -v /var/run/docker.sock:/var/run/docker.sock \
-          {{ docker_image }} bash && \
+          --entrypoint /bin/bash {{ docker_image }} && \
         docker exec {{ container }} bash -c \
           "chown -R root:root ~ && \
           sed -i 's/[# ]*Port.*/Port {{ ssh_port }}/g' /etc/ssh/sshd_config && \

third_party/Makefile

@@ -8,7 +8,6 @@ MPI_HOME ?= /usr/local/mpi
 HIP_HOME ?= /opt/rocm/hip
 RCCL_HOME ?= /opt/rocm/rccl
 ROCM_VERSION ?= rocm-$(shell dpkg -l | grep 'rocm-dev ' | awk '{print $$3}' | cut -d '.' -f1-3)
-ROCM_ARCH ?= $(shell rocminfo | grep " gfx" | uniq | awk '{print $$2}')
 
 .PHONY: all cuda rocm common cuda_cutlass cuda_bandwidthTest cuda_nccl_tests cuda_perftest rocm_perftest fio rocm_rccl_tests rocm_rocblas rocm_bandwidthTest
 
@@ -81,21 +80,14 @@ rocm_rocblas: sb_micro_path
 ifeq (, $(wildcard $(SB_MICRO_PATH)/bin/rocblas-bench))
 	if [ -d rocBLAS ]; then rm -rf rocBLAS; fi
 	git clone -b ${ROCM_VERSION} https://github.com/ROCmSoftwarePlatform/rocBLAS.git ./rocBLAS
-ifeq (${ROCM_VERSION}, rocm-4.0.0)
-	sed -i '/CMAKE_MATCH_1/a\      get_filename_component(HIP_CLANG_ROOT "$${HIP_CLANG_ROOT}" DIRECTORY)'  /opt/rocm/hip/lib/cmake/hip/hip-config.cmake
-	cd ./rocBLAS && HIPCC_COMPILE_FLAGS_APPEND="-D_OPENMP=201811 -O3 -Wno-format-nonliteral -DCMAKE_HAVE_LIBC_PTHREAD -parallel-jobs=2" HIPCC_LINK_FLAGS_APPEND="-lpthread -O3 -parallel-jobs=2" ./install.sh -idc -a ${ROCM_ARCH}
-else 
-	cd ./rocBLAS && ./install.sh -idc
-endif
+	cd ./rocBLAS && ./install.sh --dependencies --clients-only
 	cp -v ./rocBLAS/build/release/clients/staging/rocblas-bench $(SB_MICRO_PATH)/bin/
 endif
 
 # Build hipBusBandwidth.
 # HIP is released with rocm, like rocm-4.2.0 and so on.
 # The version we use is the released tag which is consistent with the rocm version in the environment or docker.
-rocm_bandwidthTest:
-	cp -r -v $(shell hipconfig -p) ./
-ifneq (, $(wildcard hip/samples/1_Utils/hipBusBandwidth/CMakeLists.txt))
-	cd ./hip/samples/1_Utils/hipBusBandwidth/ && mkdir -p build && cd build && cmake .. && make
-	cp -v ./hip/samples/1_Utils/hipBusBandwidth/build/hipBusBandwidth $(SB_MICRO_PATH)/bin/
-endif
+rocm_bandwidthTest: sb_micro_path
+	cp -r -v $(shell hipconfig -p)/samples/1_Utils/hipBusBandwidth ./
+	cd ./hipBusBandwidth/ && mkdir -p build && cd build && cmake .. && make
+	cp -v ./hipBusBandwidth/build/hipBusBandwidth $(SB_MICRO_PATH)/bin/