CI #235
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Do not edit this file! It has been generated by .github/gen-workflow-ci.py | |
| name: CI | |
| on: | |
| schedule: | |
| # run a build on master (this does not publish test results or cancel concurrent builds) | |
| - cron: '0 10 * * *' # everyday at 10am | |
| push: | |
| # only consider push to master, hotfix-branches, and tags | |
| # otherwise modify job.config.outputs.push | |
| branches: [ 'master', 'hotfix-*' ] | |
| tags: [ 'v*.*.*' ] | |
| pull_request: | |
| # only consider pull requests into master | |
| branches: [ master ] | |
| workflow_dispatch: | |
| permissions: {} | |
| concurrency: | |
| # This controls which concurrent builds to cancel: | |
| # - we do not want any concurrent builds on a branch (pull_request) | |
| # - we do not want concurrent builds on the same commit on master (push) | |
| # - we do not want concurrent builds on the same commit on a tag (push) | |
| # - we allow concurrent runs on the same commit on master and its tag (push) | |
| # - we allow concurrent runs on the same commit on master (push) and a scheduled build (schedule) | |
| # | |
| # A pull_request event only runs on branch commit, a push event only on master and tag commit. | |
| # A schedule event only runs on master HEAD commit. | |
| # | |
| # Expression github.ref means something like refs/heads/master or refs/tags/v0.22.1 or the branch. | |
| # This helps to not cancel concurrent runs on master or a tag that share the same commit. | |
| # Expression github.head_ref refers to the branch of the pull request. | |
| # On master, github.head_ref is empty, so we use the SHA of the commit, this means individual | |
| # commits to master will not be cancelled, while there can only be one concurrent build on a branch. | |
| # | |
| # We include the event name to we allow for concurrent scheduled and master builds. | |
| group: ci-${{ github.event_name }}-${{ github.ref }}-${{ github.head_ref || github.sha }} | |
| cancel-in-progress: true | |
| jobs: | |
| event_file: | |
| name: "Event File" | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Upload | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: Event File | |
| path: ${{ github.event_path }} | |
| setup-py: | |
| name: "setup.py" | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.8 | |
| - name: Test setup.py | |
| env: | |
| HOROVOD_WITHOUT_TENSORFLOW: 1 | |
| HOROVOD_WITHOUT_PYTORCH: 1 | |
| HOROVOD_WITHOUT_MXNET: 1 | |
| HOROVOD_WITHOUT_GLOO: 1 | |
| HOROVOD_WITHOUT_MPI: 1 | |
| run: | | |
| python -m pip install --upgrade pip | |
| python -m pip install setuptools wheel | |
| python setup.py sdist | |
| pip -v install dist/horovod-*.tar.gz | |
| init-workflow: | |
| name: "Init Workflow" | |
| runs-on: ubuntu-latest | |
| outputs: | |
| run-at-all: ${{ github.event_name != 'schedule' || github.repository == 'horovod/horovod' }} | |
| # if we don't get a clear 'false', we fall back to building and testing | |
| run-builds-and-tests: ${{ steps.tests.outputs.needed != 'false' }} | |
| buildkite-branch-label: "${{ steps.config-buildkite.outputs.branch-label }}" | |
| buildkite-message: "${{ steps.config-buildkite.outputs.message }}" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.8 | |
| - name: Pip install dependencies | |
| run: pip install -r .github/requirements.txt | |
| - name: Check ci.yaml is up-to-date | |
| run: | | |
| python .github/gen-workflow-ci.py | |
| if [[ $(git diff .github/workflows/ci.yaml | wc -l) -gt 0 ]] | |
| then | |
| echo "::error::Workflow file .github/workflows/ci.yaml is out-dated, please run .github/gen-workflow-ci.py and commit changes" | |
| exit 1 | |
| fi | |
| shell: bash | |
| - name: Check if tests are needed | |
| id: tests | |
| env: | |
| GITHUB_BASE_SHA: ${{ github.event.pull_request.base.sha }} | |
| GITHUB_HEAD_SHA: ${{ github.event.pull_request.head.sha }} | |
| run: | | |
| if [[ "${{ github.event_name }}" == "pull_request" ]] | |
| then | |
| changes="$(python .github/get-changed-code-files.py)" | |
| if [[ -z "$changes" ]] | |
| then | |
| echo "No code changes, no need to build and test" | |
| echo "needed=false" >> $GITHUB_OUTPUT | |
| else | |
| echo "Code changes, we need to build and test:" | |
| echo "$changes" | |
| echo "needed=true" >> $GITHUB_OUTPUT | |
| fi | |
| else | |
| echo "This is not part of a pull request, we need to build and test" | |
| echo "needed=true" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Configure Buildkite Build | |
| id: config-buildkite | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| branch="${{ github.event.pull_request.head.ref || github.ref }}" | |
| branch="${branch#"refs/heads/"}" | |
| branch="${branch#"refs/tags/"}" | |
| branch_label="${branch}" | |
| if [[ "${{ github.event_name }}" == "schedule" ]] | |
| then | |
| # we add this label to the branch used by Buildkite to avoid it cancelling one of concurrent schedule and push builds on master | |
| branch_label="${branch} (schedule)" | |
| fi | |
| echo "branch-label=${branch_label}" >> $GITHUB_OUTPUT | |
| if [[ "${{ github.event_name }}" == "pull_request" ]] | |
| then | |
| head_sha="${{ github.event.pull_request.head.sha }}" | |
| message="$(gh api https://api.github.com/repos/horovod/horovod/commits/${head_sha} -q .commit.message | head -n1)" | |
| echo "message=${message}" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Provide PR meta | |
| if: github.event_name == 'pull_request' | |
| run: | | |
| rm -f pr.json | |
| echo -n "{" >> pr.json | |
| echo -n " \"merge_sha\": \"${{ github.sha }}\"," >> pr.json | |
| echo -n " \"base_sha\": \"${{ github.event.pull_request.base.sha }}\"," >> pr.json | |
| echo -n " \"head_sha\": \"${{ github.event.pull_request.head.sha }}\" " >> pr.json | |
| echo -n "}" >> pr.json | |
| cat pr.json | |
| - name: Upload PR meta | |
| uses: actions/upload-artifact@v3 | |
| if: github.event_name == 'pull_request' | |
| with: | |
| name: PR Meta | |
| path: pr.json | |
| build-and-test: | |
| name: "Build and Test (${{ matrix.image }})" | |
| needs: [init-workflow] | |
| if: > | |
| needs.init-workflow.outputs.run-at-all == 'true' && | |
| needs.init-workflow.outputs.run-builds-and-tests == 'true' | |
| runs-on: ubuntu-latest | |
| strategy: | |
| max-parallel: 10 | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - image: test-cpu-gloo-py3_7-tf1_15_5-keras2_2_4-torch1_8_1-mxnet1_5_1_p0-pyspark3_4_0 | |
| Elastic_Spark_TensorFlow_Tests_2: true | |
| Elastic_Tests_2: true | |
| Gloo_Cluster_PyTests: true | |
| Gloo_Keras_MNIST: true | |
| Gloo_MXNet_MNIST_horovodrun: true | |
| Gloo_Parallel_PyTests: true | |
| Gloo_PyTorch_MNIST_api: true | |
| Gloo_PyTorch_MNIST_horovodrun: true | |
| Gloo_Single_PyTests: true | |
| Gloo_TensorFlow_MNIST: true | |
| Single_Keras_MNIST: true | |
| Single_MXNet_MNIST: true | |
| Single_PyTorch_MNIST: true | |
| Spark_Keras_MNIST: true | |
| Spark_Keras_Rossmann_Estimator: true | |
| Spark_Keras_Rossmann_Run: true | |
| Spark_Lightning_MNIST: true | |
| Spark_PyTests: true | |
| Spark_Torch_MNIST: true | |
| build_timeout: 30 | |
| - image: test-cpu-gloo-py3_7-tf2_11_0-keras2_11_0-torch1_13_1-mxnet1_9_1-pyspark2_4_8 | |
| Elastic_Spark_TensorFlow_Tests_1: true | |
| Elastic_Spark_Torch_Tests: true | |
| Elastic_Tests_1: true | |
| Gloo_Cluster_PyTests: true | |
| Gloo_MXNet_MNIST_horovodrun: true | |
| Gloo_Parallel_PyTests: true | |
| Gloo_PyTorch_MNIST_api: true | |
| Gloo_PyTorch_MNIST_horovodrun: true | |
| Gloo_Single_PyTests: true | |
| Gloo_TensorFlow_2_0_Keras_MNIST_api: true | |
| Gloo_TensorFlow_2_0_Keras_MNIST_horovodrun: true | |
| Gloo_TensorFlow_2_0_MNIST_Elastic_api: true | |
| Gloo_TensorFlow_2_0_MNIST_Elastic_horovodrun: true | |
| Gloo_TensorFlow_2_0_MNIST_api: true | |
| Gloo_TensorFlow_2_0_MNIST_horovodrun: true | |
| Single_MXNet_MNIST: true | |
| Single_PyTorch_MNIST: true | |
| Spark_Lightning_MNIST: true | |
| Spark_PyTests: true | |
| Spark_TensorFlow_2_0_MNIST_Data_Service: true | |
| Spark_Torch_MNIST: true | |
| build_timeout: 30 | |
| - image: test-cpu-gloo-py3_8-tf2_10_1-keras2_10_0-torch1_12_1-mxnet1_7_0_p2-pyspark3_4_0 | |
| Elastic_Tests_1: true | |
| Gloo_Cluster_PyTests: true | |
| Gloo_MXNet_MNIST_horovodrun: true | |
| Gloo_Parallel_PyTests: true | |
| Gloo_PyTorch_MNIST_api: true | |
| Gloo_PyTorch_MNIST_horovodrun: true | |
| Gloo_Single_PyTests: true | |
| Gloo_TensorFlow_2_0_Keras_MNIST_api: true | |
| Gloo_TensorFlow_2_0_Keras_MNIST_horovodrun: true | |
| Gloo_TensorFlow_2_0_MNIST_Data_Service: true | |
| Gloo_TensorFlow_2_0_MNIST_Elastic_api: true | |
| Gloo_TensorFlow_2_0_MNIST_Elastic_horovodrun: true | |
| Gloo_TensorFlow_2_0_MNIST_api: true | |
| Gloo_TensorFlow_2_0_MNIST_horovodrun: true | |
| Single_MXNet_MNIST: true | |
| Single_PyTorch_MNIST: true | |
| Spark_Lightning_MNIST: true | |
| Spark_PyTests: true | |
| Spark_TensorFlow_2_0_MNIST_Data_Service: true | |
| Spark_Torch_MNIST: true | |
| build_timeout: 30 | |
| - image: test-cpu-gloo-py3_8-tf2_11_1-keras2_11_0-torch1_13_1-mxnet1_8_0_p0-pyspark3_4_0 | |
| Elastic_Tests_1: true | |
| Gloo_Cluster_PyTests: true | |
| Gloo_MXNet_MNIST_horovodrun: true | |
| Gloo_Parallel_PyTests: true | |
| Gloo_PyTorch_MNIST_api: true | |
| Gloo_PyTorch_MNIST_horovodrun: true | |
| Gloo_Single_PyTests: true | |
| Gloo_TensorFlow_2_0_Keras_MNIST_api: true | |
| Gloo_TensorFlow_2_0_Keras_MNIST_horovodrun: true | |
| Gloo_TensorFlow_2_0_MNIST_Elastic_api: true | |
| Gloo_TensorFlow_2_0_MNIST_Elastic_horovodrun: true | |
| Gloo_TensorFlow_2_0_MNIST_api: true | |
| Gloo_TensorFlow_2_0_MNIST_horovodrun: true | |
| Single_MXNet_MNIST: true | |
| Single_PyTorch_MNIST: true | |
| Spark_Lightning_MNIST: true | |
| Spark_PyTests: true | |
| Spark_TensorFlow_2_0_MNIST_Data_Service: true | |
| Spark_Torch_MNIST: true | |
| build_timeout: 30 | |
| - image: test-cpu-gloo-py3_8-tf2_12_0-keras2_12_0-torch2_0_0-mxnet1_9_1-pyspark3_3_2 | |
| Elastic_Spark_TensorFlow_Tests_1: true | |
| Elastic_Spark_Torch_Tests: true | |
| Elastic_Tests_1: true | |
| Gloo_Cluster_PyTests: true | |
| Gloo_MXNet_MNIST_horovodrun: true | |
| Gloo_Parallel_PyTests: true | |
| Gloo_PyTorch_MNIST_api: true | |
| Gloo_PyTorch_MNIST_horovodrun: true | |
| Gloo_Single_PyTests: true | |
| Gloo_TensorFlow_2_0_Keras_MNIST_api: true | |
| Gloo_TensorFlow_2_0_Keras_MNIST_horovodrun: true | |
| Gloo_TensorFlow_2_0_MNIST_Elastic_api: true | |
| Gloo_TensorFlow_2_0_MNIST_Elastic_horovodrun: true | |
| Gloo_TensorFlow_2_0_MNIST_api: true | |
| Gloo_TensorFlow_2_0_MNIST_horovodrun: true | |
| Single_MXNet_MNIST: true | |
| Single_PyTorch_MNIST: true | |
| Spark_Lightning_MNIST: true | |
| Spark_PyTests: true | |
| Spark_TensorFlow_2_0_MNIST_Data_Service: true | |
| Spark_Torch_MNIST: true | |
| build_timeout: 30 | |
| - image: test-cpu-gloo-py3_8-tf2_12_0-keras2_12_0-torch2_0_0-mxnet1_9_1-pyspark3_4_0 | |
| Elastic_Spark_TensorFlow_Tests_1: true | |
| Elastic_Spark_Torch_Tests: true | |
| Elastic_Tests_1: true | |
| Gloo_Cluster_PyTests: true | |
| Gloo_MXNet_MNIST_horovodrun: true | |
| Gloo_Parallel_PyTests: true | |
| Gloo_PyTorch_MNIST_api: true | |
| Gloo_PyTorch_MNIST_horovodrun: true | |
| Gloo_Single_PyTests: true | |
| Gloo_TensorFlow_2_0_Keras_MNIST_api: true | |
| Gloo_TensorFlow_2_0_Keras_MNIST_horovodrun: true | |
| Gloo_TensorFlow_2_0_MNIST_Elastic_api: true | |
| Gloo_TensorFlow_2_0_MNIST_Elastic_horovodrun: true | |
| Gloo_TensorFlow_2_0_MNIST_api: true | |
| Gloo_TensorFlow_2_0_MNIST_horovodrun: true | |
| Single_MXNet_MNIST: true | |
| Single_PyTorch_MNIST: true | |
| Spark_Lightning_MNIST: true | |
| Spark_PyTests: true | |
| Spark_TensorFlow_2_0_MNIST_Data_Service: true | |
| Spark_Torch_MNIST: true | |
| build_timeout: 30 | |
| - image: test-cpu-mpich-py3_8-tf2_12_0-keras2_12_0-torch2_0_0-mxnet1_9_1-pyspark3_4_0 | |
| MPI_Cluster_PyTests: true | |
| MPI_MXNet_MNIST_horovodrun: true | |
| MPI_Parallel_PyTests: true | |
| MPI_PyTorch_MNIST_api: true | |
| MPI_PyTorch_MNIST_horovodrun: true | |
| MPI_Single_PyTests: true | |
| MPI_TensorFlow_2_0_Keras_MNIST_api: true | |
| MPI_TensorFlow_2_0_Keras_MNIST_horovodrun: true | |
| MPI_TensorFlow_2_0_MNIST_api: true | |
| MPI_TensorFlow_2_0_MNIST_horovodrun: true | |
| Single_MXNet_MNIST: true | |
| Single_PyTorch_MNIST: true | |
| build_timeout: 30 | |
| - image: test-cpu-openmpi-gloo-py3_8-tf2_12_0-keras2_12_0-torch2_0_0-mxnet1_9_1-pyspark3_4_0 | |
| Elastic_Tests_1: true | |
| Gloo_Cluster_PyTests: true | |
| Gloo_MXNet_MNIST_horovodrun: true | |
| Gloo_Parallel_PyTests: true | |
| Gloo_PyTorch_MNIST_api: true | |
| Gloo_PyTorch_MNIST_horovodrun: true | |
| Gloo_Single_PyTests: true | |
| Gloo_TensorFlow_2_0_Keras_MNIST_api: true | |
| Gloo_TensorFlow_2_0_Keras_MNIST_horovodrun: true | |
| Gloo_TensorFlow_2_0_MNIST_Elastic_api: true | |
| Gloo_TensorFlow_2_0_MNIST_Elastic_horovodrun: true | |
| Gloo_TensorFlow_2_0_MNIST_api: true | |
| Gloo_TensorFlow_2_0_MNIST_horovodrun: true | |
| MPI_Cluster_PyTests: true | |
| MPI_MXNet_MNIST_horovodrun: true | |
| MPI_Parallel_PyTests: true | |
| MPI_PyTorch_MNIST_api: true | |
| MPI_PyTorch_MNIST_horovodrun: true | |
| MPI_Single_PyTests: true | |
| MPI_TensorFlow_2_0_Keras_MNIST_api: true | |
| MPI_TensorFlow_2_0_Keras_MNIST_horovodrun: true | |
| MPI_TensorFlow_2_0_MNIST_api: true | |
| MPI_TensorFlow_2_0_MNIST_horovodrun: true | |
| Run_PyTests_test_interactiverun: true | |
| Single_MXNet_MNIST: true | |
| Single_PyTorch_MNIST: true | |
| Spark_Lightning_MNIST: true | |
| Spark_PyTests: true | |
| Spark_TensorFlow_2_0_MNIST_Data_Service: true | |
| Spark_Torch_MNIST: true | |
| build_timeout: 30 | |
| - image: test-cpu-openmpi-py3_8-tf2_12_0-keras2_12_0-torch2_0_0-mxnet1_9_1-pyspark3_4_0 | |
| MPI_Cluster_PyTests: true | |
| MPI_MXNet_MNIST_horovodrun: true | |
| MPI_Parallel_PyTests: true | |
| MPI_PyTorch_MNIST_api: true | |
| MPI_PyTorch_MNIST_horovodrun: true | |
| MPI_Single_PyTests: true | |
| MPI_TensorFlow_2_0_Keras_MNIST_api: true | |
| MPI_TensorFlow_2_0_Keras_MNIST_horovodrun: true | |
| MPI_TensorFlow_2_0_MNIST_api: true | |
| MPI_TensorFlow_2_0_MNIST_horovodrun: true | |
| Run_PyTests_test_interactiverun: true | |
| Single_MXNet_MNIST: true | |
| Single_PyTorch_MNIST: true | |
| Spark_Lightning_MNIST: true | |
| Spark_PyTests: true | |
| Spark_TensorFlow_2_0_MNIST_Data_Service: true | |
| Spark_Torch_MNIST: true | |
| build_timeout: 30 | |
| - image: test-gpu-gloo-py3_7-tf1_15_5-keras2_2_4-torch1_8_1-mxnet1_5_1_p0-pyspark3_4_0 | |
| build_timeout: 40 | |
| - image: test-gpu-gloo-py3_8-tf2_10_1-keras2_10_0-torch1_12_1-mxnet1_7_0_p1-pyspark3_4_0 | |
| build_timeout: 40 | |
| - image: test-gpu-gloo-py3_8-tf2_11_1-keras2_11_0-torch1_13_1-mxnet1_8_0_p0-pyspark3_4_0 | |
| build_timeout: 40 | |
| - image: test-gpu-openmpi-gloo-py3_8-tf2_12_0-keras2_12_0-torch2_0_0-mxnet1_9_1-pyspark3_4_0 | |
| build_timeout: 40 | |
| - image: test-mixed-openmpi-gloo-py3_8-tf2_12_0-keras2_12_0-torch2_0_0-mxnet1_9_1-pyspark3_4_0 | |
| build_timeout: 40 | |
| steps: | |
| - name: Clean up disk space | |
| # deleting these paths frees 38 GB disk space: | |
| # sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc | |
| # but this sometimes takes 3-4 minutes | |
| # so we delete only some sub-paths which are known to be quick (10s) and 20 GB | |
| run: | | |
| echo ::group::Disk space before clean up | |
| df -h | |
| echo ::endgroup:: | |
| for dir in /usr/share/dotnet/sdk/\*/nuGetPackagesArchive.lzma \ | |
| /usr/share/dotnet/shared \ | |
| /usr/local/lib/android/sdk/ndk \ | |
| /usr/local/lib/android/sdk/build-tools \ | |
| /opt/ghc | |
| do | |
| echo ::group::Deleting "$dir" | |
| sudo du -hsc $dir | tail -n1 || true | |
| sudo rm -rf $dir | |
| echo ::endgroup:: | |
| done | |
| echo ::group::Disk space after clean up | |
| df -h | |
| echo ::endgroup:: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.8 | |
| - name: Setup docker-compose | |
| run: pip install docker-compose | |
| - name: Build | |
| id: build | |
| run: | | |
| .github/timeout-and-retry.sh ${{ matrix.build_timeout }}m 3 10 docker-compose -f docker-compose.test.yml build ${{ matrix.image }} | |
| env: | |
| COMPOSE_DOCKER_CLI_BUILD: 1 | |
| DOCKER_BUILDKIT: 1 | |
| - name: "Elastic Spark TensorFlow Tests 1 [attempt 1 of 3]" | |
| id: Elastic_Spark_TensorFlow_Tests_1_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Elastic_Spark_TensorFlow_Tests_1 && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Elastic_Spark_TensorFlow_Tests_1_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Elastic_Spark_TensorFlow_Tests_1_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 30m bash -c "cd /horovod/test/integration && /spark_env.sh HOROVOD_LOG_LEVEL=DEBUG pytest --forked -v --log-cli-level 10 --log-cli-format '[%(asctime)-15s %(levelname)s %(filename)s:%(lineno)d %(funcName)s()] %(message)s' --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.gloo.elastic.spark.tf.xml test_elastic_spark_tensorflow2.py" | |
| shell: bash | |
| - name: "Elastic Spark TensorFlow Tests 1 [attempt 2 of 3]" | |
| id: Elastic_Spark_TensorFlow_Tests_1_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Elastic_Spark_TensorFlow_Tests_1 && steps.Elastic_Spark_TensorFlow_Tests_1_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Elastic_Spark_TensorFlow_Tests_1_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Elastic_Spark_TensorFlow_Tests_1_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 30m bash -c "cd /horovod/test/integration && /spark_env.sh HOROVOD_LOG_LEVEL=DEBUG pytest --forked -v --log-cli-level 10 --log-cli-format '[%(asctime)-15s %(levelname)s %(filename)s:%(lineno)d %(funcName)s()] %(message)s' --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.gloo.elastic.spark.tf.xml test_elastic_spark_tensorflow2.py" | |
| shell: bash | |
| - name: "Elastic Spark TensorFlow Tests 1 [attempt 3 of 3]" | |
| id: Elastic_Spark_TensorFlow_Tests_1_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Elastic_Spark_TensorFlow_Tests_1 && steps.Elastic_Spark_TensorFlow_Tests_1_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Elastic_Spark_TensorFlow_Tests_1_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Elastic_Spark_TensorFlow_Tests_1_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 30m bash -c "cd /horovod/test/integration && /spark_env.sh HOROVOD_LOG_LEVEL=DEBUG pytest --forked -v --log-cli-level 10 --log-cli-format '[%(asctime)-15s %(levelname)s %(filename)s:%(lineno)d %(funcName)s()] %(message)s' --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.gloo.elastic.spark.tf.xml test_elastic_spark_tensorflow2.py" | |
| shell: bash | |
| - name: "Elastic Spark TensorFlow Tests 2 [attempt 1 of 3]" | |
| id: Elastic_Spark_TensorFlow_Tests_2_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Elastic_Spark_TensorFlow_Tests_2 && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Elastic_Spark_TensorFlow_Tests_2_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Elastic_Spark_TensorFlow_Tests_2_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 30m bash -c "cd /horovod/test/integration && /spark_env.sh HOROVOD_LOG_LEVEL=DEBUG pytest --forked -v --log-cli-level 10 --log-cli-format '[%(asctime)-15s %(levelname)s %(filename)s:%(lineno)d %(funcName)s()] %(message)s' --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.gloo.elastic.spark.tf.xml test_elastic_spark_tensorflow.py" | |
| shell: bash | |
| - name: "Elastic Spark TensorFlow Tests 2 [attempt 2 of 3]" | |
| id: Elastic_Spark_TensorFlow_Tests_2_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Elastic_Spark_TensorFlow_Tests_2 && steps.Elastic_Spark_TensorFlow_Tests_2_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Elastic_Spark_TensorFlow_Tests_2_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Elastic_Spark_TensorFlow_Tests_2_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 30m bash -c "cd /horovod/test/integration && /spark_env.sh HOROVOD_LOG_LEVEL=DEBUG pytest --forked -v --log-cli-level 10 --log-cli-format '[%(asctime)-15s %(levelname)s %(filename)s:%(lineno)d %(funcName)s()] %(message)s' --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.gloo.elastic.spark.tf.xml test_elastic_spark_tensorflow.py" | |
| shell: bash | |
| - name: "Elastic Spark TensorFlow Tests 2 [attempt 3 of 3]" | |
| id: Elastic_Spark_TensorFlow_Tests_2_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Elastic_Spark_TensorFlow_Tests_2 && steps.Elastic_Spark_TensorFlow_Tests_2_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Elastic_Spark_TensorFlow_Tests_2_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Elastic_Spark_TensorFlow_Tests_2_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 30m bash -c "cd /horovod/test/integration && /spark_env.sh HOROVOD_LOG_LEVEL=DEBUG pytest --forked -v --log-cli-level 10 --log-cli-format '[%(asctime)-15s %(levelname)s %(filename)s:%(lineno)d %(funcName)s()] %(message)s' --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.gloo.elastic.spark.tf.xml test_elastic_spark_tensorflow.py" | |
| shell: bash | |
| - name: "Elastic Spark Torch Tests [attempt 1 of 3]" | |
| id: Elastic_Spark_Torch_Tests_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Elastic_Spark_Torch_Tests && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Elastic_Spark_Torch_Tests_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Elastic_Spark_Torch_Tests_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 30m bash -c "cd /horovod/test/integration && /spark_env.sh HOROVOD_LOG_LEVEL=DEBUG pytest --forked -v --log-cli-level 10 --log-cli-format '[%(asctime)-15s %(levelname)s %(filename)s:%(lineno)d %(funcName)s()] %(message)s' --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.gloo.elastic.spark.torch.xml test_elastic_spark_torch.py" | |
| shell: bash | |
| - name: "Elastic Spark Torch Tests [attempt 2 of 3]" | |
| id: Elastic_Spark_Torch_Tests_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Elastic_Spark_Torch_Tests && steps.Elastic_Spark_Torch_Tests_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Elastic_Spark_Torch_Tests_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Elastic_Spark_Torch_Tests_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 30m bash -c "cd /horovod/test/integration && /spark_env.sh HOROVOD_LOG_LEVEL=DEBUG pytest --forked -v --log-cli-level 10 --log-cli-format '[%(asctime)-15s %(levelname)s %(filename)s:%(lineno)d %(funcName)s()] %(message)s' --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.gloo.elastic.spark.torch.xml test_elastic_spark_torch.py" | |
| shell: bash | |
| - name: "Elastic Spark Torch Tests [attempt 3 of 3]" | |
| id: Elastic_Spark_Torch_Tests_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Elastic_Spark_Torch_Tests && steps.Elastic_Spark_Torch_Tests_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Elastic_Spark_Torch_Tests_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Elastic_Spark_Torch_Tests_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 30m bash -c "cd /horovod/test/integration && /spark_env.sh HOROVOD_LOG_LEVEL=DEBUG pytest --forked -v --log-cli-level 10 --log-cli-format '[%(asctime)-15s %(levelname)s %(filename)s:%(lineno)d %(funcName)s()] %(message)s' --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.gloo.elastic.spark.torch.xml test_elastic_spark_torch.py" | |
| shell: bash | |
| - name: "Elastic Tests 1 [attempt 1 of 3]" | |
| id: Elastic_Tests_1_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Elastic_Tests_1 && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Elastic_Tests_1_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Elastic_Tests_1_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c "cd /horovod/test/integration && HOROVOD_LOG_LEVEL=DEBUG pytest --forked -v --log-cli-level 10 --log-cli-format '[%(asctime)-15s %(levelname)s %(filename)s:%(lineno)d %(funcName)s()] %(message)s' --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.gloo.elastic.xml test_elastic_torch.py test_elastic_tensorflow2.py" | |
| shell: bash | |
| - name: "Elastic Tests 1 [attempt 2 of 3]" | |
| id: Elastic_Tests_1_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Elastic_Tests_1 && steps.Elastic_Tests_1_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Elastic_Tests_1_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Elastic_Tests_1_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c "cd /horovod/test/integration && HOROVOD_LOG_LEVEL=DEBUG pytest --forked -v --log-cli-level 10 --log-cli-format '[%(asctime)-15s %(levelname)s %(filename)s:%(lineno)d %(funcName)s()] %(message)s' --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.gloo.elastic.xml test_elastic_torch.py test_elastic_tensorflow2.py" | |
| shell: bash | |
| - name: "Elastic Tests 1 [attempt 3 of 3]" | |
| id: Elastic_Tests_1_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Elastic_Tests_1 && steps.Elastic_Tests_1_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Elastic_Tests_1_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Elastic_Tests_1_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c "cd /horovod/test/integration && HOROVOD_LOG_LEVEL=DEBUG pytest --forked -v --log-cli-level 10 --log-cli-format '[%(asctime)-15s %(levelname)s %(filename)s:%(lineno)d %(funcName)s()] %(message)s' --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.gloo.elastic.xml test_elastic_torch.py test_elastic_tensorflow2.py" | |
| shell: bash | |
| - name: "Elastic Tests 2 [attempt 1 of 3]" | |
| id: Elastic_Tests_2_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Elastic_Tests_2 && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Elastic_Tests_2_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Elastic_Tests_2_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c "cd /horovod/test/integration && HOROVOD_LOG_LEVEL=DEBUG pytest --forked -v --log-cli-level 10 --log-cli-format '[%(asctime)-15s %(levelname)s %(filename)s:%(lineno)d %(funcName)s()] %(message)s' --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.gloo.elastic.xml test_elastic_torch.py test_elastic_tensorflow.py test_elastic_tensorflow_keras.py" | |
| shell: bash | |
| - name: "Elastic Tests 2 [attempt 2 of 3]" | |
| id: Elastic_Tests_2_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Elastic_Tests_2 && steps.Elastic_Tests_2_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Elastic_Tests_2_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Elastic_Tests_2_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c "cd /horovod/test/integration && HOROVOD_LOG_LEVEL=DEBUG pytest --forked -v --log-cli-level 10 --log-cli-format '[%(asctime)-15s %(levelname)s %(filename)s:%(lineno)d %(funcName)s()] %(message)s' --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.gloo.elastic.xml test_elastic_torch.py test_elastic_tensorflow.py test_elastic_tensorflow_keras.py" | |
| shell: bash | |
| - name: "Elastic Tests 2 [attempt 3 of 3]" | |
| id: Elastic_Tests_2_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Elastic_Tests_2 && steps.Elastic_Tests_2_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Elastic_Tests_2_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Elastic_Tests_2_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c "cd /horovod/test/integration && HOROVOD_LOG_LEVEL=DEBUG pytest --forked -v --log-cli-level 10 --log-cli-format '[%(asctime)-15s %(levelname)s %(filename)s:%(lineno)d %(funcName)s()] %(message)s' --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.gloo.elastic.xml test_elastic_torch.py test_elastic_tensorflow.py test_elastic_tensorflow_keras.py" | |
| shell: bash | |
| - name: "Gloo Cluster PyTests [attempt 1 of 3]" | |
| id: Gloo_Cluster_PyTests_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_Cluster_PyTests && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_Cluster_PyTests_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_Cluster_PyTests_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " /etc/init.d/ssh start && cd /horovod/test/integration && pytest --forked -v --capture=fd --continue-on-collection-errors --junit-xml=/artifacts/junit.gloo.static.xml test_static_run.py" | |
| shell: bash | |
| - name: "Gloo Cluster PyTests [attempt 2 of 3]" | |
| id: Gloo_Cluster_PyTests_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_Cluster_PyTests && steps.Gloo_Cluster_PyTests_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_Cluster_PyTests_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_Cluster_PyTests_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " /etc/init.d/ssh start && cd /horovod/test/integration && pytest --forked -v --capture=fd --continue-on-collection-errors --junit-xml=/artifacts/junit.gloo.static.xml test_static_run.py" | |
| shell: bash | |
| - name: "Gloo Cluster PyTests [attempt 3 of 3]" | |
| id: Gloo_Cluster_PyTests_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_Cluster_PyTests && steps.Gloo_Cluster_PyTests_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_Cluster_PyTests_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_Cluster_PyTests_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " /etc/init.d/ssh start && cd /horovod/test/integration && pytest --forked -v --capture=fd --continue-on-collection-errors --junit-xml=/artifacts/junit.gloo.static.xml test_static_run.py" | |
| shell: bash | |
| - name: "Gloo Keras MNIST [attempt 1 of 3]" | |
| id: Gloo_Keras_MNIST_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_Keras_MNIST && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_Keras_MNIST_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_Keras_MNIST_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/keras/keras_mnist_advanced.py | |
| shell: bash | |
| - name: "Gloo Keras MNIST [attempt 2 of 3]" | |
| id: Gloo_Keras_MNIST_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_Keras_MNIST && steps.Gloo_Keras_MNIST_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_Keras_MNIST_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_Keras_MNIST_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/keras/keras_mnist_advanced.py | |
| shell: bash | |
| - name: "Gloo Keras MNIST [attempt 3 of 3]" | |
| id: Gloo_Keras_MNIST_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_Keras_MNIST && steps.Gloo_Keras_MNIST_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_Keras_MNIST_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_Keras_MNIST_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/keras/keras_mnist_advanced.py | |
| shell: bash | |
| - name: "Gloo MXNet2 MNIST api [attempt 1 of 3]" | |
| id: Gloo_MXNet2_MNIST_api_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_MXNet2_MNIST_api && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_MXNet2_MNIST_api_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_MXNet2_MNIST_api_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m python /horovod/examples/mxnet/mxnet2_mnist.py --num-proc 2 --hosts localhost:2 --communication gloo | |
| shell: bash | |
| - name: "Gloo MXNet2 MNIST api [attempt 2 of 3]" | |
| id: Gloo_MXNet2_MNIST_api_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_MXNet2_MNIST_api && steps.Gloo_MXNet2_MNIST_api_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_MXNet2_MNIST_api_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_MXNet2_MNIST_api_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m python /horovod/examples/mxnet/mxnet2_mnist.py --num-proc 2 --hosts localhost:2 --communication gloo | |
| shell: bash | |
| - name: "Gloo MXNet2 MNIST api [attempt 3 of 3]" | |
| id: Gloo_MXNet2_MNIST_api_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_MXNet2_MNIST_api && steps.Gloo_MXNet2_MNIST_api_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_MXNet2_MNIST_api_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_MXNet2_MNIST_api_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m python /horovod/examples/mxnet/mxnet2_mnist.py --num-proc 2 --hosts localhost:2 --communication gloo | |
| shell: bash | |
| - name: "Gloo MXNet2 MNIST horovodrun [attempt 1 of 3]" | |
| id: Gloo_MXNet2_MNIST_horovodrun_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_MXNet2_MNIST_horovodrun && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_MXNet2_MNIST_horovodrun_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_MXNet2_MNIST_horovodrun_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/mxnet/mxnet2_mnist.py | |
| shell: bash | |
| - name: "Gloo MXNet2 MNIST horovodrun [attempt 2 of 3]" | |
| id: Gloo_MXNet2_MNIST_horovodrun_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_MXNet2_MNIST_horovodrun && steps.Gloo_MXNet2_MNIST_horovodrun_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_MXNet2_MNIST_horovodrun_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_MXNet2_MNIST_horovodrun_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/mxnet/mxnet2_mnist.py | |
| shell: bash | |
| - name: "Gloo MXNet2 MNIST horovodrun [attempt 3 of 3]" | |
| id: Gloo_MXNet2_MNIST_horovodrun_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_MXNet2_MNIST_horovodrun && steps.Gloo_MXNet2_MNIST_horovodrun_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_MXNet2_MNIST_horovodrun_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_MXNet2_MNIST_horovodrun_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/mxnet/mxnet2_mnist.py | |
| shell: bash | |
| - name: "Gloo MXNet MNIST horovodrun [attempt 1 of 3]" | |
| id: Gloo_MXNet_MNIST_horovodrun_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_MXNet_MNIST_horovodrun && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_MXNet_MNIST_horovodrun_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_MXNet_MNIST_horovodrun_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/mxnet/mxnet_mnist.py | |
| shell: bash | |
| - name: "Gloo MXNet MNIST horovodrun [attempt 2 of 3]" | |
| id: Gloo_MXNet_MNIST_horovodrun_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_MXNet_MNIST_horovodrun && steps.Gloo_MXNet_MNIST_horovodrun_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_MXNet_MNIST_horovodrun_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_MXNet_MNIST_horovodrun_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/mxnet/mxnet_mnist.py | |
| shell: bash | |
| - name: "Gloo MXNet MNIST horovodrun [attempt 3 of 3]" | |
| id: Gloo_MXNet_MNIST_horovodrun_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_MXNet_MNIST_horovodrun && steps.Gloo_MXNet_MNIST_horovodrun_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_MXNet_MNIST_horovodrun_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_MXNet_MNIST_horovodrun_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/mxnet/mxnet_mnist.py | |
| shell: bash | |
| - name: "Gloo Parallel PyTests [attempt 1 of 3]" | |
| id: Gloo_Parallel_PyTests_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_Parallel_PyTests && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_Parallel_PyTests_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_Parallel_PyTests_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c " cd /horovod/test/parallel && (ls -1 test_*.py | xargs -n 1 horovodrun -np 2 -H localhost:2 --gloo /bin/bash /pytest.sh gloo)" | |
| shell: bash | |
| - name: "Gloo Parallel PyTests [attempt 2 of 3]" | |
| id: Gloo_Parallel_PyTests_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_Parallel_PyTests && steps.Gloo_Parallel_PyTests_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_Parallel_PyTests_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_Parallel_PyTests_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c " cd /horovod/test/parallel && (ls -1 test_*.py | xargs -n 1 horovodrun -np 2 -H localhost:2 --gloo /bin/bash /pytest.sh gloo)" | |
| shell: bash | |
| - name: "Gloo Parallel PyTests [attempt 3 of 3]" | |
| id: Gloo_Parallel_PyTests_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_Parallel_PyTests && steps.Gloo_Parallel_PyTests_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_Parallel_PyTests_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_Parallel_PyTests_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c " cd /horovod/test/parallel && (ls -1 test_*.py | xargs -n 1 horovodrun -np 2 -H localhost:2 --gloo /bin/bash /pytest.sh gloo)" | |
| shell: bash | |
| - name: "Gloo PyTorch MNIST api [attempt 1 of 3]" | |
| id: Gloo_PyTorch_MNIST_api_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_PyTorch_MNIST_api && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_PyTorch_MNIST_api_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_PyTorch_MNIST_api_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets --num-proc 2 --hosts localhost:2 --communication gloo | |
| shell: bash | |
| - name: "Gloo PyTorch MNIST api [attempt 2 of 3]" | |
| id: Gloo_PyTorch_MNIST_api_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_PyTorch_MNIST_api && steps.Gloo_PyTorch_MNIST_api_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_PyTorch_MNIST_api_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_PyTorch_MNIST_api_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets --num-proc 2 --hosts localhost:2 --communication gloo | |
| shell: bash | |
| - name: "Gloo PyTorch MNIST api [attempt 3 of 3]" | |
| id: Gloo_PyTorch_MNIST_api_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_PyTorch_MNIST_api && steps.Gloo_PyTorch_MNIST_api_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_PyTorch_MNIST_api_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_PyTorch_MNIST_api_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets --num-proc 2 --hosts localhost:2 --communication gloo | |
| shell: bash | |
| - name: "Gloo PyTorch MNIST horovodrun [attempt 1 of 3]" | |
| id: Gloo_PyTorch_MNIST_horovodrun_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_PyTorch_MNIST_horovodrun && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_PyTorch_MNIST_horovodrun_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_PyTorch_MNIST_horovodrun_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets | |
| shell: bash | |
| - name: "Gloo PyTorch MNIST horovodrun [attempt 2 of 3]" | |
| id: Gloo_PyTorch_MNIST_horovodrun_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_PyTorch_MNIST_horovodrun && steps.Gloo_PyTorch_MNIST_horovodrun_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_PyTorch_MNIST_horovodrun_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_PyTorch_MNIST_horovodrun_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets | |
| shell: bash | |
| - name: "Gloo PyTorch MNIST horovodrun [attempt 3 of 3]" | |
| id: Gloo_PyTorch_MNIST_horovodrun_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_PyTorch_MNIST_horovodrun && steps.Gloo_PyTorch_MNIST_horovodrun_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_PyTorch_MNIST_horovodrun_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_PyTorch_MNIST_horovodrun_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets | |
| shell: bash | |
| - name: "Gloo Single PyTests [attempt 1 of 3]" | |
| id: Gloo_Single_PyTests_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_Single_PyTests && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_Single_PyTests_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_Single_PyTests_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c " cd /horovod/test/single && (ls -1 test_*.py | xargs -n 1 /bin/bash /pytest_standalone.sh gloo)" | |
| shell: bash | |
| - name: "Gloo Single PyTests [attempt 2 of 3]" | |
| id: Gloo_Single_PyTests_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_Single_PyTests && steps.Gloo_Single_PyTests_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_Single_PyTests_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_Single_PyTests_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c " cd /horovod/test/single && (ls -1 test_*.py | xargs -n 1 /bin/bash /pytest_standalone.sh gloo)" | |
| shell: bash | |
| - name: "Gloo Single PyTests [attempt 3 of 3]" | |
| id: Gloo_Single_PyTests_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_Single_PyTests && steps.Gloo_Single_PyTests_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_Single_PyTests_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_Single_PyTests_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c " cd /horovod/test/single && (ls -1 test_*.py | xargs -n 1 /bin/bash /pytest_standalone.sh gloo)" | |
| shell: bash | |
| - name: "Gloo TensorFlow 2.0 Keras MNIST api [attempt 1 of 3]" | |
| id: Gloo_TensorFlow_2_0_Keras_MNIST_api_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_2_0_Keras_MNIST_api && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_Keras_MNIST_api_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_Keras_MNIST_api_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py 2 localhost:2 gloo | |
| shell: bash | |
| - name: "Gloo TensorFlow 2.0 Keras MNIST api [attempt 2 of 3]" | |
| id: Gloo_TensorFlow_2_0_Keras_MNIST_api_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_2_0_Keras_MNIST_api && steps.Gloo_TensorFlow_2_0_Keras_MNIST_api_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_Keras_MNIST_api_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_Keras_MNIST_api_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py 2 localhost:2 gloo | |
| shell: bash | |
| - name: "Gloo TensorFlow 2.0 Keras MNIST api [attempt 3 of 3]" | |
| id: Gloo_TensorFlow_2_0_Keras_MNIST_api_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_2_0_Keras_MNIST_api && steps.Gloo_TensorFlow_2_0_Keras_MNIST_api_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_Keras_MNIST_api_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_Keras_MNIST_api_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py 2 localhost:2 gloo | |
| shell: bash | |
| - name: "Gloo TensorFlow 2.0 Keras MNIST horovodrun [attempt 1 of 3]" | |
| id: Gloo_TensorFlow_2_0_Keras_MNIST_horovodrun_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_2_0_Keras_MNIST_horovodrun && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_Keras_MNIST_horovodrun_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_Keras_MNIST_horovodrun_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py | |
| shell: bash | |
| - name: "Gloo TensorFlow 2.0 Keras MNIST horovodrun [attempt 2 of 3]" | |
| id: Gloo_TensorFlow_2_0_Keras_MNIST_horovodrun_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_2_0_Keras_MNIST_horovodrun && steps.Gloo_TensorFlow_2_0_Keras_MNIST_horovodrun_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_Keras_MNIST_horovodrun_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_Keras_MNIST_horovodrun_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py | |
| shell: bash | |
| - name: "Gloo TensorFlow 2.0 Keras MNIST horovodrun [attempt 3 of 3]" | |
| id: Gloo_TensorFlow_2_0_Keras_MNIST_horovodrun_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_2_0_Keras_MNIST_horovodrun && steps.Gloo_TensorFlow_2_0_Keras_MNIST_horovodrun_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_Keras_MNIST_horovodrun_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_Keras_MNIST_horovodrun_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py | |
| shell: bash | |
| - name: "Gloo TensorFlow 2.0 MNIST Data Service [attempt 1 of 3]" | |
| id: Gloo_TensorFlow_2_0_MNIST_Data_Service_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_2_0_MNIST_Data_Service && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_Data_Service_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_Data_Service_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "horovodrun -np 2 python -m horovod.tensorflow.data.compute_worker /tmp/compute.json & horovodrun -np 2 --gloo python /horovod/examples/tensorflow2/tensorflow2_mnist_data_service.py /tmp/compute.json" | |
| shell: bash | |
| - name: "Gloo TensorFlow 2.0 MNIST Data Service [attempt 2 of 3]" | |
| id: Gloo_TensorFlow_2_0_MNIST_Data_Service_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_2_0_MNIST_Data_Service && steps.Gloo_TensorFlow_2_0_MNIST_Data_Service_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_Data_Service_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_Data_Service_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "horovodrun -np 2 python -m horovod.tensorflow.data.compute_worker /tmp/compute.json & horovodrun -np 2 --gloo python /horovod/examples/tensorflow2/tensorflow2_mnist_data_service.py /tmp/compute.json" | |
| shell: bash | |
| - name: "Gloo TensorFlow 2.0 MNIST Data Service [attempt 3 of 3]" | |
| id: Gloo_TensorFlow_2_0_MNIST_Data_Service_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_2_0_MNIST_Data_Service && steps.Gloo_TensorFlow_2_0_MNIST_Data_Service_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_Data_Service_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_Data_Service_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "horovodrun -np 2 python -m horovod.tensorflow.data.compute_worker /tmp/compute.json & horovodrun -np 2 --gloo python /horovod/examples/tensorflow2/tensorflow2_mnist_data_service.py /tmp/compute.json" | |
| shell: bash | |
| - name: "Gloo TensorFlow 2.0 MNIST Elastic api [attempt 1 of 3]" | |
| id: Gloo_TensorFlow_2_0_MNIST_Elastic_api_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_2_0_MNIST_Elastic_api && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_Elastic_api_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_Elastic_api_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m python /horovod/examples/elastic/tensorflow2/tensorflow2_mnist_elastic.py 2 2 2 localhost:2,127.0.0.1:2 | |
| shell: bash | |
| - name: "Gloo TensorFlow 2.0 MNIST Elastic api [attempt 2 of 3]" | |
| id: Gloo_TensorFlow_2_0_MNIST_Elastic_api_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_2_0_MNIST_Elastic_api && steps.Gloo_TensorFlow_2_0_MNIST_Elastic_api_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_Elastic_api_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_Elastic_api_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m python /horovod/examples/elastic/tensorflow2/tensorflow2_mnist_elastic.py 2 2 2 localhost:2,127.0.0.1:2 | |
| shell: bash | |
| - name: "Gloo TensorFlow 2.0 MNIST Elastic api [attempt 3 of 3]" | |
| id: Gloo_TensorFlow_2_0_MNIST_Elastic_api_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_2_0_MNIST_Elastic_api && steps.Gloo_TensorFlow_2_0_MNIST_Elastic_api_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_Elastic_api_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_Elastic_api_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m python /horovod/examples/elastic/tensorflow2/tensorflow2_mnist_elastic.py 2 2 2 localhost:2,127.0.0.1:2 | |
| shell: bash | |
| - name: "Gloo TensorFlow 2.0 MNIST Elastic horovodrun [attempt 1 of 3]" | |
| id: Gloo_TensorFlow_2_0_MNIST_Elastic_horovodrun_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_2_0_MNIST_Elastic_horovodrun && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_Elastic_horovodrun_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_Elastic_horovodrun_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 --min-np 2 --max-np 2 -H localhost:2,127.0.0.1:2 --gloo python /horovod/examples/elastic/tensorflow2/tensorflow2_mnist_elastic.py | |
| shell: bash | |
| - name: "Gloo TensorFlow 2.0 MNIST Elastic horovodrun [attempt 2 of 3]" | |
| id: Gloo_TensorFlow_2_0_MNIST_Elastic_horovodrun_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_2_0_MNIST_Elastic_horovodrun && steps.Gloo_TensorFlow_2_0_MNIST_Elastic_horovodrun_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_Elastic_horovodrun_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_Elastic_horovodrun_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 --min-np 2 --max-np 2 -H localhost:2,127.0.0.1:2 --gloo python /horovod/examples/elastic/tensorflow2/tensorflow2_mnist_elastic.py | |
| shell: bash | |
| - name: "Gloo TensorFlow 2.0 MNIST Elastic horovodrun [attempt 3 of 3]" | |
| id: Gloo_TensorFlow_2_0_MNIST_Elastic_horovodrun_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_2_0_MNIST_Elastic_horovodrun && steps.Gloo_TensorFlow_2_0_MNIST_Elastic_horovodrun_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_Elastic_horovodrun_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_Elastic_horovodrun_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 --min-np 2 --max-np 2 -H localhost:2,127.0.0.1:2 --gloo python /horovod/examples/elastic/tensorflow2/tensorflow2_mnist_elastic.py | |
| shell: bash | |
| - name: "Gloo TensorFlow 2.0 MNIST api [attempt 1 of 3]" | |
| id: Gloo_TensorFlow_2_0_MNIST_api_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_2_0_MNIST_api && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_api_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_api_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m python /horovod/examples/tensorflow2/tensorflow2_mnist.py 2 localhost:2 gloo | |
| shell: bash | |
| - name: "Gloo TensorFlow 2.0 MNIST api [attempt 2 of 3]" | |
| id: Gloo_TensorFlow_2_0_MNIST_api_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_2_0_MNIST_api && steps.Gloo_TensorFlow_2_0_MNIST_api_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_api_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_api_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m python /horovod/examples/tensorflow2/tensorflow2_mnist.py 2 localhost:2 gloo | |
| shell: bash | |
| - name: "Gloo TensorFlow 2.0 MNIST api [attempt 3 of 3]" | |
| id: Gloo_TensorFlow_2_0_MNIST_api_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_2_0_MNIST_api && steps.Gloo_TensorFlow_2_0_MNIST_api_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_api_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_api_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m python /horovod/examples/tensorflow2/tensorflow2_mnist.py 2 localhost:2 gloo | |
| shell: bash | |
| - name: "Gloo TensorFlow 2.0 MNIST horovodrun [attempt 1 of 3]" | |
| id: Gloo_TensorFlow_2_0_MNIST_horovodrun_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_2_0_MNIST_horovodrun && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_horovodrun_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_horovodrun_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/tensorflow2/tensorflow2_mnist.py | |
| shell: bash | |
| - name: "Gloo TensorFlow 2.0 MNIST horovodrun [attempt 2 of 3]" | |
| id: Gloo_TensorFlow_2_0_MNIST_horovodrun_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_2_0_MNIST_horovodrun && steps.Gloo_TensorFlow_2_0_MNIST_horovodrun_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_horovodrun_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_horovodrun_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/tensorflow2/tensorflow2_mnist.py | |
| shell: bash | |
| - name: "Gloo TensorFlow 2.0 MNIST horovodrun [attempt 3 of 3]" | |
| id: Gloo_TensorFlow_2_0_MNIST_horovodrun_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_2_0_MNIST_horovodrun && steps.Gloo_TensorFlow_2_0_MNIST_horovodrun_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_horovodrun_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_horovodrun_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/tensorflow2/tensorflow2_mnist.py | |
| shell: bash | |
| - name: "Gloo TensorFlow MNIST [attempt 1 of 3]" | |
| id: Gloo_TensorFlow_MNIST_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_MNIST && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_MNIST_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_MNIST_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/tensorflow/tensorflow_mnist.py | |
| shell: bash | |
| - name: "Gloo TensorFlow MNIST [attempt 2 of 3]" | |
| id: Gloo_TensorFlow_MNIST_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_MNIST && steps.Gloo_TensorFlow_MNIST_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_MNIST_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_MNIST_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/tensorflow/tensorflow_mnist.py | |
| shell: bash | |
| - name: "Gloo TensorFlow MNIST [attempt 3 of 3]" | |
| id: Gloo_TensorFlow_MNIST_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_MNIST && steps.Gloo_TensorFlow_MNIST_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_MNIST_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_MNIST_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/tensorflow/tensorflow_mnist.py | |
| shell: bash | |
| - name: "MPI Cluster PyTests [attempt 1 of 3]" | |
| id: MPI_Cluster_PyTests_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Cluster_PyTests && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Cluster_PyTests_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Cluster_PyTests_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " /etc/init.d/ssh start && cd /horovod/test/integration && pytest --forked -v --capture=fd --continue-on-collection-errors --junit-xml=/artifacts/junit.mpi.static.xml test_static_run.py" | |
| shell: bash | |
| - name: "MPI Cluster PyTests [attempt 2 of 3]" | |
| id: MPI_Cluster_PyTests_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Cluster_PyTests && steps.MPI_Cluster_PyTests_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Cluster_PyTests_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Cluster_PyTests_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " /etc/init.d/ssh start && cd /horovod/test/integration && pytest --forked -v --capture=fd --continue-on-collection-errors --junit-xml=/artifacts/junit.mpi.static.xml test_static_run.py" | |
| shell: bash | |
| - name: "MPI Cluster PyTests [attempt 3 of 3]" | |
| id: MPI_Cluster_PyTests_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Cluster_PyTests && steps.MPI_Cluster_PyTests_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Cluster_PyTests_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Cluster_PyTests_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " /etc/init.d/ssh start && cd /horovod/test/integration && pytest --forked -v --capture=fd --continue-on-collection-errors --junit-xml=/artifacts/junit.mpi.static.xml test_static_run.py" | |
| shell: bash | |
| - name: "MPI Cluster PyTests [ONECCL MPI] [attempt 1 of 3]" | |
| id: MPI_Cluster_PyTests_ONECCL_MPI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Cluster_PyTests_ONECCL_MPI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Cluster_PyTests_ONECCL_MPI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Cluster_PyTests_ONECCL_MPI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && /etc/init.d/ssh start && cd /horovod/test/integration && pytest --forked -v --capture=fd --continue-on-collection-errors --junit-xml=/artifacts/junit.mpi.static.xml test_static_run.py" | |
| shell: bash | |
| - name: "MPI Cluster PyTests [ONECCL MPI] [attempt 2 of 3]" | |
| id: MPI_Cluster_PyTests_ONECCL_MPI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Cluster_PyTests_ONECCL_MPI && steps.MPI_Cluster_PyTests_ONECCL_MPI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Cluster_PyTests_ONECCL_MPI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Cluster_PyTests_ONECCL_MPI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && /etc/init.d/ssh start && cd /horovod/test/integration && pytest --forked -v --capture=fd --continue-on-collection-errors --junit-xml=/artifacts/junit.mpi.static.xml test_static_run.py" | |
| shell: bash | |
| - name: "MPI Cluster PyTests [ONECCL MPI] [attempt 3 of 3]" | |
| id: MPI_Cluster_PyTests_ONECCL_MPI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Cluster_PyTests_ONECCL_MPI && steps.MPI_Cluster_PyTests_ONECCL_MPI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Cluster_PyTests_ONECCL_MPI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Cluster_PyTests_ONECCL_MPI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && /etc/init.d/ssh start && cd /horovod/test/integration && pytest --forked -v --capture=fd --continue-on-collection-errors --junit-xml=/artifacts/junit.mpi.static.xml test_static_run.py" | |
| shell: bash | |
| - name: "MPI Cluster PyTests [ONECCL OFI] [attempt 1 of 3]" | |
| id: MPI_Cluster_PyTests_ONECCL_OFI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Cluster_PyTests_ONECCL_OFI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Cluster_PyTests_ONECCL_OFI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Cluster_PyTests_ONECCL_OFI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && /etc/init.d/ssh start && cd /horovod/test/integration && pytest --forked -v --capture=fd --continue-on-collection-errors --junit-xml=/artifacts/junit.mpi.static.xml test_static_run.py" | |
| shell: bash | |
| - name: "MPI Cluster PyTests [ONECCL OFI] [attempt 2 of 3]" | |
| id: MPI_Cluster_PyTests_ONECCL_OFI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Cluster_PyTests_ONECCL_OFI && steps.MPI_Cluster_PyTests_ONECCL_OFI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Cluster_PyTests_ONECCL_OFI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Cluster_PyTests_ONECCL_OFI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && /etc/init.d/ssh start && cd /horovod/test/integration && pytest --forked -v --capture=fd --continue-on-collection-errors --junit-xml=/artifacts/junit.mpi.static.xml test_static_run.py" | |
| shell: bash | |
| - name: "MPI Cluster PyTests [ONECCL OFI] [attempt 3 of 3]" | |
| id: MPI_Cluster_PyTests_ONECCL_OFI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Cluster_PyTests_ONECCL_OFI && steps.MPI_Cluster_PyTests_ONECCL_OFI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Cluster_PyTests_ONECCL_OFI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Cluster_PyTests_ONECCL_OFI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && /etc/init.d/ssh start && cd /horovod/test/integration && pytest --forked -v --capture=fd --continue-on-collection-errors --junit-xml=/artifacts/junit.mpi.static.xml test_static_run.py" | |
| shell: bash | |
| - name: "MPI MXNet2 MNIST api [attempt 1 of 3]" | |
| id: MPI_MXNet2_MNIST_api_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_MXNet2_MNIST_api && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_MXNet2_MNIST_api_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_MXNet2_MNIST_api_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/mxnet/mxnet2_mnist.py --num-proc 2 --hosts localhost:2 --communication mpi" | |
| shell: bash | |
| - name: "MPI MXNet2 MNIST api [attempt 2 of 3]" | |
| id: MPI_MXNet2_MNIST_api_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_MXNet2_MNIST_api && steps.MPI_MXNet2_MNIST_api_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_MXNet2_MNIST_api_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_MXNet2_MNIST_api_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/mxnet/mxnet2_mnist.py --num-proc 2 --hosts localhost:2 --communication mpi" | |
| shell: bash | |
| - name: "MPI MXNet2 MNIST api [attempt 3 of 3]" | |
| id: MPI_MXNet2_MNIST_api_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_MXNet2_MNIST_api && steps.MPI_MXNet2_MNIST_api_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_MXNet2_MNIST_api_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_MXNet2_MNIST_api_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/mxnet/mxnet2_mnist.py --num-proc 2 --hosts localhost:2 --communication mpi" | |
| shell: bash | |
| - name: "MPI MXNet2 MNIST horovodrun [attempt 1 of 3]" | |
| id: MPI_MXNet2_MNIST_horovodrun_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_MXNet2_MNIST_horovodrun && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_MXNet2_MNIST_horovodrun_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_MXNet2_MNIST_horovodrun_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " OMP_NUM_THREADS=1 \$(cat /mpirun_command) python /horovod/examples/mxnet/mxnet2_mnist.py" | |
| shell: bash | |
| - name: "MPI MXNet2 MNIST horovodrun [attempt 2 of 3]" | |
| id: MPI_MXNet2_MNIST_horovodrun_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_MXNet2_MNIST_horovodrun && steps.MPI_MXNet2_MNIST_horovodrun_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_MXNet2_MNIST_horovodrun_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_MXNet2_MNIST_horovodrun_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " OMP_NUM_THREADS=1 \$(cat /mpirun_command) python /horovod/examples/mxnet/mxnet2_mnist.py" | |
| shell: bash | |
| - name: "MPI MXNet2 MNIST horovodrun [attempt 3 of 3]" | |
| id: MPI_MXNet2_MNIST_horovodrun_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_MXNet2_MNIST_horovodrun && steps.MPI_MXNet2_MNIST_horovodrun_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_MXNet2_MNIST_horovodrun_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_MXNet2_MNIST_horovodrun_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " OMP_NUM_THREADS=1 \$(cat /mpirun_command) python /horovod/examples/mxnet/mxnet2_mnist.py" | |
| shell: bash | |
| - name: "MPI MXNet MNIST horovodrun [attempt 1 of 3]" | |
| id: MPI_MXNet_MNIST_horovodrun_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_MXNet_MNIST_horovodrun && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_MXNet_MNIST_horovodrun_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_MXNet_MNIST_horovodrun_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " OMP_NUM_THREADS=1 \$(cat /mpirun_command) python /horovod/examples/mxnet/mxnet_mnist.py" | |
| shell: bash | |
| - name: "MPI MXNet MNIST horovodrun [attempt 2 of 3]" | |
| id: MPI_MXNet_MNIST_horovodrun_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_MXNet_MNIST_horovodrun && steps.MPI_MXNet_MNIST_horovodrun_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_MXNet_MNIST_horovodrun_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_MXNet_MNIST_horovodrun_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " OMP_NUM_THREADS=1 \$(cat /mpirun_command) python /horovod/examples/mxnet/mxnet_mnist.py" | |
| shell: bash | |
| - name: "MPI MXNet MNIST horovodrun [attempt 3 of 3]" | |
| id: MPI_MXNet_MNIST_horovodrun_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_MXNet_MNIST_horovodrun && steps.MPI_MXNet_MNIST_horovodrun_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_MXNet_MNIST_horovodrun_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_MXNet_MNIST_horovodrun_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " OMP_NUM_THREADS=1 \$(cat /mpirun_command) python /horovod/examples/mxnet/mxnet_mnist.py" | |
| shell: bash | |
| - name: "MPI MXNet MNIST horovodrun [ONECCL MPI] [attempt 1 of 3]" | |
| id: MPI_MXNet_MNIST_horovodrun_ONECCL_MPI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_MXNet_MNIST_horovodrun_ONECCL_MPI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_MXNet_MNIST_horovodrun_ONECCL_MPI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_MXNet_MNIST_horovodrun_ONECCL_MPI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && OMP_NUM_THREADS=1 \$(cat /mpirun_command) python /horovod/examples/mxnet/mxnet_mnist.py" | |
| shell: bash | |
| - name: "MPI MXNet MNIST horovodrun [ONECCL MPI] [attempt 2 of 3]" | |
| id: MPI_MXNet_MNIST_horovodrun_ONECCL_MPI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_MXNet_MNIST_horovodrun_ONECCL_MPI && steps.MPI_MXNet_MNIST_horovodrun_ONECCL_MPI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_MXNet_MNIST_horovodrun_ONECCL_MPI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_MXNet_MNIST_horovodrun_ONECCL_MPI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && OMP_NUM_THREADS=1 \$(cat /mpirun_command) python /horovod/examples/mxnet/mxnet_mnist.py" | |
| shell: bash | |
| - name: "MPI MXNet MNIST horovodrun [ONECCL MPI] [attempt 3 of 3]" | |
| id: MPI_MXNet_MNIST_horovodrun_ONECCL_MPI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_MXNet_MNIST_horovodrun_ONECCL_MPI && steps.MPI_MXNet_MNIST_horovodrun_ONECCL_MPI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_MXNet_MNIST_horovodrun_ONECCL_MPI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_MXNet_MNIST_horovodrun_ONECCL_MPI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && OMP_NUM_THREADS=1 \$(cat /mpirun_command) python /horovod/examples/mxnet/mxnet_mnist.py" | |
| shell: bash | |
| - name: "MPI MXNet MNIST horovodrun [ONECCL OFI] [attempt 1 of 3]" | |
| id: MPI_MXNet_MNIST_horovodrun_ONECCL_OFI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_MXNet_MNIST_horovodrun_ONECCL_OFI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_MXNet_MNIST_horovodrun_ONECCL_OFI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_MXNet_MNIST_horovodrun_ONECCL_OFI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && OMP_NUM_THREADS=1 \$(cat /mpirun_command) python /horovod/examples/mxnet/mxnet_mnist.py" | |
| shell: bash | |
| - name: "MPI MXNet MNIST horovodrun [ONECCL OFI] [attempt 2 of 3]" | |
| id: MPI_MXNet_MNIST_horovodrun_ONECCL_OFI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_MXNet_MNIST_horovodrun_ONECCL_OFI && steps.MPI_MXNet_MNIST_horovodrun_ONECCL_OFI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_MXNet_MNIST_horovodrun_ONECCL_OFI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_MXNet_MNIST_horovodrun_ONECCL_OFI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && OMP_NUM_THREADS=1 \$(cat /mpirun_command) python /horovod/examples/mxnet/mxnet_mnist.py" | |
| shell: bash | |
| - name: "MPI MXNet MNIST horovodrun [ONECCL OFI] [attempt 3 of 3]" | |
| id: MPI_MXNet_MNIST_horovodrun_ONECCL_OFI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_MXNet_MNIST_horovodrun_ONECCL_OFI && steps.MPI_MXNet_MNIST_horovodrun_ONECCL_OFI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_MXNet_MNIST_horovodrun_ONECCL_OFI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_MXNet_MNIST_horovodrun_ONECCL_OFI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && OMP_NUM_THREADS=1 \$(cat /mpirun_command) python /horovod/examples/mxnet/mxnet_mnist.py" | |
| shell: bash | |
| - name: "MPI Parallel PyTests [attempt 1 of 3]" | |
| id: MPI_Parallel_PyTests_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Parallel_PyTests && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Parallel_PyTests_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Parallel_PyTests_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c " cd /horovod/test/parallel && (ls -1 test_*.py | xargs -n 1 \$(cat /mpirun_command) /bin/bash /pytest.sh mpi)" | |
| shell: bash | |
| - name: "MPI Parallel PyTests [attempt 2 of 3]" | |
| id: MPI_Parallel_PyTests_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Parallel_PyTests && steps.MPI_Parallel_PyTests_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Parallel_PyTests_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Parallel_PyTests_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c " cd /horovod/test/parallel && (ls -1 test_*.py | xargs -n 1 \$(cat /mpirun_command) /bin/bash /pytest.sh mpi)" | |
| shell: bash | |
| - name: "MPI Parallel PyTests [attempt 3 of 3]" | |
| id: MPI_Parallel_PyTests_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Parallel_PyTests && steps.MPI_Parallel_PyTests_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Parallel_PyTests_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Parallel_PyTests_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c " cd /horovod/test/parallel && (ls -1 test_*.py | xargs -n 1 \$(cat /mpirun_command) /bin/bash /pytest.sh mpi)" | |
| shell: bash | |
| - name: "MPI Parallel PyTests [ONECCL MPI] [attempt 1 of 3]" | |
| id: MPI_Parallel_PyTests_ONECCL_MPI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Parallel_PyTests_ONECCL_MPI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Parallel_PyTests_ONECCL_MPI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Parallel_PyTests_ONECCL_MPI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && cd /horovod/test/parallel && (ls -1 test_*.py | xargs -n 1 \$(cat /mpirun_command) /bin/bash /pytest.sh mpi)" | |
| shell: bash | |
| - name: "MPI Parallel PyTests [ONECCL MPI] [attempt 2 of 3]" | |
| id: MPI_Parallel_PyTests_ONECCL_MPI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Parallel_PyTests_ONECCL_MPI && steps.MPI_Parallel_PyTests_ONECCL_MPI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Parallel_PyTests_ONECCL_MPI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Parallel_PyTests_ONECCL_MPI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && cd /horovod/test/parallel && (ls -1 test_*.py | xargs -n 1 \$(cat /mpirun_command) /bin/bash /pytest.sh mpi)" | |
| shell: bash | |
| - name: "MPI Parallel PyTests [ONECCL MPI] [attempt 3 of 3]" | |
| id: MPI_Parallel_PyTests_ONECCL_MPI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Parallel_PyTests_ONECCL_MPI && steps.MPI_Parallel_PyTests_ONECCL_MPI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Parallel_PyTests_ONECCL_MPI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Parallel_PyTests_ONECCL_MPI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && cd /horovod/test/parallel && (ls -1 test_*.py | xargs -n 1 \$(cat /mpirun_command) /bin/bash /pytest.sh mpi)" | |
| shell: bash | |
| - name: "MPI Parallel PyTests [ONECCL OFI] [attempt 1 of 3]" | |
| id: MPI_Parallel_PyTests_ONECCL_OFI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Parallel_PyTests_ONECCL_OFI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Parallel_PyTests_ONECCL_OFI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Parallel_PyTests_ONECCL_OFI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && cd /horovod/test/parallel && (ls -1 test_*.py | xargs -n 1 \$(cat /mpirun_command) /bin/bash /pytest.sh mpi)" | |
| shell: bash | |
| - name: "MPI Parallel PyTests [ONECCL OFI] [attempt 2 of 3]" | |
| id: MPI_Parallel_PyTests_ONECCL_OFI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Parallel_PyTests_ONECCL_OFI && steps.MPI_Parallel_PyTests_ONECCL_OFI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Parallel_PyTests_ONECCL_OFI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Parallel_PyTests_ONECCL_OFI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && cd /horovod/test/parallel && (ls -1 test_*.py | xargs -n 1 \$(cat /mpirun_command) /bin/bash /pytest.sh mpi)" | |
| shell: bash | |
| - name: "MPI Parallel PyTests [ONECCL OFI] [attempt 3 of 3]" | |
| id: MPI_Parallel_PyTests_ONECCL_OFI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Parallel_PyTests_ONECCL_OFI && steps.MPI_Parallel_PyTests_ONECCL_OFI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Parallel_PyTests_ONECCL_OFI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Parallel_PyTests_ONECCL_OFI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && cd /horovod/test/parallel && (ls -1 test_*.py | xargs -n 1 \$(cat /mpirun_command) /bin/bash /pytest.sh mpi)" | |
| shell: bash | |
| - name: "MPI PyTorch MNIST api [attempt 1 of 3]" | |
| id: MPI_PyTorch_MNIST_api_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_PyTorch_MNIST_api && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_api_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_api_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets --num-proc 2 --hosts localhost:2 --communication mpi" | |
| shell: bash | |
| - name: "MPI PyTorch MNIST api [attempt 2 of 3]" | |
| id: MPI_PyTorch_MNIST_api_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_PyTorch_MNIST_api && steps.MPI_PyTorch_MNIST_api_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_api_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_api_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets --num-proc 2 --hosts localhost:2 --communication mpi" | |
| shell: bash | |
| - name: "MPI PyTorch MNIST api [attempt 3 of 3]" | |
| id: MPI_PyTorch_MNIST_api_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_PyTorch_MNIST_api && steps.MPI_PyTorch_MNIST_api_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_api_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_api_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets --num-proc 2 --hosts localhost:2 --communication mpi" | |
| shell: bash | |
| - name: "MPI PyTorch MNIST api [ONECCL MPI] [attempt 1 of 3]" | |
| id: MPI_PyTorch_MNIST_api_ONECCL_MPI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_PyTorch_MNIST_api_ONECCL_MPI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_api_ONECCL_MPI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_api_ONECCL_MPI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets --num-proc 2 --hosts localhost:2 --communication mpi" | |
| shell: bash | |
| - name: "MPI PyTorch MNIST api [ONECCL MPI] [attempt 2 of 3]" | |
| id: MPI_PyTorch_MNIST_api_ONECCL_MPI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_PyTorch_MNIST_api_ONECCL_MPI && steps.MPI_PyTorch_MNIST_api_ONECCL_MPI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_api_ONECCL_MPI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_api_ONECCL_MPI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets --num-proc 2 --hosts localhost:2 --communication mpi" | |
| shell: bash | |
| - name: "MPI PyTorch MNIST api [ONECCL MPI] [attempt 3 of 3]" | |
| id: MPI_PyTorch_MNIST_api_ONECCL_MPI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_PyTorch_MNIST_api_ONECCL_MPI && steps.MPI_PyTorch_MNIST_api_ONECCL_MPI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_api_ONECCL_MPI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_api_ONECCL_MPI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets --num-proc 2 --hosts localhost:2 --communication mpi" | |
| shell: bash | |
| - name: "MPI PyTorch MNIST api [ONECCL OFI] [attempt 1 of 3]" | |
| id: MPI_PyTorch_MNIST_api_ONECCL_OFI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_PyTorch_MNIST_api_ONECCL_OFI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_api_ONECCL_OFI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_api_ONECCL_OFI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets --num-proc 2 --hosts localhost:2 --communication mpi" | |
| shell: bash | |
| - name: "MPI PyTorch MNIST api [ONECCL OFI] [attempt 2 of 3]" | |
| id: MPI_PyTorch_MNIST_api_ONECCL_OFI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_PyTorch_MNIST_api_ONECCL_OFI && steps.MPI_PyTorch_MNIST_api_ONECCL_OFI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_api_ONECCL_OFI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_api_ONECCL_OFI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets --num-proc 2 --hosts localhost:2 --communication mpi" | |
| shell: bash | |
| - name: "MPI PyTorch MNIST api [ONECCL OFI] [attempt 3 of 3]" | |
| id: MPI_PyTorch_MNIST_api_ONECCL_OFI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_PyTorch_MNIST_api_ONECCL_OFI && steps.MPI_PyTorch_MNIST_api_ONECCL_OFI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_api_ONECCL_OFI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_api_ONECCL_OFI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets --num-proc 2 --hosts localhost:2 --communication mpi" | |
| shell: bash | |
| - name: "MPI PyTorch MNIST horovodrun [attempt 1 of 3]" | |
| id: MPI_PyTorch_MNIST_horovodrun_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_PyTorch_MNIST_horovodrun && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_horovodrun_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_horovodrun_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " \$(cat /mpirun_command) python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets" | |
| shell: bash | |
| - name: "MPI PyTorch MNIST horovodrun [attempt 2 of 3]" | |
| id: MPI_PyTorch_MNIST_horovodrun_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_PyTorch_MNIST_horovodrun && steps.MPI_PyTorch_MNIST_horovodrun_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_horovodrun_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_horovodrun_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " \$(cat /mpirun_command) python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets" | |
| shell: bash | |
| - name: "MPI PyTorch MNIST horovodrun [attempt 3 of 3]" | |
| id: MPI_PyTorch_MNIST_horovodrun_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_PyTorch_MNIST_horovodrun && steps.MPI_PyTorch_MNIST_horovodrun_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_horovodrun_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_horovodrun_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " \$(cat /mpirun_command) python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets" | |
| shell: bash | |
| - name: "MPI PyTorch MNIST horovodrun [ONECCL MPI] [attempt 1 of 3]" | |
| id: MPI_PyTorch_MNIST_horovodrun_ONECCL_MPI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_PyTorch_MNIST_horovodrun_ONECCL_MPI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_horovodrun_ONECCL_MPI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_horovodrun_ONECCL_MPI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && \$(cat /mpirun_command) python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets" | |
| shell: bash | |
| - name: "MPI PyTorch MNIST horovodrun [ONECCL MPI] [attempt 2 of 3]" | |
| id: MPI_PyTorch_MNIST_horovodrun_ONECCL_MPI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_PyTorch_MNIST_horovodrun_ONECCL_MPI && steps.MPI_PyTorch_MNIST_horovodrun_ONECCL_MPI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_horovodrun_ONECCL_MPI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_horovodrun_ONECCL_MPI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && \$(cat /mpirun_command) python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets" | |
| shell: bash | |
| - name: "MPI PyTorch MNIST horovodrun [ONECCL MPI] [attempt 3 of 3]" | |
| id: MPI_PyTorch_MNIST_horovodrun_ONECCL_MPI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_PyTorch_MNIST_horovodrun_ONECCL_MPI && steps.MPI_PyTorch_MNIST_horovodrun_ONECCL_MPI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_horovodrun_ONECCL_MPI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_horovodrun_ONECCL_MPI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && \$(cat /mpirun_command) python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets" | |
| shell: bash | |
| - name: "MPI PyTorch MNIST horovodrun [ONECCL OFI] [attempt 1 of 3]" | |
| id: MPI_PyTorch_MNIST_horovodrun_ONECCL_OFI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_PyTorch_MNIST_horovodrun_ONECCL_OFI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_horovodrun_ONECCL_OFI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_horovodrun_ONECCL_OFI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && \$(cat /mpirun_command) python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets" | |
| shell: bash | |
| - name: "MPI PyTorch MNIST horovodrun [ONECCL OFI] [attempt 2 of 3]" | |
| id: MPI_PyTorch_MNIST_horovodrun_ONECCL_OFI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_PyTorch_MNIST_horovodrun_ONECCL_OFI && steps.MPI_PyTorch_MNIST_horovodrun_ONECCL_OFI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_horovodrun_ONECCL_OFI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_horovodrun_ONECCL_OFI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && \$(cat /mpirun_command) python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets" | |
| shell: bash | |
| - name: "MPI PyTorch MNIST horovodrun [ONECCL OFI] [attempt 3 of 3]" | |
| id: MPI_PyTorch_MNIST_horovodrun_ONECCL_OFI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_PyTorch_MNIST_horovodrun_ONECCL_OFI && steps.MPI_PyTorch_MNIST_horovodrun_ONECCL_OFI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_horovodrun_ONECCL_OFI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_horovodrun_ONECCL_OFI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && \$(cat /mpirun_command) python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets" | |
| shell: bash | |
| - name: "MPI Single PyTests [attempt 1 of 3]" | |
| id: MPI_Single_PyTests_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Single_PyTests && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Single_PyTests_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Single_PyTests_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c " cd /horovod/test/single && (ls -1 test_*.py | xargs -n 1 /bin/bash /pytest_standalone.sh mpi)" | |
| shell: bash | |
| - name: "MPI Single PyTests [attempt 2 of 3]" | |
| id: MPI_Single_PyTests_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Single_PyTests && steps.MPI_Single_PyTests_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Single_PyTests_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Single_PyTests_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c " cd /horovod/test/single && (ls -1 test_*.py | xargs -n 1 /bin/bash /pytest_standalone.sh mpi)" | |
| shell: bash | |
| - name: "MPI Single PyTests [attempt 3 of 3]" | |
| id: MPI_Single_PyTests_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Single_PyTests && steps.MPI_Single_PyTests_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Single_PyTests_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Single_PyTests_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c " cd /horovod/test/single && (ls -1 test_*.py | xargs -n 1 /bin/bash /pytest_standalone.sh mpi)" | |
| shell: bash | |
| - name: "MPI Single PyTests [ONECCL MPI] [attempt 1 of 3]" | |
| id: MPI_Single_PyTests_ONECCL_MPI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Single_PyTests_ONECCL_MPI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Single_PyTests_ONECCL_MPI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Single_PyTests_ONECCL_MPI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && cd /horovod/test/single && (ls -1 test_*.py | xargs -n 1 /bin/bash /pytest_standalone.sh mpi)" | |
| shell: bash | |
| - name: "MPI Single PyTests [ONECCL MPI] [attempt 2 of 3]" | |
| id: MPI_Single_PyTests_ONECCL_MPI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Single_PyTests_ONECCL_MPI && steps.MPI_Single_PyTests_ONECCL_MPI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Single_PyTests_ONECCL_MPI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Single_PyTests_ONECCL_MPI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && cd /horovod/test/single && (ls -1 test_*.py | xargs -n 1 /bin/bash /pytest_standalone.sh mpi)" | |
| shell: bash | |
| - name: "MPI Single PyTests [ONECCL MPI] [attempt 3 of 3]" | |
| id: MPI_Single_PyTests_ONECCL_MPI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Single_PyTests_ONECCL_MPI && steps.MPI_Single_PyTests_ONECCL_MPI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Single_PyTests_ONECCL_MPI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Single_PyTests_ONECCL_MPI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && cd /horovod/test/single && (ls -1 test_*.py | xargs -n 1 /bin/bash /pytest_standalone.sh mpi)" | |
| shell: bash | |
| - name: "MPI Single PyTests [ONECCL OFI] [attempt 1 of 3]" | |
| id: MPI_Single_PyTests_ONECCL_OFI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Single_PyTests_ONECCL_OFI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Single_PyTests_ONECCL_OFI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Single_PyTests_ONECCL_OFI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && cd /horovod/test/single && (ls -1 test_*.py | xargs -n 1 /bin/bash /pytest_standalone.sh mpi)" | |
| shell: bash | |
| - name: "MPI Single PyTests [ONECCL OFI] [attempt 2 of 3]" | |
| id: MPI_Single_PyTests_ONECCL_OFI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Single_PyTests_ONECCL_OFI && steps.MPI_Single_PyTests_ONECCL_OFI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Single_PyTests_ONECCL_OFI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Single_PyTests_ONECCL_OFI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && cd /horovod/test/single && (ls -1 test_*.py | xargs -n 1 /bin/bash /pytest_standalone.sh mpi)" | |
| shell: bash | |
| - name: "MPI Single PyTests [ONECCL OFI] [attempt 3 of 3]" | |
| id: MPI_Single_PyTests_ONECCL_OFI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Single_PyTests_ONECCL_OFI && steps.MPI_Single_PyTests_ONECCL_OFI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Single_PyTests_ONECCL_OFI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Single_PyTests_ONECCL_OFI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && cd /horovod/test/single && (ls -1 test_*.py | xargs -n 1 /bin/bash /pytest_standalone.sh mpi)" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 Keras MNIST api [attempt 1 of 3]" | |
| id: MPI_TensorFlow_2_0_Keras_MNIST_api_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_Keras_MNIST_api && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_api_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_api_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py 2 localhost:2 mpi" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 Keras MNIST api [attempt 2 of 3]" | |
| id: MPI_TensorFlow_2_0_Keras_MNIST_api_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_Keras_MNIST_api && steps.MPI_TensorFlow_2_0_Keras_MNIST_api_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_api_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_api_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py 2 localhost:2 mpi" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 Keras MNIST api [attempt 3 of 3]" | |
| id: MPI_TensorFlow_2_0_Keras_MNIST_api_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_Keras_MNIST_api && steps.MPI_TensorFlow_2_0_Keras_MNIST_api_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_api_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_api_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py 2 localhost:2 mpi" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 Keras MNIST api [ONECCL MPI] [attempt 1 of 3]" | |
| id: MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_MPI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_MPI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_MPI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_MPI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py 2 localhost:2 mpi" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 Keras MNIST api [ONECCL MPI] [attempt 2 of 3]" | |
| id: MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_MPI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_MPI && steps.MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_MPI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_MPI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_MPI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py 2 localhost:2 mpi" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 Keras MNIST api [ONECCL MPI] [attempt 3 of 3]" | |
| id: MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_MPI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_MPI && steps.MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_MPI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_MPI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_MPI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py 2 localhost:2 mpi" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 Keras MNIST api [ONECCL OFI] [attempt 1 of 3]" | |
| id: MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_OFI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_OFI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_OFI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_OFI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py 2 localhost:2 mpi" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 Keras MNIST api [ONECCL OFI] [attempt 2 of 3]" | |
| id: MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_OFI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_OFI && steps.MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_OFI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_OFI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_OFI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py 2 localhost:2 mpi" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 Keras MNIST api [ONECCL OFI] [attempt 3 of 3]" | |
| id: MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_OFI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_OFI && steps.MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_OFI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_OFI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_OFI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py 2 localhost:2 mpi" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 Keras MNIST horovodrun [attempt 1 of 3]" | |
| id: MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_Keras_MNIST_horovodrun && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " \$(cat /mpirun_command) python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 Keras MNIST horovodrun [attempt 2 of 3]" | |
| id: MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_Keras_MNIST_horovodrun && steps.MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " \$(cat /mpirun_command) python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 Keras MNIST horovodrun [attempt 3 of 3]" | |
| id: MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_Keras_MNIST_horovodrun && steps.MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " \$(cat /mpirun_command) python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 Keras MNIST horovodrun [ONECCL MPI] [attempt 1 of 3]" | |
| id: MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_MPI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_MPI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_MPI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_MPI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && \$(cat /mpirun_command) python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 Keras MNIST horovodrun [ONECCL MPI] [attempt 2 of 3]" | |
| id: MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_MPI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_MPI && steps.MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_MPI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_MPI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_MPI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && \$(cat /mpirun_command) python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 Keras MNIST horovodrun [ONECCL MPI] [attempt 3 of 3]" | |
| id: MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_MPI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_MPI && steps.MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_MPI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_MPI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_MPI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && \$(cat /mpirun_command) python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 Keras MNIST horovodrun [ONECCL OFI] [attempt 1 of 3]" | |
| id: MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_OFI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_OFI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_OFI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_OFI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && \$(cat /mpirun_command) python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 Keras MNIST horovodrun [ONECCL OFI] [attempt 2 of 3]" | |
| id: MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_OFI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_OFI && steps.MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_OFI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_OFI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_OFI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && \$(cat /mpirun_command) python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 Keras MNIST horovodrun [ONECCL OFI] [attempt 3 of 3]" | |
| id: MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_OFI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_OFI && steps.MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_OFI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_OFI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_OFI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && \$(cat /mpirun_command) python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 MNIST api [attempt 1 of 3]" | |
| id: MPI_TensorFlow_2_0_MNIST_api_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_MNIST_api && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_api_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_api_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/tensorflow2/tensorflow2_mnist.py 2 localhost:2 mpi" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 MNIST api [attempt 2 of 3]" | |
| id: MPI_TensorFlow_2_0_MNIST_api_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_MNIST_api && steps.MPI_TensorFlow_2_0_MNIST_api_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_api_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_api_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/tensorflow2/tensorflow2_mnist.py 2 localhost:2 mpi" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 MNIST api [attempt 3 of 3]" | |
| id: MPI_TensorFlow_2_0_MNIST_api_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_MNIST_api && steps.MPI_TensorFlow_2_0_MNIST_api_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_api_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_api_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/tensorflow2/tensorflow2_mnist.py 2 localhost:2 mpi" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 MNIST api [ONECCL MPI] [attempt 1 of 3]" | |
| id: MPI_TensorFlow_2_0_MNIST_api_ONECCL_MPI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_MNIST_api_ONECCL_MPI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_api_ONECCL_MPI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_api_ONECCL_MPI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && python /horovod/examples/tensorflow2/tensorflow2_mnist.py 2 localhost:2 mpi" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 MNIST api [ONECCL MPI] [attempt 2 of 3]" | |
| id: MPI_TensorFlow_2_0_MNIST_api_ONECCL_MPI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_MNIST_api_ONECCL_MPI && steps.MPI_TensorFlow_2_0_MNIST_api_ONECCL_MPI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_api_ONECCL_MPI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_api_ONECCL_MPI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && python /horovod/examples/tensorflow2/tensorflow2_mnist.py 2 localhost:2 mpi" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 MNIST api [ONECCL MPI] [attempt 3 of 3]" | |
| id: MPI_TensorFlow_2_0_MNIST_api_ONECCL_MPI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_MNIST_api_ONECCL_MPI && steps.MPI_TensorFlow_2_0_MNIST_api_ONECCL_MPI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_api_ONECCL_MPI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_api_ONECCL_MPI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && python /horovod/examples/tensorflow2/tensorflow2_mnist.py 2 localhost:2 mpi" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 MNIST api [ONECCL OFI] [attempt 1 of 3]" | |
| id: MPI_TensorFlow_2_0_MNIST_api_ONECCL_OFI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_MNIST_api_ONECCL_OFI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_api_ONECCL_OFI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_api_ONECCL_OFI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && python /horovod/examples/tensorflow2/tensorflow2_mnist.py 2 localhost:2 mpi" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 MNIST api [ONECCL OFI] [attempt 2 of 3]" | |
| id: MPI_TensorFlow_2_0_MNIST_api_ONECCL_OFI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_MNIST_api_ONECCL_OFI && steps.MPI_TensorFlow_2_0_MNIST_api_ONECCL_OFI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_api_ONECCL_OFI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_api_ONECCL_OFI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && python /horovod/examples/tensorflow2/tensorflow2_mnist.py 2 localhost:2 mpi" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 MNIST api [ONECCL OFI] [attempt 3 of 3]" | |
| id: MPI_TensorFlow_2_0_MNIST_api_ONECCL_OFI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_MNIST_api_ONECCL_OFI && steps.MPI_TensorFlow_2_0_MNIST_api_ONECCL_OFI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_api_ONECCL_OFI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_api_ONECCL_OFI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && python /horovod/examples/tensorflow2/tensorflow2_mnist.py 2 localhost:2 mpi" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 MNIST horovodrun [attempt 1 of 3]" | |
| id: MPI_TensorFlow_2_0_MNIST_horovodrun_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_MNIST_horovodrun && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_horovodrun_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_horovodrun_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " \$(cat /mpirun_command) python /horovod/examples/tensorflow2/tensorflow2_mnist.py" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 MNIST horovodrun [attempt 2 of 3]" | |
| id: MPI_TensorFlow_2_0_MNIST_horovodrun_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_MNIST_horovodrun && steps.MPI_TensorFlow_2_0_MNIST_horovodrun_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_horovodrun_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_horovodrun_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " \$(cat /mpirun_command) python /horovod/examples/tensorflow2/tensorflow2_mnist.py" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 MNIST horovodrun [attempt 3 of 3]" | |
| id: MPI_TensorFlow_2_0_MNIST_horovodrun_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_MNIST_horovodrun && steps.MPI_TensorFlow_2_0_MNIST_horovodrun_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_horovodrun_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_horovodrun_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " \$(cat /mpirun_command) python /horovod/examples/tensorflow2/tensorflow2_mnist.py" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 MNIST horovodrun [ONECCL MPI] [attempt 1 of 3]" | |
| id: MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_MPI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_MPI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_MPI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_MPI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && \$(cat /mpirun_command) python /horovod/examples/tensorflow2/tensorflow2_mnist.py" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 MNIST horovodrun [ONECCL MPI] [attempt 2 of 3]" | |
| id: MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_MPI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_MPI && steps.MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_MPI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_MPI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_MPI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && \$(cat /mpirun_command) python /horovod/examples/tensorflow2/tensorflow2_mnist.py" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 MNIST horovodrun [ONECCL MPI] [attempt 3 of 3]" | |
| id: MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_MPI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_MPI && steps.MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_MPI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_MPI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_MPI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && \$(cat /mpirun_command) python /horovod/examples/tensorflow2/tensorflow2_mnist.py" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 MNIST horovodrun [ONECCL OFI] [attempt 1 of 3]" | |
| id: MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_OFI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_OFI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_OFI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_OFI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && \$(cat /mpirun_command) python /horovod/examples/tensorflow2/tensorflow2_mnist.py" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 MNIST horovodrun [ONECCL OFI] [attempt 2 of 3]" | |
| id: MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_OFI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_OFI && steps.MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_OFI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_OFI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_OFI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && \$(cat /mpirun_command) python /horovod/examples/tensorflow2/tensorflow2_mnist.py" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 MNIST horovodrun [ONECCL OFI] [attempt 3 of 3]" | |
| id: MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_OFI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_OFI && steps.MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_OFI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_OFI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_OFI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && \$(cat /mpirun_command) python /horovod/examples/tensorflow2/tensorflow2_mnist.py" | |
| shell: bash | |
| - name: "Run PyTests test_interactiverun [attempt 1 of 3]" | |
| id: Run_PyTests_test_interactiverun_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Run_PyTests_test_interactiverun && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Run_PyTests_test_interactiverun_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Run_PyTests_test_interactiverun_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "cd /horovod/test && pytest -v --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.mpi.integration.xml integration/test_interactiverun.py" | |
| shell: bash | |
| - name: "Run PyTests test_interactiverun [attempt 2 of 3]" | |
| id: Run_PyTests_test_interactiverun_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Run_PyTests_test_interactiverun && steps.Run_PyTests_test_interactiverun_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Run_PyTests_test_interactiverun_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Run_PyTests_test_interactiverun_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "cd /horovod/test && pytest -v --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.mpi.integration.xml integration/test_interactiverun.py" | |
| shell: bash | |
| - name: "Run PyTests test_interactiverun [attempt 3 of 3]" | |
| id: Run_PyTests_test_interactiverun_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Run_PyTests_test_interactiverun && steps.Run_PyTests_test_interactiverun_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Run_PyTests_test_interactiverun_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Run_PyTests_test_interactiverun_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "cd /horovod/test && pytest -v --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.mpi.integration.xml integration/test_interactiverun.py" | |
| shell: bash | |
| - name: "Single Keras MNIST [attempt 1 of 3]" | |
| id: Single_Keras_MNIST_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_Keras_MNIST && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_Keras_MNIST_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_Keras_MNIST_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/keras/keras_mnist_advanced.py --epochs 3 --batch-size 64" | |
| shell: bash | |
| - name: "Single Keras MNIST [attempt 2 of 3]" | |
| id: Single_Keras_MNIST_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_Keras_MNIST && steps.Single_Keras_MNIST_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_Keras_MNIST_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_Keras_MNIST_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/keras/keras_mnist_advanced.py --epochs 3 --batch-size 64" | |
| shell: bash | |
| - name: "Single Keras MNIST [attempt 3 of 3]" | |
| id: Single_Keras_MNIST_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_Keras_MNIST && steps.Single_Keras_MNIST_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_Keras_MNIST_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_Keras_MNIST_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/keras/keras_mnist_advanced.py --epochs 3 --batch-size 64" | |
| shell: bash | |
| - name: "Single MXNet2 MNIST [attempt 1 of 3]" | |
| id: Single_MXNet2_MNIST_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_MXNet2_MNIST && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_MXNet2_MNIST_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_MXNet2_MNIST_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/mxnet/mxnet2_mnist.py --epochs 3" | |
| shell: bash | |
| - name: "Single MXNet2 MNIST [attempt 2 of 3]" | |
| id: Single_MXNet2_MNIST_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_MXNet2_MNIST && steps.Single_MXNet2_MNIST_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_MXNet2_MNIST_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_MXNet2_MNIST_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/mxnet/mxnet2_mnist.py --epochs 3" | |
| shell: bash | |
| - name: "Single MXNet2 MNIST [attempt 3 of 3]" | |
| id: Single_MXNet2_MNIST_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_MXNet2_MNIST && steps.Single_MXNet2_MNIST_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_MXNet2_MNIST_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_MXNet2_MNIST_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/mxnet/mxnet2_mnist.py --epochs 3" | |
| shell: bash | |
| - name: "Single MXNet MNIST [attempt 1 of 3]" | |
| id: Single_MXNet_MNIST_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_MXNet_MNIST && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_MXNet_MNIST_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_MXNet_MNIST_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/mxnet/mxnet_mnist.py --epochs 3" | |
| shell: bash | |
| - name: "Single MXNet MNIST [attempt 2 of 3]" | |
| id: Single_MXNet_MNIST_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_MXNet_MNIST && steps.Single_MXNet_MNIST_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_MXNet_MNIST_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_MXNet_MNIST_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/mxnet/mxnet_mnist.py --epochs 3" | |
| shell: bash | |
| - name: "Single MXNet MNIST [attempt 3 of 3]" | |
| id: Single_MXNet_MNIST_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_MXNet_MNIST && steps.Single_MXNet_MNIST_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_MXNet_MNIST_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_MXNet_MNIST_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/mxnet/mxnet_mnist.py --epochs 3" | |
| shell: bash | |
| - name: "Single MXNet MNIST [ONECCL MPI] [attempt 1 of 3]" | |
| id: Single_MXNet_MNIST_ONECCL_MPI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_MXNet_MNIST_ONECCL_MPI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_MXNet_MNIST_ONECCL_MPI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_MXNet_MNIST_ONECCL_MPI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && python /horovod/examples/mxnet/mxnet_mnist.py --epochs 3" | |
| shell: bash | |
| - name: "Single MXNet MNIST [ONECCL MPI] [attempt 2 of 3]" | |
| id: Single_MXNet_MNIST_ONECCL_MPI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_MXNet_MNIST_ONECCL_MPI && steps.Single_MXNet_MNIST_ONECCL_MPI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_MXNet_MNIST_ONECCL_MPI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_MXNet_MNIST_ONECCL_MPI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && python /horovod/examples/mxnet/mxnet_mnist.py --epochs 3" | |
| shell: bash | |
| - name: "Single MXNet MNIST [ONECCL MPI] [attempt 3 of 3]" | |
| id: Single_MXNet_MNIST_ONECCL_MPI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_MXNet_MNIST_ONECCL_MPI && steps.Single_MXNet_MNIST_ONECCL_MPI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_MXNet_MNIST_ONECCL_MPI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_MXNet_MNIST_ONECCL_MPI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && python /horovod/examples/mxnet/mxnet_mnist.py --epochs 3" | |
| shell: bash | |
| - name: "Single MXNet MNIST [ONECCL OFI] [attempt 1 of 3]" | |
| id: Single_MXNet_MNIST_ONECCL_OFI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_MXNet_MNIST_ONECCL_OFI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_MXNet_MNIST_ONECCL_OFI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_MXNet_MNIST_ONECCL_OFI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && python /horovod/examples/mxnet/mxnet_mnist.py --epochs 3" | |
| shell: bash | |
| - name: "Single MXNet MNIST [ONECCL OFI] [attempt 2 of 3]" | |
| id: Single_MXNet_MNIST_ONECCL_OFI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_MXNet_MNIST_ONECCL_OFI && steps.Single_MXNet_MNIST_ONECCL_OFI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_MXNet_MNIST_ONECCL_OFI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_MXNet_MNIST_ONECCL_OFI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && python /horovod/examples/mxnet/mxnet_mnist.py --epochs 3" | |
| shell: bash | |
| - name: "Single MXNet MNIST [ONECCL OFI] [attempt 3 of 3]" | |
| id: Single_MXNet_MNIST_ONECCL_OFI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_MXNet_MNIST_ONECCL_OFI && steps.Single_MXNet_MNIST_ONECCL_OFI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_MXNet_MNIST_ONECCL_OFI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_MXNet_MNIST_ONECCL_OFI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && python /horovod/examples/mxnet/mxnet_mnist.py --epochs 3" | |
| shell: bash | |
| - name: "Single PyTorch MNIST [attempt 1 of 3]" | |
| id: Single_PyTorch_MNIST_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_PyTorch_MNIST && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_PyTorch_MNIST_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_PyTorch_MNIST_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/pytorch/pytorch_mnist.py --epochs 3 --data-dir /data/pytorch_datasets" | |
| shell: bash | |
| - name: "Single PyTorch MNIST [attempt 2 of 3]" | |
| id: Single_PyTorch_MNIST_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_PyTorch_MNIST && steps.Single_PyTorch_MNIST_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_PyTorch_MNIST_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_PyTorch_MNIST_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/pytorch/pytorch_mnist.py --epochs 3 --data-dir /data/pytorch_datasets" | |
| shell: bash | |
| - name: "Single PyTorch MNIST [attempt 3 of 3]" | |
| id: Single_PyTorch_MNIST_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_PyTorch_MNIST && steps.Single_PyTorch_MNIST_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_PyTorch_MNIST_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_PyTorch_MNIST_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/pytorch/pytorch_mnist.py --epochs 3 --data-dir /data/pytorch_datasets" | |
| shell: bash | |
| - name: "Single PyTorch MNIST [ONECCL MPI] [attempt 1 of 3]" | |
| id: Single_PyTorch_MNIST_ONECCL_MPI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_PyTorch_MNIST_ONECCL_MPI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_PyTorch_MNIST_ONECCL_MPI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_PyTorch_MNIST_ONECCL_MPI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && python /horovod/examples/pytorch/pytorch_mnist.py --epochs 3 --data-dir /data/pytorch_datasets" | |
| shell: bash | |
| - name: "Single PyTorch MNIST [ONECCL MPI] [attempt 2 of 3]" | |
| id: Single_PyTorch_MNIST_ONECCL_MPI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_PyTorch_MNIST_ONECCL_MPI && steps.Single_PyTorch_MNIST_ONECCL_MPI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_PyTorch_MNIST_ONECCL_MPI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_PyTorch_MNIST_ONECCL_MPI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && python /horovod/examples/pytorch/pytorch_mnist.py --epochs 3 --data-dir /data/pytorch_datasets" | |
| shell: bash | |
| - name: "Single PyTorch MNIST [ONECCL MPI] [attempt 3 of 3]" | |
| id: Single_PyTorch_MNIST_ONECCL_MPI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_PyTorch_MNIST_ONECCL_MPI && steps.Single_PyTorch_MNIST_ONECCL_MPI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_PyTorch_MNIST_ONECCL_MPI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_PyTorch_MNIST_ONECCL_MPI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && python /horovod/examples/pytorch/pytorch_mnist.py --epochs 3 --data-dir /data/pytorch_datasets" | |
| shell: bash | |
| - name: "Single PyTorch MNIST [ONECCL OFI] [attempt 1 of 3]" | |
| id: Single_PyTorch_MNIST_ONECCL_OFI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_PyTorch_MNIST_ONECCL_OFI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_PyTorch_MNIST_ONECCL_OFI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_PyTorch_MNIST_ONECCL_OFI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && python /horovod/examples/pytorch/pytorch_mnist.py --epochs 3 --data-dir /data/pytorch_datasets" | |
| shell: bash | |
| - name: "Single PyTorch MNIST [ONECCL OFI] [attempt 2 of 3]" | |
| id: Single_PyTorch_MNIST_ONECCL_OFI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_PyTorch_MNIST_ONECCL_OFI && steps.Single_PyTorch_MNIST_ONECCL_OFI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_PyTorch_MNIST_ONECCL_OFI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_PyTorch_MNIST_ONECCL_OFI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && python /horovod/examples/pytorch/pytorch_mnist.py --epochs 3 --data-dir /data/pytorch_datasets" | |
| shell: bash | |
| - name: "Single PyTorch MNIST [ONECCL OFI] [attempt 3 of 3]" | |
| id: Single_PyTorch_MNIST_ONECCL_OFI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_PyTorch_MNIST_ONECCL_OFI && steps.Single_PyTorch_MNIST_ONECCL_OFI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_PyTorch_MNIST_ONECCL_OFI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_PyTorch_MNIST_ONECCL_OFI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && python /horovod/examples/pytorch/pytorch_mnist.py --epochs 3 --data-dir /data/pytorch_datasets" | |
| shell: bash | |
| - name: "Spark Keras MNIST [attempt 1 of 3]" | |
| id: Spark_Keras_MNIST_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Spark_Keras_MNIST && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Spark_Keras_MNIST_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Spark_Keras_MNIST_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "OMP_NUM_THREADS=1 /spark_env.sh python /horovod/examples/spark/keras/keras_spark_mnist.py --num-proc 2 --work-dir /work --data-dir /data --epochs 3" | |
| shell: bash | |
| - name: "Spark Keras MNIST [attempt 2 of 3]" | |
| id: Spark_Keras_MNIST_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Spark_Keras_MNIST && steps.Spark_Keras_MNIST_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Spark_Keras_MNIST_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Spark_Keras_MNIST_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "OMP_NUM_THREADS=1 /spark_env.sh python /horovod/examples/spark/keras/keras_spark_mnist.py --num-proc 2 --work-dir /work --data-dir /data --epochs 3" | |
| shell: bash | |
| - name: "Spark Keras MNIST [attempt 3 of 3]" | |
| id: Spark_Keras_MNIST_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Spark_Keras_MNIST && steps.Spark_Keras_MNIST_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Spark_Keras_MNIST_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Spark_Keras_MNIST_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "OMP_NUM_THREADS=1 /spark_env.sh python /horovod/examples/spark/keras/keras_spark_mnist.py --num-proc 2 --work-dir /work --data-dir /data --epochs 3" | |
| shell: bash | |
| - name: "Spark Keras Rossmann Estimator [attempt 1 of 3]" | |
| id: Spark_Keras_Rossmann_Estimator_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Spark_Keras_Rossmann_Estimator && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Spark_Keras_Rossmann_Estimator_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Spark_Keras_Rossmann_Estimator_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "OMP_NUM_THREADS=1 /spark_env.sh python /horovod/examples/spark/keras/keras_spark_rossmann_estimator.py --num-proc 2 --work-dir /work --data-dir file:///data --epochs 3 --sample-rate 0.1" | |
| shell: bash | |
| - name: "Spark Keras Rossmann Estimator [attempt 2 of 3]" | |
| id: Spark_Keras_Rossmann_Estimator_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Spark_Keras_Rossmann_Estimator && steps.Spark_Keras_Rossmann_Estimator_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Spark_Keras_Rossmann_Estimator_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Spark_Keras_Rossmann_Estimator_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "OMP_NUM_THREADS=1 /spark_env.sh python /horovod/examples/spark/keras/keras_spark_rossmann_estimator.py --num-proc 2 --work-dir /work --data-dir file:///data --epochs 3 --sample-rate 0.1" | |
| shell: bash | |
| - name: "Spark Keras Rossmann Estimator [attempt 3 of 3]" | |
| id: Spark_Keras_Rossmann_Estimator_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Spark_Keras_Rossmann_Estimator && steps.Spark_Keras_Rossmann_Estimator_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Spark_Keras_Rossmann_Estimator_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Spark_Keras_Rossmann_Estimator_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "OMP_NUM_THREADS=1 /spark_env.sh python /horovod/examples/spark/keras/keras_spark_rossmann_estimator.py --num-proc 2 --work-dir /work --data-dir file:///data --epochs 3 --sample-rate 0.1" | |
| shell: bash | |
| - name: "Spark Keras Rossmann Run [attempt 1 of 3]" | |
| id: Spark_Keras_Rossmann_Run_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Spark_Keras_Rossmann_Run && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Spark_Keras_Rossmann_Run_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Spark_Keras_Rossmann_Run_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "OMP_NUM_THREADS=1 /spark_env.sh python /horovod/examples/spark/keras/keras_spark_rossmann_run.py --num-proc 2 --data-dir file:///data --epochs 3 --sample-rate 0.1" | |
| shell: bash | |
| - name: "Spark Keras Rossmann Run [attempt 2 of 3]" | |
| id: Spark_Keras_Rossmann_Run_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Spark_Keras_Rossmann_Run && steps.Spark_Keras_Rossmann_Run_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Spark_Keras_Rossmann_Run_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Spark_Keras_Rossmann_Run_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "OMP_NUM_THREADS=1 /spark_env.sh python /horovod/examples/spark/keras/keras_spark_rossmann_run.py --num-proc 2 --data-dir file:///data --epochs 3 --sample-rate 0.1" | |
| shell: bash | |
| - name: "Spark Keras Rossmann Run [attempt 3 of 3]" | |
| id: Spark_Keras_Rossmann_Run_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Spark_Keras_Rossmann_Run && steps.Spark_Keras_Rossmann_Run_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Spark_Keras_Rossmann_Run_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Spark_Keras_Rossmann_Run_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "OMP_NUM_THREADS=1 /spark_env.sh python /horovod/examples/spark/keras/keras_spark_rossmann_run.py --num-proc 2 --data-dir file:///data --epochs 3 --sample-rate 0.1" | |
| shell: bash | |
| - name: "Spark Lightning MNIST [attempt 1 of 3]" | |
| id: Spark_Lightning_MNIST_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Spark_Lightning_MNIST && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Spark_Lightning_MNIST_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Spark_Lightning_MNIST_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "OMP_NUM_THREADS=1 /spark_env.sh python /horovod/examples/spark/pytorch/pytorch_lightning_spark_mnist.py --num-proc 2 --work-dir /work --data-dir /data --epochs 3" | |
| shell: bash | |
| - name: "Spark Lightning MNIST [attempt 2 of 3]" | |
| id: Spark_Lightning_MNIST_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Spark_Lightning_MNIST && steps.Spark_Lightning_MNIST_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Spark_Lightning_MNIST_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Spark_Lightning_MNIST_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "OMP_NUM_THREADS=1 /spark_env.sh python /horovod/examples/spark/pytorch/pytorch_lightning_spark_mnist.py --num-proc 2 --work-dir /work --data-dir /data --epochs 3" | |
| shell: bash | |
| - name: "Spark Lightning MNIST [attempt 3 of 3]" | |
| id: Spark_Lightning_MNIST_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Spark_Lightning_MNIST && steps.Spark_Lightning_MNIST_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Spark_Lightning_MNIST_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Spark_Lightning_MNIST_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "OMP_NUM_THREADS=1 /spark_env.sh python /horovod/examples/spark/pytorch/pytorch_lightning_spark_mnist.py --num-proc 2 --work-dir /work --data-dir /data --epochs 3" | |
| shell: bash | |
| - name: "Spark PyTests [attempt 1 of 3]" | |
| id: Spark_PyTests_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Spark_PyTests && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Spark_PyTests_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Spark_PyTests_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 30m bash -c "cd /horovod/test/integration && (ls -1 test_spark*.py | xargs -n 1 /bin/bash /pytest_standalone.sh spark)" | |
| shell: bash | |
| - name: "Spark PyTests [attempt 2 of 3]" | |
| id: Spark_PyTests_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Spark_PyTests && steps.Spark_PyTests_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Spark_PyTests_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Spark_PyTests_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 30m bash -c "cd /horovod/test/integration && (ls -1 test_spark*.py | xargs -n 1 /bin/bash /pytest_standalone.sh spark)" | |
| shell: bash | |
| - name: "Spark PyTests [attempt 3 of 3]" | |
| id: Spark_PyTests_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Spark_PyTests && steps.Spark_PyTests_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Spark_PyTests_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Spark_PyTests_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 30m bash -c "cd /horovod/test/integration && (ls -1 test_spark*.py | xargs -n 1 /bin/bash /pytest_standalone.sh spark)" | |
| shell: bash | |
| - name: "Spark TensorFlow 2.0 MNIST Data Service [attempt 1 of 3]" | |
| id: Spark_TensorFlow_2_0_MNIST_Data_Service_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Spark_TensorFlow_2_0_MNIST_Data_Service && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Spark_TensorFlow_2_0_MNIST_Data_Service_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Spark_TensorFlow_2_0_MNIST_Data_Service_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "cd /horovod/examples/spark/tensorflow2; spark-submit --master \"local[2]\" \"/horovod/horovod/spark/tensorflow/compute_worker.py\" /tmp/compute.json & OMP_NUM_THREADS=1 /spark_env.sh spark-submit --master \"local[2]\" --py-files tensorflow2_mnist_data_service_train_fn_compute_side_dispatcher.py,tensorflow2_mnist_data_service_train_fn_training_side_dispatcher.py tensorflow2_mnist_data_service.py /tmp/compute.json" | |
| shell: bash | |
| - name: "Spark TensorFlow 2.0 MNIST Data Service [attempt 2 of 3]" | |
| id: Spark_TensorFlow_2_0_MNIST_Data_Service_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Spark_TensorFlow_2_0_MNIST_Data_Service && steps.Spark_TensorFlow_2_0_MNIST_Data_Service_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Spark_TensorFlow_2_0_MNIST_Data_Service_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Spark_TensorFlow_2_0_MNIST_Data_Service_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "cd /horovod/examples/spark/tensorflow2; spark-submit --master \"local[2]\" \"/horovod/horovod/spark/tensorflow/compute_worker.py\" /tmp/compute.json & OMP_NUM_THREADS=1 /spark_env.sh spark-submit --master \"local[2]\" --py-files tensorflow2_mnist_data_service_train_fn_compute_side_dispatcher.py,tensorflow2_mnist_data_service_train_fn_training_side_dispatcher.py tensorflow2_mnist_data_service.py /tmp/compute.json" | |
| shell: bash | |
| - name: "Spark TensorFlow 2.0 MNIST Data Service [attempt 3 of 3]" | |
| id: Spark_TensorFlow_2_0_MNIST_Data_Service_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Spark_TensorFlow_2_0_MNIST_Data_Service && steps.Spark_TensorFlow_2_0_MNIST_Data_Service_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Spark_TensorFlow_2_0_MNIST_Data_Service_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Spark_TensorFlow_2_0_MNIST_Data_Service_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "cd /horovod/examples/spark/tensorflow2; spark-submit --master \"local[2]\" \"/horovod/horovod/spark/tensorflow/compute_worker.py\" /tmp/compute.json & OMP_NUM_THREADS=1 /spark_env.sh spark-submit --master \"local[2]\" --py-files tensorflow2_mnist_data_service_train_fn_compute_side_dispatcher.py,tensorflow2_mnist_data_service_train_fn_training_side_dispatcher.py tensorflow2_mnist_data_service.py /tmp/compute.json" | |
| shell: bash | |
| - name: "Spark Torch MNIST [attempt 1 of 3]" | |
| id: Spark_Torch_MNIST_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Spark_Torch_MNIST && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Spark_Torch_MNIST_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Spark_Torch_MNIST_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "OMP_NUM_THREADS=1 /spark_env.sh python /horovod/examples/spark/pytorch/pytorch_spark_mnist.py --num-proc 2 --work-dir /work --data-dir /data --epochs 3" | |
| shell: bash | |
| - name: "Spark Torch MNIST [attempt 2 of 3]" | |
| id: Spark_Torch_MNIST_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Spark_Torch_MNIST && steps.Spark_Torch_MNIST_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Spark_Torch_MNIST_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Spark_Torch_MNIST_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "OMP_NUM_THREADS=1 /spark_env.sh python /horovod/examples/spark/pytorch/pytorch_spark_mnist.py --num-proc 2 --work-dir /work --data-dir /data --epochs 3" | |
| shell: bash | |
| - name: "Spark Torch MNIST [attempt 3 of 3]" | |
| id: Spark_Torch_MNIST_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Spark_Torch_MNIST && steps.Spark_Torch_MNIST_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Spark_Torch_MNIST_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Spark_Torch_MNIST_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "OMP_NUM_THREADS=1 /spark_env.sh python /horovod/examples/spark/pytorch/pytorch_spark_mnist.py --num-proc 2 --work-dir /work --data-dir /data --epochs 3" | |
| shell: bash | |
| - name: Upload Test Results | |
| uses: actions/upload-artifact@v3 | |
| if: always() && contains(matrix.image, '-cpu-') | |
| with: | |
| name: Unit Test Results - ${{ matrix.image }} | |
| path: artifacts/${{ matrix.image }}/**/*.xml | |
| build-and-test-heads: | |
| name: "Build and Test heads (${{ matrix.image }})" | |
| needs: [init-workflow, build-and-test] | |
| if: > | |
| needs.init-workflow.outputs.run-at-all == 'true' && | |
| needs.init-workflow.outputs.run-builds-and-tests == 'true' | |
| runs-on: ubuntu-latest | |
| strategy: | |
| max-parallel: 2 | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - image: test-cpu-openmpi-gloo-py3_8-tfhead-keras_none-torchhead-mxnethead-pyspark3_4_0 | |
| Elastic_Tests_1: true | |
| Gloo_Cluster_PyTests: true | |
| Gloo_MXNet2_MNIST_api: true | |
| Gloo_MXNet2_MNIST_horovodrun: true | |
| Gloo_Parallel_PyTests: true | |
| Gloo_PyTorch_MNIST_api: true | |
| Gloo_PyTorch_MNIST_horovodrun: true | |
| Gloo_Single_PyTests: true | |
| Gloo_TensorFlow_2_0_Keras_MNIST_api: true | |
| Gloo_TensorFlow_2_0_Keras_MNIST_horovodrun: true | |
| Gloo_TensorFlow_2_0_MNIST_Elastic_api: true | |
| Gloo_TensorFlow_2_0_MNIST_Elastic_horovodrun: true | |
| Gloo_TensorFlow_2_0_MNIST_api: true | |
| Gloo_TensorFlow_2_0_MNIST_horovodrun: true | |
| MPI_Cluster_PyTests: true | |
| MPI_MXNet2_MNIST_api: true | |
| MPI_MXNet2_MNIST_horovodrun: true | |
| MPI_Parallel_PyTests: true | |
| MPI_PyTorch_MNIST_api: true | |
| MPI_PyTorch_MNIST_horovodrun: true | |
| MPI_Single_PyTests: true | |
| MPI_TensorFlow_2_0_Keras_MNIST_api: true | |
| MPI_TensorFlow_2_0_Keras_MNIST_horovodrun: true | |
| MPI_TensorFlow_2_0_MNIST_api: true | |
| MPI_TensorFlow_2_0_MNIST_horovodrun: true | |
| Run_PyTests_test_interactiverun: true | |
| Single_MXNet2_MNIST: true | |
| Single_PyTorch_MNIST: true | |
| Spark_Lightning_MNIST: true | |
| Spark_PyTests: true | |
| Spark_TensorFlow_2_0_MNIST_Data_Service: true | |
| Spark_Torch_MNIST: true | |
| build_timeout: 30 | |
| - image: test-gpu-openmpi-gloo-py3_8-tfhead-keras_none-torchhead-mxnethead-pyspark3_4_0 | |
| build_timeout: 40 | |
| steps: | |
| - name: Clean up disk space | |
| # deleting these paths frees 38 GB disk space: | |
| # sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc | |
| # but this sometimes takes 3-4 minutes | |
| # so we delete only some sub-paths which are known to be quick (10s) and 20 GB | |
| run: | | |
| echo ::group::Disk space before clean up | |
| df -h | |
| echo ::endgroup:: | |
| for dir in /usr/share/dotnet/sdk/\*/nuGetPackagesArchive.lzma \ | |
| /usr/share/dotnet/shared \ | |
| /usr/local/lib/android/sdk/ndk \ | |
| /usr/local/lib/android/sdk/build-tools \ | |
| /opt/ghc | |
| do | |
| echo ::group::Deleting "$dir" | |
| sudo du -hsc $dir | tail -n1 || true | |
| sudo rm -rf $dir | |
| echo ::endgroup:: | |
| done | |
| echo ::group::Disk space after clean up | |
| df -h | |
| echo ::endgroup:: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.8 | |
| - name: Setup docker-compose | |
| run: pip install docker-compose | |
| - name: Build | |
| id: build | |
| run: | | |
| .github/timeout-and-retry.sh ${{ matrix.build_timeout }}m 3 10 docker-compose -f docker-compose.test.yml build ${{ matrix.image }} | |
| env: | |
| COMPOSE_DOCKER_CLI_BUILD: 1 | |
| DOCKER_BUILDKIT: 1 | |
| - name: "Elastic Spark TensorFlow Tests 1 [attempt 1 of 3]" | |
| id: Elastic_Spark_TensorFlow_Tests_1_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Elastic_Spark_TensorFlow_Tests_1 && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Elastic_Spark_TensorFlow_Tests_1_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Elastic_Spark_TensorFlow_Tests_1_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 30m bash -c "cd /horovod/test/integration && /spark_env.sh HOROVOD_LOG_LEVEL=DEBUG pytest --forked -v --log-cli-level 10 --log-cli-format '[%(asctime)-15s %(levelname)s %(filename)s:%(lineno)d %(funcName)s()] %(message)s' --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.gloo.elastic.spark.tf.xml test_elastic_spark_tensorflow2.py" | |
| shell: bash | |
| - name: "Elastic Spark TensorFlow Tests 1 [attempt 2 of 3]" | |
| id: Elastic_Spark_TensorFlow_Tests_1_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Elastic_Spark_TensorFlow_Tests_1 && steps.Elastic_Spark_TensorFlow_Tests_1_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Elastic_Spark_TensorFlow_Tests_1_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Elastic_Spark_TensorFlow_Tests_1_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 30m bash -c "cd /horovod/test/integration && /spark_env.sh HOROVOD_LOG_LEVEL=DEBUG pytest --forked -v --log-cli-level 10 --log-cli-format '[%(asctime)-15s %(levelname)s %(filename)s:%(lineno)d %(funcName)s()] %(message)s' --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.gloo.elastic.spark.tf.xml test_elastic_spark_tensorflow2.py" | |
| shell: bash | |
| - name: "Elastic Spark TensorFlow Tests 1 [attempt 3 of 3]" | |
| id: Elastic_Spark_TensorFlow_Tests_1_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Elastic_Spark_TensorFlow_Tests_1 && steps.Elastic_Spark_TensorFlow_Tests_1_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Elastic_Spark_TensorFlow_Tests_1_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Elastic_Spark_TensorFlow_Tests_1_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 30m bash -c "cd /horovod/test/integration && /spark_env.sh HOROVOD_LOG_LEVEL=DEBUG pytest --forked -v --log-cli-level 10 --log-cli-format '[%(asctime)-15s %(levelname)s %(filename)s:%(lineno)d %(funcName)s()] %(message)s' --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.gloo.elastic.spark.tf.xml test_elastic_spark_tensorflow2.py" | |
| shell: bash | |
| - name: "Elastic Spark TensorFlow Tests 2 [attempt 1 of 3]" | |
| id: Elastic_Spark_TensorFlow_Tests_2_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Elastic_Spark_TensorFlow_Tests_2 && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Elastic_Spark_TensorFlow_Tests_2_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Elastic_Spark_TensorFlow_Tests_2_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 30m bash -c "cd /horovod/test/integration && /spark_env.sh HOROVOD_LOG_LEVEL=DEBUG pytest --forked -v --log-cli-level 10 --log-cli-format '[%(asctime)-15s %(levelname)s %(filename)s:%(lineno)d %(funcName)s()] %(message)s' --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.gloo.elastic.spark.tf.xml test_elastic_spark_tensorflow.py" | |
| shell: bash | |
| - name: "Elastic Spark TensorFlow Tests 2 [attempt 2 of 3]" | |
| id: Elastic_Spark_TensorFlow_Tests_2_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Elastic_Spark_TensorFlow_Tests_2 && steps.Elastic_Spark_TensorFlow_Tests_2_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Elastic_Spark_TensorFlow_Tests_2_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Elastic_Spark_TensorFlow_Tests_2_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 30m bash -c "cd /horovod/test/integration && /spark_env.sh HOROVOD_LOG_LEVEL=DEBUG pytest --forked -v --log-cli-level 10 --log-cli-format '[%(asctime)-15s %(levelname)s %(filename)s:%(lineno)d %(funcName)s()] %(message)s' --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.gloo.elastic.spark.tf.xml test_elastic_spark_tensorflow.py" | |
| shell: bash | |
| - name: "Elastic Spark TensorFlow Tests 2 [attempt 3 of 3]" | |
| id: Elastic_Spark_TensorFlow_Tests_2_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Elastic_Spark_TensorFlow_Tests_2 && steps.Elastic_Spark_TensorFlow_Tests_2_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Elastic_Spark_TensorFlow_Tests_2_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Elastic_Spark_TensorFlow_Tests_2_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 30m bash -c "cd /horovod/test/integration && /spark_env.sh HOROVOD_LOG_LEVEL=DEBUG pytest --forked -v --log-cli-level 10 --log-cli-format '[%(asctime)-15s %(levelname)s %(filename)s:%(lineno)d %(funcName)s()] %(message)s' --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.gloo.elastic.spark.tf.xml test_elastic_spark_tensorflow.py" | |
| shell: bash | |
| - name: "Elastic Spark Torch Tests [attempt 1 of 3]" | |
| id: Elastic_Spark_Torch_Tests_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Elastic_Spark_Torch_Tests && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Elastic_Spark_Torch_Tests_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Elastic_Spark_Torch_Tests_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 30m bash -c "cd /horovod/test/integration && /spark_env.sh HOROVOD_LOG_LEVEL=DEBUG pytest --forked -v --log-cli-level 10 --log-cli-format '[%(asctime)-15s %(levelname)s %(filename)s:%(lineno)d %(funcName)s()] %(message)s' --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.gloo.elastic.spark.torch.xml test_elastic_spark_torch.py" | |
| shell: bash | |
| - name: "Elastic Spark Torch Tests [attempt 2 of 3]" | |
| id: Elastic_Spark_Torch_Tests_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Elastic_Spark_Torch_Tests && steps.Elastic_Spark_Torch_Tests_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Elastic_Spark_Torch_Tests_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Elastic_Spark_Torch_Tests_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 30m bash -c "cd /horovod/test/integration && /spark_env.sh HOROVOD_LOG_LEVEL=DEBUG pytest --forked -v --log-cli-level 10 --log-cli-format '[%(asctime)-15s %(levelname)s %(filename)s:%(lineno)d %(funcName)s()] %(message)s' --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.gloo.elastic.spark.torch.xml test_elastic_spark_torch.py" | |
| shell: bash | |
| - name: "Elastic Spark Torch Tests [attempt 3 of 3]" | |
| id: Elastic_Spark_Torch_Tests_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Elastic_Spark_Torch_Tests && steps.Elastic_Spark_Torch_Tests_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Elastic_Spark_Torch_Tests_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Elastic_Spark_Torch_Tests_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 30m bash -c "cd /horovod/test/integration && /spark_env.sh HOROVOD_LOG_LEVEL=DEBUG pytest --forked -v --log-cli-level 10 --log-cli-format '[%(asctime)-15s %(levelname)s %(filename)s:%(lineno)d %(funcName)s()] %(message)s' --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.gloo.elastic.spark.torch.xml test_elastic_spark_torch.py" | |
| shell: bash | |
| - name: "Elastic Tests 1 [attempt 1 of 3]" | |
| id: Elastic_Tests_1_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Elastic_Tests_1 && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Elastic_Tests_1_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Elastic_Tests_1_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c "cd /horovod/test/integration && HOROVOD_LOG_LEVEL=DEBUG pytest --forked -v --log-cli-level 10 --log-cli-format '[%(asctime)-15s %(levelname)s %(filename)s:%(lineno)d %(funcName)s()] %(message)s' --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.gloo.elastic.xml test_elastic_torch.py test_elastic_tensorflow2.py" | |
| shell: bash | |
| - name: "Elastic Tests 1 [attempt 2 of 3]" | |
| id: Elastic_Tests_1_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Elastic_Tests_1 && steps.Elastic_Tests_1_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Elastic_Tests_1_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Elastic_Tests_1_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c "cd /horovod/test/integration && HOROVOD_LOG_LEVEL=DEBUG pytest --forked -v --log-cli-level 10 --log-cli-format '[%(asctime)-15s %(levelname)s %(filename)s:%(lineno)d %(funcName)s()] %(message)s' --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.gloo.elastic.xml test_elastic_torch.py test_elastic_tensorflow2.py" | |
| shell: bash | |
| - name: "Elastic Tests 1 [attempt 3 of 3]" | |
| id: Elastic_Tests_1_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Elastic_Tests_1 && steps.Elastic_Tests_1_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Elastic_Tests_1_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Elastic_Tests_1_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c "cd /horovod/test/integration && HOROVOD_LOG_LEVEL=DEBUG pytest --forked -v --log-cli-level 10 --log-cli-format '[%(asctime)-15s %(levelname)s %(filename)s:%(lineno)d %(funcName)s()] %(message)s' --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.gloo.elastic.xml test_elastic_torch.py test_elastic_tensorflow2.py" | |
| shell: bash | |
| - name: "Elastic Tests 2 [attempt 1 of 3]" | |
| id: Elastic_Tests_2_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Elastic_Tests_2 && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Elastic_Tests_2_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Elastic_Tests_2_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c "cd /horovod/test/integration && HOROVOD_LOG_LEVEL=DEBUG pytest --forked -v --log-cli-level 10 --log-cli-format '[%(asctime)-15s %(levelname)s %(filename)s:%(lineno)d %(funcName)s()] %(message)s' --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.gloo.elastic.xml test_elastic_torch.py test_elastic_tensorflow.py test_elastic_tensorflow_keras.py" | |
| shell: bash | |
| - name: "Elastic Tests 2 [attempt 2 of 3]" | |
| id: Elastic_Tests_2_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Elastic_Tests_2 && steps.Elastic_Tests_2_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Elastic_Tests_2_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Elastic_Tests_2_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c "cd /horovod/test/integration && HOROVOD_LOG_LEVEL=DEBUG pytest --forked -v --log-cli-level 10 --log-cli-format '[%(asctime)-15s %(levelname)s %(filename)s:%(lineno)d %(funcName)s()] %(message)s' --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.gloo.elastic.xml test_elastic_torch.py test_elastic_tensorflow.py test_elastic_tensorflow_keras.py" | |
| shell: bash | |
| - name: "Elastic Tests 2 [attempt 3 of 3]" | |
| id: Elastic_Tests_2_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Elastic_Tests_2 && steps.Elastic_Tests_2_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Elastic_Tests_2_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Elastic_Tests_2_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c "cd /horovod/test/integration && HOROVOD_LOG_LEVEL=DEBUG pytest --forked -v --log-cli-level 10 --log-cli-format '[%(asctime)-15s %(levelname)s %(filename)s:%(lineno)d %(funcName)s()] %(message)s' --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.gloo.elastic.xml test_elastic_torch.py test_elastic_tensorflow.py test_elastic_tensorflow_keras.py" | |
| shell: bash | |
| - name: "Gloo Cluster PyTests [attempt 1 of 3]" | |
| id: Gloo_Cluster_PyTests_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_Cluster_PyTests && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_Cluster_PyTests_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_Cluster_PyTests_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " /etc/init.d/ssh start && cd /horovod/test/integration && pytest --forked -v --capture=fd --continue-on-collection-errors --junit-xml=/artifacts/junit.gloo.static.xml test_static_run.py" | |
| shell: bash | |
| - name: "Gloo Cluster PyTests [attempt 2 of 3]" | |
| id: Gloo_Cluster_PyTests_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_Cluster_PyTests && steps.Gloo_Cluster_PyTests_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_Cluster_PyTests_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_Cluster_PyTests_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " /etc/init.d/ssh start && cd /horovod/test/integration && pytest --forked -v --capture=fd --continue-on-collection-errors --junit-xml=/artifacts/junit.gloo.static.xml test_static_run.py" | |
| shell: bash | |
| - name: "Gloo Cluster PyTests [attempt 3 of 3]" | |
| id: Gloo_Cluster_PyTests_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_Cluster_PyTests && steps.Gloo_Cluster_PyTests_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_Cluster_PyTests_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_Cluster_PyTests_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " /etc/init.d/ssh start && cd /horovod/test/integration && pytest --forked -v --capture=fd --continue-on-collection-errors --junit-xml=/artifacts/junit.gloo.static.xml test_static_run.py" | |
| shell: bash | |
| - name: "Gloo Keras MNIST [attempt 1 of 3]" | |
| id: Gloo_Keras_MNIST_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_Keras_MNIST && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_Keras_MNIST_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_Keras_MNIST_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/keras/keras_mnist_advanced.py | |
| shell: bash | |
| - name: "Gloo Keras MNIST [attempt 2 of 3]" | |
| id: Gloo_Keras_MNIST_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_Keras_MNIST && steps.Gloo_Keras_MNIST_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_Keras_MNIST_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_Keras_MNIST_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/keras/keras_mnist_advanced.py | |
| shell: bash | |
| - name: "Gloo Keras MNIST [attempt 3 of 3]" | |
| id: Gloo_Keras_MNIST_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_Keras_MNIST && steps.Gloo_Keras_MNIST_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_Keras_MNIST_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_Keras_MNIST_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/keras/keras_mnist_advanced.py | |
| shell: bash | |
| - name: "Gloo MXNet2 MNIST api [attempt 1 of 3]" | |
| id: Gloo_MXNet2_MNIST_api_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_MXNet2_MNIST_api && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_MXNet2_MNIST_api_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_MXNet2_MNIST_api_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m python /horovod/examples/mxnet/mxnet2_mnist.py --num-proc 2 --hosts localhost:2 --communication gloo | |
| shell: bash | |
| - name: "Gloo MXNet2 MNIST api [attempt 2 of 3]" | |
| id: Gloo_MXNet2_MNIST_api_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_MXNet2_MNIST_api && steps.Gloo_MXNet2_MNIST_api_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_MXNet2_MNIST_api_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_MXNet2_MNIST_api_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m python /horovod/examples/mxnet/mxnet2_mnist.py --num-proc 2 --hosts localhost:2 --communication gloo | |
| shell: bash | |
| - name: "Gloo MXNet2 MNIST api [attempt 3 of 3]" | |
| id: Gloo_MXNet2_MNIST_api_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_MXNet2_MNIST_api && steps.Gloo_MXNet2_MNIST_api_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_MXNet2_MNIST_api_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_MXNet2_MNIST_api_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m python /horovod/examples/mxnet/mxnet2_mnist.py --num-proc 2 --hosts localhost:2 --communication gloo | |
| shell: bash | |
| - name: "Gloo MXNet2 MNIST horovodrun [attempt 1 of 3]" | |
| id: Gloo_MXNet2_MNIST_horovodrun_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_MXNet2_MNIST_horovodrun && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_MXNet2_MNIST_horovodrun_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_MXNet2_MNIST_horovodrun_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/mxnet/mxnet2_mnist.py | |
| shell: bash | |
| - name: "Gloo MXNet2 MNIST horovodrun [attempt 2 of 3]" | |
| id: Gloo_MXNet2_MNIST_horovodrun_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_MXNet2_MNIST_horovodrun && steps.Gloo_MXNet2_MNIST_horovodrun_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_MXNet2_MNIST_horovodrun_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_MXNet2_MNIST_horovodrun_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/mxnet/mxnet2_mnist.py | |
| shell: bash | |
| - name: "Gloo MXNet2 MNIST horovodrun [attempt 3 of 3]" | |
| id: Gloo_MXNet2_MNIST_horovodrun_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_MXNet2_MNIST_horovodrun && steps.Gloo_MXNet2_MNIST_horovodrun_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_MXNet2_MNIST_horovodrun_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_MXNet2_MNIST_horovodrun_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/mxnet/mxnet2_mnist.py | |
| shell: bash | |
| - name: "Gloo MXNet MNIST horovodrun [attempt 1 of 3]" | |
| id: Gloo_MXNet_MNIST_horovodrun_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_MXNet_MNIST_horovodrun && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_MXNet_MNIST_horovodrun_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_MXNet_MNIST_horovodrun_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/mxnet/mxnet_mnist.py | |
| shell: bash | |
| - name: "Gloo MXNet MNIST horovodrun [attempt 2 of 3]" | |
| id: Gloo_MXNet_MNIST_horovodrun_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_MXNet_MNIST_horovodrun && steps.Gloo_MXNet_MNIST_horovodrun_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_MXNet_MNIST_horovodrun_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_MXNet_MNIST_horovodrun_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/mxnet/mxnet_mnist.py | |
| shell: bash | |
| - name: "Gloo MXNet MNIST horovodrun [attempt 3 of 3]" | |
| id: Gloo_MXNet_MNIST_horovodrun_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_MXNet_MNIST_horovodrun && steps.Gloo_MXNet_MNIST_horovodrun_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_MXNet_MNIST_horovodrun_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_MXNet_MNIST_horovodrun_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/mxnet/mxnet_mnist.py | |
| shell: bash | |
| - name: "Gloo Parallel PyTests [attempt 1 of 3]" | |
| id: Gloo_Parallel_PyTests_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_Parallel_PyTests && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_Parallel_PyTests_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_Parallel_PyTests_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c " cd /horovod/test/parallel && (ls -1 test_*.py | xargs -n 1 horovodrun -np 2 -H localhost:2 --gloo /bin/bash /pytest.sh gloo)" | |
| shell: bash | |
| - name: "Gloo Parallel PyTests [attempt 2 of 3]" | |
| id: Gloo_Parallel_PyTests_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_Parallel_PyTests && steps.Gloo_Parallel_PyTests_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_Parallel_PyTests_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_Parallel_PyTests_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c " cd /horovod/test/parallel && (ls -1 test_*.py | xargs -n 1 horovodrun -np 2 -H localhost:2 --gloo /bin/bash /pytest.sh gloo)" | |
| shell: bash | |
| - name: "Gloo Parallel PyTests [attempt 3 of 3]" | |
| id: Gloo_Parallel_PyTests_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_Parallel_PyTests && steps.Gloo_Parallel_PyTests_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_Parallel_PyTests_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_Parallel_PyTests_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c " cd /horovod/test/parallel && (ls -1 test_*.py | xargs -n 1 horovodrun -np 2 -H localhost:2 --gloo /bin/bash /pytest.sh gloo)" | |
| shell: bash | |
| - name: "Gloo PyTorch MNIST api [attempt 1 of 3]" | |
| id: Gloo_PyTorch_MNIST_api_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_PyTorch_MNIST_api && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_PyTorch_MNIST_api_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_PyTorch_MNIST_api_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets --num-proc 2 --hosts localhost:2 --communication gloo | |
| shell: bash | |
| - name: "Gloo PyTorch MNIST api [attempt 2 of 3]" | |
| id: Gloo_PyTorch_MNIST_api_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_PyTorch_MNIST_api && steps.Gloo_PyTorch_MNIST_api_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_PyTorch_MNIST_api_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_PyTorch_MNIST_api_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets --num-proc 2 --hosts localhost:2 --communication gloo | |
| shell: bash | |
| - name: "Gloo PyTorch MNIST api [attempt 3 of 3]" | |
| id: Gloo_PyTorch_MNIST_api_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_PyTorch_MNIST_api && steps.Gloo_PyTorch_MNIST_api_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_PyTorch_MNIST_api_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_PyTorch_MNIST_api_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets --num-proc 2 --hosts localhost:2 --communication gloo | |
| shell: bash | |
| - name: "Gloo PyTorch MNIST horovodrun [attempt 1 of 3]" | |
| id: Gloo_PyTorch_MNIST_horovodrun_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_PyTorch_MNIST_horovodrun && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_PyTorch_MNIST_horovodrun_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_PyTorch_MNIST_horovodrun_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets | |
| shell: bash | |
| - name: "Gloo PyTorch MNIST horovodrun [attempt 2 of 3]" | |
| id: Gloo_PyTorch_MNIST_horovodrun_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_PyTorch_MNIST_horovodrun && steps.Gloo_PyTorch_MNIST_horovodrun_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_PyTorch_MNIST_horovodrun_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_PyTorch_MNIST_horovodrun_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets | |
| shell: bash | |
| - name: "Gloo PyTorch MNIST horovodrun [attempt 3 of 3]" | |
| id: Gloo_PyTorch_MNIST_horovodrun_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_PyTorch_MNIST_horovodrun && steps.Gloo_PyTorch_MNIST_horovodrun_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_PyTorch_MNIST_horovodrun_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_PyTorch_MNIST_horovodrun_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets | |
| shell: bash | |
| - name: "Gloo Single PyTests [attempt 1 of 3]" | |
| id: Gloo_Single_PyTests_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_Single_PyTests && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_Single_PyTests_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_Single_PyTests_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c " cd /horovod/test/single && (ls -1 test_*.py | xargs -n 1 /bin/bash /pytest_standalone.sh gloo)" | |
| shell: bash | |
| - name: "Gloo Single PyTests [attempt 2 of 3]" | |
| id: Gloo_Single_PyTests_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_Single_PyTests && steps.Gloo_Single_PyTests_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_Single_PyTests_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_Single_PyTests_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c " cd /horovod/test/single && (ls -1 test_*.py | xargs -n 1 /bin/bash /pytest_standalone.sh gloo)" | |
| shell: bash | |
| - name: "Gloo Single PyTests [attempt 3 of 3]" | |
| id: Gloo_Single_PyTests_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_Single_PyTests && steps.Gloo_Single_PyTests_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_Single_PyTests_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_Single_PyTests_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c " cd /horovod/test/single && (ls -1 test_*.py | xargs -n 1 /bin/bash /pytest_standalone.sh gloo)" | |
| shell: bash | |
| - name: "Gloo TensorFlow 2.0 Keras MNIST api [attempt 1 of 3]" | |
| id: Gloo_TensorFlow_2_0_Keras_MNIST_api_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_2_0_Keras_MNIST_api && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_Keras_MNIST_api_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_Keras_MNIST_api_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py 2 localhost:2 gloo | |
| shell: bash | |
| - name: "Gloo TensorFlow 2.0 Keras MNIST api [attempt 2 of 3]" | |
| id: Gloo_TensorFlow_2_0_Keras_MNIST_api_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_2_0_Keras_MNIST_api && steps.Gloo_TensorFlow_2_0_Keras_MNIST_api_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_Keras_MNIST_api_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_Keras_MNIST_api_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py 2 localhost:2 gloo | |
| shell: bash | |
| - name: "Gloo TensorFlow 2.0 Keras MNIST api [attempt 3 of 3]" | |
| id: Gloo_TensorFlow_2_0_Keras_MNIST_api_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_2_0_Keras_MNIST_api && steps.Gloo_TensorFlow_2_0_Keras_MNIST_api_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_Keras_MNIST_api_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_Keras_MNIST_api_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py 2 localhost:2 gloo | |
| shell: bash | |
| - name: "Gloo TensorFlow 2.0 Keras MNIST horovodrun [attempt 1 of 3]" | |
| id: Gloo_TensorFlow_2_0_Keras_MNIST_horovodrun_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_2_0_Keras_MNIST_horovodrun && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_Keras_MNIST_horovodrun_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_Keras_MNIST_horovodrun_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py | |
| shell: bash | |
| - name: "Gloo TensorFlow 2.0 Keras MNIST horovodrun [attempt 2 of 3]" | |
| id: Gloo_TensorFlow_2_0_Keras_MNIST_horovodrun_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_2_0_Keras_MNIST_horovodrun && steps.Gloo_TensorFlow_2_0_Keras_MNIST_horovodrun_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_Keras_MNIST_horovodrun_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_Keras_MNIST_horovodrun_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py | |
| shell: bash | |
| - name: "Gloo TensorFlow 2.0 Keras MNIST horovodrun [attempt 3 of 3]" | |
| id: Gloo_TensorFlow_2_0_Keras_MNIST_horovodrun_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_2_0_Keras_MNIST_horovodrun && steps.Gloo_TensorFlow_2_0_Keras_MNIST_horovodrun_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_Keras_MNIST_horovodrun_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_Keras_MNIST_horovodrun_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py | |
| shell: bash | |
| - name: "Gloo TensorFlow 2.0 MNIST Data Service [attempt 1 of 3]" | |
| id: Gloo_TensorFlow_2_0_MNIST_Data_Service_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_2_0_MNIST_Data_Service && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_Data_Service_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_Data_Service_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "horovodrun -np 2 python -m horovod.tensorflow.data.compute_worker /tmp/compute.json & horovodrun -np 2 --gloo python /horovod/examples/tensorflow2/tensorflow2_mnist_data_service.py /tmp/compute.json" | |
| shell: bash | |
| - name: "Gloo TensorFlow 2.0 MNIST Data Service [attempt 2 of 3]" | |
| id: Gloo_TensorFlow_2_0_MNIST_Data_Service_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_2_0_MNIST_Data_Service && steps.Gloo_TensorFlow_2_0_MNIST_Data_Service_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_Data_Service_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_Data_Service_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "horovodrun -np 2 python -m horovod.tensorflow.data.compute_worker /tmp/compute.json & horovodrun -np 2 --gloo python /horovod/examples/tensorflow2/tensorflow2_mnist_data_service.py /tmp/compute.json" | |
| shell: bash | |
| - name: "Gloo TensorFlow 2.0 MNIST Data Service [attempt 3 of 3]" | |
| id: Gloo_TensorFlow_2_0_MNIST_Data_Service_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_2_0_MNIST_Data_Service && steps.Gloo_TensorFlow_2_0_MNIST_Data_Service_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_Data_Service_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_Data_Service_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "horovodrun -np 2 python -m horovod.tensorflow.data.compute_worker /tmp/compute.json & horovodrun -np 2 --gloo python /horovod/examples/tensorflow2/tensorflow2_mnist_data_service.py /tmp/compute.json" | |
| shell: bash | |
| - name: "Gloo TensorFlow 2.0 MNIST Elastic api [attempt 1 of 3]" | |
| id: Gloo_TensorFlow_2_0_MNIST_Elastic_api_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_2_0_MNIST_Elastic_api && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_Elastic_api_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_Elastic_api_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m python /horovod/examples/elastic/tensorflow2/tensorflow2_mnist_elastic.py 2 2 2 localhost:2,127.0.0.1:2 | |
| shell: bash | |
| - name: "Gloo TensorFlow 2.0 MNIST Elastic api [attempt 2 of 3]" | |
| id: Gloo_TensorFlow_2_0_MNIST_Elastic_api_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_2_0_MNIST_Elastic_api && steps.Gloo_TensorFlow_2_0_MNIST_Elastic_api_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_Elastic_api_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_Elastic_api_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m python /horovod/examples/elastic/tensorflow2/tensorflow2_mnist_elastic.py 2 2 2 localhost:2,127.0.0.1:2 | |
| shell: bash | |
| - name: "Gloo TensorFlow 2.0 MNIST Elastic api [attempt 3 of 3]" | |
| id: Gloo_TensorFlow_2_0_MNIST_Elastic_api_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_2_0_MNIST_Elastic_api && steps.Gloo_TensorFlow_2_0_MNIST_Elastic_api_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_Elastic_api_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_Elastic_api_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m python /horovod/examples/elastic/tensorflow2/tensorflow2_mnist_elastic.py 2 2 2 localhost:2,127.0.0.1:2 | |
| shell: bash | |
| - name: "Gloo TensorFlow 2.0 MNIST Elastic horovodrun [attempt 1 of 3]" | |
| id: Gloo_TensorFlow_2_0_MNIST_Elastic_horovodrun_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_2_0_MNIST_Elastic_horovodrun && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_Elastic_horovodrun_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_Elastic_horovodrun_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 --min-np 2 --max-np 2 -H localhost:2,127.0.0.1:2 --gloo python /horovod/examples/elastic/tensorflow2/tensorflow2_mnist_elastic.py | |
| shell: bash | |
| - name: "Gloo TensorFlow 2.0 MNIST Elastic horovodrun [attempt 2 of 3]" | |
| id: Gloo_TensorFlow_2_0_MNIST_Elastic_horovodrun_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_2_0_MNIST_Elastic_horovodrun && steps.Gloo_TensorFlow_2_0_MNIST_Elastic_horovodrun_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_Elastic_horovodrun_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_Elastic_horovodrun_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 --min-np 2 --max-np 2 -H localhost:2,127.0.0.1:2 --gloo python /horovod/examples/elastic/tensorflow2/tensorflow2_mnist_elastic.py | |
| shell: bash | |
| - name: "Gloo TensorFlow 2.0 MNIST Elastic horovodrun [attempt 3 of 3]" | |
| id: Gloo_TensorFlow_2_0_MNIST_Elastic_horovodrun_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_2_0_MNIST_Elastic_horovodrun && steps.Gloo_TensorFlow_2_0_MNIST_Elastic_horovodrun_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_Elastic_horovodrun_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_Elastic_horovodrun_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 --min-np 2 --max-np 2 -H localhost:2,127.0.0.1:2 --gloo python /horovod/examples/elastic/tensorflow2/tensorflow2_mnist_elastic.py | |
| shell: bash | |
| - name: "Gloo TensorFlow 2.0 MNIST api [attempt 1 of 3]" | |
| id: Gloo_TensorFlow_2_0_MNIST_api_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_2_0_MNIST_api && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_api_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_api_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m python /horovod/examples/tensorflow2/tensorflow2_mnist.py 2 localhost:2 gloo | |
| shell: bash | |
| - name: "Gloo TensorFlow 2.0 MNIST api [attempt 2 of 3]" | |
| id: Gloo_TensorFlow_2_0_MNIST_api_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_2_0_MNIST_api && steps.Gloo_TensorFlow_2_0_MNIST_api_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_api_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_api_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m python /horovod/examples/tensorflow2/tensorflow2_mnist.py 2 localhost:2 gloo | |
| shell: bash | |
| - name: "Gloo TensorFlow 2.0 MNIST api [attempt 3 of 3]" | |
| id: Gloo_TensorFlow_2_0_MNIST_api_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_2_0_MNIST_api && steps.Gloo_TensorFlow_2_0_MNIST_api_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_api_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_api_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m python /horovod/examples/tensorflow2/tensorflow2_mnist.py 2 localhost:2 gloo | |
| shell: bash | |
| - name: "Gloo TensorFlow 2.0 MNIST horovodrun [attempt 1 of 3]" | |
| id: Gloo_TensorFlow_2_0_MNIST_horovodrun_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_2_0_MNIST_horovodrun && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_horovodrun_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_horovodrun_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/tensorflow2/tensorflow2_mnist.py | |
| shell: bash | |
| - name: "Gloo TensorFlow 2.0 MNIST horovodrun [attempt 2 of 3]" | |
| id: Gloo_TensorFlow_2_0_MNIST_horovodrun_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_2_0_MNIST_horovodrun && steps.Gloo_TensorFlow_2_0_MNIST_horovodrun_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_horovodrun_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_horovodrun_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/tensorflow2/tensorflow2_mnist.py | |
| shell: bash | |
| - name: "Gloo TensorFlow 2.0 MNIST horovodrun [attempt 3 of 3]" | |
| id: Gloo_TensorFlow_2_0_MNIST_horovodrun_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_2_0_MNIST_horovodrun && steps.Gloo_TensorFlow_2_0_MNIST_horovodrun_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_horovodrun_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_2_0_MNIST_horovodrun_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/tensorflow2/tensorflow2_mnist.py | |
| shell: bash | |
| - name: "Gloo TensorFlow MNIST [attempt 1 of 3]" | |
| id: Gloo_TensorFlow_MNIST_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_MNIST && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_MNIST_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_MNIST_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/tensorflow/tensorflow_mnist.py | |
| shell: bash | |
| - name: "Gloo TensorFlow MNIST [attempt 2 of 3]" | |
| id: Gloo_TensorFlow_MNIST_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_MNIST && steps.Gloo_TensorFlow_MNIST_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_MNIST_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_MNIST_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/tensorflow/tensorflow_mnist.py | |
| shell: bash | |
| - name: "Gloo TensorFlow MNIST [attempt 3 of 3]" | |
| id: Gloo_TensorFlow_MNIST_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Gloo_TensorFlow_MNIST && steps.Gloo_TensorFlow_MNIST_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Gloo_TensorFlow_MNIST_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Gloo_TensorFlow_MNIST_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/tensorflow/tensorflow_mnist.py | |
| shell: bash | |
| - name: "MPI Cluster PyTests [attempt 1 of 3]" | |
| id: MPI_Cluster_PyTests_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Cluster_PyTests && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Cluster_PyTests_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Cluster_PyTests_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " /etc/init.d/ssh start && cd /horovod/test/integration && pytest --forked -v --capture=fd --continue-on-collection-errors --junit-xml=/artifacts/junit.mpi.static.xml test_static_run.py" | |
| shell: bash | |
| - name: "MPI Cluster PyTests [attempt 2 of 3]" | |
| id: MPI_Cluster_PyTests_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Cluster_PyTests && steps.MPI_Cluster_PyTests_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Cluster_PyTests_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Cluster_PyTests_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " /etc/init.d/ssh start && cd /horovod/test/integration && pytest --forked -v --capture=fd --continue-on-collection-errors --junit-xml=/artifacts/junit.mpi.static.xml test_static_run.py" | |
| shell: bash | |
| - name: "MPI Cluster PyTests [attempt 3 of 3]" | |
| id: MPI_Cluster_PyTests_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Cluster_PyTests && steps.MPI_Cluster_PyTests_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Cluster_PyTests_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Cluster_PyTests_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " /etc/init.d/ssh start && cd /horovod/test/integration && pytest --forked -v --capture=fd --continue-on-collection-errors --junit-xml=/artifacts/junit.mpi.static.xml test_static_run.py" | |
| shell: bash | |
| - name: "MPI Cluster PyTests [ONECCL MPI] [attempt 1 of 3]" | |
| id: MPI_Cluster_PyTests_ONECCL_MPI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Cluster_PyTests_ONECCL_MPI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Cluster_PyTests_ONECCL_MPI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Cluster_PyTests_ONECCL_MPI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && /etc/init.d/ssh start && cd /horovod/test/integration && pytest --forked -v --capture=fd --continue-on-collection-errors --junit-xml=/artifacts/junit.mpi.static.xml test_static_run.py" | |
| shell: bash | |
| - name: "MPI Cluster PyTests [ONECCL MPI] [attempt 2 of 3]" | |
| id: MPI_Cluster_PyTests_ONECCL_MPI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Cluster_PyTests_ONECCL_MPI && steps.MPI_Cluster_PyTests_ONECCL_MPI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Cluster_PyTests_ONECCL_MPI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Cluster_PyTests_ONECCL_MPI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && /etc/init.d/ssh start && cd /horovod/test/integration && pytest --forked -v --capture=fd --continue-on-collection-errors --junit-xml=/artifacts/junit.mpi.static.xml test_static_run.py" | |
| shell: bash | |
| - name: "MPI Cluster PyTests [ONECCL MPI] [attempt 3 of 3]" | |
| id: MPI_Cluster_PyTests_ONECCL_MPI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Cluster_PyTests_ONECCL_MPI && steps.MPI_Cluster_PyTests_ONECCL_MPI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Cluster_PyTests_ONECCL_MPI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Cluster_PyTests_ONECCL_MPI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && /etc/init.d/ssh start && cd /horovod/test/integration && pytest --forked -v --capture=fd --continue-on-collection-errors --junit-xml=/artifacts/junit.mpi.static.xml test_static_run.py" | |
| shell: bash | |
| - name: "MPI Cluster PyTests [ONECCL OFI] [attempt 1 of 3]" | |
| id: MPI_Cluster_PyTests_ONECCL_OFI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Cluster_PyTests_ONECCL_OFI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Cluster_PyTests_ONECCL_OFI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Cluster_PyTests_ONECCL_OFI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && /etc/init.d/ssh start && cd /horovod/test/integration && pytest --forked -v --capture=fd --continue-on-collection-errors --junit-xml=/artifacts/junit.mpi.static.xml test_static_run.py" | |
| shell: bash | |
| - name: "MPI Cluster PyTests [ONECCL OFI] [attempt 2 of 3]" | |
| id: MPI_Cluster_PyTests_ONECCL_OFI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Cluster_PyTests_ONECCL_OFI && steps.MPI_Cluster_PyTests_ONECCL_OFI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Cluster_PyTests_ONECCL_OFI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Cluster_PyTests_ONECCL_OFI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && /etc/init.d/ssh start && cd /horovod/test/integration && pytest --forked -v --capture=fd --continue-on-collection-errors --junit-xml=/artifacts/junit.mpi.static.xml test_static_run.py" | |
| shell: bash | |
| - name: "MPI Cluster PyTests [ONECCL OFI] [attempt 3 of 3]" | |
| id: MPI_Cluster_PyTests_ONECCL_OFI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Cluster_PyTests_ONECCL_OFI && steps.MPI_Cluster_PyTests_ONECCL_OFI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Cluster_PyTests_ONECCL_OFI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Cluster_PyTests_ONECCL_OFI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && /etc/init.d/ssh start && cd /horovod/test/integration && pytest --forked -v --capture=fd --continue-on-collection-errors --junit-xml=/artifacts/junit.mpi.static.xml test_static_run.py" | |
| shell: bash | |
| - name: "MPI MXNet2 MNIST api [attempt 1 of 3]" | |
| id: MPI_MXNet2_MNIST_api_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_MXNet2_MNIST_api && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_MXNet2_MNIST_api_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_MXNet2_MNIST_api_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/mxnet/mxnet2_mnist.py --num-proc 2 --hosts localhost:2 --communication mpi" | |
| shell: bash | |
| - name: "MPI MXNet2 MNIST api [attempt 2 of 3]" | |
| id: MPI_MXNet2_MNIST_api_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_MXNet2_MNIST_api && steps.MPI_MXNet2_MNIST_api_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_MXNet2_MNIST_api_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_MXNet2_MNIST_api_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/mxnet/mxnet2_mnist.py --num-proc 2 --hosts localhost:2 --communication mpi" | |
| shell: bash | |
| - name: "MPI MXNet2 MNIST api [attempt 3 of 3]" | |
| id: MPI_MXNet2_MNIST_api_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_MXNet2_MNIST_api && steps.MPI_MXNet2_MNIST_api_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_MXNet2_MNIST_api_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_MXNet2_MNIST_api_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/mxnet/mxnet2_mnist.py --num-proc 2 --hosts localhost:2 --communication mpi" | |
| shell: bash | |
| - name: "MPI MXNet2 MNIST horovodrun [attempt 1 of 3]" | |
| id: MPI_MXNet2_MNIST_horovodrun_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_MXNet2_MNIST_horovodrun && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_MXNet2_MNIST_horovodrun_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_MXNet2_MNIST_horovodrun_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " OMP_NUM_THREADS=1 \$(cat /mpirun_command) python /horovod/examples/mxnet/mxnet2_mnist.py" | |
| shell: bash | |
| - name: "MPI MXNet2 MNIST horovodrun [attempt 2 of 3]" | |
| id: MPI_MXNet2_MNIST_horovodrun_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_MXNet2_MNIST_horovodrun && steps.MPI_MXNet2_MNIST_horovodrun_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_MXNet2_MNIST_horovodrun_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_MXNet2_MNIST_horovodrun_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " OMP_NUM_THREADS=1 \$(cat /mpirun_command) python /horovod/examples/mxnet/mxnet2_mnist.py" | |
| shell: bash | |
| - name: "MPI MXNet2 MNIST horovodrun [attempt 3 of 3]" | |
| id: MPI_MXNet2_MNIST_horovodrun_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_MXNet2_MNIST_horovodrun && steps.MPI_MXNet2_MNIST_horovodrun_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_MXNet2_MNIST_horovodrun_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_MXNet2_MNIST_horovodrun_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " OMP_NUM_THREADS=1 \$(cat /mpirun_command) python /horovod/examples/mxnet/mxnet2_mnist.py" | |
| shell: bash | |
| - name: "MPI MXNet MNIST horovodrun [attempt 1 of 3]" | |
| id: MPI_MXNet_MNIST_horovodrun_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_MXNet_MNIST_horovodrun && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_MXNet_MNIST_horovodrun_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_MXNet_MNIST_horovodrun_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " OMP_NUM_THREADS=1 \$(cat /mpirun_command) python /horovod/examples/mxnet/mxnet_mnist.py" | |
| shell: bash | |
| - name: "MPI MXNet MNIST horovodrun [attempt 2 of 3]" | |
| id: MPI_MXNet_MNIST_horovodrun_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_MXNet_MNIST_horovodrun && steps.MPI_MXNet_MNIST_horovodrun_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_MXNet_MNIST_horovodrun_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_MXNet_MNIST_horovodrun_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " OMP_NUM_THREADS=1 \$(cat /mpirun_command) python /horovod/examples/mxnet/mxnet_mnist.py" | |
| shell: bash | |
| - name: "MPI MXNet MNIST horovodrun [attempt 3 of 3]" | |
| id: MPI_MXNet_MNIST_horovodrun_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_MXNet_MNIST_horovodrun && steps.MPI_MXNet_MNIST_horovodrun_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_MXNet_MNIST_horovodrun_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_MXNet_MNIST_horovodrun_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " OMP_NUM_THREADS=1 \$(cat /mpirun_command) python /horovod/examples/mxnet/mxnet_mnist.py" | |
| shell: bash | |
| - name: "MPI MXNet MNIST horovodrun [ONECCL MPI] [attempt 1 of 3]" | |
| id: MPI_MXNet_MNIST_horovodrun_ONECCL_MPI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_MXNet_MNIST_horovodrun_ONECCL_MPI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_MXNet_MNIST_horovodrun_ONECCL_MPI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_MXNet_MNIST_horovodrun_ONECCL_MPI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && OMP_NUM_THREADS=1 \$(cat /mpirun_command) python /horovod/examples/mxnet/mxnet_mnist.py" | |
| shell: bash | |
| - name: "MPI MXNet MNIST horovodrun [ONECCL MPI] [attempt 2 of 3]" | |
| id: MPI_MXNet_MNIST_horovodrun_ONECCL_MPI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_MXNet_MNIST_horovodrun_ONECCL_MPI && steps.MPI_MXNet_MNIST_horovodrun_ONECCL_MPI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_MXNet_MNIST_horovodrun_ONECCL_MPI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_MXNet_MNIST_horovodrun_ONECCL_MPI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && OMP_NUM_THREADS=1 \$(cat /mpirun_command) python /horovod/examples/mxnet/mxnet_mnist.py" | |
| shell: bash | |
| - name: "MPI MXNet MNIST horovodrun [ONECCL MPI] [attempt 3 of 3]" | |
| id: MPI_MXNet_MNIST_horovodrun_ONECCL_MPI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_MXNet_MNIST_horovodrun_ONECCL_MPI && steps.MPI_MXNet_MNIST_horovodrun_ONECCL_MPI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_MXNet_MNIST_horovodrun_ONECCL_MPI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_MXNet_MNIST_horovodrun_ONECCL_MPI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && OMP_NUM_THREADS=1 \$(cat /mpirun_command) python /horovod/examples/mxnet/mxnet_mnist.py" | |
| shell: bash | |
| - name: "MPI MXNet MNIST horovodrun [ONECCL OFI] [attempt 1 of 3]" | |
| id: MPI_MXNet_MNIST_horovodrun_ONECCL_OFI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_MXNet_MNIST_horovodrun_ONECCL_OFI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_MXNet_MNIST_horovodrun_ONECCL_OFI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_MXNet_MNIST_horovodrun_ONECCL_OFI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && OMP_NUM_THREADS=1 \$(cat /mpirun_command) python /horovod/examples/mxnet/mxnet_mnist.py" | |
| shell: bash | |
| - name: "MPI MXNet MNIST horovodrun [ONECCL OFI] [attempt 2 of 3]" | |
| id: MPI_MXNet_MNIST_horovodrun_ONECCL_OFI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_MXNet_MNIST_horovodrun_ONECCL_OFI && steps.MPI_MXNet_MNIST_horovodrun_ONECCL_OFI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_MXNet_MNIST_horovodrun_ONECCL_OFI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_MXNet_MNIST_horovodrun_ONECCL_OFI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && OMP_NUM_THREADS=1 \$(cat /mpirun_command) python /horovod/examples/mxnet/mxnet_mnist.py" | |
| shell: bash | |
| - name: "MPI MXNet MNIST horovodrun [ONECCL OFI] [attempt 3 of 3]" | |
| id: MPI_MXNet_MNIST_horovodrun_ONECCL_OFI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_MXNet_MNIST_horovodrun_ONECCL_OFI && steps.MPI_MXNet_MNIST_horovodrun_ONECCL_OFI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_MXNet_MNIST_horovodrun_ONECCL_OFI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_MXNet_MNIST_horovodrun_ONECCL_OFI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && OMP_NUM_THREADS=1 \$(cat /mpirun_command) python /horovod/examples/mxnet/mxnet_mnist.py" | |
| shell: bash | |
| - name: "MPI Parallel PyTests [attempt 1 of 3]" | |
| id: MPI_Parallel_PyTests_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Parallel_PyTests && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Parallel_PyTests_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Parallel_PyTests_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c " cd /horovod/test/parallel && (ls -1 test_*.py | xargs -n 1 \$(cat /mpirun_command) /bin/bash /pytest.sh mpi)" | |
| shell: bash | |
| - name: "MPI Parallel PyTests [attempt 2 of 3]" | |
| id: MPI_Parallel_PyTests_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Parallel_PyTests && steps.MPI_Parallel_PyTests_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Parallel_PyTests_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Parallel_PyTests_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c " cd /horovod/test/parallel && (ls -1 test_*.py | xargs -n 1 \$(cat /mpirun_command) /bin/bash /pytest.sh mpi)" | |
| shell: bash | |
| - name: "MPI Parallel PyTests [attempt 3 of 3]" | |
| id: MPI_Parallel_PyTests_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Parallel_PyTests && steps.MPI_Parallel_PyTests_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Parallel_PyTests_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Parallel_PyTests_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c " cd /horovod/test/parallel && (ls -1 test_*.py | xargs -n 1 \$(cat /mpirun_command) /bin/bash /pytest.sh mpi)" | |
| shell: bash | |
| - name: "MPI Parallel PyTests [ONECCL MPI] [attempt 1 of 3]" | |
| id: MPI_Parallel_PyTests_ONECCL_MPI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Parallel_PyTests_ONECCL_MPI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Parallel_PyTests_ONECCL_MPI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Parallel_PyTests_ONECCL_MPI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && cd /horovod/test/parallel && (ls -1 test_*.py | xargs -n 1 \$(cat /mpirun_command) /bin/bash /pytest.sh mpi)" | |
| shell: bash | |
| - name: "MPI Parallel PyTests [ONECCL MPI] [attempt 2 of 3]" | |
| id: MPI_Parallel_PyTests_ONECCL_MPI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Parallel_PyTests_ONECCL_MPI && steps.MPI_Parallel_PyTests_ONECCL_MPI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Parallel_PyTests_ONECCL_MPI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Parallel_PyTests_ONECCL_MPI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && cd /horovod/test/parallel && (ls -1 test_*.py | xargs -n 1 \$(cat /mpirun_command) /bin/bash /pytest.sh mpi)" | |
| shell: bash | |
| - name: "MPI Parallel PyTests [ONECCL MPI] [attempt 3 of 3]" | |
| id: MPI_Parallel_PyTests_ONECCL_MPI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Parallel_PyTests_ONECCL_MPI && steps.MPI_Parallel_PyTests_ONECCL_MPI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Parallel_PyTests_ONECCL_MPI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Parallel_PyTests_ONECCL_MPI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && cd /horovod/test/parallel && (ls -1 test_*.py | xargs -n 1 \$(cat /mpirun_command) /bin/bash /pytest.sh mpi)" | |
| shell: bash | |
| - name: "MPI Parallel PyTests [ONECCL OFI] [attempt 1 of 3]" | |
| id: MPI_Parallel_PyTests_ONECCL_OFI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Parallel_PyTests_ONECCL_OFI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Parallel_PyTests_ONECCL_OFI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Parallel_PyTests_ONECCL_OFI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && cd /horovod/test/parallel && (ls -1 test_*.py | xargs -n 1 \$(cat /mpirun_command) /bin/bash /pytest.sh mpi)" | |
| shell: bash | |
| - name: "MPI Parallel PyTests [ONECCL OFI] [attempt 2 of 3]" | |
| id: MPI_Parallel_PyTests_ONECCL_OFI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Parallel_PyTests_ONECCL_OFI && steps.MPI_Parallel_PyTests_ONECCL_OFI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Parallel_PyTests_ONECCL_OFI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Parallel_PyTests_ONECCL_OFI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && cd /horovod/test/parallel && (ls -1 test_*.py | xargs -n 1 \$(cat /mpirun_command) /bin/bash /pytest.sh mpi)" | |
| shell: bash | |
| - name: "MPI Parallel PyTests [ONECCL OFI] [attempt 3 of 3]" | |
| id: MPI_Parallel_PyTests_ONECCL_OFI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Parallel_PyTests_ONECCL_OFI && steps.MPI_Parallel_PyTests_ONECCL_OFI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Parallel_PyTests_ONECCL_OFI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Parallel_PyTests_ONECCL_OFI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && cd /horovod/test/parallel && (ls -1 test_*.py | xargs -n 1 \$(cat /mpirun_command) /bin/bash /pytest.sh mpi)" | |
| shell: bash | |
| - name: "MPI PyTorch MNIST api [attempt 1 of 3]" | |
| id: MPI_PyTorch_MNIST_api_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_PyTorch_MNIST_api && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_api_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_api_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets --num-proc 2 --hosts localhost:2 --communication mpi" | |
| shell: bash | |
| - name: "MPI PyTorch MNIST api [attempt 2 of 3]" | |
| id: MPI_PyTorch_MNIST_api_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_PyTorch_MNIST_api && steps.MPI_PyTorch_MNIST_api_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_api_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_api_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets --num-proc 2 --hosts localhost:2 --communication mpi" | |
| shell: bash | |
| - name: "MPI PyTorch MNIST api [attempt 3 of 3]" | |
| id: MPI_PyTorch_MNIST_api_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_PyTorch_MNIST_api && steps.MPI_PyTorch_MNIST_api_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_api_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_api_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets --num-proc 2 --hosts localhost:2 --communication mpi" | |
| shell: bash | |
| - name: "MPI PyTorch MNIST api [ONECCL MPI] [attempt 1 of 3]" | |
| id: MPI_PyTorch_MNIST_api_ONECCL_MPI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_PyTorch_MNIST_api_ONECCL_MPI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_api_ONECCL_MPI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_api_ONECCL_MPI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets --num-proc 2 --hosts localhost:2 --communication mpi" | |
| shell: bash | |
| - name: "MPI PyTorch MNIST api [ONECCL MPI] [attempt 2 of 3]" | |
| id: MPI_PyTorch_MNIST_api_ONECCL_MPI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_PyTorch_MNIST_api_ONECCL_MPI && steps.MPI_PyTorch_MNIST_api_ONECCL_MPI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_api_ONECCL_MPI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_api_ONECCL_MPI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets --num-proc 2 --hosts localhost:2 --communication mpi" | |
| shell: bash | |
| - name: "MPI PyTorch MNIST api [ONECCL MPI] [attempt 3 of 3]" | |
| id: MPI_PyTorch_MNIST_api_ONECCL_MPI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_PyTorch_MNIST_api_ONECCL_MPI && steps.MPI_PyTorch_MNIST_api_ONECCL_MPI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_api_ONECCL_MPI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_api_ONECCL_MPI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets --num-proc 2 --hosts localhost:2 --communication mpi" | |
| shell: bash | |
| - name: "MPI PyTorch MNIST api [ONECCL OFI] [attempt 1 of 3]" | |
| id: MPI_PyTorch_MNIST_api_ONECCL_OFI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_PyTorch_MNIST_api_ONECCL_OFI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_api_ONECCL_OFI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_api_ONECCL_OFI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets --num-proc 2 --hosts localhost:2 --communication mpi" | |
| shell: bash | |
| - name: "MPI PyTorch MNIST api [ONECCL OFI] [attempt 2 of 3]" | |
| id: MPI_PyTorch_MNIST_api_ONECCL_OFI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_PyTorch_MNIST_api_ONECCL_OFI && steps.MPI_PyTorch_MNIST_api_ONECCL_OFI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_api_ONECCL_OFI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_api_ONECCL_OFI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets --num-proc 2 --hosts localhost:2 --communication mpi" | |
| shell: bash | |
| - name: "MPI PyTorch MNIST api [ONECCL OFI] [attempt 3 of 3]" | |
| id: MPI_PyTorch_MNIST_api_ONECCL_OFI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_PyTorch_MNIST_api_ONECCL_OFI && steps.MPI_PyTorch_MNIST_api_ONECCL_OFI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_api_ONECCL_OFI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_api_ONECCL_OFI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets --num-proc 2 --hosts localhost:2 --communication mpi" | |
| shell: bash | |
| - name: "MPI PyTorch MNIST horovodrun [attempt 1 of 3]" | |
| id: MPI_PyTorch_MNIST_horovodrun_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_PyTorch_MNIST_horovodrun && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_horovodrun_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_horovodrun_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " \$(cat /mpirun_command) python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets" | |
| shell: bash | |
| - name: "MPI PyTorch MNIST horovodrun [attempt 2 of 3]" | |
| id: MPI_PyTorch_MNIST_horovodrun_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_PyTorch_MNIST_horovodrun && steps.MPI_PyTorch_MNIST_horovodrun_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_horovodrun_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_horovodrun_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " \$(cat /mpirun_command) python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets" | |
| shell: bash | |
| - name: "MPI PyTorch MNIST horovodrun [attempt 3 of 3]" | |
| id: MPI_PyTorch_MNIST_horovodrun_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_PyTorch_MNIST_horovodrun && steps.MPI_PyTorch_MNIST_horovodrun_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_horovodrun_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_horovodrun_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " \$(cat /mpirun_command) python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets" | |
| shell: bash | |
| - name: "MPI PyTorch MNIST horovodrun [ONECCL MPI] [attempt 1 of 3]" | |
| id: MPI_PyTorch_MNIST_horovodrun_ONECCL_MPI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_PyTorch_MNIST_horovodrun_ONECCL_MPI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_horovodrun_ONECCL_MPI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_horovodrun_ONECCL_MPI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && \$(cat /mpirun_command) python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets" | |
| shell: bash | |
| - name: "MPI PyTorch MNIST horovodrun [ONECCL MPI] [attempt 2 of 3]" | |
| id: MPI_PyTorch_MNIST_horovodrun_ONECCL_MPI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_PyTorch_MNIST_horovodrun_ONECCL_MPI && steps.MPI_PyTorch_MNIST_horovodrun_ONECCL_MPI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_horovodrun_ONECCL_MPI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_horovodrun_ONECCL_MPI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && \$(cat /mpirun_command) python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets" | |
| shell: bash | |
| - name: "MPI PyTorch MNIST horovodrun [ONECCL MPI] [attempt 3 of 3]" | |
| id: MPI_PyTorch_MNIST_horovodrun_ONECCL_MPI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_PyTorch_MNIST_horovodrun_ONECCL_MPI && steps.MPI_PyTorch_MNIST_horovodrun_ONECCL_MPI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_horovodrun_ONECCL_MPI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_horovodrun_ONECCL_MPI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && \$(cat /mpirun_command) python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets" | |
| shell: bash | |
| - name: "MPI PyTorch MNIST horovodrun [ONECCL OFI] [attempt 1 of 3]" | |
| id: MPI_PyTorch_MNIST_horovodrun_ONECCL_OFI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_PyTorch_MNIST_horovodrun_ONECCL_OFI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_horovodrun_ONECCL_OFI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_horovodrun_ONECCL_OFI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && \$(cat /mpirun_command) python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets" | |
| shell: bash | |
| - name: "MPI PyTorch MNIST horovodrun [ONECCL OFI] [attempt 2 of 3]" | |
| id: MPI_PyTorch_MNIST_horovodrun_ONECCL_OFI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_PyTorch_MNIST_horovodrun_ONECCL_OFI && steps.MPI_PyTorch_MNIST_horovodrun_ONECCL_OFI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_horovodrun_ONECCL_OFI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_horovodrun_ONECCL_OFI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && \$(cat /mpirun_command) python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets" | |
| shell: bash | |
| - name: "MPI PyTorch MNIST horovodrun [ONECCL OFI] [attempt 3 of 3]" | |
| id: MPI_PyTorch_MNIST_horovodrun_ONECCL_OFI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_PyTorch_MNIST_horovodrun_ONECCL_OFI && steps.MPI_PyTorch_MNIST_horovodrun_ONECCL_OFI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_horovodrun_ONECCL_OFI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_PyTorch_MNIST_horovodrun_ONECCL_OFI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && \$(cat /mpirun_command) python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets" | |
| shell: bash | |
| - name: "MPI Single PyTests [attempt 1 of 3]" | |
| id: MPI_Single_PyTests_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Single_PyTests && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Single_PyTests_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Single_PyTests_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c " cd /horovod/test/single && (ls -1 test_*.py | xargs -n 1 /bin/bash /pytest_standalone.sh mpi)" | |
| shell: bash | |
| - name: "MPI Single PyTests [attempt 2 of 3]" | |
| id: MPI_Single_PyTests_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Single_PyTests && steps.MPI_Single_PyTests_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Single_PyTests_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Single_PyTests_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c " cd /horovod/test/single && (ls -1 test_*.py | xargs -n 1 /bin/bash /pytest_standalone.sh mpi)" | |
| shell: bash | |
| - name: "MPI Single PyTests [attempt 3 of 3]" | |
| id: MPI_Single_PyTests_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Single_PyTests && steps.MPI_Single_PyTests_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Single_PyTests_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Single_PyTests_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c " cd /horovod/test/single && (ls -1 test_*.py | xargs -n 1 /bin/bash /pytest_standalone.sh mpi)" | |
| shell: bash | |
| - name: "MPI Single PyTests [ONECCL MPI] [attempt 1 of 3]" | |
| id: MPI_Single_PyTests_ONECCL_MPI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Single_PyTests_ONECCL_MPI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Single_PyTests_ONECCL_MPI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Single_PyTests_ONECCL_MPI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && cd /horovod/test/single && (ls -1 test_*.py | xargs -n 1 /bin/bash /pytest_standalone.sh mpi)" | |
| shell: bash | |
| - name: "MPI Single PyTests [ONECCL MPI] [attempt 2 of 3]" | |
| id: MPI_Single_PyTests_ONECCL_MPI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Single_PyTests_ONECCL_MPI && steps.MPI_Single_PyTests_ONECCL_MPI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Single_PyTests_ONECCL_MPI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Single_PyTests_ONECCL_MPI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && cd /horovod/test/single && (ls -1 test_*.py | xargs -n 1 /bin/bash /pytest_standalone.sh mpi)" | |
| shell: bash | |
| - name: "MPI Single PyTests [ONECCL MPI] [attempt 3 of 3]" | |
| id: MPI_Single_PyTests_ONECCL_MPI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Single_PyTests_ONECCL_MPI && steps.MPI_Single_PyTests_ONECCL_MPI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Single_PyTests_ONECCL_MPI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Single_PyTests_ONECCL_MPI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && cd /horovod/test/single && (ls -1 test_*.py | xargs -n 1 /bin/bash /pytest_standalone.sh mpi)" | |
| shell: bash | |
| - name: "MPI Single PyTests [ONECCL OFI] [attempt 1 of 3]" | |
| id: MPI_Single_PyTests_ONECCL_OFI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Single_PyTests_ONECCL_OFI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Single_PyTests_ONECCL_OFI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Single_PyTests_ONECCL_OFI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && cd /horovod/test/single && (ls -1 test_*.py | xargs -n 1 /bin/bash /pytest_standalone.sh mpi)" | |
| shell: bash | |
| - name: "MPI Single PyTests [ONECCL OFI] [attempt 2 of 3]" | |
| id: MPI_Single_PyTests_ONECCL_OFI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Single_PyTests_ONECCL_OFI && steps.MPI_Single_PyTests_ONECCL_OFI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Single_PyTests_ONECCL_OFI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Single_PyTests_ONECCL_OFI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && cd /horovod/test/single && (ls -1 test_*.py | xargs -n 1 /bin/bash /pytest_standalone.sh mpi)" | |
| shell: bash | |
| - name: "MPI Single PyTests [ONECCL OFI] [attempt 3 of 3]" | |
| id: MPI_Single_PyTests_ONECCL_OFI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_Single_PyTests_ONECCL_OFI && steps.MPI_Single_PyTests_ONECCL_OFI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_Single_PyTests_ONECCL_OFI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_Single_PyTests_ONECCL_OFI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 15m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && cd /horovod/test/single && (ls -1 test_*.py | xargs -n 1 /bin/bash /pytest_standalone.sh mpi)" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 Keras MNIST api [attempt 1 of 3]" | |
| id: MPI_TensorFlow_2_0_Keras_MNIST_api_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_Keras_MNIST_api && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_api_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_api_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py 2 localhost:2 mpi" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 Keras MNIST api [attempt 2 of 3]" | |
| id: MPI_TensorFlow_2_0_Keras_MNIST_api_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_Keras_MNIST_api && steps.MPI_TensorFlow_2_0_Keras_MNIST_api_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_api_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_api_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py 2 localhost:2 mpi" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 Keras MNIST api [attempt 3 of 3]" | |
| id: MPI_TensorFlow_2_0_Keras_MNIST_api_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_Keras_MNIST_api && steps.MPI_TensorFlow_2_0_Keras_MNIST_api_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_api_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_api_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py 2 localhost:2 mpi" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 Keras MNIST api [ONECCL MPI] [attempt 1 of 3]" | |
| id: MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_MPI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_MPI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_MPI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_MPI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py 2 localhost:2 mpi" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 Keras MNIST api [ONECCL MPI] [attempt 2 of 3]" | |
| id: MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_MPI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_MPI && steps.MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_MPI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_MPI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_MPI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py 2 localhost:2 mpi" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 Keras MNIST api [ONECCL MPI] [attempt 3 of 3]" | |
| id: MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_MPI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_MPI && steps.MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_MPI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_MPI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_MPI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py 2 localhost:2 mpi" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 Keras MNIST api [ONECCL OFI] [attempt 1 of 3]" | |
| id: MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_OFI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_OFI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_OFI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_OFI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py 2 localhost:2 mpi" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 Keras MNIST api [ONECCL OFI] [attempt 2 of 3]" | |
| id: MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_OFI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_OFI && steps.MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_OFI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_OFI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_OFI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py 2 localhost:2 mpi" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 Keras MNIST api [ONECCL OFI] [attempt 3 of 3]" | |
| id: MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_OFI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_OFI && steps.MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_OFI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_OFI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_api_ONECCL_OFI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py 2 localhost:2 mpi" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 Keras MNIST horovodrun [attempt 1 of 3]" | |
| id: MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_Keras_MNIST_horovodrun && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " \$(cat /mpirun_command) python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 Keras MNIST horovodrun [attempt 2 of 3]" | |
| id: MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_Keras_MNIST_horovodrun && steps.MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " \$(cat /mpirun_command) python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 Keras MNIST horovodrun [attempt 3 of 3]" | |
| id: MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_Keras_MNIST_horovodrun && steps.MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " \$(cat /mpirun_command) python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 Keras MNIST horovodrun [ONECCL MPI] [attempt 1 of 3]" | |
| id: MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_MPI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_MPI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_MPI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_MPI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && \$(cat /mpirun_command) python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 Keras MNIST horovodrun [ONECCL MPI] [attempt 2 of 3]" | |
| id: MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_MPI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_MPI && steps.MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_MPI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_MPI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_MPI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && \$(cat /mpirun_command) python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 Keras MNIST horovodrun [ONECCL MPI] [attempt 3 of 3]" | |
| id: MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_MPI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_MPI && steps.MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_MPI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_MPI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_MPI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && \$(cat /mpirun_command) python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 Keras MNIST horovodrun [ONECCL OFI] [attempt 1 of 3]" | |
| id: MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_OFI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_OFI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_OFI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_OFI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && \$(cat /mpirun_command) python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 Keras MNIST horovodrun [ONECCL OFI] [attempt 2 of 3]" | |
| id: MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_OFI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_OFI && steps.MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_OFI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_OFI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_OFI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && \$(cat /mpirun_command) python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 Keras MNIST horovodrun [ONECCL OFI] [attempt 3 of 3]" | |
| id: MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_OFI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_OFI && steps.MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_OFI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_OFI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_Keras_MNIST_horovodrun_ONECCL_OFI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && \$(cat /mpirun_command) python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 MNIST api [attempt 1 of 3]" | |
| id: MPI_TensorFlow_2_0_MNIST_api_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_MNIST_api && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_api_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_api_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/tensorflow2/tensorflow2_mnist.py 2 localhost:2 mpi" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 MNIST api [attempt 2 of 3]" | |
| id: MPI_TensorFlow_2_0_MNIST_api_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_MNIST_api && steps.MPI_TensorFlow_2_0_MNIST_api_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_api_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_api_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/tensorflow2/tensorflow2_mnist.py 2 localhost:2 mpi" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 MNIST api [attempt 3 of 3]" | |
| id: MPI_TensorFlow_2_0_MNIST_api_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_MNIST_api && steps.MPI_TensorFlow_2_0_MNIST_api_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_api_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_api_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/tensorflow2/tensorflow2_mnist.py 2 localhost:2 mpi" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 MNIST api [ONECCL MPI] [attempt 1 of 3]" | |
| id: MPI_TensorFlow_2_0_MNIST_api_ONECCL_MPI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_MNIST_api_ONECCL_MPI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_api_ONECCL_MPI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_api_ONECCL_MPI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && python /horovod/examples/tensorflow2/tensorflow2_mnist.py 2 localhost:2 mpi" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 MNIST api [ONECCL MPI] [attempt 2 of 3]" | |
| id: MPI_TensorFlow_2_0_MNIST_api_ONECCL_MPI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_MNIST_api_ONECCL_MPI && steps.MPI_TensorFlow_2_0_MNIST_api_ONECCL_MPI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_api_ONECCL_MPI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_api_ONECCL_MPI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && python /horovod/examples/tensorflow2/tensorflow2_mnist.py 2 localhost:2 mpi" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 MNIST api [ONECCL MPI] [attempt 3 of 3]" | |
| id: MPI_TensorFlow_2_0_MNIST_api_ONECCL_MPI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_MNIST_api_ONECCL_MPI && steps.MPI_TensorFlow_2_0_MNIST_api_ONECCL_MPI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_api_ONECCL_MPI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_api_ONECCL_MPI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && python /horovod/examples/tensorflow2/tensorflow2_mnist.py 2 localhost:2 mpi" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 MNIST api [ONECCL OFI] [attempt 1 of 3]" | |
| id: MPI_TensorFlow_2_0_MNIST_api_ONECCL_OFI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_MNIST_api_ONECCL_OFI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_api_ONECCL_OFI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_api_ONECCL_OFI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && python /horovod/examples/tensorflow2/tensorflow2_mnist.py 2 localhost:2 mpi" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 MNIST api [ONECCL OFI] [attempt 2 of 3]" | |
| id: MPI_TensorFlow_2_0_MNIST_api_ONECCL_OFI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_MNIST_api_ONECCL_OFI && steps.MPI_TensorFlow_2_0_MNIST_api_ONECCL_OFI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_api_ONECCL_OFI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_api_ONECCL_OFI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && python /horovod/examples/tensorflow2/tensorflow2_mnist.py 2 localhost:2 mpi" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 MNIST api [ONECCL OFI] [attempt 3 of 3]" | |
| id: MPI_TensorFlow_2_0_MNIST_api_ONECCL_OFI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_MNIST_api_ONECCL_OFI && steps.MPI_TensorFlow_2_0_MNIST_api_ONECCL_OFI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_api_ONECCL_OFI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_api_ONECCL_OFI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && python /horovod/examples/tensorflow2/tensorflow2_mnist.py 2 localhost:2 mpi" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 MNIST horovodrun [attempt 1 of 3]" | |
| id: MPI_TensorFlow_2_0_MNIST_horovodrun_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_MNIST_horovodrun && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_horovodrun_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_horovodrun_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " \$(cat /mpirun_command) python /horovod/examples/tensorflow2/tensorflow2_mnist.py" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 MNIST horovodrun [attempt 2 of 3]" | |
| id: MPI_TensorFlow_2_0_MNIST_horovodrun_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_MNIST_horovodrun && steps.MPI_TensorFlow_2_0_MNIST_horovodrun_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_horovodrun_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_horovodrun_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " \$(cat /mpirun_command) python /horovod/examples/tensorflow2/tensorflow2_mnist.py" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 MNIST horovodrun [attempt 3 of 3]" | |
| id: MPI_TensorFlow_2_0_MNIST_horovodrun_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_MNIST_horovodrun && steps.MPI_TensorFlow_2_0_MNIST_horovodrun_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_horovodrun_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_horovodrun_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " \$(cat /mpirun_command) python /horovod/examples/tensorflow2/tensorflow2_mnist.py" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 MNIST horovodrun [ONECCL MPI] [attempt 1 of 3]" | |
| id: MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_MPI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_MPI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_MPI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_MPI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && \$(cat /mpirun_command) python /horovod/examples/tensorflow2/tensorflow2_mnist.py" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 MNIST horovodrun [ONECCL MPI] [attempt 2 of 3]" | |
| id: MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_MPI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_MPI && steps.MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_MPI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_MPI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_MPI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && \$(cat /mpirun_command) python /horovod/examples/tensorflow2/tensorflow2_mnist.py" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 MNIST horovodrun [ONECCL MPI] [attempt 3 of 3]" | |
| id: MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_MPI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_MPI && steps.MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_MPI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_MPI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_MPI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && \$(cat /mpirun_command) python /horovod/examples/tensorflow2/tensorflow2_mnist.py" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 MNIST horovodrun [ONECCL OFI] [attempt 1 of 3]" | |
| id: MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_OFI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_OFI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_OFI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_OFI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && \$(cat /mpirun_command) python /horovod/examples/tensorflow2/tensorflow2_mnist.py" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 MNIST horovodrun [ONECCL OFI] [attempt 2 of 3]" | |
| id: MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_OFI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_OFI && steps.MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_OFI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_OFI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_OFI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && \$(cat /mpirun_command) python /horovod/examples/tensorflow2/tensorflow2_mnist.py" | |
| shell: bash | |
| - name: "MPI TensorFlow 2.0 MNIST horovodrun [ONECCL OFI] [attempt 3 of 3]" | |
| id: MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_OFI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_OFI && steps.MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_OFI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_OFI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/MPI_TensorFlow_2_0_MNIST_horovodrun_ONECCL_OFI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && \$(cat /mpirun_command) python /horovod/examples/tensorflow2/tensorflow2_mnist.py" | |
| shell: bash | |
| - name: "Run PyTests test_interactiverun [attempt 1 of 3]" | |
| id: Run_PyTests_test_interactiverun_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Run_PyTests_test_interactiverun && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Run_PyTests_test_interactiverun_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Run_PyTests_test_interactiverun_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "cd /horovod/test && pytest -v --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.mpi.integration.xml integration/test_interactiverun.py" | |
| shell: bash | |
| - name: "Run PyTests test_interactiverun [attempt 2 of 3]" | |
| id: Run_PyTests_test_interactiverun_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Run_PyTests_test_interactiverun && steps.Run_PyTests_test_interactiverun_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Run_PyTests_test_interactiverun_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Run_PyTests_test_interactiverun_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "cd /horovod/test && pytest -v --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.mpi.integration.xml integration/test_interactiverun.py" | |
| shell: bash | |
| - name: "Run PyTests test_interactiverun [attempt 3 of 3]" | |
| id: Run_PyTests_test_interactiverun_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Run_PyTests_test_interactiverun && steps.Run_PyTests_test_interactiverun_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Run_PyTests_test_interactiverun_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Run_PyTests_test_interactiverun_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "cd /horovod/test && pytest -v --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.mpi.integration.xml integration/test_interactiverun.py" | |
| shell: bash | |
| - name: "Single Keras MNIST [attempt 1 of 3]" | |
| id: Single_Keras_MNIST_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_Keras_MNIST && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_Keras_MNIST_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_Keras_MNIST_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/keras/keras_mnist_advanced.py --epochs 3 --batch-size 64" | |
| shell: bash | |
| - name: "Single Keras MNIST [attempt 2 of 3]" | |
| id: Single_Keras_MNIST_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_Keras_MNIST && steps.Single_Keras_MNIST_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_Keras_MNIST_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_Keras_MNIST_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/keras/keras_mnist_advanced.py --epochs 3 --batch-size 64" | |
| shell: bash | |
| - name: "Single Keras MNIST [attempt 3 of 3]" | |
| id: Single_Keras_MNIST_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_Keras_MNIST && steps.Single_Keras_MNIST_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_Keras_MNIST_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_Keras_MNIST_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/keras/keras_mnist_advanced.py --epochs 3 --batch-size 64" | |
| shell: bash | |
| - name: "Single MXNet2 MNIST [attempt 1 of 3]" | |
| id: Single_MXNet2_MNIST_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_MXNet2_MNIST && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_MXNet2_MNIST_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_MXNet2_MNIST_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/mxnet/mxnet2_mnist.py --epochs 3" | |
| shell: bash | |
| - name: "Single MXNet2 MNIST [attempt 2 of 3]" | |
| id: Single_MXNet2_MNIST_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_MXNet2_MNIST && steps.Single_MXNet2_MNIST_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_MXNet2_MNIST_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_MXNet2_MNIST_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/mxnet/mxnet2_mnist.py --epochs 3" | |
| shell: bash | |
| - name: "Single MXNet2 MNIST [attempt 3 of 3]" | |
| id: Single_MXNet2_MNIST_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_MXNet2_MNIST && steps.Single_MXNet2_MNIST_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_MXNet2_MNIST_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_MXNet2_MNIST_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/mxnet/mxnet2_mnist.py --epochs 3" | |
| shell: bash | |
| - name: "Single MXNet MNIST [attempt 1 of 3]" | |
| id: Single_MXNet_MNIST_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_MXNet_MNIST && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_MXNet_MNIST_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_MXNet_MNIST_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/mxnet/mxnet_mnist.py --epochs 3" | |
| shell: bash | |
| - name: "Single MXNet MNIST [attempt 2 of 3]" | |
| id: Single_MXNet_MNIST_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_MXNet_MNIST && steps.Single_MXNet_MNIST_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_MXNet_MNIST_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_MXNet_MNIST_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/mxnet/mxnet_mnist.py --epochs 3" | |
| shell: bash | |
| - name: "Single MXNet MNIST [attempt 3 of 3]" | |
| id: Single_MXNet_MNIST_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_MXNet_MNIST && steps.Single_MXNet_MNIST_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_MXNet_MNIST_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_MXNet_MNIST_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/mxnet/mxnet_mnist.py --epochs 3" | |
| shell: bash | |
| - name: "Single MXNet MNIST [ONECCL MPI] [attempt 1 of 3]" | |
| id: Single_MXNet_MNIST_ONECCL_MPI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_MXNet_MNIST_ONECCL_MPI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_MXNet_MNIST_ONECCL_MPI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_MXNet_MNIST_ONECCL_MPI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && python /horovod/examples/mxnet/mxnet_mnist.py --epochs 3" | |
| shell: bash | |
| - name: "Single MXNet MNIST [ONECCL MPI] [attempt 2 of 3]" | |
| id: Single_MXNet_MNIST_ONECCL_MPI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_MXNet_MNIST_ONECCL_MPI && steps.Single_MXNet_MNIST_ONECCL_MPI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_MXNet_MNIST_ONECCL_MPI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_MXNet_MNIST_ONECCL_MPI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && python /horovod/examples/mxnet/mxnet_mnist.py --epochs 3" | |
| shell: bash | |
| - name: "Single MXNet MNIST [ONECCL MPI] [attempt 3 of 3]" | |
| id: Single_MXNet_MNIST_ONECCL_MPI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_MXNet_MNIST_ONECCL_MPI && steps.Single_MXNet_MNIST_ONECCL_MPI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_MXNet_MNIST_ONECCL_MPI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_MXNet_MNIST_ONECCL_MPI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && python /horovod/examples/mxnet/mxnet_mnist.py --epochs 3" | |
| shell: bash | |
| - name: "Single MXNet MNIST [ONECCL OFI] [attempt 1 of 3]" | |
| id: Single_MXNet_MNIST_ONECCL_OFI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_MXNet_MNIST_ONECCL_OFI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_MXNet_MNIST_ONECCL_OFI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_MXNet_MNIST_ONECCL_OFI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && python /horovod/examples/mxnet/mxnet_mnist.py --epochs 3" | |
| shell: bash | |
| - name: "Single MXNet MNIST [ONECCL OFI] [attempt 2 of 3]" | |
| id: Single_MXNet_MNIST_ONECCL_OFI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_MXNet_MNIST_ONECCL_OFI && steps.Single_MXNet_MNIST_ONECCL_OFI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_MXNet_MNIST_ONECCL_OFI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_MXNet_MNIST_ONECCL_OFI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && python /horovod/examples/mxnet/mxnet_mnist.py --epochs 3" | |
| shell: bash | |
| - name: "Single MXNet MNIST [ONECCL OFI] [attempt 3 of 3]" | |
| id: Single_MXNet_MNIST_ONECCL_OFI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_MXNet_MNIST_ONECCL_OFI && steps.Single_MXNet_MNIST_ONECCL_OFI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_MXNet_MNIST_ONECCL_OFI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_MXNet_MNIST_ONECCL_OFI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && python /horovod/examples/mxnet/mxnet_mnist.py --epochs 3" | |
| shell: bash | |
| - name: "Single PyTorch MNIST [attempt 1 of 3]" | |
| id: Single_PyTorch_MNIST_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_PyTorch_MNIST && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_PyTorch_MNIST_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_PyTorch_MNIST_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/pytorch/pytorch_mnist.py --epochs 3 --data-dir /data/pytorch_datasets" | |
| shell: bash | |
| - name: "Single PyTorch MNIST [attempt 2 of 3]" | |
| id: Single_PyTorch_MNIST_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_PyTorch_MNIST && steps.Single_PyTorch_MNIST_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_PyTorch_MNIST_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_PyTorch_MNIST_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/pytorch/pytorch_mnist.py --epochs 3 --data-dir /data/pytorch_datasets" | |
| shell: bash | |
| - name: "Single PyTorch MNIST [attempt 3 of 3]" | |
| id: Single_PyTorch_MNIST_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_PyTorch_MNIST && steps.Single_PyTorch_MNIST_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_PyTorch_MNIST_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_PyTorch_MNIST_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c " python /horovod/examples/pytorch/pytorch_mnist.py --epochs 3 --data-dir /data/pytorch_datasets" | |
| shell: bash | |
| - name: "Single PyTorch MNIST [ONECCL MPI] [attempt 1 of 3]" | |
| id: Single_PyTorch_MNIST_ONECCL_MPI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_PyTorch_MNIST_ONECCL_MPI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_PyTorch_MNIST_ONECCL_MPI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_PyTorch_MNIST_ONECCL_MPI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && python /horovod/examples/pytorch/pytorch_mnist.py --epochs 3 --data-dir /data/pytorch_datasets" | |
| shell: bash | |
| - name: "Single PyTorch MNIST [ONECCL MPI] [attempt 2 of 3]" | |
| id: Single_PyTorch_MNIST_ONECCL_MPI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_PyTorch_MNIST_ONECCL_MPI && steps.Single_PyTorch_MNIST_ONECCL_MPI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_PyTorch_MNIST_ONECCL_MPI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_PyTorch_MNIST_ONECCL_MPI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && python /horovod/examples/pytorch/pytorch_mnist.py --epochs 3 --data-dir /data/pytorch_datasets" | |
| shell: bash | |
| - name: "Single PyTorch MNIST [ONECCL MPI] [attempt 3 of 3]" | |
| id: Single_PyTorch_MNIST_ONECCL_MPI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_PyTorch_MNIST_ONECCL_MPI && steps.Single_PyTorch_MNIST_ONECCL_MPI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_PyTorch_MNIST_ONECCL_MPI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_PyTorch_MNIST_ONECCL_MPI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_mpi' > /mpirun_command && python /horovod/examples/pytorch/pytorch_mnist.py --epochs 3 --data-dir /data/pytorch_datasets" | |
| shell: bash | |
| - name: "Single PyTorch MNIST [ONECCL OFI] [attempt 1 of 3]" | |
| id: Single_PyTorch_MNIST_ONECCL_OFI_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_PyTorch_MNIST_ONECCL_OFI && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_PyTorch_MNIST_ONECCL_OFI_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_PyTorch_MNIST_ONECCL_OFI_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && python /horovod/examples/pytorch/pytorch_mnist.py --epochs 3 --data-dir /data/pytorch_datasets" | |
| shell: bash | |
| - name: "Single PyTorch MNIST [ONECCL OFI] [attempt 2 of 3]" | |
| id: Single_PyTorch_MNIST_ONECCL_OFI_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_PyTorch_MNIST_ONECCL_OFI && steps.Single_PyTorch_MNIST_ONECCL_OFI_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_PyTorch_MNIST_ONECCL_OFI_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_PyTorch_MNIST_ONECCL_OFI_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && python /horovod/examples/pytorch/pytorch_mnist.py --epochs 3 --data-dir /data/pytorch_datasets" | |
| shell: bash | |
| - name: "Single PyTorch MNIST [ONECCL OFI] [attempt 3 of 3]" | |
| id: Single_PyTorch_MNIST_ONECCL_OFI_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Single_PyTorch_MNIST_ONECCL_OFI && steps.Single_PyTorch_MNIST_ONECCL_OFI_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Single_PyTorch_MNIST_ONECCL_OFI_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Single_PyTorch_MNIST_ONECCL_OFI_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "\$(cat /oneccl_env) && echo '/mpirun_command_ofi' > /mpirun_command && python /horovod/examples/pytorch/pytorch_mnist.py --epochs 3 --data-dir /data/pytorch_datasets" | |
| shell: bash | |
| - name: "Spark Keras MNIST [attempt 1 of 3]" | |
| id: Spark_Keras_MNIST_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Spark_Keras_MNIST && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Spark_Keras_MNIST_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Spark_Keras_MNIST_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "OMP_NUM_THREADS=1 /spark_env.sh python /horovod/examples/spark/keras/keras_spark_mnist.py --num-proc 2 --work-dir /work --data-dir /data --epochs 3" | |
| shell: bash | |
| - name: "Spark Keras MNIST [attempt 2 of 3]" | |
| id: Spark_Keras_MNIST_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Spark_Keras_MNIST && steps.Spark_Keras_MNIST_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Spark_Keras_MNIST_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Spark_Keras_MNIST_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "OMP_NUM_THREADS=1 /spark_env.sh python /horovod/examples/spark/keras/keras_spark_mnist.py --num-proc 2 --work-dir /work --data-dir /data --epochs 3" | |
| shell: bash | |
| - name: "Spark Keras MNIST [attempt 3 of 3]" | |
| id: Spark_Keras_MNIST_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Spark_Keras_MNIST && steps.Spark_Keras_MNIST_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Spark_Keras_MNIST_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Spark_Keras_MNIST_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "OMP_NUM_THREADS=1 /spark_env.sh python /horovod/examples/spark/keras/keras_spark_mnist.py --num-proc 2 --work-dir /work --data-dir /data --epochs 3" | |
| shell: bash | |
| - name: "Spark Keras Rossmann Estimator [attempt 1 of 3]" | |
| id: Spark_Keras_Rossmann_Estimator_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Spark_Keras_Rossmann_Estimator && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Spark_Keras_Rossmann_Estimator_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Spark_Keras_Rossmann_Estimator_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "OMP_NUM_THREADS=1 /spark_env.sh python /horovod/examples/spark/keras/keras_spark_rossmann_estimator.py --num-proc 2 --work-dir /work --data-dir file:///data --epochs 3 --sample-rate 0.1" | |
| shell: bash | |
| - name: "Spark Keras Rossmann Estimator [attempt 2 of 3]" | |
| id: Spark_Keras_Rossmann_Estimator_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Spark_Keras_Rossmann_Estimator && steps.Spark_Keras_Rossmann_Estimator_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Spark_Keras_Rossmann_Estimator_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Spark_Keras_Rossmann_Estimator_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "OMP_NUM_THREADS=1 /spark_env.sh python /horovod/examples/spark/keras/keras_spark_rossmann_estimator.py --num-proc 2 --work-dir /work --data-dir file:///data --epochs 3 --sample-rate 0.1" | |
| shell: bash | |
| - name: "Spark Keras Rossmann Estimator [attempt 3 of 3]" | |
| id: Spark_Keras_Rossmann_Estimator_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Spark_Keras_Rossmann_Estimator && steps.Spark_Keras_Rossmann_Estimator_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Spark_Keras_Rossmann_Estimator_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Spark_Keras_Rossmann_Estimator_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "OMP_NUM_THREADS=1 /spark_env.sh python /horovod/examples/spark/keras/keras_spark_rossmann_estimator.py --num-proc 2 --work-dir /work --data-dir file:///data --epochs 3 --sample-rate 0.1" | |
| shell: bash | |
| - name: "Spark Keras Rossmann Run [attempt 1 of 3]" | |
| id: Spark_Keras_Rossmann_Run_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Spark_Keras_Rossmann_Run && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Spark_Keras_Rossmann_Run_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Spark_Keras_Rossmann_Run_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "OMP_NUM_THREADS=1 /spark_env.sh python /horovod/examples/spark/keras/keras_spark_rossmann_run.py --num-proc 2 --data-dir file:///data --epochs 3 --sample-rate 0.1" | |
| shell: bash | |
| - name: "Spark Keras Rossmann Run [attempt 2 of 3]" | |
| id: Spark_Keras_Rossmann_Run_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Spark_Keras_Rossmann_Run && steps.Spark_Keras_Rossmann_Run_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Spark_Keras_Rossmann_Run_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Spark_Keras_Rossmann_Run_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "OMP_NUM_THREADS=1 /spark_env.sh python /horovod/examples/spark/keras/keras_spark_rossmann_run.py --num-proc 2 --data-dir file:///data --epochs 3 --sample-rate 0.1" | |
| shell: bash | |
| - name: "Spark Keras Rossmann Run [attempt 3 of 3]" | |
| id: Spark_Keras_Rossmann_Run_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Spark_Keras_Rossmann_Run && steps.Spark_Keras_Rossmann_Run_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Spark_Keras_Rossmann_Run_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Spark_Keras_Rossmann_Run_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "OMP_NUM_THREADS=1 /spark_env.sh python /horovod/examples/spark/keras/keras_spark_rossmann_run.py --num-proc 2 --data-dir file:///data --epochs 3 --sample-rate 0.1" | |
| shell: bash | |
| - name: "Spark Lightning MNIST [attempt 1 of 3]" | |
| id: Spark_Lightning_MNIST_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Spark_Lightning_MNIST && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Spark_Lightning_MNIST_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Spark_Lightning_MNIST_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "OMP_NUM_THREADS=1 /spark_env.sh python /horovod/examples/spark/pytorch/pytorch_lightning_spark_mnist.py --num-proc 2 --work-dir /work --data-dir /data --epochs 3" | |
| shell: bash | |
| - name: "Spark Lightning MNIST [attempt 2 of 3]" | |
| id: Spark_Lightning_MNIST_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Spark_Lightning_MNIST && steps.Spark_Lightning_MNIST_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Spark_Lightning_MNIST_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Spark_Lightning_MNIST_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "OMP_NUM_THREADS=1 /spark_env.sh python /horovod/examples/spark/pytorch/pytorch_lightning_spark_mnist.py --num-proc 2 --work-dir /work --data-dir /data --epochs 3" | |
| shell: bash | |
| - name: "Spark Lightning MNIST [attempt 3 of 3]" | |
| id: Spark_Lightning_MNIST_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Spark_Lightning_MNIST && steps.Spark_Lightning_MNIST_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Spark_Lightning_MNIST_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Spark_Lightning_MNIST_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "OMP_NUM_THREADS=1 /spark_env.sh python /horovod/examples/spark/pytorch/pytorch_lightning_spark_mnist.py --num-proc 2 --work-dir /work --data-dir /data --epochs 3" | |
| shell: bash | |
| - name: "Spark PyTests [attempt 1 of 3]" | |
| id: Spark_PyTests_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Spark_PyTests && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Spark_PyTests_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Spark_PyTests_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 30m bash -c "cd /horovod/test/integration && (ls -1 test_spark*.py | xargs -n 1 /bin/bash /pytest_standalone.sh spark)" | |
| shell: bash | |
| - name: "Spark PyTests [attempt 2 of 3]" | |
| id: Spark_PyTests_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Spark_PyTests && steps.Spark_PyTests_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Spark_PyTests_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Spark_PyTests_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 30m bash -c "cd /horovod/test/integration && (ls -1 test_spark*.py | xargs -n 1 /bin/bash /pytest_standalone.sh spark)" | |
| shell: bash | |
| - name: "Spark PyTests [attempt 3 of 3]" | |
| id: Spark_PyTests_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Spark_PyTests && steps.Spark_PyTests_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Spark_PyTests_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Spark_PyTests_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 30m bash -c "cd /horovod/test/integration && (ls -1 test_spark*.py | xargs -n 1 /bin/bash /pytest_standalone.sh spark)" | |
| shell: bash | |
| - name: "Spark TensorFlow 2.0 MNIST Data Service [attempt 1 of 3]" | |
| id: Spark_TensorFlow_2_0_MNIST_Data_Service_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Spark_TensorFlow_2_0_MNIST_Data_Service && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Spark_TensorFlow_2_0_MNIST_Data_Service_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Spark_TensorFlow_2_0_MNIST_Data_Service_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "cd /horovod/examples/spark/tensorflow2; spark-submit --master \"local[2]\" \"/horovod/horovod/spark/tensorflow/compute_worker.py\" /tmp/compute.json & OMP_NUM_THREADS=1 /spark_env.sh spark-submit --master \"local[2]\" --py-files tensorflow2_mnist_data_service_train_fn_compute_side_dispatcher.py,tensorflow2_mnist_data_service_train_fn_training_side_dispatcher.py tensorflow2_mnist_data_service.py /tmp/compute.json" | |
| shell: bash | |
| - name: "Spark TensorFlow 2.0 MNIST Data Service [attempt 2 of 3]" | |
| id: Spark_TensorFlow_2_0_MNIST_Data_Service_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Spark_TensorFlow_2_0_MNIST_Data_Service && steps.Spark_TensorFlow_2_0_MNIST_Data_Service_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Spark_TensorFlow_2_0_MNIST_Data_Service_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Spark_TensorFlow_2_0_MNIST_Data_Service_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "cd /horovod/examples/spark/tensorflow2; spark-submit --master \"local[2]\" \"/horovod/horovod/spark/tensorflow/compute_worker.py\" /tmp/compute.json & OMP_NUM_THREADS=1 /spark_env.sh spark-submit --master \"local[2]\" --py-files tensorflow2_mnist_data_service_train_fn_compute_side_dispatcher.py,tensorflow2_mnist_data_service_train_fn_training_side_dispatcher.py tensorflow2_mnist_data_service.py /tmp/compute.json" | |
| shell: bash | |
| - name: "Spark TensorFlow 2.0 MNIST Data Service [attempt 3 of 3]" | |
| id: Spark_TensorFlow_2_0_MNIST_Data_Service_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Spark_TensorFlow_2_0_MNIST_Data_Service && steps.Spark_TensorFlow_2_0_MNIST_Data_Service_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Spark_TensorFlow_2_0_MNIST_Data_Service_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Spark_TensorFlow_2_0_MNIST_Data_Service_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "cd /horovod/examples/spark/tensorflow2; spark-submit --master \"local[2]\" \"/horovod/horovod/spark/tensorflow/compute_worker.py\" /tmp/compute.json & OMP_NUM_THREADS=1 /spark_env.sh spark-submit --master \"local[2]\" --py-files tensorflow2_mnist_data_service_train_fn_compute_side_dispatcher.py,tensorflow2_mnist_data_service_train_fn_training_side_dispatcher.py tensorflow2_mnist_data_service.py /tmp/compute.json" | |
| shell: bash | |
| - name: "Spark Torch MNIST [attempt 1 of 3]" | |
| id: Spark_Torch_MNIST_run_1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Spark_Torch_MNIST && true | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Spark_Torch_MNIST_run_1 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Spark_Torch_MNIST_run_1:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "OMP_NUM_THREADS=1 /spark_env.sh python /horovod/examples/spark/pytorch/pytorch_spark_mnist.py --num-proc 2 --work-dir /work --data-dir /data --epochs 3" | |
| shell: bash | |
| - name: "Spark Torch MNIST [attempt 2 of 3]" | |
| id: Spark_Torch_MNIST_run_2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && matrix.Spark_Torch_MNIST && steps.Spark_Torch_MNIST_run_1.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Spark_Torch_MNIST_run_2 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Spark_Torch_MNIST_run_2:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "OMP_NUM_THREADS=1 /spark_env.sh python /horovod/examples/spark/pytorch/pytorch_spark_mnist.py --num-proc 2 --work-dir /work --data-dir /data --epochs 3" | |
| shell: bash | |
| - name: "Spark Torch MNIST [attempt 3 of 3]" | |
| id: Spark_Torch_MNIST_run_3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && matrix.Spark_Torch_MNIST && steps.Spark_Torch_MNIST_run_2.outcome == 'failure' | |
| run: | | |
| mkdir -p artifacts/${{ matrix.image }}/Spark_Torch_MNIST_run_3 | |
| docker-compose -f docker-compose.test.yml run -e GITHUB_ACTIONS --rm --volume "$(pwd)/artifacts/${{ matrix.image }}/Spark_Torch_MNIST_run_3:/artifacts" ${{ matrix.image }} /usr/bin/timeout 10m bash -c "OMP_NUM_THREADS=1 /spark_env.sh python /horovod/examples/spark/pytorch/pytorch_spark_mnist.py --num-proc 2 --work-dir /work --data-dir /data --epochs 3" | |
| shell: bash | |
| - name: Upload Test Results | |
| uses: actions/upload-artifact@v3 | |
| if: always() && contains(matrix.image, '-cpu-') | |
| with: | |
| name: Unit Test Results - ${{ matrix.image }} | |
| path: artifacts/${{ matrix.image }}/**/*.xml | |
| build-mins: | |
| name: "Build mins (${{ matrix.image }})" | |
| needs: [init-workflow, build-and-test] | |
| if: > | |
| needs.init-workflow.outputs.run-at-all == 'true' && | |
| needs.init-workflow.outputs.run-builds-and-tests == 'true' | |
| runs-on: ubuntu-latest | |
| strategy: | |
| max-parallel: 2 | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - image: test-cpu-openmpi-gloo-py3_7-tfmin-kerasmin-torchmin-mxnetmin-pysparkmin | |
| build_timeout: 30 | |
| - image: test-gpu-openmpi-gloo-py3_7-tfmin-kerasmin-torchmin-mxnetmin-pysparkmin | |
| build_timeout: 40 | |
| steps: | |
| - name: Clean up disk space | |
| # deleting these paths frees 38 GB disk space: | |
| # sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc | |
| # but this sometimes takes 3-4 minutes | |
| # so we delete only some sub-paths which are known to be quick (10s) and 20 GB | |
| run: | | |
| echo ::group::Disk space before clean up | |
| df -h | |
| echo ::endgroup:: | |
| for dir in /usr/share/dotnet/sdk/\*/nuGetPackagesArchive.lzma \ | |
| /usr/share/dotnet/shared \ | |
| /usr/local/lib/android/sdk/ndk \ | |
| /usr/local/lib/android/sdk/build-tools \ | |
| /opt/ghc | |
| do | |
| echo ::group::Deleting "$dir" | |
| sudo du -hsc $dir | tail -n1 || true | |
| sudo rm -rf $dir | |
| echo ::endgroup:: | |
| done | |
| echo ::group::Disk space after clean up | |
| df -h | |
| echo ::endgroup:: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.8 | |
| - name: Setup docker-compose | |
| run: pip install docker-compose | |
| - name: Build | |
| id: build | |
| run: | | |
| .github/timeout-and-retry.sh ${{ matrix.build_timeout }}m 3 10 docker-compose -f docker-compose.test.yml build ${{ matrix.image }} | |
| env: | |
| COMPOSE_DOCKER_CLI_BUILD: 1 | |
| DOCKER_BUILDKIT: 1 | |
| - name: Upload Test Results | |
| uses: actions/upload-artifact@v3 | |
| if: always() && contains(matrix.image, '-cpu-') | |
| with: | |
| name: Unit Test Results - ${{ matrix.image }} | |
| path: artifacts/${{ matrix.image }}/**/*.xml | |
| build-and-test-macos: | |
| name: "Build and Test macOS (${{ matrix.image }}-macos)" | |
| needs: [init-workflow, build-and-test] | |
| if: > | |
| needs.init-workflow.outputs.run-at-all == 'true' && | |
| needs.init-workflow.outputs.run-builds-and-tests == 'true' | |
| runs-on: macos-11 | |
| strategy: | |
| max-parallel: 3 | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - image: test-cpu-openmpi-py3_7-tf1_15_5-keras2_2_4-torch1_6_0-mxnet1_5_1_p0 | |
| HOROVOD_WITH_MPI: 1 | |
| HOROVOD_WITHOUT_GLOO: 1 | |
| TENSORFLOW: 1.15.0 | |
| KERAS: 2.2.4 | |
| PYTORCH: 1.6.0 | |
| PYTORCH_LIGHTNING: 1.3.8 | |
| TORCHVISION: 0.7.0 | |
| MXNET: 1.5.1.post0 | |
| - image: test-cpu-gloo-py3_8-tf2_9_2-keras2_9_0-torch1_11_0-mxnet1_7_0_p2 | |
| HOROVOD_WITHOUT_MPI: 1 | |
| HOROVOD_WITH_GLOO: 1 | |
| TENSORFLOW: 2.9.2 | |
| KERAS: 2.9.0 | |
| PYTORCH: 1.11.0 | |
| PYTORCH_LIGHTNING: 1.5.9 | |
| TORCHVISION: 0.12.0 | |
| MXNET: 1.7.0.post2 | |
| - image: test-openmpi-cpu-gloo-py3_8-tf2_10_0-keras2_10_0-torch1_12_1-mxnet1_9_1 | |
| HOROVOD_WITH_MPI: 1 | |
| HOROVOD_WITH_GLOO: 1 | |
| TENSORFLOW: 2.10.0 | |
| KERAS: 2.10.0 | |
| PYTORCH: 1.12.1 | |
| PYTORCH_LIGHTNING: 1.5.9 | |
| TORCHVISION: 0.13.1 | |
| MXNET: 1.9.1 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| - name: Build | |
| id: build | |
| env: | |
| HOROVOD_WITH_MPI: ${{ matrix.HOROVOD_WITH_MPI }} | |
| HOROVOD_WITHOUT_MPI: ${{ matrix.HOROVOD_WITHOUT_MPI }} | |
| HOROVOD_WITH_GLOO: ${{ matrix.HOROVOD_WITH_GLOO }} | |
| HOROVOD_WITHOUT_GLOO: ${{ matrix.HOROVOD_WITHOUT_GLOO }} | |
| TENSORFLOW: ${{ matrix.TENSORFLOW }} | |
| KERAS: ${{ matrix.KERAS }} | |
| PYTORCH: ${{ matrix.PYTORCH }} | |
| PYTORCH_LIGHTNING: ${{ matrix.PYTORCH_LIGHTNING }} | |
| TORCHVISION: ${{ matrix.TORCHVISION }} | |
| MXNET: ${{ matrix.MXNET }} | |
| # The python patch in the pyenv install step is to work around an incompatibility introduced in new xcode version in macOS Big Sur. The patch is provided by python team. | |
| # The original discussion is here https://github.com/pyenv/pyenv/issues/1737 | |
| run: | | |
| brew reinstall -f zlib bzip2 | |
| brew install -f openmpi cmake libuv pyenv coreutils curl | |
| export PATH=$(pyenv root)/shims:$PATH | |
| pyenv uninstall -f 3.7.7 | |
| CFLAGS="-I$(brew --prefix bzip2)/include -I$(brew --prefix zlib)/include" LDFLAGS="-L$(brew --prefix zlib)/lib -L$(brew --prefix bzip2)/lib" pyenv install --patch 3.7.7 < <(curl -sSL https://github.com/python/cpython/commit/8ea6353.patch) | |
| pyenv global 3.7.7 | |
| python --version | |
| python -m pip install -U pip | |
| pip install tensorflow==${TENSORFLOW} keras==${KERAS} | |
| if [[ ${TENSORFLOW} == 1.* ]] || [[ ${TENSORFLOW} == 2.[012345].* ]]; then pip install "h5py<3" "protobuf~=3.20"; fi | |
| pip install torch==${PYTORCH} pytorch_lightning==${PYTORCH_LIGHTNING} torchvision==${TORCHVISION} | |
| pip install mxnet==${MXNET} | |
| HOROVOD_WITH_TENSORFLOW=1 HOROVOD_WITH_PYTORCH=1 HOROVOD_WITH_MXNET=1 pip install --no-cache-dir .[test] | |
| horovodrun --check-build | |
| - name: Test [attempt 1 of 3] | |
| id: test-1 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && true | |
| run: | | |
| export PATH=$(pyenv root)/shims:$PATH | |
| pyenv global 3.7.7 | |
| python --version | |
| artifacts_path="$(pwd)/artifacts/${{ matrix.image }}-macos-run-1" | |
| mkdir -p "$artifacts_path" | |
| echo "artifacts-path=$artifacts_path" >> $GITHUB_OUTPUT | |
| echo pytest -v --capture=no --continue-on-collection-errors --junit-xml=$artifacts_path/junit.\$1.\${HOROVOD_RANK:-\${OMPI_COMM_WORLD_RANK:-\${PMI_RANK}}}.\$2.xml \${@:2} > pytest.sh | |
| chmod u+x pytest.sh | |
| cd test/parallel | |
| ls test_*.py | gtimeout 10m xargs -n 1 horovodrun -np 2 /bin/bash ../../pytest.sh macos | |
| - name: Test [attempt 2 of 3] | |
| id: test-2 | |
| continue-on-error: true | |
| if: always() && steps.build.outcome == 'success' && steps.test-1.outcome == 'failure' | |
| run: | | |
| export PATH=$(pyenv root)/shims:$PATH | |
| pyenv global 3.7.7 | |
| python --version | |
| artifacts_path="$(pwd)/artifacts/${{ matrix.image }}-macos-run-2" | |
| mkdir -p "$artifacts_path" | |
| echo "artifacts-path=$artifacts_path" >> $GITHUB_OUTPUT | |
| echo pytest -v --capture=no --continue-on-collection-errors --junit-xml=$artifacts_path/junit.\$1.\${HOROVOD_RANK:-\${OMPI_COMM_WORLD_RANK:-\${PMI_RANK}}}.\$2.xml \${@:2} > pytest.sh | |
| chmod u+x pytest.sh | |
| cd test/parallel | |
| ls test_*.py | gtimeout 10m xargs -n 1 horovodrun -np 2 /bin/bash ../../pytest.sh macos | |
| - name: Test [attempt 3 of 3] | |
| id: test-3 | |
| continue-on-error: false | |
| if: always() && steps.build.outcome == 'success' && steps.test-2.outcome == 'failure' | |
| run: | | |
| export PATH=$(pyenv root)/shims:$PATH | |
| pyenv global 3.7.7 | |
| python --version | |
| artifacts_path="$(pwd)/artifacts/${{ matrix.image }}-macos-run-3" | |
| mkdir -p "$artifacts_path" | |
| echo "artifacts-path=$artifacts_path" >> $GITHUB_OUTPUT | |
| echo pytest -v --capture=no --continue-on-collection-errors --junit-xml=$artifacts_path/junit.\$1.\${HOROVOD_RANK:-\${OMPI_COMM_WORLD_RANK:-\${PMI_RANK}}}.\$2.xml \${@:2} > pytest.sh | |
| chmod u+x pytest.sh | |
| cd test/parallel | |
| ls test_*.py | gtimeout 10m xargs -n 1 horovodrun -np 2 /bin/bash ../../pytest.sh macos | |
| - name: Upload Test Results | |
| uses: actions/upload-artifact@v3 | |
| if: always() | |
| with: | |
| name: Unit Test Results - ${{ matrix.image }}-macos | |
| path: | | |
| ${{ steps.test-1.outputs.artifacts-path }} | |
| ${{ steps.test-2.outputs.artifacts-path }} | |
| ${{ steps.test-3.outputs.artifacts-path }} | |
| buildkite-trigger: | |
| name: "Build and Test GPU (trigger Builtkite)" | |
| needs: [init-workflow, build-and-test] | |
| runs-on: ubuntu-latest | |
| if: > | |
| github.repository == 'horovod/horovod' && | |
| needs.init-workflow.outputs.run-at-all == 'true' && | |
| needs.init-workflow.outputs.run-builds-and-tests == 'true' && | |
| ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository ) | |
| outputs: | |
| url: ${{ steps.build.outputs.url }} | |
| steps: | |
| - name: Trigger Buildkite Pipeline | |
| id: build | |
| uses: buildkite/trigger-pipeline-action@v1.3.1 | |
| env: | |
| PIPELINE: "horovod/horovod" | |
| # COMMIT is taken from GITHUB_SHA | |
| BRANCH: "${{ needs.init-workflow.outputs.buildkite-branch-label }} (GPU NON HEADS)" | |
| # empty MESSAGE will be filled by Buildkite from commit message | |
| MESSAGE: "${{ needs.init-workflow.outputs.buildkite-message }}" | |
| BUILDKITE_API_ACCESS_TOKEN: ${{ secrets.BUILDKITE_TOKEN }} | |
| BUILD_ENV_VARS: "{\"PIPELINE_MODE\": \"GPU NON HEADS\"}" | |
| buildkite: | |
| name: "Build and Test GPU (download Builtkite)" | |
| needs: [buildkite-trigger] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Download Buildkite Artifacts | |
| id: download | |
| uses: EnricoMi/download-buildkite-artifact-action@v1 | |
| with: | |
| buildkite_token: ${{ secrets.BUILDKITE_TOKEN }} | |
| buildkite_build_url: ${{ needs.buildkite-trigger.outputs.url }} | |
| ignore_build_states: blocked,canceled,skipped,not_run | |
| ignore_job_states: timed_out | |
| output_path: artifacts/Unit Test Results - GPU NON HEADS on Builtkite | |
| - name: Upload Test Results | |
| uses: actions/upload-artifact@v3 | |
| if: always() | |
| with: | |
| name: Unit Test Results - GPU NON HEADS on Builtkite | |
| path: artifacts/Unit Test Results - GPU NON HEADS on Builtkite/**/*.xml | |
| - name: Check Buildkite job state | |
| if: > | |
| always() && | |
| steps.download.conclusion == 'success' && | |
| steps.download.outputs.build-state != 'passed' | |
| run: | | |
| echo "::warning::Buildkite pipeline did not pass: ${{ needs.buildkite-trigger.outputs.url }}" | |
| exit 1 | |
| buildkite-heads-trigger: | |
| name: "Build and Test GPU heads (trigger Builtkite)" | |
| needs: [init-workflow, build-and-test] | |
| runs-on: ubuntu-latest | |
| if: > | |
| github.repository == 'horovod/horovod' && | |
| needs.init-workflow.outputs.run-at-all == 'true' && | |
| needs.init-workflow.outputs.run-builds-and-tests == 'true' && | |
| ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository ) | |
| outputs: | |
| url: ${{ steps.build.outputs.url }} | |
| steps: | |
| - name: Trigger Buildkite Pipeline | |
| id: build | |
| uses: buildkite/trigger-pipeline-action@v1.3.1 | |
| env: | |
| PIPELINE: "horovod/horovod" | |
| # COMMIT is taken from GITHUB_SHA | |
| BRANCH: "${{ needs.init-workflow.outputs.buildkite-branch-label }} (GPU HEADS)" | |
| # empty MESSAGE will be filled by Buildkite from commit message | |
| MESSAGE: "${{ needs.init-workflow.outputs.buildkite-message }}" | |
| BUILDKITE_API_ACCESS_TOKEN: ${{ secrets.BUILDKITE_TOKEN }} | |
| BUILD_ENV_VARS: "{\"PIPELINE_MODE\": \"GPU HEADS\"}" | |
| buildkite-heads: | |
| name: "Build and Test GPU heads (download Builtkite)" | |
| needs: [buildkite-heads-trigger] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Download Buildkite Artifacts | |
| id: download | |
| uses: EnricoMi/download-buildkite-artifact-action@v1 | |
| with: | |
| buildkite_token: ${{ secrets.BUILDKITE_TOKEN }} | |
| buildkite_build_url: ${{ needs.buildkite-heads-trigger.outputs.url }} | |
| ignore_build_states: blocked,canceled,skipped,not_run | |
| ignore_job_states: timed_out | |
| output_path: artifacts/Unit Test Results - GPU HEADS on Builtkite | |
| - name: Upload Test Results | |
| uses: actions/upload-artifact@v3 | |
| if: always() | |
| with: | |
| name: Unit Test Results - GPU HEADS on Builtkite | |
| path: artifacts/Unit Test Results - GPU HEADS on Builtkite/**/*.xml | |
| - name: Check Buildkite job state | |
| if: > | |
| always() && | |
| steps.download.conclusion == 'success' && | |
| steps.download.outputs.build-state != 'passed' | |
| run: | | |
| echo "::warning::Buildkite pipeline did not pass: ${{ needs.buildkite-heads-trigger.outputs.url }}" | |
| exit 1 | |
| docker-config: | |
| name: Configure docker build | |
| needs: [init-workflow, build-and-test, buildkite] | |
| # build-and-test and buildkite might have been skipped (! needs.init-workflow.outputs.run-builds-and-tests) | |
| # buildkite might have been skipped (workflow runs for a fork PR), | |
| # we still want to build docker images (though we might not want to push them) | |
| if: > | |
| always() && | |
| needs.init-workflow.outputs.run-at-all == 'true' && | |
| needs.init-workflow.outputs.run-builds-and-tests == 'true' && | |
| needs.build-and-test.result == 'success' && | |
| ( needs.buildkite.result == 'success' || needs.buildkite.result == 'skipped' ) | |
| runs-on: ubuntu-latest | |
| outputs: | |
| run: ${{ steps.config.outputs.run }} | |
| push: ${{ steps.config.outputs.push }} | |
| steps: | |
| - name: Config | |
| id: config | |
| env: | |
| # run workflow for all events on Horovod repo and non-schedule events on forks | |
| run: ${{ github.repository == 'horovod/horovod' || github.event_name != 'schedule' }} | |
| # push images only from Horovod repo and for schedule and push events | |
| push: ${{ github.repository == 'horovod/horovod' && contains('schedule,push', github.event_name) }} | |
| run: | | |
| echo Repository: ${{ github.repository }} | |
| echo Event: ${{ github.event_name }} | |
| echo Run: $run | |
| echo "run=$run" >> $GITHUB_OUTPUT | |
| echo Push: $push | |
| echo "push=$push" >> $GITHUB_OUTPUT | |
| docker-build: | |
| name: Build docker image ${{ matrix.docker-image }} (push=${{ needs.docker-config.outputs.push }}) | |
| needs: docker-config | |
| if: always() && needs.docker-config.outputs.run == 'true' | |
| runs-on: ubuntu-latest | |
| # we want an ongoing run of this workflow to be canceled by a later commit | |
| # so that there is only one concurrent run of this workflow for each branch | |
| concurrency: | |
| # github.ref means something like refs/heads/master or refs/tags/v0.22.1 or the branch. | |
| # This helps to not cancel concurrent runs on master and a tag that share the same commit | |
| # head_ref refers to the pull request branch so we run only one workflow for the given pull request. | |
| # On master, head_ref is empty, so we use the SHA of the commit, this means | |
| # commits to master will not be cancelled, which is important to ensure | |
| # that every commit to master is full tested and deployed. | |
| group: docker-${{ matrix.docker-image }}-${{ github.ref }}-${{ github.head_ref || github.sha }} | |
| cancel-in-progress: true | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| docker-image: | |
| - horovod | |
| - horovod-cpu | |
| - horovod-nvtabular | |
| - horovod-ray | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: 'recursive' | |
| - name: Docker meta | |
| id: meta | |
| uses: crazy-max/ghaction-docker-meta@v2 | |
| with: | |
| # list of Docker images to use as base name for tags | |
| images: | | |
| horovod/${{ matrix.docker-image }} | |
| # generate Docker tags based on the following events/attributes | |
| tags: | | |
| type=schedule | |
| type=ref,event=branch | |
| type=ref,event=pr | |
| type=semver,pattern={{version}} | |
| type=semver,pattern={{major}}.{{minor}} | |
| type=semver,pattern={{major}} | |
| type=sha | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| with: | |
| driver: docker | |
| - name: Login to DockerHub | |
| if: needs.docker-config.outputs.push == 'true' | |
| uses: docker/login-action@v2 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Clean up disk space | |
| # deleting these paths frees 38 GB disk space: | |
| # sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc | |
| # but this sometimes takes 3-4 minutes | |
| # so we delete only some sub-paths which are known to be quick (10s) and 20 GB | |
| run: | | |
| echo ::group::Disk space before clean up | |
| df -h | |
| echo ::endgroup:: | |
| for dir in /usr/share/dotnet/sdk/\*/nuGetPackagesArchive.lzma \ | |
| /usr/share/dotnet/shared \ | |
| /usr/local/lib/android/sdk/ndk \ | |
| /usr/local/lib/android/sdk/build-tools \ | |
| /opt/ghc | |
| do | |
| echo ::group::Deleting "$dir" | |
| sudo du -hsc $dir | tail -n1 || true | |
| sudo rm -rf $dir | |
| echo ::endgroup:: | |
| done | |
| echo ::group::Disk space after clean up | |
| df -h | |
| echo ::endgroup:: | |
| - name: Build image | |
| id: build | |
| uses: docker/build-push-action@v3 | |
| timeout-minutes: 60 | |
| with: | |
| context: . | |
| file: ./docker/${{ matrix.docker-image }}/Dockerfile | |
| pull: true | |
| push: false | |
| load: true | |
| tags: horovod-test | |
| outputs: type=docker | |
| - name: List image | |
| run: | | |
| docker image ls horovod-test | |
| - name: Prepare container for test | |
| run: | | |
| grep "RUN sed" Dockerfile.test.cpu | sed "s/^RUN //" | docker run -i --name horovod-test horovod-test:latest /bin/bash | |
| - name: Test image (pytorch gloo) | |
| if: always() && steps.build.outcome == 'success' | |
| run: | | |
| docker start -ai horovod-test <<<"python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets --num-proc 2 --hosts localhost:2 --communication gloo" | |
| - name: Test image (tensorflow2 gloo) | |
| if: always() && steps.build.outcome == 'success' | |
| run: | | |
| docker start -ai horovod-test <<<"python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py 2 localhost:2 gloo" | |
| - name: Test image (pytorch mpi) | |
| if: always() && steps.build.outcome == 'success' && matrix.docker-image != 'horovod-ray' | |
| run: | | |
| docker start -ai horovod-test <<<"python /horovod/examples/pytorch/pytorch_mnist.py --data-dir /data/pytorch_datasets --num-proc 2 --hosts localhost:2 --communication mpi" | |
| - name: Test image (tensorflow2 mpi) | |
| if: always() && steps.build.outcome == 'success' && matrix.docker-image != 'horovod-ray' | |
| run: | | |
| docker start -ai horovod-test <<<"python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py 2 localhost:2 mpi" | |
| - name: Push image | |
| if: needs.docker-config.outputs.push == 'true' | |
| uses: docker/build-push-action@v3 | |
| timeout-minutes: 60 | |
| with: | |
| context: . | |
| file: ./docker/${{ matrix.docker-image }}/Dockerfile | |
| push: ${{ needs.docker-config.outputs.push }} | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| - name: Show free space | |
| if: always() | |
| run: | | |
| echo ::group::Disk Space | |
| df -h | |
| echo ::endgroup:: | |
| echo ::group::Docker Space | |
| docker system df | |
| echo ::endgroup:: | |
| echo ::group::Docker Images | |
| docker images -a | |
| echo ::endgroup:: | |
| echo ::group::Docker Container | |
| docker container list -a | |
| echo ::endgroup:: | |
| sync-files: | |
| name: "Sync Files (${{ matrix.name }})" | |
| needs: [init-workflow] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: Docs Summary | |
| left_file: README.rst | |
| right_file: docs/summary.rst | |
| init: sed -i -e s/docs\///g README.rst | |
| - name: Examples Keras Spark3 | |
| left_file: examples/spark/keras/keras_spark_rossmann_run.py | |
| right_file: examples/spark/keras/keras_spark3_rossmann.py | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Diffing ${{ matrix.left_file }} with ${{ matrix.right_file }} | |
| env: | |
| LEFT: ${{ matrix.left_file }} | |
| RIGHT: ${{ matrix.right_file }} | |
| INIT: ${{ matrix.init }} | |
| run: | | |
| $INIT | |
| patch --quiet -p0 $LEFT ${RIGHT}.patch -o ${LEFT}.expected | |
| if ! diff -q ${LEFT}.expected --label $LEFT $RIGHT | |
| then | |
| echo | |
| echo "::error::Files are out-of-sync: $LEFT vs. $RIGHT" | |
| echo "Unexpected differences are:" | |
| diff ${LEFT}.expected --label $LEFT $RIGHT || true | |
| echo | |
| echo "Use the following as ${RIGHT}.patch to accept those changes:" | |
| diff $LEFT $RIGHT || true | |
| false | |
| fi |