From 6249feaa7d9e555f02aff1bdd7a83fc6550b31b1 Mon Sep 17 00:00:00 2001 From: atalman Date: Thu, 16 May 2024 13:56:43 -0700 Subject: [PATCH 1/5] Pin rocm docker images --- .github/workflows/inductor.yml | 2 +- .github/workflows/periodic.yml | 2 +- .github/workflows/pull.yml | 2 +- .github/workflows/rocm.yml | 2 +- .github/workflows/slow.yml | 2 +- .github/workflows/trunk.yml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/inductor.yml b/.github/workflows/inductor.yml index 1934d7fd8642..81f72275cff7 100644 --- a/.github/workflows/inductor.yml +++ b/.github/workflows/inductor.yml @@ -21,7 +21,7 @@ jobs: uses: ./.github/workflows/_linux-build.yml with: build-environment: linux-focal-rocm6.0-py3.8 - docker-image-name: pytorch-linux-focal-rocm-n-py3 + docker-image-name: pytorch-linux-focal-rocm-n-py3:cea4be730564c18dd285a12828c7c449490b10b9 test-matrix: | { include: [ { config: "inductor", shard: 1, num_shards: 1, runner: "linux.rocm.gpu.2" }, diff --git a/.github/workflows/periodic.yml b/.github/workflows/periodic.yml index 99f4dd99395f..bfa4ccc78e33 100644 --- a/.github/workflows/periodic.yml +++ b/.github/workflows/periodic.yml @@ -214,7 +214,7 @@ jobs: uses: ./.github/workflows/_linux-build.yml with: build-environment: linux-focal-rocm6.0-py3.8 - docker-image-name: pytorch-linux-focal-rocm-n-py3 + docker-image-name: pytorch-linux-focal-rocm-n-py3:cea4be730564c18dd285a12828c7c449490b10b9 test-matrix: | { include: [ { config: "distributed", shard: 1, num_shards: 2, runner: "linux.rocm.gpu" }, diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index 887902bb39ee..a2df75c4a291 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -408,7 +408,7 @@ jobs: uses: ./.github/workflows/_linux-build.yml with: build-environment: linux-focal-rocm6.0-py3.8 - docker-image-name: pytorch-linux-focal-rocm-n-py3 + docker-image-name: pytorch-linux-focal-rocm-n-py3:cea4be730564c18dd285a12828c7c449490b10b9 sync-tag: rocm-build test-matrix: | { include: [ diff --git a/.github/workflows/rocm.yml b/.github/workflows/rocm.yml index 24542c3ddc47..0db1fd42f1c5 100644 --- a/.github/workflows/rocm.yml +++ b/.github/workflows/rocm.yml @@ -30,7 +30,7 @@ jobs: uses: ./.github/workflows/_linux-build.yml with: build-environment: linux-focal-rocm6.0-py3.8 - docker-image-name: pytorch-linux-focal-rocm-n-py3 + docker-image-name: pytorch-linux-focal-rocm-n-py3:cea4be730564c18dd285a12828c7c449490b10b9 sync-tag: rocm-build test-matrix: | { include: [ diff --git a/.github/workflows/slow.yml b/.github/workflows/slow.yml index 33577986f643..307c6e8f7095 100644 --- a/.github/workflows/slow.yml +++ b/.github/workflows/slow.yml @@ -106,7 +106,7 @@ jobs: uses: ./.github/workflows/_linux-build.yml with: build-environment: linux-focal-rocm6.0-py3.8 - docker-image-name: pytorch-linux-focal-rocm-n-py3 + docker-image-name: pytorch-linux-focal-rocm-n-py3:cea4be730564c18dd285a12828c7c449490b10b9 test-matrix: | { include: [ { config: "slow", shard: 1, num_shards: 1, runner: "linux.rocm.gpu" }, diff --git a/.github/workflows/trunk.yml b/.github/workflows/trunk.yml index c0538a8600d9..2898874a551f 100644 --- a/.github/workflows/trunk.yml +++ b/.github/workflows/trunk.yml @@ -195,7 +195,7 @@ jobs: uses: ./.github/workflows/_linux-build.yml with: build-environment: linux-focal-rocm6.0-py3.8 - docker-image-name: pytorch-linux-focal-rocm-n-py3 + docker-image-name: pytorch-linux-focal-rocm-n-py3:cea4be730564c18dd285a12828c7c449490b10b9 sync-tag: rocm-build test-matrix: | { include: [ From 74d0d79e7171ed916c1532ce9a7db4d014901b37 Mon Sep 17 00:00:00 2001 From: atalman Date: Thu, 16 May 2024 14:57:55 -0700 Subject: [PATCH 2/5] test --- .github/workflows/_linux-build.yml | 27 +++++++++++++++++++++++---- .github/workflows/inductor.yml | 3 ++- .github/workflows/periodic.yml | 3 ++- .github/workflows/pull.yml | 3 ++- .github/workflows/rocm.yml | 3 ++- .github/workflows/slow.yml | 3 ++- .github/workflows/trunk.yml | 3 ++- 7 files changed, 35 insertions(+), 10 deletions(-) diff --git a/.github/workflows/_linux-build.yml b/.github/workflows/_linux-build.yml index 6af95927ae1d..f114d123d611 100644 --- a/.github/workflows/_linux-build.yml +++ b/.github/workflows/_linux-build.yml @@ -11,6 +11,11 @@ on: required: true type: string description: Name of the base docker image to build with. + docker-image-tag: + required: false + type: string + description: Name of the base docker image tag + default: "" build-generates-artifacts: required: false type: boolean @@ -69,7 +74,7 @@ jobs: runs-on: ${{ inputs.runner }} timeout-minutes: 240 outputs: - docker-image: ${{ steps.calculate-docker-image.outputs.docker-image }} + docker-image: ${{ steps.calculate-docker.outputs.docker-image }} test-matrix: ${{ steps.filter.outputs.test-matrix }} steps: - name: Setup SSH (Click me for login details) @@ -93,19 +98,33 @@ jobs: with: docker-image-name: ${{ inputs.docker-image-name }} + - name: Override docker image tag if pinned + id: calculate-docker + env: + ECR_DOCKER_IMAGE: ${{ steps.calculate-docker-image.outputs.docker-image }} + shell: bash | + export NEW_TAG=${{ inputs.docker-image-tag }} + if[[ ${NEW_TAG} != '' ]]; then + IMAGE=${ECR_DOCKER_IMAGE%:*} + echo "docker-image=${IMAGE}:${NEW_TAG}" >> "${GITHUB_OUTPUT}" + else + echo "docker-image=${ECR_DOCKER_IMAGE}" >> "${GITHUB_OUTPUT}" + fi + - name: Use following to pull public copy of the image id: print-ghcr-mirror env: - ECR_DOCKER_IMAGE: ${{ steps.calculate-docker-image.outputs.docker-image }} + ECR_DOCKER_IMAGE: ${{ steps.calculate-docker.outputs.docker-image }} shell: bash run: | tag=${ECR_DOCKER_IMAGE##*/} echo "docker pull ghcr.io/pytorch/ci-image:${tag/:/-}" + echo "docker-image=${DOCKER_IMAGE_NAME}" >> "${GITHUB_OUTPUT}" - name: Pull docker image uses: pytorch/test-infra/.github/actions/pull-docker-image@release/2.3 with: - docker-image: ${{ steps.calculate-docker-image.outputs.docker-image }} + docker-image: ${{ steps.calculate-docker.outputs.docker-image }} - name: Parse ref id: parse-ref @@ -149,7 +168,7 @@ jobs: XLA_CLANG_CACHE_S3_BUCKET_NAME: ossci-compiler-clang-cache-circleci-xla PR_LABELS: ${{ toJson(github.event.pull_request.labels.*.name) }} TORCH_CUDA_ARCH_LIST: ${{ inputs.cuda-arch-list }} - DOCKER_IMAGE: ${{ steps.calculate-docker-image.outputs.docker-image }} + DOCKER_IMAGE: ${{ steps.calculate-docker.outputs.docker-image }} XLA_CUDA: ${{ contains(inputs.build-environment, 'xla') && '0' || '' }} DEBUG: ${{ inputs.build-with-debug && '1' || '0' }} OUR_GITHUB_JOB_ID: ${{ steps.get-job-id.outputs.job-id }} diff --git a/.github/workflows/inductor.yml b/.github/workflows/inductor.yml index 81f72275cff7..af08fc3e9947 100644 --- a/.github/workflows/inductor.yml +++ b/.github/workflows/inductor.yml @@ -21,7 +21,8 @@ jobs: uses: ./.github/workflows/_linux-build.yml with: build-environment: linux-focal-rocm6.0-py3.8 - docker-image-name: pytorch-linux-focal-rocm-n-py3:cea4be730564c18dd285a12828c7c449490b10b9 + docker-image-name: pytorch-linux-focal-rocm-n-py3 + docker-image-tag: cea4be730564c18dd285a12828c7c449490b10b9 test-matrix: | { include: [ { config: "inductor", shard: 1, num_shards: 1, runner: "linux.rocm.gpu.2" }, diff --git a/.github/workflows/periodic.yml b/.github/workflows/periodic.yml index bfa4ccc78e33..b6a4eab8762c 100644 --- a/.github/workflows/periodic.yml +++ b/.github/workflows/periodic.yml @@ -214,7 +214,8 @@ jobs: uses: ./.github/workflows/_linux-build.yml with: build-environment: linux-focal-rocm6.0-py3.8 - docker-image-name: pytorch-linux-focal-rocm-n-py3:cea4be730564c18dd285a12828c7c449490b10b9 + docker-image-name: pytorch-linux-focal-rocm-n-py3 + docker-image-tag: cea4be730564c18dd285a12828c7c449490b10b9 test-matrix: | { include: [ { config: "distributed", shard: 1, num_shards: 2, runner: "linux.rocm.gpu" }, diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index a2df75c4a291..a604b095158c 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -408,7 +408,8 @@ jobs: uses: ./.github/workflows/_linux-build.yml with: build-environment: linux-focal-rocm6.0-py3.8 - docker-image-name: pytorch-linux-focal-rocm-n-py3:cea4be730564c18dd285a12828c7c449490b10b9 + docker-image-name: pytorch-linux-focal-rocm-n-py3 + docker-image-tag: cea4be730564c18dd285a12828c7c449490b10b9 sync-tag: rocm-build test-matrix: | { include: [ diff --git a/.github/workflows/rocm.yml b/.github/workflows/rocm.yml index 0db1fd42f1c5..1d3b5a53569e 100644 --- a/.github/workflows/rocm.yml +++ b/.github/workflows/rocm.yml @@ -30,7 +30,8 @@ jobs: uses: ./.github/workflows/_linux-build.yml with: build-environment: linux-focal-rocm6.0-py3.8 - docker-image-name: pytorch-linux-focal-rocm-n-py3:cea4be730564c18dd285a12828c7c449490b10b9 + docker-image-name: pytorch-linux-focal-rocm-n-py3 + docker-image-tag: cea4be730564c18dd285a12828c7c449490b10b9 sync-tag: rocm-build test-matrix: | { include: [ diff --git a/.github/workflows/slow.yml b/.github/workflows/slow.yml index 307c6e8f7095..04858e89b375 100644 --- a/.github/workflows/slow.yml +++ b/.github/workflows/slow.yml @@ -106,7 +106,8 @@ jobs: uses: ./.github/workflows/_linux-build.yml with: build-environment: linux-focal-rocm6.0-py3.8 - docker-image-name: pytorch-linux-focal-rocm-n-py3:cea4be730564c18dd285a12828c7c449490b10b9 + docker-image-name: pytorch-linux-focal-rocm-n-py3 + docker-image-tag: cea4be730564c18dd285a12828c7c449490b10b9 test-matrix: | { include: [ { config: "slow", shard: 1, num_shards: 1, runner: "linux.rocm.gpu" }, diff --git a/.github/workflows/trunk.yml b/.github/workflows/trunk.yml index 2898874a551f..8f5596700a96 100644 --- a/.github/workflows/trunk.yml +++ b/.github/workflows/trunk.yml @@ -195,7 +195,8 @@ jobs: uses: ./.github/workflows/_linux-build.yml with: build-environment: linux-focal-rocm6.0-py3.8 - docker-image-name: pytorch-linux-focal-rocm-n-py3:cea4be730564c18dd285a12828c7c449490b10b9 + docker-image-name: pytorch-linux-focal-rocm-n-py3 + docker-image-tag: cea4be730564c18dd285a12828c7c449490b10b9 sync-tag: rocm-build test-matrix: | { include: [ From 661afa8b71a5462dcb6c4b8e1dfacb53e13b3976 Mon Sep 17 00:00:00 2001 From: atalman Date: Thu, 16 May 2024 15:00:57 -0700 Subject: [PATCH 3/5] test --- .github/workflows/_linux-build.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/_linux-build.yml b/.github/workflows/_linux-build.yml index f114d123d611..fc7b6015379f 100644 --- a/.github/workflows/_linux-build.yml +++ b/.github/workflows/_linux-build.yml @@ -119,7 +119,6 @@ jobs: run: | tag=${ECR_DOCKER_IMAGE##*/} echo "docker pull ghcr.io/pytorch/ci-image:${tag/:/-}" - echo "docker-image=${DOCKER_IMAGE_NAME}" >> "${GITHUB_OUTPUT}" - name: Pull docker image uses: pytorch/test-infra/.github/actions/pull-docker-image@release/2.3 From 82010d8261952a1b7bb035cc7049947f5a901a28 Mon Sep 17 00:00:00 2001 From: atalman Date: Thu, 16 May 2024 17:44:12 -0700 Subject: [PATCH 4/5] test --- .github/workflows/_linux-build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/_linux-build.yml b/.github/workflows/_linux-build.yml index fc7b6015379f..a171effee415 100644 --- a/.github/workflows/_linux-build.yml +++ b/.github/workflows/_linux-build.yml @@ -102,7 +102,8 @@ jobs: id: calculate-docker env: ECR_DOCKER_IMAGE: ${{ steps.calculate-docker-image.outputs.docker-image }} - shell: bash | + shell: bash + run: | export NEW_TAG=${{ inputs.docker-image-tag }} if[[ ${NEW_TAG} != '' ]]; then IMAGE=${ECR_DOCKER_IMAGE%:*} From f85309d1f5d443730ba84967b58b98b0c291bd3d Mon Sep 17 00:00:00 2001 From: atalman Date: Thu, 16 May 2024 17:50:55 -0700 Subject: [PATCH 5/5] test --- .github/workflows/_linux-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_linux-build.yml b/.github/workflows/_linux-build.yml index a171effee415..9bd7a6dfd1c7 100644 --- a/.github/workflows/_linux-build.yml +++ b/.github/workflows/_linux-build.yml @@ -105,7 +105,7 @@ jobs: shell: bash run: | export NEW_TAG=${{ inputs.docker-image-tag }} - if[[ ${NEW_TAG} != '' ]]; then + if [[ ${NEW_TAG} != '' ]]; then IMAGE=${ECR_DOCKER_IMAGE%:*} echo "docker-image=${IMAGE}:${NEW_TAG}" >> "${GITHUB_OUTPUT}" else