From b8f4a172f740128588037456efd880ed57783b59 Mon Sep 17 00:00:00 2001 From: Zain Rizvi Date: Thu, 4 May 2023 14:42:53 -0500 Subject: [PATCH] pass down secrets --- .../actions/pytest-cache-download/action.yml | 8 +++++++ .../actions/pytest-cache-upload/action.yml | 8 +++++++ .github/scripts/pytest_cache.py | 14 ++++++++++- .github/workflows/_linux-test.yml | 10 ++++++++ .github/workflows/pull.yml | 24 +++++++++++++++++++ 5 files changed, 63 insertions(+), 1 deletion(-) diff --git a/.github/actions/pytest-cache-download/action.yml b/.github/actions/pytest-cache-download/action.yml index 7daf4cf67bc9..ed3cd6cdb8af 100644 --- a/.github/actions/pytest-cache-download/action.yml +++ b/.github/actions/pytest-cache-download/action.yml @@ -9,6 +9,12 @@ inputs: github-token: description: GITHUB_TOKEN required: true + aws-access-key-id: + description: AWS access key id + required: true + aws-secret-access-key: + description: AWS secret access key + required: true runs: using: composite @@ -33,6 +39,8 @@ runs: WORKFLOW: ${{ github.workflow }} JOB: ${{ github.job }} SHARD: ${{ inputs.shard }} + AWS_ACCESS_KEY_ID: ${{ inputs.aws-access-key-id }} + AWS_SECRET_ACCESS_KEY: ${{ inputs.aws-secret-access-key }} run: | set +x diff --git a/.github/actions/pytest-cache-upload/action.yml b/.github/actions/pytest-cache-upload/action.yml index 9aef8e1977fe..1758aee60c05 100644 --- a/.github/actions/pytest-cache-upload/action.yml +++ b/.github/actions/pytest-cache-upload/action.yml @@ -13,6 +13,12 @@ inputs: github-token: description: GITHUB_TOKEN required: true + aws-access-key-id: + description: AWS access key id + required: true + aws-secret-access-key: + description: AWS secret access key + required: true runs: using: composite @@ -37,6 +43,8 @@ runs: WORKFLOW: ${{ github.workflow }} JOB: ${{ github.job }} SHARD: ${{ inputs.shard }} + AWS_ACCESS_KEY_ID: ${{ inputs.aws-access-key-id }} + AWS_SECRET_ACCESS_KEY: ${{ inputs.aws-secret-access-key }} run: | set +x diff --git a/.github/scripts/pytest_cache.py b/.github/scripts/pytest_cache.py index 59bbeaa5a42e..006d499dbfd8 100644 --- a/.github/scripts/pytest_cache.py +++ b/.github/scripts/pytest_cache.py @@ -49,7 +49,19 @@ def main(): # TODO: First check if it's even worth uploading a new cache: # Does the cache even mark any failed tests? - print(os.getenv("AWS_ACCESS_KEY_ID")) + id = os.getenv("AWS_ACCESS_KEY_ID") + # get the first three chars if it's not none + if id: + id = id[:3] + print(f"Access key id prefix: {id}xxxxxxxxxx") + else: + print("No access key id found") + + if os.getenv("AWS_SECRET_ACCESS_KEY"): + print("Secret access key found") + else: + print("No secret access key found") + upload_pytest_cache( pr_identifier=PRIdentifier(args.pr_identifier), workflow=args.workflow, diff --git a/.github/workflows/_linux-test.yml b/.github/workflows/_linux-test.yml index b178766eb4fc..61f3491880ef 100644 --- a/.github/workflows/_linux-test.yml +++ b/.github/workflows/_linux-test.yml @@ -33,6 +33,14 @@ on: type: string default: "" description: If set to any value, upload to GHA. Otherwise upload to S3. + secrets: + aws-access-key-id: + required: false + description: AWS access key id + aws-pytorch-uploader-secret-access-key: + required: false + description: | + Secret access key for uploading to S3. env: GIT_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} @@ -228,6 +236,8 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} cache_dir: .pytest_cache shard: ${{ matrix.shard }} + aws-access-key-id: ${{ secrets.aws-access-key-id }} + aws-secret-access-key: ${{ secrets.aws-pytorch-uploader-secret-access-key }} - name: Print remaining test logs shell: bash diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index d477971f81b8..578e88652d52 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -42,6 +42,9 @@ jobs: build-environment: linux-focal-py3.8-gcc7 docker-image: ${{ needs.linux-focal-py3_8-gcc7-build.outputs.docker-image }} test-matrix: ${{ needs.linux-focal-py3_8-gcc7-build.outputs.test-matrix }} + secrets: + aws-access-key-id: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} + aws-pytorch-uploader-secret-access-key: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} linux-docs: name: linux-docs @@ -99,6 +102,9 @@ jobs: docker-image: ${{ needs.linux-focal-py3_9-clang7-asan-build.outputs.docker-image }} test-matrix: ${{ needs.linux-focal-py3_9-clang7-asan-build.outputs.test-matrix }} sync-tag: asan-test + secrets: + aws-access-key-id: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} + aws-pytorch-uploader-secret-access-key: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} linux-focal-py3_8-clang10-onnx-build: name: linux-focal-py3.8-clang10-onnx @@ -120,6 +126,9 @@ jobs: build-environment: linux-focal-py3.8-clang10-onnx docker-image: ${{ needs.linux-focal-py3_8-clang10-onnx-build.outputs.docker-image }} test-matrix: ${{ needs.linux-focal-py3_8-clang10-onnx-build.outputs.test-matrix }} + secrets: + aws-access-key-id: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} + aws-pytorch-uploader-secret-access-key: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} linux-bionic-py3_8-clang9-build: name: linux-bionic-py3.8-clang9 @@ -146,6 +155,9 @@ jobs: build-environment: linux-bionic-py3.8-clang9 docker-image: ${{ needs.linux-bionic-py3_8-clang9-build.outputs.docker-image }} test-matrix: ${{ needs.linux-bionic-py3_8-clang9-build.outputs.test-matrix }} + secrets: + aws-access-key-id: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} + aws-pytorch-uploader-secret-access-key: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} linux-bionic-py3_11-clang9-build: name: linux-bionic-py3.11-clang9 @@ -172,6 +184,9 @@ jobs: build-environment: linux-bionic-py3.11-clang9 docker-image: ${{ needs.linux-bionic-py3_11-clang9-build.outputs.docker-image }} test-matrix: ${{ needs.linux-bionic-py3_11-clang9-build.outputs.test-matrix }} + secrets: + aws-access-key-id: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} + aws-pytorch-uploader-secret-access-key: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} linux-bionic-cuda11_8-py3_10-gcc7-build: name: linux-bionic-cuda11.8-py3.10-gcc7 @@ -200,6 +215,9 @@ jobs: build-environment: linux-bionic-cuda11.8-py3.10-gcc7 docker-image: ${{ needs.linux-bionic-cuda11_8-py3_10-gcc7-build.outputs.docker-image }} test-matrix: ${{ needs.linux-bionic-cuda11_8-py3_10-gcc7-build.outputs.test-matrix }} + secrets: + aws-access-key-id: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} + aws-pytorch-uploader-secret-access-key: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} linux-focal-py3-clang7-mobile-build: name: linux-focal-py3-clang7-mobile-build @@ -255,6 +273,9 @@ jobs: build-environment: linux-bionic-py3_8-clang8-xla docker-image: ${{ needs.linux-bionic-py3_8-clang8-xla-build.outputs.docker-image }} test-matrix: ${{ needs.linux-bionic-py3_8-clang8-xla-build.outputs.test-matrix }} + secrets: + aws-access-key-id: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} + aws-pytorch-uploader-secret-access-key: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} win-vs2019-cpu-py3-build: name: win-vs2019-cpu-py3 @@ -367,3 +388,6 @@ jobs: build-environment: linux-bionic-cuda11.8-py3.10-gcc7-sm86 docker-image: ${{ needs.linux-bionic-cuda11_8-py3_10-gcc7-sm86-build.outputs.docker-image }} test-matrix: ${{ needs.linux-bionic-cuda11_8-py3_10-gcc7-sm86-build.outputs.test-matrix }} + secrets: + aws-access-key-id: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} + aws-pytorch-uploader-secret-access-key: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }}