From 8d8425b6af5818ba44de343eb82cc3d0ecda3015 Mon Sep 17 00:00:00 2001 From: Tristan Rice Date: Thu, 14 Oct 2021 13:22:32 -0700 Subject: [PATCH] CI: use OIDC --- .../components-integration-tests.yaml | 17 +++++++------- .github/workflows/kfp-integration-tests.yaml | 17 +++++++------- ...bernetes-dist-train-integration-tests.yaml | 23 ++++++++++--------- 3 files changed, 30 insertions(+), 27 deletions(-) diff --git a/.github/workflows/components-integration-tests.yaml b/.github/workflows/components-integration-tests.yaml index 006812636..9357583d4 100644 --- a/.github/workflows/components-integration-tests.yaml +++ b/.github/workflows/components-integration-tests.yaml @@ -9,6 +9,9 @@ on: jobs: components-launch: runs-on: ubuntu-18.04 + permissions: + id-token: write + contents: read steps: - name: Setup Python uses: actions/setup-python@v2 @@ -17,19 +20,19 @@ jobs: architecture: x64 - name: Checkout TorchX uses: actions/checkout@v2 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@master + with: + aws-region: us-west-2 + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + role-session-name: component-integration-tests - name: Configure Kube Config - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} run: | set -eux if [ -n "$AWS_ACCESS_KEY_ID" ]; then aws eks update-kubeconfig --region=us-west-2 --name=${{ secrets.EKS_CLUSTER_NAME }} fi - name: Configure Docker - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} run: | set -eux if [ -n "$AWS_ACCESS_KEY_ID" ]; then @@ -42,8 +45,6 @@ jobs: pip install -e .[kubernetes] - name: Run Components Integration Tests env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} INTEGRATION_TEST_STORAGE: ${{ secrets.INTEGRATION_TEST_STORAGE }} CONTAINER_REPO: ${{ secrets.CONTAINER_REPO }} run: scripts/component_integration_tests.py diff --git a/.github/workflows/kfp-integration-tests.yaml b/.github/workflows/kfp-integration-tests.yaml index 53591852e..828b397da 100644 --- a/.github/workflows/kfp-integration-tests.yaml +++ b/.github/workflows/kfp-integration-tests.yaml @@ -9,6 +9,9 @@ on: jobs: kfp-launch: runs-on: ubuntu-18.04 + permissions: + id-token: write + contents: read steps: - name: Install kubectl # More info: https://kubernetes.io/docs/tasks/tools/install-kubectl-linux/ @@ -18,10 +21,13 @@ jobs: mkdir -p ~/.local/bin/kubectl mv ./kubectl ~/.local/bin/kubectl export PATH=$PATH:~/.local/bin/kubectl + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@master + with: + aws-region: us-west-2 + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + role-session-name: kfp-integration-tests - name: Configure Kube Config - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} run: | set -eux if [ -n "$AWS_ACCESS_KEY_ID" ]; then @@ -35,9 +41,6 @@ jobs: - name: Checkout TorchX uses: actions/checkout@v2 - name: Configure Docker - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} run: | set -eux if [ -n "$AWS_ACCESS_KEY_ID" ]; then @@ -50,8 +53,6 @@ jobs: python setup.py install - name: Run KFP Integration Tests env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} KFP_NAMESPACE: ${{ secrets.KFP_NAMESPACE }} INTEGRATION_TEST_STORAGE: ${{ secrets.INTEGRATION_TEST_STORAGE }} CONTAINER_REPO: ${{ secrets.CONTAINER_REPO }} diff --git a/.github/workflows/kubernetes-dist-train-integration-tests.yaml b/.github/workflows/kubernetes-dist-train-integration-tests.yaml index 8088d5baa..b66fc4ea6 100644 --- a/.github/workflows/kubernetes-dist-train-integration-tests.yaml +++ b/.github/workflows/kubernetes-dist-train-integration-tests.yaml @@ -9,6 +9,9 @@ on: jobs: kubernetes-launch: runs-on: ubuntu-18.04 + permissions: + id-token: write + contents: read steps: - name: Setup Python uses: actions/setup-python@v2 @@ -17,22 +20,22 @@ jobs: architecture: x64 - name: Checkout TorchX uses: actions/checkout@v2 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@master + with: + aws-region: us-west-2 + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + role-session-name: kubernetes-dist-train-integration-tests - name: Configure Kube Config - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} run: | set -eux - if [ -n "$AWS_ACCESS_KEY_ID" ]; then + if [ -n "$AWS_ROLE_ARN" ]; then aws eks update-kubeconfig --region=us-west-2 --name=${{ secrets.EKS_CLUSTER_NAME }} fi - name: Configure Docker - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} run: | set -eux - if [ -n "$AWS_ACCESS_KEY_ID" ]; then + if [ -n "$AWS_ROLE_ARN" ]; then aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 495572122715.dkr.ecr.us-west-2.amazonaws.com fi - name: Install dependencies @@ -41,12 +44,10 @@ jobs: pip install -e .[kubernetes] - name: Run Kubernetes Integration Tests env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} INTEGRATION_TEST_STORAGE: ${{ secrets.INTEGRATION_TEST_STORAGE }} CONTAINER_REPO: ${{ secrets.CONTAINER_REPO }} run: | - if [ -z "$AWS_ACCESS_KEY_ID" ]; then + if [ -z "$AWS_ROLE_ARN" ]; then # only dryrun if no secrets ARGS="--dryrun" else