From 6df23beec4749bde9c081a721c7d17251b632c33 Mon Sep 17 00:00:00 2001 From: Kian-Tat Lim Date: Mon, 7 Mar 2022 15:13:32 -0800 Subject: [PATCH 01/10] Remove null workflow. --- .github/workflows/null.yml | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 .github/workflows/null.yml diff --git a/.github/workflows/null.yml b/.github/workflows/null.yml deleted file mode 100644 index be262c5a..00000000 --- a/.github/workflows/null.yml +++ /dev/null @@ -1,13 +0,0 @@ -name: Empty check for branch protection - -on: - push: - branches: - - master - pull_request: - -jobs: - null_check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 From 3865b2e8c675256bf37c9a9f4b599c4bfed1d26c Mon Sep 17 00:00:00 2001 From: Kian-Tat Lim Date: Mon, 7 Mar 2022 15:42:37 -0800 Subject: [PATCH 02/10] Add base container build GHA. --- .github/workflows/build-base.yml | 40 ++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 .github/workflows/build-base.yml diff --git a/.github/workflows/build-base.yml b/.github/workflows/build-base.yml new file mode 100644 index 00000000..d592cb03 --- /dev/null +++ b/.github/workflows/build-base.yml @@ -0,0 +1,40 @@ +--- +name: "Build" +on: + push: + pull_request: + branches: + - main + workflow_dispatch: + +permissions: + packages: write + +jobs: + update-base-image: + name: Update base image + runs-on: ubuntu-latest + env: + IMAGE_NAME: prompt-proto-base + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Build image + run: docker build python/base --tag $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}" + - name: Login to GitHub Container Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Push image + run: | + IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME + VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') + [ "$VERSION" == "main" ] && VERSION=latest + echo IMAGE_ID=$IMAGE_ID + echo VERSION=$VERSION + docker tag $IMAGE_NAME $IMAGE_ID:$VERSION + docker push $IMAGE_ID:$VERSION From ba3281e3b88d92bf4e40c5367860b208453da8a8 Mon Sep 17 00:00:00 2001 From: Kian-Tat Lim Date: Mon, 7 Mar 2022 16:26:12 -0800 Subject: [PATCH 03/10] Add Google Artifact Registry. --- .github/workflows/build-base.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/build-base.yml b/.github/workflows/build-base.yml index d592cb03..13a8559b 100644 --- a/.github/workflows/build-base.yml +++ b/.github/workflows/build-base.yml @@ -38,3 +38,18 @@ jobs: echo VERSION=$VERSION docker tag $IMAGE_NAME $IMAGE_ID:$VERSION docker push $IMAGE_ID:$VERSION + - name: Login to Google Artifact Registry + uses: docker/login-action@v1 + with: + registry: us-central1-docker.pkg.dev + username: _json_key_base64 + password: ${{ secrets.GAR_JSON_B64 }} + - name: Push image + run: | + IMAGE_ID=us-central1-docker.pkg.dev/prompt-proto/prompt/$IMAGE_NAME + VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') + [ "$VERSION" == "main" ] && VERSION=latest + echo IMAGE_ID=$IMAGE_ID + echo VERSION=$VERSION + docker tag $IMAGE_NAME $IMAGE_ID:$VERSION + docker push $IMAGE_ID:$VERSION From aae20e3a76f2255189866c580d8f0f84bc8389a2 Mon Sep 17 00:00:00 2001 From: Kian-Tat Lim Date: Mon, 7 Mar 2022 16:35:08 -0800 Subject: [PATCH 04/10] Rebuild only if source modified. --- .github/workflows/build-base.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-base.yml b/.github/workflows/build-base.yml index 13a8559b..3804f355 100644 --- a/.github/workflows/build-base.yml +++ b/.github/workflows/build-base.yml @@ -1,8 +1,14 @@ --- -name: "Build" +name: "Build base" on: push: + paths: + - '.github/workflows/build-base.yml' + - 'python/base/**' pull_request: + paths: + - '.github/workflows/build-base.yml' + - 'python/base/**' branches: - main workflow_dispatch: From 4696e98e2c0bfa1587caaf76f815d7c154b64a83 Mon Sep 17 00:00:00 2001 From: Kian-Tat Lim Date: Mon, 7 Mar 2022 16:36:53 -0800 Subject: [PATCH 05/10] Add service build workflow. --- .github/workflows/build-service.yml | 61 +++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 .github/workflows/build-service.yml diff --git a/.github/workflows/build-service.yml b/.github/workflows/build-service.yml new file mode 100644 index 00000000..619dba88 --- /dev/null +++ b/.github/workflows/build-service.yml @@ -0,0 +1,61 @@ +--- +name: "Build service" +on: + push: + paths: + - '.github/workflows/build-service.yml' + - 'python/activator/**' + pull_request: + paths: + - '.github/workflows/build-service.yml' + - 'python/activator/**' + branches: + - main + workflow_dispatch: + +permissions: + packages: write + +jobs: + update-service-image: + name: Update service image + runs-on: ubuntu-latest + env: + IMAGE_NAME: prompt-proto-service + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Build image + run: docker build python/activator --tag $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}" + - name: Login to GitHub Container Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Push image + run: | + IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME + VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') + [ "$VERSION" == "main" ] && VERSION=latest + echo IMAGE_ID=$IMAGE_ID + echo VERSION=$VERSION + docker tag $IMAGE_NAME $IMAGE_ID:$VERSION + docker push $IMAGE_ID:$VERSION + - name: Login to Google Artifact Registry + uses: docker/login-action@v1 + with: + registry: us-central1-docker.pkg.dev + username: _json_key_base64 + password: ${{ secrets.GAR_JSON_B64 }} + - name: Push image + run: | + IMAGE_ID=us-central1-docker.pkg.dev/prompt-proto/prompt/$IMAGE_NAME + VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') + [ "$VERSION" == "main" ] && VERSION=latest + echo IMAGE_ID=$IMAGE_ID + echo VERSION=$VERSION + docker tag $IMAGE_NAME $IMAGE_ID:$VERSION + docker push $IMAGE_ID:$VERSION From 8aa18d9d448573fbd6f2a08741bbaa2957f85eb2 Mon Sep 17 00:00:00 2001 From: Kian-Tat Lim Date: Mon, 7 Mar 2022 16:40:33 -0800 Subject: [PATCH 06/10] Reorder to get GAR credentials. Credentials are needed to pull the container for testing. --- .github/workflows/build-service.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build-service.yml b/.github/workflows/build-service.yml index 619dba88..c05f0577 100644 --- a/.github/workflows/build-service.yml +++ b/.github/workflows/build-service.yml @@ -27,32 +27,32 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Build image - run: docker build python/activator --tag $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}" - - name: Login to GitHub Container Registry + - name: Login to Google Artifact Registry uses: docker/login-action@v1 with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Push image + registry: us-central1-docker.pkg.dev + username: _json_key_base64 + password: ${{ secrets.GAR_JSON_B64 }} + - name: Build image + run: docker build python/activator --tag $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}" + - name: Push image to Google Artifact Registry run: | - IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME + IMAGE_ID=us-central1-docker.pkg.dev/prompt-proto/prompt/$IMAGE_NAME VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') [ "$VERSION" == "main" ] && VERSION=latest echo IMAGE_ID=$IMAGE_ID echo VERSION=$VERSION docker tag $IMAGE_NAME $IMAGE_ID:$VERSION docker push $IMAGE_ID:$VERSION - - name: Login to Google Artifact Registry + - name: Login to GitHub Container Registry uses: docker/login-action@v1 with: - registry: us-central1-docker.pkg.dev - username: _json_key_base64 - password: ${{ secrets.GAR_JSON_B64 }} - - name: Push image + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Push image to GitHub Container Registry run: | - IMAGE_ID=us-central1-docker.pkg.dev/prompt-proto/prompt/$IMAGE_NAME + IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') [ "$VERSION" == "main" ] && VERSION=latest echo IMAGE_ID=$IMAGE_ID From 381bd126ea806991be6975c2fddf902c4f44083f Mon Sep 17 00:00:00 2001 From: Kian-Tat Lim Date: Mon, 7 Mar 2022 17:00:06 -0800 Subject: [PATCH 07/10] Use scons to test in separate job. --- .github/workflows/build-base.yml | 6 ++--- .github/workflows/build-service.yml | 34 ++++++++++++++++++++++++----- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-base.yml b/.github/workflows/build-base.yml index 3804f355..3e846c77 100644 --- a/.github/workflows/build-base.yml +++ b/.github/workflows/build-base.yml @@ -5,12 +5,12 @@ on: paths: - '.github/workflows/build-base.yml' - 'python/base/**' + branches: + - main pull_request: paths: - '.github/workflows/build-base.yml' - 'python/base/**' - branches: - - main workflow_dispatch: permissions: @@ -25,8 +25,6 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 - with: - fetch-depth: 0 - name: Build image run: docker build python/base --tag $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}" - name: Login to GitHub Container Registry diff --git a/.github/workflows/build-service.yml b/.github/workflows/build-service.yml index c05f0577..6fabc9f9 100644 --- a/.github/workflows/build-service.yml +++ b/.github/workflows/build-service.yml @@ -1,7 +1,9 @@ --- -name: "Build service" +name: "Build and test service" on: push: + branches: + - main paths: - '.github/workflows/build-service.yml' - 'python/activator/**' @@ -9,24 +11,46 @@ on: paths: - '.github/workflows/build-service.yml' - 'python/activator/**' - branches: - - main workflow_dispatch: permissions: packages: write jobs: + test-service: + name: Test service + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Fix permissions + run: chmod -R a+rwX $GITHUB_WORKSPACE + - name: Login to Google Artifact Registry + uses: docker/login-action@v1 + with: + registry: us-central1-docker.pkg.dev + username: _json_key_base64 + password: ${{ secrets.GAR_JSON_B64 }} + - name: Run tests + run: | + docker run \ + -v $GITHUB_WORKSPACE:/home/lsst/prompt_prototype \ + us-central1-docker.pkg.dev/prompt-proto/prompt/prompt-proto-base:latest \ + bash -c ' + cd /home/lsst/prompt_prototype + source /opt/lsst/software/stack/loadLSST.bash + setup -r . + scons' + update-service-image: name: Update service image + needs: test-service runs-on: ubuntu-latest env: IMAGE_NAME: prompt-proto-service steps: - name: Checkout code uses: actions/checkout@v2 - with: - fetch-depth: 0 - name: Login to Google Artifact Registry uses: docker/login-action@v1 with: From 420962276a11cec80635ec07afe32c9f35acff21 Mon Sep 17 00:00:00 2001 From: Kian-Tat Lim Date: Wed, 9 Mar 2022 20:35:48 -0800 Subject: [PATCH 08/10] Document push destination. --- .github/workflows/build-base.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-base.yml b/.github/workflows/build-base.yml index 3e846c77..2437e8a1 100644 --- a/.github/workflows/build-base.yml +++ b/.github/workflows/build-base.yml @@ -33,7 +33,7 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Push image + - name: Push image to GitHub Container Registry run: | IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') @@ -48,7 +48,7 @@ jobs: registry: us-central1-docker.pkg.dev username: _json_key_base64 password: ${{ secrets.GAR_JSON_B64 }} - - name: Push image + - name: Push image to Google Artifact Registry run: | IMAGE_ID=us-central1-docker.pkg.dev/prompt-proto/prompt/$IMAGE_NAME VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') From 864f9d696ee6c8399f92258b048e49b096ec90da Mon Sep 17 00:00:00 2001 From: Kian-Tat Lim Date: Wed, 9 Mar 2022 20:37:14 -0800 Subject: [PATCH 09/10] Use GitHub as source rather than Google. --- .github/workflows/build-service.yml | 34 ++++++++++++++--------------- python/activator/Dockerfile | 2 +- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build-service.yml b/.github/workflows/build-service.yml index 6fabc9f9..cd5f6b79 100644 --- a/.github/workflows/build-service.yml +++ b/.github/workflows/build-service.yml @@ -25,17 +25,17 @@ jobs: uses: actions/checkout@v2 - name: Fix permissions run: chmod -R a+rwX $GITHUB_WORKSPACE - - name: Login to Google Artifact Registry + - name: Login to GitHub Container Registry uses: docker/login-action@v1 with: - registry: us-central1-docker.pkg.dev - username: _json_key_base64 - password: ${{ secrets.GAR_JSON_B64 }} + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Run tests run: | docker run \ -v $GITHUB_WORKSPACE:/home/lsst/prompt_prototype \ - us-central1-docker.pkg.dev/prompt-proto/prompt/prompt-proto-base:latest \ + ghcr.io/${{ github.repository_owner }}/prompt-proto-base:latest \ bash -c ' cd /home/lsst/prompt_prototype source /opt/lsst/software/stack/loadLSST.bash @@ -51,32 +51,32 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 - - name: Login to Google Artifact Registry + - name: Login to GitHub Container Registry uses: docker/login-action@v1 with: - registry: us-central1-docker.pkg.dev - username: _json_key_base64 - password: ${{ secrets.GAR_JSON_B64 }} + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Build image run: docker build python/activator --tag $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}" - - name: Push image to Google Artifact Registry + - name: Push image to GitHub Container Registry run: | - IMAGE_ID=us-central1-docker.pkg.dev/prompt-proto/prompt/$IMAGE_NAME + IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') [ "$VERSION" == "main" ] && VERSION=latest echo IMAGE_ID=$IMAGE_ID echo VERSION=$VERSION docker tag $IMAGE_NAME $IMAGE_ID:$VERSION docker push $IMAGE_ID:$VERSION - - name: Login to GitHub Container Registry + - name: Login to Google Artifact Registry uses: docker/login-action@v1 with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Push image to GitHub Container Registry + registry: us-central1-docker.pkg.dev + username: _json_key_base64 + password: ${{ secrets.GAR_JSON_B64 }} + - name: Push image to Google Artifact Registry run: | - IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME + IMAGE_ID=us-central1-docker.pkg.dev/prompt-proto/prompt/$IMAGE_NAME VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') [ "$VERSION" == "main" ] && VERSION=latest echo IMAGE_ID=$IMAGE_ID diff --git a/python/activator/Dockerfile b/python/activator/Dockerfile index db37c539..a2fbef9c 100644 --- a/python/activator/Dockerfile +++ b/python/activator/Dockerfile @@ -1,4 +1,4 @@ -FROM us-central1-docker.pkg.dev/prompt-proto/prompt/prompt-proto-base:latest +FROM ghcr.io/lsst-dm/prompt-proto-base:latest ENV PYTHONUNBUFFERED True ENV APP_HOME /app ARG RUBIN_INSTRUMENT From 81c84b3ec51cf52feca92107cdb254c444f38a0d Mon Sep 17 00:00:00 2001 From: Kian-Tat Lim Date: Wed, 9 Mar 2022 21:21:15 -0800 Subject: [PATCH 10/10] Use branch name for PRs instead of merge. --- .github/workflows/build-base.yml | 2 ++ .github/workflows/build-service.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/build-base.yml b/.github/workflows/build-base.yml index 2437e8a1..d6fa0310 100644 --- a/.github/workflows/build-base.yml +++ b/.github/workflows/build-base.yml @@ -37,6 +37,7 @@ jobs: run: | IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') + [ "$VERSION" == "merge" ] && VERSION=$(echo "${{ github.head_ref }}" | sed -e 's,.*/\(.*\),\1,') [ "$VERSION" == "main" ] && VERSION=latest echo IMAGE_ID=$IMAGE_ID echo VERSION=$VERSION @@ -52,6 +53,7 @@ jobs: run: | IMAGE_ID=us-central1-docker.pkg.dev/prompt-proto/prompt/$IMAGE_NAME VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') + [ "$VERSION" == "merge" ] && VERSION=$(echo "${{ github.head_ref }}" | sed -e 's,.*/\(.*\),\1,') [ "$VERSION" == "main" ] && VERSION=latest echo IMAGE_ID=$IMAGE_ID echo VERSION=$VERSION diff --git a/.github/workflows/build-service.yml b/.github/workflows/build-service.yml index cd5f6b79..ea34e727 100644 --- a/.github/workflows/build-service.yml +++ b/.github/workflows/build-service.yml @@ -63,6 +63,7 @@ jobs: run: | IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') + [ "$VERSION" == "merge" ] && VERSION=$(echo "${{ github.head_ref }}" | sed -e 's,.*/\(.*\),\1,') [ "$VERSION" == "main" ] && VERSION=latest echo IMAGE_ID=$IMAGE_ID echo VERSION=$VERSION @@ -78,6 +79,7 @@ jobs: run: | IMAGE_ID=us-central1-docker.pkg.dev/prompt-proto/prompt/$IMAGE_NAME VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') + [ "$VERSION" == "merge" ] && VERSION=$(echo "${{ github.head_ref }}" | sed -e 's,.*/\(.*\),\1,') [ "$VERSION" == "main" ] && VERSION=latest echo IMAGE_ID=$IMAGE_ID echo VERSION=$VERSION