diff --git a/.github/workflows/build-base.yml b/.github/workflows/build-base.yml index b901103b..0a87a09a 100644 --- a/.github/workflows/build-base.yml +++ b/.github/workflows/build-base.yml @@ -12,6 +12,17 @@ on: - '.github/workflows/build-base.yml' - 'Dockerfile.main' workflow_dispatch: + inputs: + stackTag: + description: 'Science Pipelines tag (default: d_latest)' + required: true + default: 'd_latest' + type: string + makeLatest: + description: 'Push container with "latest" tag' + required: false + type: boolean + permissions: packages: write @@ -22,41 +33,89 @@ jobs: runs-on: ubuntu-latest env: IMAGE_NAME: prompt-proto-base + STACK_TAG: ${{ inputs.stackTag }} steps: - name: Checkout code uses: actions/checkout@v3 - - name: Build image - # Context-frree build - run: docker build - --tag $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}" < Dockerfile.main - name: Login to GitHub Container Registry uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Push image to GitHub Container Registry - run: | - IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME - VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') - [ "$VERSION" == "merge" ] && VERSION=$(echo "${{ github.head_ref }}" | sed -e 's,.*/\(.*\),\1,') - [ "$VERSION" == "main" ] && VERSION=latest - echo IMAGE_ID=$IMAGE_ID - echo VERSION=$VERSION - docker tag $IMAGE_NAME $IMAGE_ID:$VERSION - docker push $IMAGE_ID:$VERSION - name: Login to Google Artifact Registry uses: docker/login-action@v2 with: registry: us-central1-docker.pkg.dev username: _json_key_base64 password: ${{ secrets.GAR_JSON_B64 }} - - name: Push image to Google Artifact Registry + - name: Determine base image eups tag + run: | + if [[ -n "$STACK_TAG" ]]; then + if [[ "$STACK_TAG" == "*_latest" ]]; then + echo "$STACK_TAG" > lsst.docker.tag + else + echo "7-stack-lsst_distrib-$STACK_TAG" > lsst.docker.tag + fi + echo "$STACK_TAG" > stack.tag + else + echo "d_latest" > lsst.docker.tag + echo "d_latest" > stack.tag + fi + docker run lsstsqre/centos:"$(< lsst.docker.tag)" bash -c "cat stack/miniconda*/ups_db/global.tags" > eups.tag + echo "Eups tag = $(< eups.tag)" + - name: Build image + # Context-free build run: | - IMAGE_ID=us-central1-docker.pkg.dev/prompt-proto/prompt/$IMAGE_NAME - VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') - [ "$VERSION" == "merge" ] && VERSION=$(echo "${{ github.head_ref }}" | sed -e 's,.*/\(.*\),\1,') - [ "$VERSION" == "main" ] && VERSION=latest - echo IMAGE_ID=$IMAGE_ID - echo VERSION=$VERSION - docker tag $IMAGE_NAME $IMAGE_ID:$VERSION - docker push $IMAGE_ID:$VERSION + docker build - \ + --build-arg "STACK_TAG=$(< lsst.docker.tag)" \ + --tag $IMAGE_NAME \ + --label "runnumber=${GITHUB_RUN_ID}" \ + --label "stacktag=$(< stack.tag)" \ + --label "eupstag=$(< eups.tag)" \ + < Dockerfile.main + - name: Push image to registries + run: | + MAKE_LATEST="${{ inputs.makeLatest }}" + [[ -n "$MAKE_LATEST" ]] || MAKE_LATEST="false" + BRANCH=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') + [ "$BRANCH" == "merge" ] && BRANCH=$(echo "${{ github.head_ref }}" | sed -e 's,.*/\(.*\),\1,') + + for IMAGE_ID in "ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME" \ + "us-central1-docker.pkg.dev/prompt-proto/prompt/$IMAGE_NAME"; do + + STACK_TAG="$(< stack.tag)" + if [ "$BRANCH" == "main" ]; then + VERSION="$STACK_TAG" + else + VERSION="${BRANCH}-$STACK_TAG" + fi + echo IMAGE_ID=$IMAGE_ID + echo VERSION=$VERSION + docker tag $IMAGE_NAME $IMAGE_ID:$VERSION + docker push $IMAGE_ID:$VERSION + + EUPS_TAG=$(< eups.tag) + if [ "$STACK_TAG" != "$EUPS_TAG" ]; then + # Also push actual eups tag if not the same (e.g. d_latest) + if [ "$BRANCH" == "main" ]; then + VERSION="$EUPS_TAG" + else + VERSION="${BRANCH}-$EUPS_TAG" + fi + echo VERSION=$VERSION + docker tag $IMAGE_NAME $IMAGE_ID:$VERSION + docker push $IMAGE_ID:$VERSION + fi + if [ "$MAKE_LATEST" == "true" ]; then + # Push latest if requested + if [ "$BRANCH" == "main" ]; then + VERSION="latest" + else + VERSION="${BRANCH}-latest" + fi + echo VERSION=$VERSION + docker tag $IMAGE_NAME $IMAGE_ID:$VERSION + docker push $IMAGE_ID:$VERSION + fi + done diff --git a/.github/workflows/build-service.yml b/.github/workflows/build-service.yml index 4d00b5e1..27865f67 100644 --- a/.github/workflows/build-service.yml +++ b/.github/workflows/build-service.yml @@ -18,8 +18,31 @@ on: permissions: packages: write +env: + # This is a bracketed, comma-separated list of double-quoted base container + # tags that will be used to build service containers on each branch + # (including "main"). Typically, any tags listed beyond "latest" would be + # relatively stable Pipelines containers that are needed to avoid issues with + # the "latest" version; they would remain in this list until "latest" becomes + # usable for all building and testing. + BASE_TAG_LIST: '["latest"]' + jobs: + matrix-gen: + # This job exists solely because fromJSON() cannot directly accept + # env.BASE_TAG_LIST for some reason. + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - id: set-matrix + run: echo "matrix=${BASE_TAG_LIST}" >> $GITHUB_OUTPUT + test-service: + needs: matrix-gen + strategy: + matrix: + baseTag: ${{ fromJSON(needs.matrix-gen.outputs.matrix) }} name: Test service runs-on: ubuntu-latest steps: @@ -37,7 +60,7 @@ jobs: run: | docker run \ -v $GITHUB_WORKSPACE:/home/lsst/prompt_prototype \ - ghcr.io/${{ github.repository_owner }}/prompt-proto-base:latest \ + ghcr.io/${{ github.repository_owner }}/prompt-proto-base:${{ matrix.baseTag }} \ bash -c ' cd /home/lsst/prompt_prototype source /opt/lsst/software/stack/loadLSST.bash @@ -48,10 +71,16 @@ jobs: update-service-image: name: Update service image - needs: test-service + needs: + - matrix-gen + - test-service runs-on: ubuntu-latest + strategy: + matrix: + baseTag: ${{ fromJSON(needs.matrix-gen.outputs.matrix) }} env: IMAGE_NAME: prompt-proto-service + BASE_TAG: ${{ matrix.baseTag }} steps: - name: Checkout code uses: actions/checkout@v3 @@ -61,31 +90,49 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build image - run: docker build . -f Dockerfile.activator --tag $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}" - - name: Push image to GitHub Container Registry - run: | - IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME - VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') - [ "$VERSION" == "merge" ] && VERSION=$(echo "${{ github.head_ref }}" | sed -e 's,.*/\(.*\),\1,') - [ "$VERSION" == "main" ] && VERSION=latest - echo IMAGE_ID=$IMAGE_ID - echo VERSION=$VERSION - docker tag $IMAGE_NAME $IMAGE_ID:$VERSION - docker push $IMAGE_ID:$VERSION - name: Login to Google Artifact Registry uses: docker/login-action@v2 with: registry: us-central1-docker.pkg.dev username: _json_key_base64 password: ${{ secrets.GAR_JSON_B64 }} - - name: Push image to Google Artifact Registry + - name: Determine eups tag + run: | + docker run ghcr.io/${{ github.repository_owner }}/prompt-proto-base:"$BASE_TAG" bash -c "cat stack/miniconda*/ups_db/global.tags" > eups.tag + echo "Eups tag = $(< eups.tag)" + - name: Build image run: | - IMAGE_ID=us-central1-docker.pkg.dev/prompt-proto/prompt/$IMAGE_NAME - VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') - [ "$VERSION" == "merge" ] && VERSION=$(echo "${{ github.head_ref }}" | sed -e 's,.*/\(.*\),\1,') - [ "$VERSION" == "main" ] && VERSION=latest - echo IMAGE_ID=$IMAGE_ID - echo VERSION=$VERSION - docker tag $IMAGE_NAME $IMAGE_ID:$VERSION - docker push $IMAGE_ID:$VERSION + docker build . -f Dockerfile.activator \ + --build-arg "BASE_TAG=$BASE_TAG" \ + --tag $IMAGE_NAME \ + --label "runnumber=${GITHUB_RUN_ID}" \ + --label "basetag=${BASE_TAG}" \ + --label "eupstag=$(< eups.tag)" + - name: Push image to container registries + run: | + BRANCH=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') + [ "$BRANCH" == "merge" ] && BRANCH=$(echo "${{ github.head_ref }}" | sed -e 's,.*/\(.*\),\1,') + + for IMAGE_ID in "ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME" \ + "us-central1-docker.pkg.dev/prompt-proto/prompt/$IMAGE_NAME"; do + if [ "$BRANCH" == "main" ]; then + VERSION="$BASE_TAG" + else + VERSION="${BRANCH}-$BASE_TAG" + fi + echo IMAGE_ID=$IMAGE_ID + echo VERSION=$VERSION + docker tag $IMAGE_NAME $IMAGE_ID:$VERSION + docker push $IMAGE_ID:$VERSION + EUPS_TAG=$(< eups.tag) + if [ "$BASE_TAG" != "$EUPS_TAG" ]; then + if [ "$BRANCH" == "main" ]; then + VERSION="$EUPS_TAG" + else + VERSION="${BRANCH}-$EUPS_TAG" + fi + echo VERSION=$VERSION + docker tag $IMAGE_NAME $IMAGE_ID:$VERSION + docker push $IMAGE_ID:$VERSION + fi + done diff --git a/Dockerfile.activator b/Dockerfile.activator index 1d818a50..eff3fbeb 100644 --- a/Dockerfile.activator +++ b/Dockerfile.activator @@ -1,4 +1,5 @@ -FROM ghcr.io/lsst-dm/prompt-proto-base:latest +ARG BASE_TAG=latest +FROM ghcr.io/lsst-dm/prompt-proto-base:${BASE_TAG} ENV PYTHONUNBUFFERED True ENV APP_HOME /app ENV PROMPT_PROTOTYPE_DIR $APP_HOME diff --git a/Dockerfile.main b/Dockerfile.main index abe24441..f26f0cfb 100644 --- a/Dockerfile.main +++ b/Dockerfile.main @@ -1,4 +1,5 @@ -FROM lsstsqre/centos:d_latest +ARG STACK_TAG=d_latest +FROM lsstsqre/centos:${STACK_TAG} ENV PYTHONUNBUFFERED True RUN source /opt/lsst/software/stack/loadLSST.bash \ && mamba install -y \