From 1424635d4f30a28c873e6d3e6154a7435e0d6937 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Sun, 20 Sep 2020 20:33:20 +0200 Subject: [PATCH 01/11] Add GitHub Action for basic Docker test This just invokes "wetterdienst about parameters", which is probably better than nothing. While it currently builds the whole Docker container, it can be used in the future to verify Docker images published to Docker Hub or the GitHub docker registry. --- .github/workflows/README.md | 2 ++ .github/workflows/docker_test.yml | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 .github/workflows/docker_test.yml diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 418e2a9f8..a79c83b52 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -2,3 +2,5 @@ Check out the guide that was used to create the CI environment including setting - https://medium.com/@cjolowicz/hypermodern-python-d44485d9d769 - https://cjolowicz.github.io/posts/hypermodern-python-01-setup/ +- https://docs.github.com/en/actions/creating-actions/creating-a-docker-container-action +- https://docs.github.com/en/actions/creating-actions/dockerfile-support-for-github-actions diff --git a/.github/workflows/docker_test.yml b/.github/workflows/docker_test.yml new file mode 100644 index 000000000..fad779ee0 --- /dev/null +++ b/.github/workflows/docker_test.yml @@ -0,0 +1,18 @@ +# https://docs.github.com/en/actions/creating-actions/creating-a-docker-container-action +# https://docs.github.com/en/actions/creating-actions/dockerfile-support-for-github-actions +name: Docker Test +on: + push: + branches: [ master, test-docker ] + pull_request: + branches: [ master ] +jobs: + test_docker: + name: Test Docker image + runs-on: ubuntu-latest + steps: + - name: Invoke + id: invoke + uses: earthobservations/test-wetterdienst-docker@main + - name: Display runtime + run: echo "The time was ${{ steps.invoke.outputs.time }}" From 6dbad61d975404624d81d9f2237a52d468ce0c2d Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Sun, 20 Sep 2020 20:52:26 +0200 Subject: [PATCH 02/11] Add GitHub Action for Docker image publishing --- .github/workflows/README.md | 2 ++ .github/workflows/docker_publish.yml | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 .github/workflows/docker_publish.yml diff --git a/.github/workflows/README.md b/.github/workflows/README.md index a79c83b52..208cd5d25 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -2,5 +2,7 @@ Check out the guide that was used to create the CI environment including setting - https://medium.com/@cjolowicz/hypermodern-python-d44485d9d769 - https://cjolowicz.github.io/posts/hypermodern-python-01-setup/ + - https://docs.github.com/en/actions/creating-actions/creating-a-docker-container-action - https://docs.github.com/en/actions/creating-actions/dockerfile-support-for-github-actions +- https://docs.github.com/en/actions/guides/publishing-docker-images diff --git a/.github/workflows/docker_publish.yml b/.github/workflows/docker_publish.yml new file mode 100644 index 000000000..18235dc5b --- /dev/null +++ b/.github/workflows/docker_publish.yml @@ -0,0 +1,20 @@ +# https://docs.github.com/en/actions/guides/publishing-docker-images +name: Publish Docker image +on: + release: + types: [published] +jobs: + push_to_registry: + name: Push Docker image to GitHub Packages + runs-on: ubuntu-latest + steps: + - name: Check out the repo + uses: actions/checkout@v2 + - name: Push to GitHub Packages + uses: docker/build-push-action@v1 + with: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + registry: docker.pkg.github.com + repository: earthobservations/wetterdienst/wetterdienst + tag_with_ref: true From 13ac7908d87db4893e3ccf837ccbb93610514a81 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Sun, 20 Sep 2020 21:51:51 +0200 Subject: [PATCH 03/11] Create docker-publish.yml --- .github/workflows/docker-publish.yml | 76 ++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 .github/workflows/docker-publish.yml diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 000000000..8849b17b0 --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,76 @@ +name: Docker + +on: + push: + # Publish `master` as Docker `latest` image. + branches: + - master + + # Publish `v1.2.3` tags as releases. + tags: + - v* + + # Run tests for any PRs. + pull_request: + +env: + IMAGE_NAME: wetterdienst + +jobs: + # Run tests. + # See also https://docs.docker.com/docker-hub/builds/automated-testing/ + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Run tests + run: | + if [ -f docker-compose.test.yml ]; then + docker-compose --file docker-compose.test.yml build + docker-compose --file docker-compose.test.yml run sut + else + docker build . --file Dockerfile + fi + + # Push image to GitHub Packages. + # See also https://docs.docker.com/docker-hub/builds/ + push: + # Ensure test job passes before pushing image. + needs: test + + runs-on: ubuntu-latest + if: github.event_name == 'push' + + steps: + - uses: actions/checkout@v2 + + - name: Build image + run: docker build . --file Dockerfile --tag $IMAGE_NAME + + - name: Log into GitHub Container Registry + # TODO: Create a PAT with `read:packages` and `write:packages` scopes and save it as an Actions secret `CR_PAT` + run: echo "${{ secrets.CR_PAT }}" | docker login https://ghcr.io -u ${{ github.actor }} --password-stdin + + - name: Push image to GitHub Container Registry + run: | + IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME + + # Change all uppercase to lowercase + IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') + + # Strip git ref prefix from version + VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') + + # Strip "v" prefix from tag name + [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') + + # Use Docker `latest` tag convention + [ "$VERSION" == "master" ] && VERSION=latest + + echo IMAGE_ID=$IMAGE_ID + echo VERSION=$VERSION + + docker tag $IMAGE_NAME $IMAGE_ID:$VERSION + docker push $IMAGE_ID:$VERSION From 6d5e9b630858ef6888eb71fb91c51366965ecd31 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Sun, 20 Sep 2020 21:52:27 +0200 Subject: [PATCH 04/11] Remove docker_publish.yml --- .github/workflows/docker_publish.yml | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 .github/workflows/docker_publish.yml diff --git a/.github/workflows/docker_publish.yml b/.github/workflows/docker_publish.yml deleted file mode 100644 index 18235dc5b..000000000 --- a/.github/workflows/docker_publish.yml +++ /dev/null @@ -1,20 +0,0 @@ -# https://docs.github.com/en/actions/guides/publishing-docker-images -name: Publish Docker image -on: - release: - types: [published] -jobs: - push_to_registry: - name: Push Docker image to GitHub Packages - runs-on: ubuntu-latest - steps: - - name: Check out the repo - uses: actions/checkout@v2 - - name: Push to GitHub Packages - uses: docker/build-push-action@v1 - with: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - registry: docker.pkg.github.com - repository: earthobservations/wetterdienst/wetterdienst - tag_with_ref: true From 929986fdcc38a05c9c7d4e3d25cafa0ab5d49f8a Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Mon, 21 Sep 2020 00:06:43 +0200 Subject: [PATCH 05/11] Discriminate between wetterdienst-standard and wetterdienst-full pkgs wetterdienst-full will include GDAL and wradlib. --- .github/release/full.test.yml | 3 + .github/release/full/Dockerfile | 28 +++++++ .github/release/standard.test.yml | 3 + .github/release/standard/Dockerfile | 6 ++ ...er-publish.yml => docker-publish-full.yml} | 12 ++- .github/workflows/docker-publish-standard.yml | 74 +++++++++++++++++++ 6 files changed, 119 insertions(+), 7 deletions(-) create mode 100644 .github/release/full.test.yml create mode 100644 .github/release/full/Dockerfile create mode 100644 .github/release/standard.test.yml create mode 100644 .github/release/standard/Dockerfile rename .github/workflows/{docker-publish.yml => docker-publish-full.yml} (84%) create mode 100644 .github/workflows/docker-publish-standard.yml diff --git a/.github/release/full.test.yml b/.github/release/full.test.yml new file mode 100644 index 000000000..8b7a6eb23 --- /dev/null +++ b/.github/release/full.test.yml @@ -0,0 +1,3 @@ +sut: + build: full + command: python -c 'import wradlib; print(wradlib.__version__)' diff --git a/.github/release/full/Dockerfile b/.github/release/full/Dockerfile new file mode 100644 index 000000000..46c93aa81 --- /dev/null +++ b/.github/release/full/Dockerfile @@ -0,0 +1,28 @@ +FROM python:3.8.5-slim + +# TODO: This currently installs GDAL==2.4.0. For a more recent version, ... +# +# - Maybe use "ubuntu:focal" already? +# https://github.com/thinkWhere/GDAL-Docker/blob/develop/3.8-ubuntu/Dockerfile +# +# - See also: +# https://github.com/andrejreznik/docker-python-gdal +# + +ENV DEBIAN_FRONTEND noninteractive +ENV TERM linux + +# Install GDAL. +RUN apt-get update +RUN apt-get --yes install build-essential libgdal-dev + +# Make sure you have numpy installed before you attempt to install the GDAL Python bindings. +# https://gis.stackexchange.com/a/274328 +RUN pip install numpy +RUN pip install GDAL==$(gdal-config --version) + +# Install wradlib. +RUN pip install wradlib + +# Install Wetterdienst. +RUN pip install wetterdienst[excel] diff --git a/.github/release/standard.test.yml b/.github/release/standard.test.yml new file mode 100644 index 000000000..e382fad41 --- /dev/null +++ b/.github/release/standard.test.yml @@ -0,0 +1,3 @@ +sut: + build: standard + command: wetterdienst about parameters diff --git a/.github/release/standard/Dockerfile b/.github/release/standard/Dockerfile new file mode 100644 index 000000000..0df25b0f3 --- /dev/null +++ b/.github/release/standard/Dockerfile @@ -0,0 +1,6 @@ +FROM python:3.8.5-slim + +ENV DEBIAN_FRONTEND noninteractive +ENV TERM linux + +RUN pip install wetterdienst[excel] diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish-full.yml similarity index 84% rename from .github/workflows/docker-publish.yml rename to .github/workflows/docker-publish-full.yml index 8849b17b0..c969aa598 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish-full.yml @@ -14,7 +14,7 @@ on: pull_request: env: - IMAGE_NAME: wetterdienst + IMAGE_NAME: wetterdienst-full jobs: # Run tests. @@ -27,11 +27,9 @@ jobs: - name: Run tests run: | - if [ -f docker-compose.test.yml ]; then - docker-compose --file docker-compose.test.yml build - docker-compose --file docker-compose.test.yml run sut - else - docker build . --file Dockerfile + if [ -f .github/release/full.test.yml ]; then + docker-compose --file .github/release/full.test.yml build + docker-compose --file .github/release/full.test.yml run sut fi # Push image to GitHub Packages. @@ -47,7 +45,7 @@ jobs: - uses: actions/checkout@v2 - name: Build image - run: docker build . --file Dockerfile --tag $IMAGE_NAME + run: docker build . --file .github/release/Dockerfile.full --tag $IMAGE_NAME - name: Log into GitHub Container Registry # TODO: Create a PAT with `read:packages` and `write:packages` scopes and save it as an Actions secret `CR_PAT` diff --git a/.github/workflows/docker-publish-standard.yml b/.github/workflows/docker-publish-standard.yml new file mode 100644 index 000000000..ed2ed86a2 --- /dev/null +++ b/.github/workflows/docker-publish-standard.yml @@ -0,0 +1,74 @@ +name: Docker + +on: + push: + # Publish `master` as Docker `latest` image. + branches: + - master + + # Publish `v1.2.3` tags as releases. + tags: + - v* + + # Run tests for any PRs. + pull_request: + +env: + IMAGE_NAME: wetterdienst-standard + +jobs: + # Run tests. + # See also https://docs.docker.com/docker-hub/builds/automated-testing/ + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Run tests + run: | + if [ -f .github/release/standard.test.yml]; then + docker-compose --file .github/release/standard.test.yml build + docker-compose --file .github/release/standard.test.yml run sut + fi + + # Push image to GitHub Packages. + # See also https://docs.docker.com/docker-hub/builds/ + push: + # Ensure test job passes before pushing image. + needs: test + + runs-on: ubuntu-latest + if: github.event_name == 'push' + + steps: + - uses: actions/checkout@v2 + + - name: Build image + run: docker build . --file .github/release/Dockerfile.standard --tag $IMAGE_NAME + + - name: Log into GitHub Container Registry + # TODO: Create a PAT with `read:packages` and `write:packages` scopes and save it as an Actions secret `CR_PAT` + run: echo "${{ secrets.CR_PAT }}" | docker login https://ghcr.io -u ${{ github.actor }} --password-stdin + + - name: Push image to GitHub Container Registry + run: | + IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME + + # Change all uppercase to lowercase + IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') + + # Strip git ref prefix from version + VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') + + # Strip "v" prefix from tag name + [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') + + # Use Docker `latest` tag convention + [ "$VERSION" == "master" ] && VERSION=latest + + echo IMAGE_ID=$IMAGE_ID + echo VERSION=$VERSION + + docker tag $IMAGE_NAME $IMAGE_ID:$VERSION + docker push $IMAGE_ID:$VERSION From 870b1a03a47c4cdbdce63c39e16cb1ee15060732 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Mon, 21 Sep 2020 00:09:16 +0200 Subject: [PATCH 06/11] Also run on "test-docker" --- .github/workflows/docker-publish-full.yml | 1 + .github/workflows/docker-publish-standard.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/docker-publish-full.yml b/.github/workflows/docker-publish-full.yml index c969aa598..8de97672f 100644 --- a/.github/workflows/docker-publish-full.yml +++ b/.github/workflows/docker-publish-full.yml @@ -5,6 +5,7 @@ on: # Publish `master` as Docker `latest` image. branches: - master + - test-docker # Publish `v1.2.3` tags as releases. tags: diff --git a/.github/workflows/docker-publish-standard.yml b/.github/workflows/docker-publish-standard.yml index ed2ed86a2..c4c6c5bd2 100644 --- a/.github/workflows/docker-publish-standard.yml +++ b/.github/workflows/docker-publish-standard.yml @@ -5,6 +5,7 @@ on: # Publish `master` as Docker `latest` image. branches: - master + - test-docker # Publish `v1.2.3` tags as releases. tags: From 20a8da7d5fea33a2242ac9d87c0ae263865732ae Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Mon, 21 Sep 2020 00:15:04 +0200 Subject: [PATCH 07/11] More fixes --- .github/release/full.test.yml | 1 + .github/workflows/docker-publish-full.yml | 4 ++-- .github/workflows/docker-publish-standard.yml | 4 ++-- .github/workflows/{docker_test.yml => docker-test.yml} | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) rename .github/workflows/{docker_test.yml => docker-test.yml} (93%) diff --git a/.github/release/full.test.yml b/.github/release/full.test.yml index 8b7a6eb23..34879b440 100644 --- a/.github/release/full.test.yml +++ b/.github/release/full.test.yml @@ -1,3 +1,4 @@ sut: build: full + # TODO: We might went to improve this. Just say "wetterdienst about system" here. command: python -c 'import wradlib; print(wradlib.__version__)' diff --git a/.github/workflows/docker-publish-full.yml b/.github/workflows/docker-publish-full.yml index 8de97672f..67de30548 100644 --- a/.github/workflows/docker-publish-full.yml +++ b/.github/workflows/docker-publish-full.yml @@ -1,4 +1,4 @@ -name: Docker +name: Docker Full on: push: @@ -46,7 +46,7 @@ jobs: - uses: actions/checkout@v2 - name: Build image - run: docker build . --file .github/release/Dockerfile.full --tag $IMAGE_NAME + run: docker build . --file .github/release/full/Dockerfile --tag $IMAGE_NAME - name: Log into GitHub Container Registry # TODO: Create a PAT with `read:packages` and `write:packages` scopes and save it as an Actions secret `CR_PAT` diff --git a/.github/workflows/docker-publish-standard.yml b/.github/workflows/docker-publish-standard.yml index c4c6c5bd2..989a8dc63 100644 --- a/.github/workflows/docker-publish-standard.yml +++ b/.github/workflows/docker-publish-standard.yml @@ -1,4 +1,4 @@ -name: Docker +name: Docker Standard on: push: @@ -46,7 +46,7 @@ jobs: - uses: actions/checkout@v2 - name: Build image - run: docker build . --file .github/release/Dockerfile.standard --tag $IMAGE_NAME + run: docker build . --file .github/release/standard/Dockerfile --tag $IMAGE_NAME - name: Log into GitHub Container Registry # TODO: Create a PAT with `read:packages` and `write:packages` scopes and save it as an Actions secret `CR_PAT` diff --git a/.github/workflows/docker_test.yml b/.github/workflows/docker-test.yml similarity index 93% rename from .github/workflows/docker_test.yml rename to .github/workflows/docker-test.yml index fad779ee0..870c93f40 100644 --- a/.github/workflows/docker_test.yml +++ b/.github/workflows/docker-test.yml @@ -3,7 +3,7 @@ name: Docker Test on: push: - branches: [ master, test-docker ] + #branches: [ master, test-docker ] pull_request: branches: [ master ] jobs: From 8964404fced13429310a07b051682294b7352a8b Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Mon, 21 Sep 2020 00:17:49 +0200 Subject: [PATCH 08/11] Fix test command --- .github/workflows/docker-publish-full.yml | 2 +- .github/workflows/docker-publish-standard.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish-full.yml b/.github/workflows/docker-publish-full.yml index 67de30548..c939f6f44 100644 --- a/.github/workflows/docker-publish-full.yml +++ b/.github/workflows/docker-publish-full.yml @@ -28,7 +28,7 @@ jobs: - name: Run tests run: | - if [ -f .github/release/full.test.yml ]; then + if [[ -f .github/release/full.test.yml ]]; then docker-compose --file .github/release/full.test.yml build docker-compose --file .github/release/full.test.yml run sut fi diff --git a/.github/workflows/docker-publish-standard.yml b/.github/workflows/docker-publish-standard.yml index 989a8dc63..f939e3dce 100644 --- a/.github/workflows/docker-publish-standard.yml +++ b/.github/workflows/docker-publish-standard.yml @@ -28,7 +28,7 @@ jobs: - name: Run tests run: | - if [ -f .github/release/standard.test.yml]; then + if [[ -f .github/release/standard.test.yml ]]; then docker-compose --file .github/release/standard.test.yml build docker-compose --file .github/release/standard.test.yml run sut fi From 7c92d636995a812fd7f14ac4e96d5c9641c31f81 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Mon, 21 Sep 2020 02:40:53 +0200 Subject: [PATCH 09/11] Remove container action again --- .github/workflows/docker-test.yml | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100644 .github/workflows/docker-test.yml diff --git a/.github/workflows/docker-test.yml b/.github/workflows/docker-test.yml deleted file mode 100644 index 870c93f40..000000000 --- a/.github/workflows/docker-test.yml +++ /dev/null @@ -1,18 +0,0 @@ -# https://docs.github.com/en/actions/creating-actions/creating-a-docker-container-action -# https://docs.github.com/en/actions/creating-actions/dockerfile-support-for-github-actions -name: Docker Test -on: - push: - #branches: [ master, test-docker ] - pull_request: - branches: [ master ] -jobs: - test_docker: - name: Test Docker image - runs-on: ubuntu-latest - steps: - - name: Invoke - id: invoke - uses: earthobservations/test-wetterdienst-docker@main - - name: Display runtime - run: echo "The time was ${{ steps.invoke.outputs.time }}" From be3670633c4bfe017bf748889933d4336888b55b Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Mon, 21 Sep 2020 03:07:20 +0200 Subject: [PATCH 10/11] Reduce Docker image size of wetterdienst-full from 1.5G to 900MB --- .github/release/full/Dockerfile | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/release/full/Dockerfile b/.github/release/full/Dockerfile index 46c93aa81..af48946aa 100644 --- a/.github/release/full/Dockerfile +++ b/.github/release/full/Dockerfile @@ -1,4 +1,5 @@ -FROM python:3.8.5-slim +# 1. Build GDAL-python +FROM python:3.8.5-slim as build-step # TODO: This currently installs GDAL==2.4.0. For a more recent version, ... # @@ -24,5 +25,16 @@ RUN pip install GDAL==$(gdal-config --version) # Install wradlib. RUN pip install wradlib + +# 2. Main +FROM python:3.8.5-slim + +# Install libgdal. +RUN apt-get update +RUN apt-get --yes install libgdal20 + +# Copy build artefacts from first build step. +COPY --from=build-step /usr/local/lib /usr/local/lib + # Install Wetterdienst. RUN pip install wetterdienst[excel] From d473584431a9c40227c0b31302086564dd871fb5 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Mon, 21 Sep 2020 03:16:53 +0200 Subject: [PATCH 11/11] More logging --- .github/workflows/docker-publish-full.yml | 2 ++ .github/workflows/docker-publish-standard.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/docker-publish-full.yml b/.github/workflows/docker-publish-full.yml index c939f6f44..954a53928 100644 --- a/.github/workflows/docker-publish-full.yml +++ b/.github/workflows/docker-publish-full.yml @@ -73,3 +73,5 @@ jobs: docker tag $IMAGE_NAME $IMAGE_ID:$VERSION docker push $IMAGE_ID:$VERSION + + echo "Published image $IMAGE_ID:$VERSION" diff --git a/.github/workflows/docker-publish-standard.yml b/.github/workflows/docker-publish-standard.yml index f939e3dce..3d2d71eae 100644 --- a/.github/workflows/docker-publish-standard.yml +++ b/.github/workflows/docker-publish-standard.yml @@ -73,3 +73,5 @@ jobs: docker tag $IMAGE_NAME $IMAGE_ID:$VERSION docker push $IMAGE_ID:$VERSION + + echo "Published image $IMAGE_ID:$VERSION"