From 6790f8dd03699efbd606b9d82375309a7d63e0c1 Mon Sep 17 00:00:00 2001 From: "Simeon H.K. Fitch" Date: Mon, 27 Sep 2021 13:56:27 -0400 Subject: [PATCH 1/3] GitHub actions build. --- .circleci/README.md | 6 ------ .github/image/.dockerignore | 3 +++ .github/image/Dockerfile | 28 ++++++++++++++++++++++++++++ .github/image/Makefile | 27 +++++++++++++++++++++++++++ .github/image/requirements-conda.txt | 5 +++++ .github/workflows/build-test.yml | 2 +- 6 files changed, 64 insertions(+), 7 deletions(-) delete mode 100644 .circleci/README.md create mode 100644 .github/image/.dockerignore create mode 100644 .github/image/Dockerfile create mode 100644 .github/image/Makefile create mode 100644 .github/image/requirements-conda.txt diff --git a/.circleci/README.md b/.circleci/README.md deleted file mode 100644 index 6a507cc5f..000000000 --- a/.circleci/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# CircleCI Dockerfile Build file - -```bash -make -docker push s22s/rasterframes-circleci:latest -``` diff --git a/.github/image/.dockerignore b/.github/image/.dockerignore new file mode 100644 index 000000000..dbe9a91d7 --- /dev/null +++ b/.github/image/.dockerignore @@ -0,0 +1,3 @@ +* +!requirements-conda.txt +!fix-permissions diff --git a/.github/image/Dockerfile b/.github/image/Dockerfile new file mode 100644 index 000000000..27cd7a1aa --- /dev/null +++ b/.github/image/Dockerfile @@ -0,0 +1,28 @@ +FROM adoptopenjdk/openjdk11:debian-slim + +# See: https://docs.conda.io/projects/conda/en/latest/user-guide/install/rpm-debian.html +RUN \ + apt-get update && \ + apt-get install -yq gpg && \ + curl -s https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc | gpg --dearmor > conda.gpg && \ + install -o root -g root -m 644 conda.gpg /usr/share/keyrings/conda-archive-keyring.gpg && \ + gpg --keyring /usr/share/keyrings/conda-archive-keyring.gpg --no-default-keyring --fingerprint 34161F5BF5EB1D4BFBBB8F0A8AEB4F8B29D82806 && \ + echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" > /etc/apt/sources.list.d/conda.list && \ + apt-get update && \ + apt-get install -yq --no-install-recommends conda && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +ENV CONDA_DIR=/opt/conda +ENV PATH=$CONDA_DIR/bin:$PATH + +COPY requirements-conda.txt /tmp +RUN \ + conda install --quiet --yes --channel=conda-forge --file=/tmp/requirements-conda.txt && \ + echo "$CONDA_DIR/lib" > /etc/ld.so.conf.d/conda.conf && \ + ldconfig && \ + conda clean --all --force-pkgs-dirs --yes --quiet + +# Work-around for pyproj issue https://github.com/pyproj4/pyproj/issues/415 +ENV PROJ_LIB=/opt/conda/share/proj + diff --git a/.github/image/Makefile b/.github/image/Makefile new file mode 100644 index 000000000..1dab66b65 --- /dev/null +++ b/.github/image/Makefile @@ -0,0 +1,27 @@ +IMAGE_NAME=debian-openjdk-conda-gdal +SHA=$(shell git log -n1 --format=format:"%H" | cut -c 1-7) +VERSION?=$(SHA) +HOST=docker.io +REPO=$(HOST)/s22s +FULL_NAME=$(REPO)/$(IMAGE_NAME):$(VERSION) + +.DEFAULT_GOAL := help +help: +# http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html + @echo "Usage: make [target]" + @echo "Targets: " + @grep -E '^[a-zA-Z0-9_%/-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\t\033[36m%-20s\033[0m %s\n", $$1, $$2}' + +all: build push ## Build and then push image + +build: ## Build the docker image + docker build . -t ${FULL_NAME} + +login: ## Login to the docker registry + docker login + +push: login ## Push docker image to registry + docker push ${FULL_NAME} + +run: build ## Build image and launch shell + docker run --rm -it ${FULL_NAME} bash diff --git a/.github/image/requirements-conda.txt b/.github/image/requirements-conda.txt new file mode 100644 index 000000000..a8ebfd56b --- /dev/null +++ b/.github/image/requirements-conda.txt @@ -0,0 +1,5 @@ +python==3.8 +gdal==3.1.2 +libspatialindex +rasterio[s3] +rtree \ No newline at end of file diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 9e205bc5f..6096a17cb 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -13,7 +13,7 @@ jobs: build: runs-on: ubuntu-latest container: - image: s22s/circleci-openjdk-conda-gdal:b8e30ee + image: s22s/debian-openjdk-conda-gdal:d7e065d steps: - uses: actions/checkout@v2 From 62e099a35759f7877668fed27c488dfbeacb5c5c Mon Sep 17 00:00:00 2001 From: "Simeon H.K. Fitch" Date: Mon, 27 Sep 2021 14:13:08 -0400 Subject: [PATCH 2/3] Split out docs build into separate workflow. --- .github/workflows/build-test.yml | 64 ++--------------------------- .github/workflows/docs.yml | 70 ++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 61 deletions(-) create mode 100644 .github/workflows/docs.yml diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 6096a17cb..eacd631b5 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -13,7 +13,7 @@ jobs: build: runs-on: ubuntu-latest container: - image: s22s/debian-openjdk-conda-gdal:d7e065d + image: s22s/debian-openjdk-conda-gdal:6790f8d steps: - uses: actions/checkout@v2 @@ -31,8 +31,8 @@ jobs: - name: Install Conda dependencies run: | - # $CONDA is an environment variable pointing to the root of the miniconda directory - $CONDA/bin/conda install -c conda-forge --yes --file pyrasterframes/src/main/python/requirements-condaforge.txt + # $CONDA_DIR is an environment variable pointing to the root of the miniconda directory + $CONDA_DIR/bin/conda install -c conda-forge --yes --file pyrasterframes/src/main/python/requirements-condaforge.txt - run: ulimit -c unlimited -S @@ -71,61 +71,3 @@ jobs: with: name: core-dumps path: /tmp/core_dumps - - docs: - runs-on: ubuntu-latest - container: - image: s22s/circleci-openjdk-conda-gdal:b8e30ee - - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - uses: coursier/cache-action@v6 - - uses: olafurpg/setup-scala@v13 - with: - java-version: adopt@1.11 - - - name: Set up Python 3.8 - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - - name: Install Conda dependencies - run: | - # $CONDA is an environment variable pointing to the root of the miniconda directory - $CONDA/bin/conda install -c conda-forge --yes --file pyrasterframes/src/main/python/requirements-condaforge.txt - - - run: ulimit -c unlimited -S - - - name: Build documentation - run: sbt makeSite - - - name: Collect artifacts - if: ${{ failure() }} - run: | - mkdir -p /tmp/core_dumps - cp core.* *.hs /tmp/core_dumps 2> /dev/null || true - mkdir -p /tmp/markdown - cp pyrasterframes/target/python/docs/*.md /tmp/markdown 2> /dev/null || true - - - name: Upload core dumps - if: ${{ failure() }} - uses: actions/upload-artifact@v2 - with: - name: core-dumps - path: /tmp/core_dumps - - - name: Upload markdown - if: ${{ failure() }} - uses: actions/upload-artifact@v2 - with: - name: markdown - path: /tmp/markdown - - - name: Upload rf-site - if: ${{ failure() }} - uses: actions/upload-artifact@v2 - with: - name: rf-site - path: docs/target/site \ No newline at end of file diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 000000000..7e12e10a6 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,70 @@ +name: Compile documentation + +on: + workflow_dispatch: + + pull_request: + branches: ['**docs*'] + push: + branches: ['master', 'release/*'] + release: + types: [published] + +jobs: + docs: + runs-on: ubuntu-latest + container: + image: s22s/debian-openjdk-conda-gdal:6790f8d + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - uses: coursier/cache-action@v6 + - uses: olafurpg/setup-scala@v13 + with: + java-version: adopt@1.11 + + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Install Conda dependencies + run: | + # $CONDA_DIR is an environment variable pointing to the root of the miniconda directory + $CONDA_DIR/bin/conda install -c conda-forge --yes --file pyrasterframes/src/main/python/requirements-condaforge.txt + + - run: ulimit -c unlimited -S + + - name: Build documentation + run: sbt makeSite + + - name: Collect artifacts + if: ${{ failure() }} + run: | + mkdir -p /tmp/core_dumps + cp core.* *.hs /tmp/core_dumps 2> /dev/null || true + mkdir -p /tmp/markdown + cp pyrasterframes/target/python/docs/*.md /tmp/markdown 2> /dev/null || true + + - name: Upload core dumps + if: ${{ failure() }} + uses: actions/upload-artifact@v2 + with: + name: core-dumps + path: /tmp/core_dumps + + - name: Upload markdown + if: ${{ failure() }} + uses: actions/upload-artifact@v2 + with: + name: markdown + path: /tmp/markdown + + - name: Upload rf-site + if: ${{ failure() }} + uses: actions/upload-artifact@v2 + with: + name: rf-site + path: docs/target/site \ No newline at end of file From 938638b9723605b87f5f88079e32ee4c9f9f4611 Mon Sep 17 00:00:00 2001 From: "Simeon H.K. Fitch" Date: Mon, 27 Sep 2021 14:17:20 -0400 Subject: [PATCH 3/3] Removed umlimit call (not needed) --- .github/workflows/build-test.yml | 2 -- .github/workflows/docs.yml | 2 -- 2 files changed, 4 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index eacd631b5..50b87ad5d 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -34,8 +34,6 @@ jobs: # $CONDA_DIR is an environment variable pointing to the root of the miniconda directory $CONDA_DIR/bin/conda install -c conda-forge --yes --file pyrasterframes/src/main/python/requirements-condaforge.txt - - run: ulimit -c unlimited -S - # Do just the compilation stage to minimize sbt memory footprint - name: Compile run: sbt -v -batch compile test:compile it:compile diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 7e12e10a6..100b78d4f 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -35,8 +35,6 @@ jobs: # $CONDA_DIR is an environment variable pointing to the root of the miniconda directory $CONDA_DIR/bin/conda install -c conda-forge --yes --file pyrasterframes/src/main/python/requirements-condaforge.txt - - run: ulimit -c unlimited -S - - name: Build documentation run: sbt makeSite