diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index c1582e4..d7fa2e8 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,4 +1,4 @@ -name: Docker +name: Push Docker images on Docker Hub on: push: @@ -43,6 +43,43 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max + build_scala_images: + needs: build_base_images + strategy: + matrix: + sbt_version: [1.8.2, 1.8.3, 1.9.0, 1.9.1, 1.9.2] + environment: docker-hub + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + + - name: Build corretto with sbt installed + id: docker_build_corretto_sbt + uses: docker/build-push-action@v4 + with: + builder: ${{ steps.buildx.outputs.name }} + context: ./corretto-emr-dbs-universal-spark-scala + file: ./corretto-emr-dbs-universal-spark-scala/Dockerfile + # Only Java 8 is supported for Spark + build-args: | + JDK_VERSION=8 + SBT_VERSION=${{ matrix.sbt_version }} + push: true + tags: infrahelpers/dpp:jdk8-sbt${{ matrix.sbt_version }} + cache-from: type=gha + cache-to: type=gha,mode=max + build_python_images: needs: build_base_images strategy: diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..485dee6 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.idea diff --git a/corretto-emr-dbs-universal-spark-scala/Dockerfile b/corretto-emr-dbs-universal-spark-scala/Dockerfile new file mode 100644 index 0000000..5e77f7f --- /dev/null +++ b/corretto-emr-dbs-universal-spark-scala/Dockerfile @@ -0,0 +1,21 @@ +# +# Source: https://github.com/data-engineering-helpers/dpp-images/tree/main/pyspark-py310/Dockerfile +# On Docker Hub: https://hub.docker.com/repository/docker/infrahelpers/dpp/general +# Usual Docker tags: +# * infrahelpers/dpp:jdk8-sbt{SBT_VERSION} +# +# Image containing python installation, to be accessed by EMR and Databricks (for spark scala) +# See https://github.com/data-engineering-helpers/dpp-images/tree/main/coretto-emr-dbs-universal-base/Dockerfile +# for more details about the base image (tag: infrahelpers/dpp:jdk{JDK_VERSION}) +# +FROM infrahelpers/dpp:jdk8 + +ARG SBT_VERSION + +LABEL authors="Antoine Chenon" + +# Update the OS +RUN yum -y update && yum clean all + +# Install sbt +RUN rm -f /etc/yum.repos.d/bintray-rpm.repo; curl -L https://www.scala-sbt.org/sbt-rpm.repo > sbt-rpm.repo; mv sbt-rpm.repo /etc/yum.repos.d/; yum -y install sbt-${SBT_VERSION}-0 \ No newline at end of file