wip: Refactor CI to use GitHub actions and generic bash scripts (#64)

.github/workflows/main.yaml

@@ -50,9 +50,45 @@ jobs:
           SOPS_AGE_KEY: ${{ secrets.SOPS_AGE_KEY }}
         run: bash ./scripts/run_tests.sh -n ${{ matrix.node }} -m ${{ matrix.mongo }}
 
-  build:
-    name: Build jobs
-    if: ${{ !contains(github.event.head_commit.message, 'skip app') }}
+  build_models:
+    strategy:
+      fail-fast: false
+      matrix:
+        model: [ 'gfs', 'arpege', 'arome' ]
+    name: Build models
+    if: ${{ !contains(github.event.head_commit.message, 'skip build') }}
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+      - name: Init runner
+        run: bash ./scripts/init_runner.sh ${{ github.job }}
+      - name: Setup workspace
+        env:
+          GITHUB_DEVELOPMENT_PAT: ${{ secrets.GH_DEVELOPMENT_PAT }}
+        run: bash ./scripts/setup_workspace.sh
+      - name: Build image
+        env:
+          SOPS_AGE_KEY: ${{ secrets.SOPS_AGE_KEY }}
+        run: bash ./scripts/build.sh -m ${{ matrix.model }} -p -r ${{ github.job }}
+
+  build_loaders:
+    strategy:
+      fail-fast: false
+      matrix:
+        model: [ 'gfs', 'arpege', 'arome' ]
+        loader: [ 'world', 'isobaric-world', 'europe', 'isobaric-europe', 'france', 'france-high' ]
+        exclude:
+          - model: 'gfs'
+            loader: [ 'world', 'isobaric-world' ]
+          - model: 'arpege'
+            loader: [ 'france', 'france-high' ]
+          - model: 'arome'
+            loader: [ 'world', 'isobaric-world', 'europe', 'isobaric-europe' ]
+    name: Build loaders
+    if: ${{ !contains(github.event.head_commit.message, 'skip build') }}
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout repo
@@ -65,7 +101,7 @@ jobs:
         env:
           GITHUB_DEVELOPMENT_PAT: ${{ secrets.GH_DEVELOPMENT_PAT }}
         run: bash ./scripts/setup_workspace.sh
-      - name: Build jobs
+      - name: Build image
         env:
           SOPS_AGE_KEY: ${{ secrets.SOPS_AGE_KEY }}
-        run: bash ./scripts/build.sh -p -r ${{ github.job }}
+        run: bash ./scripts/build.sh -m ${{ matrix.model }} -l ${{ matrix.loader }} -p -r ${{ github.job }}

dockerfile.gfs

@@ -14,7 +14,7 @@ RUN \
   DEBIAN_FRONTEND=noninteractive && \
   apt-get update && \
   apt-get --no-install-recommends --yes install \
-    openjdk-11-jre-headless \
+    openjdk-17-jre-headless \
     ca-certificates \
     gdal-bin && \
   apt-get clean && \
@@ -26,7 +26,7 @@ RUN \
   chmod a+x /usr/local/share/.config/yarn/global/node_modules/@weacast/grib2json/bin/grib2json && \
   yarn cache clean
 
-ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64
+ENV JAVA_HOME /usr/lib/jvm/java-17-openjdk-amd64
 RUN $JAVA_HOME/bin/java -version
 
 USER node

scripts/build.sh

@@ -13,7 +13,8 @@ ROOT_DIR=$(dirname "$THIS_DIR")
 
 PUBLISH=false
 CI_STEP_NAME="Build"
-while getopts "prml:" option; do
+LOADER=
+while getopts "pr:m:l:" option; do
     case $option in
         p) # publish
             PUBLISH=true
@@ -22,7 +23,7 @@ while getopts "prml:" option; do
             CI_STEP_NAME=$OPTARG
             trap 'slack_ci_report "$ROOT_DIR" "$CI_STEP_NAME" "$?" "$SLACK_WEBHOOK_JOBS"' EXIT
             ;;
-        m) # weacast model image to be build: gfs, arpege
+        m) # weacast model image to be build or used (if loader build): gfs, arpege, ...
             MODEL=$OPTARG
             ;;
         l) # weacast loader image to be build: europe, isobaric-europe, ...
@@ -39,15 +40,9 @@ done
 WORKSPACE_DIR="$(dirname "$ROOT_DIR")"
 init_job_infos "$ROOT_DIR"
 
-JOB=$(get_job_name)
-KRAWLER_VERSION=$(get_job_krawler_version)
 GIT_TAG=$(get_job_tag)
-
-if [[ -z "$GIT_TAG" ]]; then
-    echo "About to build ${JOB}:${LOADER} development version based on krawler development version..."
-else
-    echo "About to build ${JOB}:${LOADER} v${VERSION} based on krawler ${KRAWLER_VERSION}..."
-fi
+GTIFF2JSON_TAG=$(node -p -e "require('./package.json').peerDependencies['@weacast/gtiff2json']")
+GRIB2JSON_TAG=$(node -p -e "require('./package.json').peerDependencies['@weacast/grib2json']")
 
 load_env_files "$WORKSPACE_DIR/development/common/kalisio_dockerhub.enc.env" "$WORKSPACE_DIR/development/common/SLACK_WEBHOOK_JOBS.enc.env"
 load_value_files "$WORKSPACE_DIR/development/common/KALISIO_DOCKERHUB_PASSWORD.enc.value"
@@ -56,20 +51,31 @@ load_value_files "$WORKSPACE_DIR/development/common/KALISIO_DOCKERHUB_PASSWORD.e
 ##
 
 IMAGE_NAME="weacast/weacast-$MODEL"
-DOCKERFILE=dockerfile.$MODEL-$LOADER
 if [[ -z "$GIT_TAG" ]]; then
     VERSION=latest
+    KRAWLER_TAG=latest
 else
     VERSION=$(get_job_version)
+    KRAWLER_TAG=$(get_job_krawler_version)
 fi
-IMAGE_TAG=$LOADER-$VERSION
+if [[ -z "$LOADER" ]]; then
+    IMAGE_TAG=$VERSION
+    DOCKERFILE=dockerfile.$MODEL
+else
+    IMAGE_TAG=$LOADER-$VERSION
+    DOCKERFILE=dockerfile.$MODEL-$LOADER
+fi
+
+echo "About to build image ${IMAGE_NAME}:${IMAGE_TAG} based on kalisio/krawler:${KRAWLER_TAG}..."
 
 begin_group "Building container ..."
 
 docker login --username "$KALISIO_DOCKERHUB_USERNAME" --password-stdin < "$KALISIO_DOCKERHUB_PASSWORD"
 # DOCKER_BUILDKIT is here to be able to use Dockerfile specific dockerginore (app.Dockerfile.dockerignore)
 DOCKER_BUILDKIT=1 docker build -f "$ROOT_DIR/$DOCKERFILE" \
-    --build-arg TAG=$VERSION -t "$IMAGE_NAME:$IMAGE_TAG" \
+    --build-arg KRAWLER_TAG=$KRAWLER_TAG --build-arg TAG=$VERSION \
+    --build-arg GTIFF2JSON_TAG=$GTIFF2JSON_TAG --build-arg GRIB2JSON_TAG=$GRIB2JSON_TAG \
+    -t "$IMAGE_NAME:$IMAGE_TAG" \
     "$ROOT_DIR"
 
 if [ "$PUBLISH" = true ]; then

scripts/run_tests.sh

@@ -43,6 +43,8 @@ done
 . "$WORKSPACE_DIR/development/workspaces/jobs/jobs.sh" weacast-loaders
 
 # Required by tests
+GTIFF2JSON_TAG=$(node -p -e "require('./package.json').peerDependencies['@weacast/gtiff2json']")
+GRIB2JSON_TAG=$(node -p -e "require('./package.json').peerDependencies['@weacast/grib2json']")
 git clone https://github.com/kalisio/krawler.git && cd krawler && yarn install && yarn link && cd ..
 yarn link @kalisio/krawler
 yarn global add @weacast/gtiff2json@${GTIFF2JSON_TAG}

scripts/setup_workspace.sh

@@ -16,9 +16,6 @@ WORKSPACE_TAG=
 
 begin_group "Setting up workspace ..."
 
-GTIFF2JSON_TAG=$(node -p -e "require('./package.json').peerDependencies['@weacast/gtiff2json']")
-GRIB2JSON_TAG=$(node -p -e "require('./package.json').peerDependencies['@weacast/grib2json']")
-
 if [ "$CI" = true ]; then
     WORKSPACE_DIR="$(dirname "$ROOT_DIR")"
     DEVELOPMENT_REPO_URL="https://$GITHUB_DEVELOPMENT_PAT@github.com/kalisio/development.git"