Implement build system using self-hosted aarch64 runners, GitHub needs jobs feature and reusable workflows

.github/actions/create-dev-env/action.yml

@@ -0,0 +1,26 @@
+name: Build environment
+description: Create build environment
+
+inputs:
+  platform:
+    description: Platform to be run on
+    required: true
+    type: string
+
+runs:
+  using: composite
+  steps:
+    # actions/setup-python doesn't suport Linux aarch64 runners
+    # See: https://github.com/actions/setup-python/issues/108
+    # python3 is manually preinstalled in the aarch64 VM self-hosted runner
+    - name: Set Up Python 🐍
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.x
+      if: ${{ inputs.platform == 'amd64' }}
+
+    - name: Install Dev Dependencies 📦
+      run: |
+        pip install --upgrade pip
+        pip install --upgrade -r requirements-dev.txt
+      shell: bash

.github/actions/download-manifests/action.yml

@@ -0,0 +1,161 @@
+name: Download manifests
+description: Download all manifests and history lines
+
+# Unfortunately, `actions/download-artifact` doesn't support wildcard download
+# To make this workflow fast, we manually list all manifests and history lines downloads
+# https://github.com/actions/download-artifact/issues/6
+
+inputs:
+  histLineDir:
+    description: Directory to store history lines
+    required: true
+    type: string
+  manifestDir:
+    description: Directory to store manifest files
+    required: true
+    type: string
+
+runs:
+  using: composite
+  steps:
+    - name: Download artifact 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: base-notebook-aarch64-history_line
+        path: ${{ inputs.histLineDir }}
+    - name: Download artifact 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: base-notebook-amd64-history_line
+        path: ${{ inputs.histLineDir }}
+    - name: Download artifact 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: minimal-notebook-aarch64-history_line
+        path: ${{ inputs.histLineDir }}
+    - name: Download artifact 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: minimal-notebook-amd64-history_line
+        path: ${{ inputs.histLineDir }}
+    - name: Download artifact 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: scipy-notebook-aarch64-history_line
+        path: ${{ inputs.histLineDir }}
+    - name: Download artifact 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: scipy-notebook-amd64-history_line
+        path: ${{ inputs.histLineDir }}
+    - name: Download artifact 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: r-notebook-aarch64-history_line
+        path: ${{ inputs.histLineDir }}
+    - name: Download artifact 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: r-notebook-amd64-history_line
+        path: ${{ inputs.histLineDir }}
+    - name: Download artifact 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: tensorflow-notebook-amd64-history_line
+        path: ${{ inputs.histLineDir }}
+    - name: Download artifact 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: datascience-notebook-amd64-history_line
+        path: ${{ inputs.histLineDir }}
+    - name: Download artifact 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: pyspark-notebook-aarch64-history_line
+        path: ${{ inputs.histLineDir }}
+    - name: Download artifact 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: pyspark-notebook-amd64-history_line
+        path: ${{ inputs.histLineDir }}
+    - name: Download artifact 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: all-spark-notebook-aarch64-history_line
+        path: ${{ inputs.histLineDir }}
+    - name: Download artifact 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: all-spark-notebook-amd64-history_line
+        path: ${{ inputs.histLineDir }}
+
+    - name: Download artifact 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: base-notebook-aarch64-manifest
+        path: ${{ inputs.manifestDir }}
+    - name: Download artifact 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: base-notebook-amd64-manifest
+        path: ${{ inputs.manifestDir }}
+    - name: Download artifact 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: minimal-notebook-aarch64-manifest
+        path: ${{ inputs.manifestDir }}
+    - name: Download artifact 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: minimal-notebook-amd64-manifest
+        path: ${{ inputs.manifestDir }}
+    - name: Download artifact 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: scipy-notebook-aarch64-manifest
+        path: ${{ inputs.manifestDir }}
+    - name: Download artifact 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: scipy-notebook-amd64-manifest
+        path: ${{ inputs.manifestDir }}
+    - name: Download artifact 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: r-notebook-aarch64-manifest
+        path: ${{ inputs.manifestDir }}
+    - name: Download artifact 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: r-notebook-amd64-manifest
+        path: ${{ inputs.manifestDir }}
+    - name: Download artifact 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: tensorflow-notebook-amd64-manifest
+        path: ${{ inputs.manifestDir }}
+    - name: Download artifact 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: datascience-notebook-amd64-manifest
+        path: ${{ inputs.manifestDir }}
+    - name: Download artifact 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: pyspark-notebook-aarch64-manifest
+        path: ${{ inputs.manifestDir }}
+    - name: Download artifact 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: pyspark-notebook-amd64-manifest
+        path: ${{ inputs.manifestDir }}
+    - name: Download artifact 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: all-spark-notebook-aarch64-manifest
+        path: ${{ inputs.manifestDir }}
+    - name: Download artifact 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: all-spark-notebook-amd64-manifest
+        path: ${{ inputs.manifestDir }}

.github/actions/load-image/action.yml

@@ -0,0 +1,35 @@
+name: Load Docker image
+description: Download image tar and load it to docker
+
+inputs:
+  image:
+    description: Image name
+    required: true
+    type: string
+  platform:
+    description: Image platform
+    required: true
+    type: string
+
+runs:
+  using: composite
+  steps:
+    # Self-hosted runners share a state (whole VM) between runs
+    - name: Reset docker state 🗑️
+      run: docker system prune --all --force
+      shell: bash
+
+    - name: Download built image 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: ${{ inputs.image }}-${{ inputs.platform }}
+        path: /tmp/
+    - name: Load downloaded image to docker 📥
+      run: |
+        docker load --input /tmp/${{ inputs.image }}-${{ inputs.platform }}.tar
+        docker image ls -a
+      shell: bash
+    - name: Delete the file 🗑️
+      run: rm -f /tmp/${{ inputs.image }}-${{ inputs.platform }}.tar
+      shell: bash
+      if: always()

.github/workflows/docker-build-test-upload.yml

@@ -0,0 +1,69 @@
+name: Download parent image, build new one, test it and upload to GitHub artifacts
+
+on:
+  workflow_call:
+    inputs:
+      parentImage:
+        description: Parent image name
+        required: true
+        type: string
+      image:
+        description: Image name
+        required: true
+        type: string
+      platform:
+        description: Image platform
+        required: true
+        type: string
+      runsOn:
+        description: GitHub Actions Runner image
+        required: true
+        type: string
+
+jobs:
+  build-test-upload:
+    runs-on: ${{ inputs.runsOn }}
+    steps:
+      - name: Checkout Repo ⚡️
+        uses: actions/checkout@v3
+      - name: Create dev environment 📦
+        uses: ./.github/actions/create-dev-env
+        with:
+          platform: ${{ inputs.platform }}
+
+      - name: Load parent built image to Docker 📥
+        if: ${{ inputs.parentImage != '' }}
+        uses: ./.github/actions/load-image
+        with:
+          image: ${{ inputs.parentImage }}
+          platform: ${{ inputs.platform }}
+
+      - name: Build image 🛠
+        run: docker build --rm --force-rm --tag jupyter/${{ inputs.image }} ${{ inputs.image }}/
+        env:
+          DOCKER_BUILDKIT: 1
+          # Full logs for CI build
+          BUILDKIT_PROGRESS: plain
+        shell: bash
+
+      - name: Run tests ✅
+        run: python3 -m tests.run_tests --short-image-name ${{ inputs.image }}
+        shell: bash
+
+      - name: Save image as a tar for later use 💾
+        run: docker save jupyter/${{ inputs.image }} -o /tmp/${{ inputs.image }}-${{ inputs.platform }}.tar
+        shell: bash
+      - name: Upload image as artifact 💾
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ inputs.image }}-${{ inputs.platform }}
+          path: /tmp/${{ inputs.image }}-${{ inputs.platform }}.tar
+          retention-days: 3
+
+      # Self-hosted runners share a state (whole VM) between runs
+      - name: Cleanup artifacts 🗑️
+        run: |
+          rm -f /tmp/${{ inputs.image }}-${{ inputs.platform }}.tar
+          docker system prune --all --force
+        shell: bash
+        if: always()