Optimize build and release pipelines

.github/workflows/build.yml

@@ -1,169 +1,231 @@
 name: Build on ubuntu
 
 on:
-  push:
   workflow_dispatch:
-  release:
-    types: [published]
+  pull_request_review:
+    types: [submitted]
+  push:
+    branches:
+      - master
 
 jobs:
   build:
+    if: github.event_name == 'workflow_dispatch' || (github.event.pull_request.base.ref == 'master' && github.event.review.state == 'approved')
     runs-on: ubuntu-latest
     container: falkordb/falkordb-build:latest
+    services:
+      registry:
+        image: registry:2
+        ports:
+          - 5000:5000
     strategy:
       fail-fast: false
       matrix:
         platform:
           - linux/amd64
           - linux/arm64
     steps:
+      - name: Safe dir
+        run: git config --global --add safe.directory '*'
 
-    - name: Safe dir
-      run: git config --global --add safe.directory '*'
-
-    - uses: actions/checkout@v3
-      with:
-        set-safe-directory: '*'
-        submodules: recursive
-
-    - name: Cache GraphBLAS ${{ matrix.platform }}
-      if: github.event_name == 'push'
-      id: cache_graphblas
-      uses: actions/cache@v3
-      with:
-        path: /FalkorDB/bin/linux-${{ matrix.platform == 'linux/amd64' && 'x64' || 'arm64v8' }}-release/GraphBLAS
-        key: graphblas-${{ matrix.platform == 'linux/amd64' && 'x64' || 'arm64v8' }}-${{ hashFiles('./deps/GraphBLAS/Include/GraphBLAS.h') }}
-
-    - name: Cache parser ${{ matrix.platform }}
-      if: github.event_name == 'push'
-      id: cache_parser
-      uses: actions/cache@v3
-      with:
-        path: /FalkorDB/bin/linux-${{ matrix.platform == 'linux/amd64' && 'x64' || 'arm64v8' }}-release/libcypher-parser
-        key: parser-${{ matrix.platform == 'linux/amd64' && 'x64' || 'arm64v8' }}-${{ hashFiles('./deps/libcypher-parser/lib/src/parser.c') }}
-
-    - name: Cache search ${{ matrix.platform }}
-      if: github.event_name == 'push'
-      id: cache_search
-      uses: actions/cache@v3
-      with:
-        path: /FalkorDB/bin/linux-${{ matrix.platform == 'linux/amd64' && 'x64' || 'arm64v8' }}-release/search-static
-        key: search-${{ matrix.platform == 'linux/amd64' && 'x64' || 'arm64v8' }}-${{ hashFiles('./deps/RediSearch/src/version.h') }}
-
-    # Make sure the working directory is static between runs 
-    # and always the same for the CMake cache
-    - name: Relocate source
-      run: |
-        mkdir -p /FalkorDB
-        mv $GITHUB_WORKSPACE/* /FalkorDB
-
-    - name: Set up QEMU
-      uses: docker/setup-qemu-action@v3
-
-    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v2
-
-    - name: Login to Docker Hub
-      uses: docker/login-action@v3
-      with:
-        username: ${{ secrets.DOCKER_USERNAME }}
-        password: ${{ secrets.DOCKER_PASSWORD }}
-
-    - name: Build and push
-      id: build
-      uses: docker/build-push-action@v5
-      with:
-        context: /FalkorDB
-        file: /FalkorDB/build/docker/Dockerfile
-        platforms: ${{ matrix.platform }}
-        outputs: type=image,name=falkordb/falkordb,push-by-digest=true,name-canonical=true,push=true
-
-    - name: Export digest
-      run: |
-          mkdir -p /tmp/digests
-          digest="${{ steps.build.outputs.digest }}"
-          touch "/tmp/digests/${digest#sha256:}"
-
-    - name: Upload digest
-      uses: actions/upload-artifact@v3
-      with:
-        name: digests
-        path: /tmp/digests/*
-        if-no-files-found: error
-        retention-days: 1
-
-    - name: Copy bin from docker on PUSH
-      if: github.event_name == 'push'
-      working-directory: /FalkorDB
-      run: |
-        id=$(docker create --platform=${{ matrix.platform }} falkordb/falkordb@${{ steps.build.outputs.digest }})
-        docker cp $id:/FalkorDB/bin .
-        docker rm -v $id
-
-  merge:
-    runs-on: ubuntu-latest
-    needs: build
-    steps:
-      - name: Download digests
-        uses: actions/download-artifact@v3
+      - uses: actions/checkout@v3
         with:
-          name: digests
-          path: /tmp/digests
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
+          set-safe-directory: "*"
+          submodules: recursive
+
+      - name: Cache GraphBLAS ${{ matrix.platform }}
+        id: cache_graphblas
+        uses: actions/cache@v3
         with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-      - name: Create manifest list and push on PUSH
-        if: github.event_name == 'push'
-        working-directory: /tmp/digests
-        run: |
-          docker buildx imagetools create -t falkordb/falkordb:${{ github.ref_name == 'master' && 'master -t falkordb/falkordb:edge' || github.ref_name }} \
-            $(printf 'falkordb/falkordb@sha256:%s ' *)
-      - name: Create manifest list and push on RELEASE
-        if: github.event_name == 'release'
-        working-directory: /tmp/digests
+          path: /FalkorDB/bin/linux-${{ matrix.platform == 'linux/amd64' && 'x64' || 'arm64v8' }}-release/GraphBLAS
+          key: graphblas-${{ matrix.platform == 'linux/amd64' && 'x64' || 'arm64v8' }}-${{ hashFiles('./deps/GraphBLAS/Include/GraphBLAS.h') }}
+
+      - name: Cache parser ${{ matrix.platform }}
+        id: cache_parser
+        uses: actions/cache@v3
+        with:
+          path: /FalkorDB/bin/linux-${{ matrix.platform == 'linux/amd64' && 'x64' || 'arm64v8' }}-release/libcypher-parser
+          key: parser-${{ matrix.platform == 'linux/amd64' && 'x64' || 'arm64v8' }}-${{ hashFiles('./deps/libcypher-parser/lib/src/parser.c') }}
+
+      - name: Cache search ${{ matrix.platform }}
+        id: cache_search
+        uses: actions/cache@v3
+        with:
+          path: /FalkorDB/bin/linux-${{ matrix.platform == 'linux/amd64' && 'x64' || 'arm64v8' }}-release/search-static
+          key: search-${{ matrix.platform == 'linux/amd64' && 'x64' || 'arm64v8' }}-${{ hashFiles('./deps/RediSearch/src/version.h') }}
+
+      # Make sure the working directory is static between runs
+      # and always the same for the CMake cache
+      - name: Relocate source
         run: |
-          docker buildx imagetools create -t falkordb/falkordb:${{ github.event.release.tag_name }} -t falkordb/falkordb:latest \
-            $(printf 'falkordb/falkordb@sha256:%s ' *)
-      - name: Inspect image
+          mkdir -p /FalkorDB
+          mv $GITHUB_WORKSPACE/* /FalkorDB
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          # network=host driver-opt needed to push to local registry
+          driver-opts: network=host
+
+      - name: Build compiler image
+        id: build_compiler
+        uses: docker/build-push-action@v5
+        with:
+          context: /FalkorDB
+          file: /FalkorDB/build/docker/Dockerfile.compiler
+          platforms: ${{ matrix.platform }}
+          tags: localhost:5000/falkordb/falkordb-compiler
+          # outputs: type=docker,dest=/tmp/falkordb-compiler.tar
+          push: true
+          build-args: |
+            TARGETPLATFORM=${{ matrix.platform }}
+
+      - name: Copy bin from docker for caching
+        working-directory: /FalkorDB
+        if: steps.cache_graphblas.outputs.cache-hit == false || steps.cache_parser.outputs.cache-hit == false || steps.cache_search.outputs.cache-hit == false
         run: |
-          docker buildx imagetools inspect falkordb/falkordb:latest
+          id=$(docker create --platform=${{ matrix.platform }} localhost:5000/falkordb/falkordb-compiler)
+          docker cp $id:/FalkorDB/bin /FalkorDB/bin
+          docker rm -v $id
+          ls -l -R /FalkorDB/bin
+
+      - name: Build tests image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: /FalkorDB/tests/Dockerfile
+          outputs: type=docker,dest=/tmp/falkordb-tests-${{ matrix.platform == 'linux/amd64' && 'x64' || 'arm64v8'}}.tar
+          tags: falkordb/falkordb-tests
+          build-args: |
+            BASE_IMAGE=localhost:5000/falkordb/falkordb-compiler
+            TARGETPLATFORM=${{ matrix.platform }}
+
+      - name: Upload tests image
+        uses: actions/upload-artifact@v3
+        with:
+          name: falkordb-tests-${{ matrix.platform == 'linux/amd64' && 'x64' || 'arm64v8'}}
+          path: /tmp/falkordb-tests-${{ matrix.platform == 'linux/amd64' && 'x64' || 'arm64v8'}}.tar
+          if-no-files-found: error
+
+      - name: Build base image
+        id: build
+        uses: docker/build-push-action@v5
+        with:
+          context: /FalkorDB
+          file: /FalkorDB/build/docker/Dockerfile
+          platforms: ${{ matrix.platform }}
+          outputs: type=docker,dest=/tmp/falkordb-${{ matrix.platform == 'linux/amd64' && 'x64' || 'arm64v8'}}.tar
+          tags: falkordb/falkordb-${{ matrix.platform == 'linux/amd64' && 'x64' || 'arm64v8'}}
+          build-args: |
+            BASE_IMAGE=localhost:5000/falkordb/falkordb-compiler
+            TARGETPLATFORM=${{ matrix.platform }}
+
+      - name: Upload image
+        uses: actions/upload-artifact@v3
+        with:
+          name: falkordb-${{ matrix.platform == 'linux/amd64' && 'x64' || 'arm64v8'}}
+          path: /tmp/falkordb-${{ matrix.platform == 'linux/amd64' && 'x64' || 'arm64v8'}}.tar
+          if-no-files-found: error
 
   unit-tests:
+    needs: build
     runs-on: ubuntu-latest
-    container: falkordb/falkordb:${{ github.ref_name }}
-    needs: merge
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - linux/amd64
+          - linux/arm64
     steps:
+      - name: Download image
+        uses: actions/download-artifact@v3
+        with:
+          name: falkordb-tests-${{ matrix.platform == 'linux/amd64' && 'x64' || 'arm64v8'}}
+          path: /tmp
+
+      - name: Load image
+        id: load_image
+        run: |
+          docker load --input /tmp/falkordb-tests-${{ matrix.platform == 'linux/amd64' && 'x64' || 'arm64v8'}}.tar
+
       - name: Unit tests
-        working-directory: /FalkorDB
-        run: make unit-tests CLEAR_LOGS=0
+        run: |
+          docker run --platform ${{ matrix.platform }} --rm falkordb/falkordb-tests make CLEAR_LOGS=0 unit-tests
 
   flow-tests:
+    needs: build
     runs-on: ubuntu-latest
-    container: falkordb/falkordb:${{ github.ref_name }}
-    needs: merge
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - linux/amd64
+          - linux/arm64
     steps:
+      - name: Download image
+        uses: actions/download-artifact@v3
+        with:
+          name: falkordb-tests-${{ matrix.platform == 'linux/amd64' && 'x64' || 'arm64v8'}}
+          path: /tmp
+
+      - name: Load image
+        id: load_image
+        run: |
+          docker load --input /tmp/falkordb-tests-${{ matrix.platform == 'linux/amd64' && 'x64' || 'arm64v8'}}.tar
+
       - name: Flow tests
-        working-directory: /FalkorDB
-        run: make CLEAR_LOGS=0 PARALLEL=4 flow-tests
+        run: |
+          docker run --platform ${{ matrix.platform }} --rm falkordb/falkordb-tests make CLEAR_LOGS=0 PARALLEL=4 flow-tests
 
   tck-tests:
+    needs: build
     runs-on: ubuntu-latest
-    container: falkordb/falkordb:${{ github.ref_name }}
-    needs: merge
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - linux/amd64
+          - linux/arm64
     steps:
+      - name: Download image
+        uses: actions/download-artifact@v3
+        with:
+          name: falkordb-tests-${{ matrix.platform == 'linux/amd64' && 'x64' || 'arm64v8'}}
+          path: /tmp
+
+      - name: Load image
+        id: load_image
+        run: |
+          docker load --input /tmp/falkordb-tests-${{ matrix.platform == 'linux/amd64' && 'x64' || 'arm64v8'}}.tar
+
       - name: TCK tests
-        working-directory: /FalkorDB
-        run: make CLEAR_LOGS=0 tck-tests
+        run: |
+          docker run --platform ${{ matrix.platform }} --rm falkordb/falkordb-tests make CLEAR_LOGS=0 tck-tests
 
   fuzz-tests:
+    needs: build
     runs-on: ubuntu-latest
-    container: falkordb/falkordb:${{ github.ref_name }}
-    needs: merge
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - linux/amd64
+          - linux/arm64
     steps:
+      - name: Download image
+        uses: actions/download-artifact@v3
+        with:
+          name: falkordb-tests-${{ matrix.platform == 'linux/amd64' && 'x64' || 'arm64v8'}}
+          path: /tmp
+
+      - name: Load image
+        id: load_image
+        run: |
+          docker load --input /tmp/falkordb-tests-${{ matrix.platform == 'linux/amd64' && 'x64' || 'arm64v8'}}.tar
+
       - name: Fuzz tests
-        working-directory: /FalkorDB
-        run: make fuzz TIMEOUT=180
+        run: |
+          docker run --platform ${{ matrix.platform }} --rm falkordb/falkordb-tests make fuzz TIMEOUT=180