Add daily platform build of artifacts

.github/workflows/daily-platform-build-new.yml

@@ -0,0 +1,46 @@
+name: Daily Platform Build
+
+on: [pull_request, workflow_dispatch]
+
+jobs:
+  build_memgraph:
+    name: "Build memgraph Debian 11"
+    runs-on: [self-hosted, DockerMgBuild, X64]
+    timeout-minutes: 60
+    steps:
+      - name: "Set up repository"
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # Required because of release/get_version.py
+          token: ${{ secrets.PAT }}
+      - name: "Build package"
+        run: |
+          git clone -b master https://github.com/memgraph/memgraph.git
+          cd memgraph
+          ./release/package/run.sh package debian-11 Release
+      - name: "Upload package"
+        uses: actions/upload-artifact@v3
+        with:
+          name: memgraph-debian-11
+          path: memgraph/build/output/debian-11/memgraph*.deb
+
+  build_memgraph_arm:
+    name: "Build memgraph Debian 11 ARM"
+    runs-on: [self-hosted, DockerMgBuild, ARM64, strange]
+    timeout-minutes: 120
+    steps:
+      - name: "Set up repository"
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # Required because of release/get_version.py
+          token: ${{ secrets.PAT }}
+      - name: "Build package"
+        run: |
+          git clone -b master https://github.com/memgraph/memgraph.git
+          cd memgraph
+          ./release/package/run.sh package debian-11-arm Release
+      - name: "Upload package"
+        uses: actions/upload-artifact@v3
+        with:
+          name: memgraph-debian-aarch64
+          path: memgraph/build/output/debian-11-arm/memgraph*.deb

.github/workflows/daily-platform-build.yml

@@ -0,0 +1,79 @@
+name: Daily Platform Build
+
+# on: [pull_request, workflow_dispatch]
+on: [workflow_dispatch]
+
+jobs:
+  artifacts_build:
+    name: "Build artifacts"
+    runs-on: [self-hosted, Linux, X64, Diff, Ubuntu20.04]
+    env:
+      THREADS: 24
+      PY_VERSION: "3.9"
+
+    steps:
+      - name: Set up repository
+        uses: actions/checkout@v3
+        with:
+          # Number of commits to fetch. `0` indicates all history for all
+          # branches and tags. (default: 1)
+          fetch-depth: 0
+          token: ${{ secrets.PAT }}
+
+      - name: Build community binaries
+        run: |
+          git clone -b master https://github.com/memgraph/memgraph.git
+
+          cd memgraph
+
+          # Activate toolchain.
+          source /opt/toolchain-v4/activate
+
+          # Initialize dependencies.
+          ./init
+
+          # Build community binaries.
+          cd build
+          cmake -DCMAKE_BUILD_TYPE=release -DMG_ENTERPRISE=OFF ..
+          make -j$THREADS && make -j$THREADS mgconsole
+
+          mkdir -p output
+          cd output
+
+          cpack -G DEB --config "../CPackConfig.cmake"
+
+          cd ../../../
+
+          mkdir -p scripts/resources
+
+          find memgraph/build/output -type f -name "*.deb" -exec mv -f {} scripts/resources/memgraph-daily-deb_amd64.deb \;
+
+          rm -rf memgraph
+
+      - name: "Upload DEB"
+        uses: actions/upload-artifact@v3
+        with:
+          name: daily-deb
+          path: scripts/resources/memgraph*.deb
+
+      # - name: Build MAGE image
+      #   run: |
+      #     cd scripts
+
+      #     ./build_mage_img.sh main daily-mage daily-deb
+
+      # - name: "Upload MAGE image"
+      #   uses: actions/upload-artifact@v3
+      #   with:
+      #     name: daily-mage
+      #     path: resources/daily-mage.tar.gz
+
+      # - name: Build platform image
+      #   run: |
+      #     ./build_platform_img.sh master daily-platform daily-deb ${{ secrets.PAT }}
+
+      # - name: "Upload platform image"
+      #   uses: actions/upload-artifact@v3
+      #   with:
+      #     name: daily-platform
+      #     path: resources/daily-platform.tar.gz

.gitignore

@@ -1,2 +1,3 @@
+scripts/resources
 scripts/dist
 *.deb

scripts/README.md

@@ -1,3 +1,72 @@
+# Scripts for creating custom docker images
+
+Each folder contains `.sh` script for creating either docker image or memgraph debian. Before running scripts, you need to change permissions on `build_` scripts with command:
+
+```
+sudo chmod 777 <SCRIPT_NAME>
+```
+
+Additional note, always check before running any scripts if your local user has the permission to read and write over the `resources` directory, which gets created in the `scripts` directory.
+
+## Creating Memgraph debian 11 package
+
+To create a Memgraph `.deb` file do the following steps:
+
+1. Run the `build_deb.sh` script as follows to create the deb from the GitHub repository:
+
+    ```console
+    $ sudo ./build_deb.sh {branch_name} {deb_name}
+    ```
+   - `{branch_name}`: any branch from the `memgraph` repo, e.g. `master` or `T1220-MG-properties-c++-api-bug`; for the script to pull and build `memgraph` from it
+   - `{deb_name}`: used to name the final  `.deb` package file: `memgraph-{deb_name}_amd64.deb`
+
+
+The script stores all `.deb` files in the `output_debian` folder.
+
+## Creating MAGE Docker image
+
+> ### This script releases docker images **only** for `amd64/linux` platform arhitectures. 
+
+1. Store the memgraph Debian package file in the `memgraph_deb_files` folder with the exact name: `memgraph-{deb_name}_amd64.deb` and run the script with the following command:
+
+    ```
+    sudo ./build_mage_img.sh {branch_name} {image_name} {deb_name}
+    ```
+    - `{branch_name}` - branch name, e.g. `main`
+    - `{deb_name}` - deb package filename 
+    - `{image_name}` - for the final image name
+
+3. The docker image will be stored in the `output` folder in the `tar.gz` format, e.g. `{image_name}.tar.gz`. To check how to load the image, jump to the [Loading custom docker image](#loading-custom-docker-image) section.
+
+
+## Creating Platform Docker image
+
+> ### Note: Lab is a private repo so this build is unavailable to public. You can still build Memgraph + MAGE above and use downloaded Lab with your image.
+
+> ### Docker image is released for platform `amd64/linux` arhitectures only.
+
+1. Make sure you added memgraph debian in `memgraph_deb_files`, configured `ssh` correctly to connect to `github` and run script with following command:
+    ```
+    ./build_platform_img.sh {branch_name} {image_name} {deb_name} {github_pat_token}
+
+    ```
+    - `{branch_name}` - branch name of memgraph-platform, e.g. `main`
+    - `{image_name}` - name the final image name of memgraph platform
+    - `{deb_name}` - middle name of the deb file you provided in previous steps, e.g. memgraph-{deb_name}_amd64.deb
+    - `{github_pat_token}` - your github pat token for downloading Memgraph Lab
+
+
+2. Platform docker image can be also built with only memgraph and lab wihout mage.  run script with following command:
+    ```
+    ./build_platform_img.sh main image_name deb_package_name github_pat_token true
+
+    ```
+
+Docker image will be stored in folder `output` in `tar.gz` format, for example `image_name.tar.gz`.
+
+
+**NOTE** Deb packages need to be stored in format `memgraph-{deb_name}_amd64.deb`, for example `memgraph-feature_amd64.deb` and you start script with `feature`.
+
 # Scripts for creating custom memgraph packages and docker images
 
 Run `{script}.sh -h` to figure the details and how to start building.
@@ -15,6 +84,16 @@ Run `{script}.sh -h` to figure the details and how to start building.
 ## Loading custom docker image
 
 Docker image can be loaded with (it takes some time):
+
+```
+docker load < <IMAGE_NAME>.tar.gz
+```
+
+and then used for example like this:
+
+```
+docker run -it --rm -p 7687:7687 --name <NAME> <IMAGE_NAME>
+```
 ```
 docker load -i <IMAGE_NAME>.tar.gz
 ```

scripts/_build_mg.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+git config --global --add safe.directory /memgraph
+
+source /opt/toolchain-v4/activate
+
+cd memgraph
+./environment/os/debian-11.sh install MEMGRAPH_BUILD_DEPS
+./init
+
+mkdir -p build
+cd build
+
+cmake -DCMAKE_BUILD_TYPE=Release .. && make -j4 memgraph && make -j4 mgconsole
+
+cd output
+cpack -G DEB --config "../CPackConfig.cmake"

scripts/build_deb.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+CURR_DIR="$PWD"
+MEMGRAPH_DIR="$CURR_DIR/memgraph"
+
+cloned=true
+branch_name=$1
+deb_name=$2
+
+git clone -b $branch_name https://github.com/memgraph/memgraph.git
+
+mkdir -p resources
+
+docker pull memgraph/memgraph-builder
+docker stop meme-build || true && docker rm meme-build || true
+
+docker run --name meme-build -itd -v $MEMGRAPH_DIR:/memgraph -v $CURR_DIR/resources/output:/memgraph/build/output --entrypoint bash memgraph/memgraph-builder 
+docker exec -i meme-build bash < $CURR_DIR/_build_mg.sh
+docker stop meme-build
+docker rm meme-build
+
+find $CURR_DIR/resources/output -type f -name "*.deb" -exec mv -f {} resources/memgraph-${deb_name}_amd64.deb \;
+
+if $cloned;then
+  rm -rf $MEMGRAPH_DIR
+fi
+rm -rf $CURR_DIR/resources/output
+

scripts/build_mage_img.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+CURR_DIR="$PWD"
+MAGE_DIR="$CURR_DIR/mage"
+
+branch=$1
+image_name=$2
+deb_name=$3
+
+git clone --recurse-submodules -b $1 https://github.com/memgraph/mage.git
+
+cp "${CURR_DIR}/resources/memgraph-${deb_name}_amd64.deb" "${MAGE_DIR}/memgraph-${deb_name}_amd64.deb"
+
+cd ${MAGE_DIR}
+
+docker buildx build --no-cache --target prod --platform=linux/amd64 -t ${image_name} --build-arg TARGETARCH=${deb_name}_amd64 -f ${MAGE_DIR}/Dockerfile.release .
+
+cd ${CURR_DIR}
+
+docker save ${image_name} | gzip -f > "${CURR_DIR}/resources/${image_name}.tar.gz"
+docker rmi ${image_name}
+
+rm -rf ${image_name}
+rm -rf ${MAGE_DIR}

scripts/build_platform_img.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+CURR_DIR="$PWD"
+PLATFORM_DIR="${CURR_DIR}/memgraph-platform"
+
+branch=$1
+image_name=$2
+target_arch=$3
+token=$4
+memgraph_and_lab=${5-false}
+
+git clone --recurse-submodules -b $1 https://github.com/memgraph/memgraph-platform.git
+
+cp "${CURR_DIR}/resources/memgraph-${target_arch}_amd64.deb" "${PLATFORM_DIR}/memgraph-${target_arch}_amd64.deb"
+cd ${PLATFORM_DIR}
+
+dockerfile=${PLATFORM_DIR}/Dockerfile
+
+if [ "${memgraph_and_lab}" == true ];
+then
+    dockerfile = ${PLATFORM_DIR}/memgraph_and_lab.Dockerfile
+fi
+
+docker buildx build --platform=linux/amd64 -t ${image_name} --build-arg TARGETARCH=${target_arch}_amd64 --build-arg NPM_PACKAGE_TOKEN=${token} -f ${dockerfile} .
+
+cd ${CURR_DIR}
+
+docker save ${image_name} | gzip -f > "${CURR_DIR}/resources/${image_name}.tar.gz"
+docker rmi ${image_name}
+
+rm -rf ${image_name}
+rm -rf ${PLATFORM_DIR}