ci: Update build workflows for linux

.github/workflows/Dockerfile-arm64-linux

@@ -0,0 +1,57 @@
+FROM ubuntu:20.04 as build-stage
+
+RUN apt-get update -y
+
+RUN DEBIAN_FRONTEND=noninteractive apt-get install -y lcov libnuma1 clang-12 llvm-12 nasm curl zip wget automake libtool unzip cmake git perl build-essential libffi-dev libffi7 libgmp-dev libgmp10 libncurses-dev libncurses5 libtinfo5
+
+RUN ln -s /usr/lib/aarch64-linux-gnu/libnuma.so.1 /usr/lib/libnuma.so
+
+# RUN echo "Setup Haskell"
+# RUN curl --proto '=https' --tlsv1.2 -sSf https://get-ghcup.haskell.org | sh
+# RUN ~/.ghcup/bin/ghcup install 8.10.7
+# RUN ~/.ghcup/bin/ghcup install cabal 3.4.0.0
+# RUN ~/.ghcup/bin/ghcup install stack 2.7.3
+# RUN ~/.ghcup/bin/ghcup set 8.10.7
+# RUN ~/.ghcup/bin/ghcup set cabal 3.4.0.0
+# RUN ~/.ghcup/bin/ghcup set stack 2.7.3
+
+# install stack
+RUN curl -sSL https://get.haskellstack.org/ | sh
+ENV PATH="/root/.local/bin:$PATH"
+
+# tell stack to use the global ghc, installing GHC with stack fails
+RUN stack config set system-ghc --global true
+ENV PATH="$(stack path --bin-path):$PATH"
+
+# setup node and rollup
+RUN curl -sL https://deb.nodesource.com/setup_14.x | bash -
+RUN apt-get install -y nodejs
+RUN npm i -g rollup @rollup/plugin-node-resolve
+
+# run the build
+COPY . .
+
+ENV LC_ALL="C.UTF-8"
+# ENV LANG="en_US.UTF-8"
+
+# note: this needs to happen here as we need the resolved field from the stack.yml or else stack
+# will download a new ghc version, which we want to avoid.
+RUN stack install alex happy
+
+RUN stack build --jobs 1
+
+# build runtime
+RUN TARGET="LINUX_ARM64" ./scripts/build-runtime-libs
+RUN AR="llvm-ar-12" TARGET="LINUX_ARM64" ./scripts/build-runtime
+
+
+# build the js tools ( test-runner and package-installer )
+RUN ./scripts/build
+
+RUN cp "$(stack path --dist-dir)/build/madlib/madlib" .
+
+
+FROM scratch AS export-stage
+COPY --from=build-stage ./madlib /
+COPY --from=build-stage ./runtime /runtime
+COPY --from=build-stage ./tools/package-installer/dist/package-installer.js /package-installer.js

.github/workflows/Dockerfile-x86_64-alpine

@@ -0,0 +1,59 @@
+FROM alpine:3.13.0 as build-stage
+
+# The only Alpine specific bit. build-base mainly installs make and a C++ compiler. Python 3 is required by LLVM for some reason.
+RUN apk add libtool ncurses ncurses-dev git curl gmp-dev wget unzip cmake linux-headers musl-dev build-base libuv-static bash nasm zlib-dev zlib-static perl automake m4 autoconf
+RUN apk add build-base cmake git python3
+
+# needed for stack to install Haskell tools
+RUN apk add --no-cache --upgrade grep
+
+# install llvm12
+RUN git clone https://github.com/llvm/llvm-project.git --branch llvmorg-12.0.0  --depth 1
+RUN cd llvm-project/
+RUN mkdir build
+RUN cd build/
+# The flag LLVM_ENABLE_PROJECTS is crucial, otherwise only llvm will be built, without clang or lld,
+# and we need all three with the exact same version since C++ does not have a stable ABI.
+RUN cmake -DCMAKE_BUILD_TYPE=Release -DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD="AVR" -DLLVM_ENABLE_LIBXML2=OFF -DLLVM_ENABLE_TERMINFO=OFF -DLLVM_ENABLE_PROJECTS="clang;lld" ../llvm
+
+RUN make -j1
+RUN make install
+
+RUN PATH="/root/.ghcup/bin:$PATH"
+
+# install Haskell
+RUN echo "Setup Haskell"
+RUN curl --proto '=https' --tlsv1.2 -sSf https://get-ghcup.haskell.org | sh
+RUN ~/.ghcup/bin/ghcup install 8.10.7
+RUN ~/.ghcup/bin/ghcup install cabal 3.6.2.0
+RUN ~/.ghcup/bin/ghcup install stack 2.11.1
+RUN ~/.ghcup/bin/ghcup set 8.10.7
+RUN ~/.ghcup/bin/ghcup set cabal 3.6.2.0
+RUN ~/.ghcup/bin/ghcup set stack 2.11.1
+
+# setup node and rollup
+RUN apk add --update nodejs npm
+RUN npm i -g rollup @rollup/plugin-node-resolve
+
+# build runtime
+RUN TARGET="MUSL_X64" sh ./scripts/build-runtime-libs
+RUN AR="llvm-ar" TARGET="MUSL_X64" sh ./scripts/build-runtime
+
+# build the js tools ( test-runner and package-installer )
+RUN ./scripts/build
+
+# install Alex and Happy
+RUN stack --system-ghc install alex happy
+
+# build Madlib executable
+RUN stack --system-ghc build --jobs 1
+RUN cp "$(stack --system-ghc path --dist-dir)/build/madlib/madlib" .
+
+# build package installer
+RUN "$(stack --system-ghc path --dist-dir)/build/madlib/madlib" compile -i tools/package-installer/src/Main.mad -o ./tools/package-installer/dist/package-installer.js --bundle
+
+FROM scratch AS export-stage
+COPY --from=build-stage ./madlib /
+COPY --from=build-stage ./runtime /runtime
+COPY --from=build-stage ./tools/package-installer/dist/package-installer.js /package-installer.js
+

.github/workflows/Dockerfile-x86_64-linux

@@ -1,13 +1,13 @@
-FROM alpine:3.13.0 as build-musl-runtime
+# FROM alpine:3.13.0 as build-musl-runtime
 
-RUN apk add --update git wget unzip cmake linux-headers musl-dev build-base libuv-static bash nasm zlib-dev zlib-static perl automake m4 autoconf
+# RUN apk add --update git wget unzip cmake linux-headers musl-dev build-base libuv-static bash nasm zlib-dev zlib-static perl automake m4 autoconf
 
-COPY . .
+# COPY . .
 
-# build runtime
-RUN TARGET="MUSL_X64" ./scripts/build-runtime-libs
-RUN cp /usr/lib/libuv.a runtime/lib/
-RUN TARGET="MUSL_X64" CXX="g++" ./scripts/build-runtime
+# # build runtime
+# RUN TARGET="MUSL_X64" ./scripts/build-runtime-libs
+# RUN cp /usr/lib/libuv.a runtime/lib/
+# RUN TARGET="MUSL_X64" CXX="g++" ./scripts/build-runtime
 
 
 
@@ -51,7 +51,7 @@ COPY . .
 # will download a new ghc version, which we want to avoid.
 RUN stack install alex happy
 
-RUN stack build --jobs 1 --flag madlib:static
+RUN stack build --jobs 1
 
 # build runtime
 RUN TARGET="LINUX_X64" ./scripts/build-runtime-libs
@@ -68,4 +68,3 @@ FROM scratch AS export-stage
 COPY --from=build-stage ./madlib /
 COPY --from=build-stage ./runtime /runtime
 COPY --from=build-stage ./tools/package-installer/dist/package-installer.js /package-installer.js
-COPY --from=build-musl-runtime ./runtime /runtime-musl

.github/workflows/build.yml

@@ -19,12 +19,9 @@ jobs:
       - name: Configure stack
         run: stack config set system-ghc --global true
 
-      - run: sudo apt-get update -y
+      - run: DEBIAN_FRONTEND=noninteractive sudo apt-get update -y
 
-      # TODO: remove llvm-9
       - run: sudo apt-get install -y lcov llvm-12 nasm
-      # - run: DEBIAN_FRONTEND=noninteractive sudo apt-get install -y curl llvm-9 zip wget automake libtool clang gcc-9 g++-9 libstdc++-9-dev libc++abi-9-dev unzip musl-tools cmake git nasm libssl-dev
-      # - run: sudo apt-get install -y build-essential curl libffi-dev libffi7 libgmp-dev libgmp10 libncurses-dev libncurses5 libtinfo5
 
       - uses: actions/cache@v2.1.3
         name: Cache ~/.stack

.github/workflows/release-x86_64-alpine.yml

@@ -0,0 +1,50 @@
+name: 'Build linux alpine target for release'
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Version to release ( ex: v0.0.1 )'
+        required: true
+
+env:
+  node-version: 14.x
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: docker-practice/actions-setup-docker@master
+        with:
+          docker_version: "20.10"
+          docker_channel: stable
+
+      - name: Build madlib
+        run: DOCKER_BUILDKIT=1 docker build -f ".github/workflows/Dockerfile-x86_64-alpine" -o . .
+
+      - name: Bundle alpine archive
+        run: |
+          mkdir madlib-x86_64-alpine-linux-musl
+          cp ./madlib ./madlib-x86_64-alpine-linux-musl/
+          cp -R runtime ./madlib-x86_64-alpine-linux-musl/
+          cp -R prelude ./madlib-x86_64-alpine-linux-musl/
+          cp ./package-installer.js ./madlib-x86_64-alpine-linux-musl/
+          tar -czvf madlib-x86_64-alpine-linux-musl.tar.gz madlib-x86_64-alpine-linux-musl
+
+      - uses: pdamianik/release-tag-to-upload-url-action@v1.0.1
+        id: release_upload_url
+        with:
+          tag: ${{ github.event.inputs.version }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Upload alpine build
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.release_upload_url.outputs.uploadUrl }}
+          asset_path: ./madlib-x86_64-alpine-linux-musl.tar.gz
+          asset_name: madlib-x86_64-alpine-linux-musl.tar.gz
+          asset_content_type: application/tar+gzip

.github/workflows/release-x86_64-linux.yml

@@ -16,8 +16,6 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
-      # TODO: need to use the Dockerfile to build
-
       - uses: docker-practice/actions-setup-docker@master
         with:
           docker_version: "20.10"
@@ -35,16 +33,6 @@ jobs:
           cp ./package-installer.js ./madlib-x86_64-unknown-linux-gnu/
           tar -czvf madlib-x86_64-unknown-linux-gnu.tar.gz madlib-x86_64-unknown-linux-gnu
 
-      - name: Bundle alpine archive
-        run: |
-          mkdir madlib-x86_64-alpine-linux-musl
-          cp ./madlib ./madlib-x86_64-alpine-linux-musl/
-          cp -R runtime-musl ./madlib-x86_64-alpine-linux-musl/
-          mv ./madlib-x86_64-alpine-linux-musl/runtime-musl ./madlib-x86_64-alpine-linux-musl/runtime
-          cp -R prelude ./madlib-x86_64-alpine-linux-musl/
-          cp ./package-installer.js ./madlib-x86_64-alpine-linux-musl/
-          tar -czvf madlib-x86_64-alpine-linux-musl.tar.gz madlib-x86_64-alpine-linux-musl
-
       - uses: pdamianik/release-tag-to-upload-url-action@v1.0.1
         id: release_upload_url
         with:
@@ -60,13 +48,3 @@ jobs:
           asset_path: ./madlib-x86_64-unknown-linux-gnu.tar.gz
           asset_name: madlib-x86_64-unknown-linux-gnu.tar.gz
           asset_content_type: application/tar+gzip
-
-      - name: Upload alpine build
-        uses: actions/upload-release-asset@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          upload_url: ${{ steps.release_upload_url.outputs.uploadUrl }}
-          asset_path: ./madlib-x86_64-alpine-linux-musl.tar.gz
-          asset_name: madlib-x86_64-alpine-linux-musl.tar.gz
-          asset_content_type: application/tar+gzip

.github/workflows/release.yml

@@ -71,6 +71,13 @@ jobs:
           token: ${{ secrets.TOKEN_GITHUB }}
           inputs: '{ "version": "${{ github.event.inputs.version }}" }'
 
+      - name: Start release for linux alpine
+        uses: benc-uk/workflow-dispatch@v1
+        with:
+          workflow: Build linux alpine target for release
+          token: ${{ secrets.TOKEN_GITHUB }}
+          inputs: '{ "version": "${{ github.event.inputs.version }}" }'
+
       - name: Start release for windows
         uses: benc-uk/workflow-dispatch@v1
         with:

runtime/Makefile

@@ -50,6 +50,8 @@ build/stack.o:
 		as "$(SRCDIR)/stack/stack-arm64-apple-darwin.s" -o "$(BUILDDIR)/stack.o";\
 	elif [ $(TARGET) = "MACOS_X64" ]; then\
 		nasm -f macho64 "$(SRCDIR)/stack/stack-x64-apple-darwin.s" -o "$(BUILDDIR)/stack.o";\
+	elif [ $(TARGET) = "LINUX_ARM64" ]; then\
+		as "$(SRCDIR)/stack/stack-arm64-linux.s" -o "$(BUILDDIR)/stack.o";\
 	elif [ $(TARGET) = "LINUX_X64" ] || [ $(TARGET) = "MUSL_X64" ]; then\
 		nasm -f elf64 "$(SRCDIR)/stack/stack-x64-linux.s" -o "$(BUILDDIR)/stack.o";\
 	elif [ $(TARGET) = "WIN_X64" ]; then\

runtime/src/stack/stack-arm64-linux.s

@@ -0,0 +1,13 @@
+.text
+.globl madlib__stack__init
+.p2align    2
+madlib__stack__init:
+  stp    x29, x30, [sp, #-16]!
+  mov    x29, sp
+
+  mov sp, x0
+  blr x1
+  mov sp, x29
+
+  ldp     x29, x30, [sp], 16
+  ret

scripts/build-runtime-libs

@@ -82,14 +82,21 @@ OPENSSL_PREFIX="$OPENSSL_FOLDER/build"
 wget "https://github.com/openssl/openssl/archive/refs/tags/openssl-$OPENSSL_VERSION.zip"
 unzip "openssl-$OPENSSL_VERSION.zip"
 cd "openssl-openssl-$OPENSSL_VERSION"
-if [ "$TARGET" = "LINUX_X64" ] || [ "$TARGET" = "MUSL_X64" ]
+if [ "$TARGET" = "LINUX_X64" ] || [ "$TARGET" = "MUSL_X64" ] || [ "$TARGET" = "LINUX_ARM64" ]
 then
   ./Configure --prefix=$OPENSSL_PREFIX
   make
   make install
   cp -R include/* ../../runtime/include/
-  cp build/lib64/libssl.a ../../runtime/lib/
-  cp build/lib64/libcrypto.a ../../runtime/lib/
+
+  if [ "$TARGET" = "LINUX_X64" ] || [ "$TARGET" = "MUSL_X64" ]
+  then
+    cp build/lib64/libssl.a ../../runtime/lib/
+    cp build/lib64/libcrypto.a ../../runtime/lib/
+  elif [ "$TARGET" = "LINUX_ARM64" ]
+    cp build/lib/libssl.a ../../runtime/lib/
+    cp build/lib/libcrypto.a ../../runtime/lib/
+  fi
   cd ..
 elif [ "$TARGET" = "MACOS_X64" ] || [ "$TARGET" = "MACOS_ARM64" ]
 then

scripts/bundle-darwin-arm64-archive

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+
+stack build -j10
+TARGET=MACOS_ARM64 ./scripts/build-runtime-libs
+TARGET=MACOS_ARM64 AR=llvm-ar ./scripts/build-runtime
+./scripts/build
+./scripts/update-pkg-build
+
+mkdir ./madlib-arm64-apple-darwin/
+cp -R ./pkg/node_modules/binary-install/node_modules/.bin/* ./madlib-arm64-apple-darwin/
+cp -R ./prelude ./madlib-arm64-apple-darwin/
+tar -czvf madlib-arm64-apple-darwin.tar.gz madlib-arm64-apple-darwin
+rm -r ./madlib-arm64-apple-darwin

scripts/bundle-linux-arm64-archive

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+
+mkdir ./madlib-arm64-unknown-linux-gnu
+
+DOCKER_BUILDKIT=1 docker build -f ".github/workflows/Dockerfile-arm64-linux" -o ./madlib-arm64-unknown-linux-gnu/ .
+
+cp -R prelude ./madlib-arm64-unknown-linux-gnu/
+# cp -R ./pkg/node_modules/binary-install/node_modules/.bin/* ./madlib-arm64-apple-darwin/
+# cp -R ./prelude ./madlib-arm64-apple-darwin/
+tar -czvf madlib-arm64-unknown-linux-gnu.tar.gz madlib-arm64-unknown-linux-gnu
+# rm -r ./madlib-arm64-unknown-linux-gnu