Scripts to create image and run reproducible builds for JSC32

This adds script to update container image and script to run container
image, create execution plan and run it. Also adds initial CI workflow
using GitHub Actions to create and push the image.

.github/workflows/webkitbuild-image-creation.yml

@@ -0,0 +1,59 @@
+name: CI
+
+on: 
+  push:
+    paths:
+      - 'containers/jsc32-base/**'
+      - '.github/workflows/webkitbuild-image-creation.yml'
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix: 
+        arch: [mips, arm]
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: Login to docker hub
+      run: docker login -u pmatos -p ${{ secrets.DOCKER_PASS }}
+    - name: Build
+      working-directory: containers/jsc32-base/
+      run: ./bimage.sh ${{ matrix.arch }}
+
+  test-image:
+    runs-on: ubuntu-latest
+    needs: build
+    container:
+      image: debian:buster
+      options: --privileged -v /var/run/docker.sock:/var/run/docker.sock
+
+    strategy:
+      fail-fast: false
+      matrix:
+        arch: [mips, arm]
+        orch: [podman, docker]
+
+    steps:
+    - name: Install qemu and binfmt support
+      run: |
+        apt-get update
+        apt-get install -y gpg software-properties-common qemu-user-static binfmt-support
+    - name: Install podman if needed
+      if: matrix.orch == 'podman'
+      run: |
+        add-apt-repository -y ppa:projectatomic/ppa
+        apt-get update
+        apt-get install -y podman
+    - name: Install docker if needed
+      if: matrix.orch == 'docker'
+      run: |
+        apt-get install -y apt-transport-https ca-certificates curl gnupg2 software-properties-common
+        curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add -
+        apt-key fingerprint 0EBFCD88
+        add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable"
+        apt-get update
+        apt-get install -y docker-ce docker-ce-cli containerd.io
+    - name: Test image architecture
+      run: ${{ matrix.orch }} run pmatos/jsc32-base:${{ matrix.arch }} /bin/bash -c "[[ \"$(uname -m)\" == \"${{ matrix.arch }}\" ]] || exit 1"  

.gitignore

@@ -0,0 +1,2 @@
+# Temp files
+*~

containers/README.md

@@ -0,0 +1,6 @@
+# WebKit Containers
+
+This folder contains the definition of several images that ease the development and reproducibility of problems in native and cross compiled JSC binaries.
+
+## Images
+

containers/jsc32-base/bimage.sh

@@ -0,0 +1,42 @@
+#! /bin/bash
+## $PROG 0.1 - Build WebKit-ready docker image for 32bits mips and arm
+## 
+## Usage: $PROG [ARCH]
+## Commands:
+##   -h, --help             Displays this help and exists
+##   -v, --version          Displays output version and exists
+## Examples:
+##   $PROG arm
+##   $PROG mips
+set -eu
+
+PROG=${0##*/}
+die() { echo $@ >&2; exit 2; }
+
+help() {
+    grep "^##" "$0" | sed -e "s/^...//" -e "s/\$PROG/$PROG/g"; exit 0
+}
+version() {
+    help | head -1
+}
+
+[ $# = 0 ] && help
+while [ $# -ne 1 ]; do
+    help
+    exit 1
+done
+
+ARCH=$1
+
+if [ "$ARCH" != "arm" ] && [ "$ARCH" != "mips" ]
+then
+    echo "Invalid architecture: $ARCH"
+    help
+    exit 1
+fi
+
+docker build -t buster-$ARCH --build-arg ARCH=$ARCH -f build-image.Dockerfile .
+docker run --rm --privileged -v `pwd`/artifacts:/artifacts buster-$ARCH
+docker import `pwd`/artifacts/buster-$ARCH.tar jsc32-base:$ARCH-raw
+docker build -t pmatos/jsc32-base:$ARCH -f jsc32-base.Dockerfile --build-arg ARCH=$ARCH .
+docker push pmatos/jsc32-base:$ARCH

containers/jsc32-base/build-image.Dockerfile

@@ -0,0 +1,9 @@
+FROM debian:stable
+
+ARG ARCH
+# Need to pass arch argument to script
+WORKDIR /
+ADD ./build-image.sh .
+ADD ./utils.sh .
+ENV ARCH=${ARCH}
+ENTRYPOINT /build-image.sh ${ARCH}

containers/jsc32-base/build-image.sh

@@ -0,0 +1,82 @@
+#! /bin/bash
+## $PROG 0.1 - Build WebKit-ready docker image for mipsel and arm
+## 
+## Usage: $PROG [ARCH]
+## Commands:
+##   -h, --help             Displays this help and exists
+##   -v, --version          Displays output version and exists
+## Examples:
+##   $PROG arm
+##   $PROG mips
+set -Eeuo pipefail
+
+source ./utils.sh
+
+PROG=${0##*/}
+die() { echo $@ >&2; exit 2; }
+
+help() {
+    grep "^##" "$0" | sed -e "s/^...//" -e "s/\$PROG/$PROG/g"; exit 0
+}
+version() {
+    help | head -1
+}
+
+[ $# = 0 ] && help
+while [ $# -ne 1 ]; do
+    help
+    exit 1
+done
+
+ARCH=$1
+
+if [ "$ARCH" != "arm" ] && [ "$ARCH" != "mips" ]
+then
+    echo "Invalid architecture: $ARCH"
+    help
+    exit 1
+fi
+
+ROOTFS=`mktemp -d`
+QEMUARCH=$(qemuarch $ARCH)
+DEBIANARCH=$(debianarch $ARCH)
+
+# Host dependencies
+apt-get update
+apt-get install -y qemu-user-static debootstrap binfmt-support 
+debootstrap --foreign --no-check-gpg --arch=$DEBIANARCH buster $ROOTFS http://httpredir.debian.org/debian/
+
+QEMU=`which qemu-$QEMUARCH-static`
+QEMUBASE=$(dirname $QEMU)
+mkdir -p $ROOTFS$QEMUBASE
+cp -v $QEMU $ROOTFS$QEMUBASE
+INTR=$(cat /proc/sys/fs/binfmt_misc/qemu-$QEMUARCH | grep interpreter | cut -d ' ' -f 2)
+if [[ "$INTR" != "$QEMU" ]]; then
+    echo -1 > /proc/sys/fs/binfmt_misc/qemu-$QEMUARCH
+    if [ "$ARCH" == "mips" ]
+    then
+	echo ":qemu-$QEMUARCH:M:0:\\x7f\\x45\\x4c\\x46\\x01\\x01\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x08\\x00:\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\x00\\xfe\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xfe\\xff\\xff\\xff:$QEMU:OC" > /proc/sys/fs/binfmt_misc/register
+    elif [ "$ARCH" == "arm" ]
+    then
+	echo ":qemu-$QEMUARCH:M:0:\\x7f\\x45\\x4c\\x46\\x01\\x01\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x28\\x00:\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\x00\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xfe\\xff\\xff\\xff:$QEMU:OCF" > /proc/sys/fs/binfmt_misc/register
+    fi
+fi
+# todo check that the interpreter exists in right place
+chroot $ROOTFS ./debootstrap/debootstrap --second-stage --verbose
+
+mount -t devpts devpts $ROOTFS/dev/pts
+mount -t proc proc $ROOTFS/proc
+mount -t sysfs sysfs $ROOTFS/sys
+
+chroot $ROOTFS apt-get update
+chroot $ROOTFS apt-get -y upgrade
+chroot $ROOTFS apt-get install -y g++ cmake libicu-dev git ruby-highline ruby-json python
+chroot $ROOTFS apt-get -y autoremove
+chroot $ROOTFS apt-get clean
+chroot $ROOTFS find /var/lib/apt/lists -type f -delete
+umount $ROOTFS/dev/pts
+umount $ROOTFS/proc
+umount $ROOTFS/sys
+
+cd /artifacts
+tar --numeric-owner -cvf buster-$ARCH.tar -C $ROOTFS .

containers/jsc32-base/jsc32-base.Dockerfile

@@ -0,0 +1,7 @@
+ARG ARCH
+FROM jsc32-base:$ARCH-raw
+
+LABEL description="Minimal Debian image to reproduce JSC dev"
+LABEL maintainer="Paulo Matos <pmatos@igalia.com>"
+
+CMD ["/bin/bash"]

containers/jsc32-base/utils.sh

@@ -0,0 +1,21 @@
+#! /bin/bash
+
+qemuarch() {
+    if [ "$1" == "arm" ]
+    then
+       echo "arm"
+    elif [ "$1" == "mips" ]
+    then
+	echo "mipsel"
+    fi
+}
+
+debianarch() {
+    if [ "$1" == "arm" ]
+    then
+       echo "armhf"
+    elif [ "$1" == "mips" ]
+    then
+	echo "mipsel"
+    fi
+}    

containers/reprojsc.sh

@@ -0,0 +1,149 @@
+#! /bin/bash
+#
+# Required Options:
+# -a : arch (mips or arm)
+# -q : qemu path to setup binfmt
+# -b : build webkit
+# -t : test webkit (assumes --build)
+# -f  <regexp>: filter to be passed to the run-javascript-tests script
+# Optional Options:
+# -i : enters a terminal 
+# Arguments:
+# <path>: path to a local webkit checkout to use in building and testing
+set -Eeuo pipefail
+
+# Command line argument handling
+
+INTERACTIVE=0
+BUILD=0
+TEST=0
+FILTER=""
+ARCH=
+QEMU=
+while getopts ":a:q:ibtf:" opt; do
+    case ${opt} in
+	a)
+	    ARCH="$OPTARG"
+	    ;;
+	q)
+	    QEMU="$OPTARG"
+	    ;;
+	i)
+	    INTERACTIVE=1
+	    ;;
+	b)
+	    BUILD=1
+	    ;;
+	t)
+	    TEST=1
+	    BUILD=1
+	    ;;
+	f)
+	    TEST=1
+	    BUILD=1
+	    FILTER="$OPTARG"
+	    ;;
+	\?)
+	    echo "Invalid option: $OPTARG" 1>&2
+	    exit
+	    ;;
+	:)
+	    echo "Invalid option: $OPTARG requires an argument" 1>&2
+	    exit
+	    ;;
+    esac
+done
+shift $((OPTIND - 1))
+
+if [[ $ARCH == '' ]]; then
+    echo "Please specify -a <arch>"
+    exit
+fi
+
+if [[ $# != 1 ]]; then
+    echo "Not enough arguments"
+    exit
+fi
+WEBKIT=$1
+
+if [ ! -d $WEBKIT ]; then
+    echo "Directory $WEBKIT does not exist"
+    exit
+fi
+
+# Deal with exit
+BINFMT_NAME=
+function finish {
+    if [[ $BINFMT_NAME != '' ]]; then
+	echo "Cleaning up binfmt support"
+	echo -1 > /proc/sys/fs/binfmt_misc/$BINFMT_NAME
+    fi
+}
+trap finish EXIT SIGINT
+
+# Output plan
+
+echo "Using WebKit path $WEBKIT"
+echo "Plan:"
+if [[ $BUILD == 1 ]]; then
+    echo " * Build;"
+fi
+if [[ $TEST == 1 ]]; then
+    echo " * Test;"
+    if [[ $FILTER != "" ]]; then
+	echo "   Using filter: \"$FILTER\""
+    fi
+fi
+if [[ $INTERACTIVE == 1 ]]; then
+    echo " * Going into interactive mode;"
+fi
+
+read -n1 -r -p "Press any key to continue or Ctrl-C to exit..." keyign
+
+if [[ $QEMU != '' ]]; then
+    echo "Setting up $ARCH binfmt to run interpreter in $QEMU"
+    BINFMT_NAME="reprojsc-qemu-$ARCH"
+    if [ -f  "/proc/sys/fs/binfmt_misc/$BINFMT_NAME" ]; then
+	echo -1 > /proc/sys/fs/binfmt_misc/$BINFMT_NAME
+    fi
+
+    if [[ $ARCH == "mips" ]]; then
+	echo ":$BINFMT_NAME:M:0:\\x7f\\x45\\x4c\\x46\\x01\\x01\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x08\\x00:\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\x00\\xfe\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xfe\\xff\\xff\\xff:$QEMU:OC" > /proc/sys/fs/binfmt_misc/register
+    elif [[ $ARCH == "arm" ]]; then
+	echo ":$BINFMT_NAME:M:0:\\x7f\\x45\\x4c\\x46\\x01\\x01\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x28\\x00:\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\x00\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xfe\\xff\\xff\\xff:$QEMU:OCF" > /proc/sys/fs/binfmt_misc/register
+    fi
+fi
+
+out=$(docker run --rm pmatos/jsc32-base:$ARCH /bin/true 2>&1 || true)
+if `echo $out | grep -q "exec format error"`; then
+    echo "binfmt/qemu are not setup for $ARCH"
+    echo "use -q <qemu-path> to setup qemu using binfmt (make sure qemu-path points to the qemu with the right arch)"
+    exit 1
+fi
+
+did=$(docker run --rm -di -v $WEBKIT:/WebKit pmatos/jsc32-base:$ARCH)
+
+UNAME=
+if [[ "$ARCH" == "arm" ]]; then
+    UNAME="armv7l"
+elif [[ "$ARCH" == "mips" ]]; then
+    UNAME="mips"
+fi
+
+
+if [[ "$(docker exec $did uname -m)" != "$UNAME" ]]; then
+    echo "Something is wrong - incorrect container architecture"
+    docker stop $did
+    exit
+fi
+
+
+set +e
+
+[[ $BUILD == 1 ]] && docker exec $did /bin/bash -c "cd /WebKit && Tools/Scripts/build-jsc --release --jsc-only --cmakeargs=\"-DENABLE_STATIC_JSC=ON -DCMAKE_VERBOSE_MAKEFILE=ON -DENABLE_JIT=OFF\" $BUILD_JSC_ARGS"
+echo "Build returned $?"
+[[ $TEST == 1 ]] && docker exec $did /bin/bash -c "cd /WebKit && Tools/Scripts/run-javascriptcore-tests --no-build --no-fail-fast --json-output=jsc_results.json --release --memory-limited --no-testmasm --no-testair --no-testb3 --no-testdfg --no-testapi --jsc-only --no-jit-stress-tests $TEST_JSC_ARGS"
+echo "Test returned $?"
+[[ $INTERACTIVE == 1 ]] && docker exec -ti $did /bin/bash
+docker stop $did
+