Skip to content
Permalink
Browse files

Scripts to create image and run reproducible builds for JSC32

This adds script to update container image and script to run container
image, create execution plan and run it. Also adds initial CI workflow
using GitHub Actions to create and push the image.
  • Loading branch information
pmatos committed Nov 20, 2019
1 parent 3f162e5 commit 94bf25ebe888ae1824113eeb269fe0c0fcb5f3d4
@@ -0,0 +1,59 @@
name: CI

on:
push:
paths:
- 'containers/jsc32-base/**'
- '.github/workflows/webkitbuild-image-creation.yml'

jobs:
build:

runs-on: ubuntu-latest
strategy:
matrix:
arch: [mips, arm]

steps:
- uses: actions/checkout@v1
- name: Login to docker hub
run: docker login -u pmatos -p ${{ secrets.DOCKER_PASS }}
- name: Build
working-directory: containers/jsc32-base/
run: ./bimage.sh ${{ matrix.arch }}

test-image:
runs-on: ubuntu-latest
needs: build
container:
image: debian:buster
options: --privileged -v /var/run/docker.sock:/var/run/docker.sock

strategy:
fail-fast: false
matrix:
arch: [mips, arm]
orch: [podman, docker]

steps:
- name: Install qemu and binfmt support
run: |
apt-get update
apt-get install -y gpg software-properties-common qemu-user-static binfmt-support
- name: Install podman if needed
if: matrix.orch == 'podman'
run: |
add-apt-repository -y ppa:projectatomic/ppa
apt-get update
apt-get install -y podman
- name: Install docker if needed
if: matrix.orch == 'docker'
run: |
apt-get install -y apt-transport-https ca-certificates curl gnupg2 software-properties-common
curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add -
apt-key fingerprint 0EBFCD88
add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable"
apt-get update
apt-get install -y docker-ce docker-ce-cli containerd.io
- name: Test image architecture
run: ${{ matrix.orch }} run pmatos/jsc32-base:${{ matrix.arch }} /bin/bash -c "[[ \"$(uname -m)\" == \"${{ matrix.arch }}\" ]] || exit 1"
@@ -0,0 +1,2 @@
# Temp files
*~
@@ -0,0 +1,6 @@
# WebKit Containers

This folder contains the definition of several images that ease the development and reproducibility of problems in native and cross compiled JSC binaries.

## Images

@@ -0,0 +1,42 @@
#! /bin/bash
## $PROG 0.1 - Build WebKit-ready docker image for 32bits mips and arm
##
## Usage: $PROG [ARCH]
## Commands:
## -h, --help Displays this help and exists
## -v, --version Displays output version and exists
## Examples:
## $PROG arm
## $PROG mips
set -eu

PROG=${0##*/}
die() { echo $@ >&2; exit 2; }

help() {
grep "^##" "$0" | sed -e "s/^...//" -e "s/\$PROG/$PROG/g"; exit 0
}
version() {
help | head -1
}

[ $# = 0 ] && help
while [ $# -ne 1 ]; do
help
exit 1
done

ARCH=$1

if [ "$ARCH" != "arm" ] && [ "$ARCH" != "mips" ]
then
echo "Invalid architecture: $ARCH"
help
exit 1
fi

docker build -t buster-$ARCH --build-arg ARCH=$ARCH -f build-image.Dockerfile .
docker run --rm --privileged -v `pwd`/artifacts:/artifacts buster-$ARCH
docker import `pwd`/artifacts/buster-$ARCH.tar jsc32-base:$ARCH-raw
docker build -t pmatos/jsc32-base:$ARCH -f jsc32-base.Dockerfile --build-arg ARCH=$ARCH .
docker push pmatos/jsc32-base:$ARCH
@@ -0,0 +1,9 @@
FROM debian:stable

ARG ARCH
# Need to pass arch argument to script
WORKDIR /
ADD ./build-image.sh .
ADD ./utils.sh .
ENV ARCH=${ARCH}
ENTRYPOINT /build-image.sh ${ARCH}
@@ -0,0 +1,82 @@
#! /bin/bash
## $PROG 0.1 - Build WebKit-ready docker image for mipsel and arm
##
## Usage: $PROG [ARCH]
## Commands:
## -h, --help Displays this help and exists
## -v, --version Displays output version and exists
## Examples:
## $PROG arm
## $PROG mips
set -Eeuo pipefail

source ./utils.sh

PROG=${0##*/}
die() { echo $@ >&2; exit 2; }

help() {
grep "^##" "$0" | sed -e "s/^...//" -e "s/\$PROG/$PROG/g"; exit 0
}
version() {
help | head -1
}

[ $# = 0 ] && help
while [ $# -ne 1 ]; do
help
exit 1
done

ARCH=$1

if [ "$ARCH" != "arm" ] && [ "$ARCH" != "mips" ]
then
echo "Invalid architecture: $ARCH"
help
exit 1
fi

ROOTFS=`mktemp -d`
QEMUARCH=$(qemuarch $ARCH)
DEBIANARCH=$(debianarch $ARCH)

# Host dependencies
apt-get update
apt-get install -y qemu-user-static debootstrap binfmt-support
debootstrap --foreign --no-check-gpg --arch=$DEBIANARCH buster $ROOTFS http://httpredir.debian.org/debian/

QEMU=`which qemu-$QEMUARCH-static`
QEMUBASE=$(dirname $QEMU)
mkdir -p $ROOTFS$QEMUBASE
cp -v $QEMU $ROOTFS$QEMUBASE
INTR=$(cat /proc/sys/fs/binfmt_misc/qemu-$QEMUARCH | grep interpreter | cut -d ' ' -f 2)
if [[ "$INTR" != "$QEMU" ]]; then
echo -1 > /proc/sys/fs/binfmt_misc/qemu-$QEMUARCH
if [ "$ARCH" == "mips" ]
then
echo ":qemu-$QEMUARCH:M:0:\\x7f\\x45\\x4c\\x46\\x01\\x01\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x08\\x00:\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\x00\\xfe\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xfe\\xff\\xff\\xff:$QEMU:OC" > /proc/sys/fs/binfmt_misc/register
elif [ "$ARCH" == "arm" ]
then
echo ":qemu-$QEMUARCH:M:0:\\x7f\\x45\\x4c\\x46\\x01\\x01\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x28\\x00:\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\x00\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xfe\\xff\\xff\\xff:$QEMU:OCF" > /proc/sys/fs/binfmt_misc/register
fi
fi
# todo check that the interpreter exists in right place
chroot $ROOTFS ./debootstrap/debootstrap --second-stage --verbose

mount -t devpts devpts $ROOTFS/dev/pts
mount -t proc proc $ROOTFS/proc
mount -t sysfs sysfs $ROOTFS/sys

chroot $ROOTFS apt-get update
chroot $ROOTFS apt-get -y upgrade
chroot $ROOTFS apt-get install -y g++ cmake libicu-dev git ruby-highline ruby-json python
chroot $ROOTFS apt-get -y autoremove
chroot $ROOTFS apt-get clean
chroot $ROOTFS find /var/lib/apt/lists -type f -delete
umount $ROOTFS/dev/pts
umount $ROOTFS/proc
umount $ROOTFS/sys

cd /artifacts
tar --numeric-owner -cvf buster-$ARCH.tar -C $ROOTFS .
@@ -0,0 +1,7 @@
ARG ARCH
FROM jsc32-base:$ARCH-raw

LABEL description="Minimal Debian image to reproduce JSC dev"
LABEL maintainer="Paulo Matos <pmatos@igalia.com>"

CMD ["/bin/bash"]
@@ -0,0 +1,21 @@
#! /bin/bash

qemuarch() {
if [ "$1" == "arm" ]
then
echo "arm"
elif [ "$1" == "mips" ]
then
echo "mipsel"
fi
}

debianarch() {
if [ "$1" == "arm" ]
then
echo "armhf"
elif [ "$1" == "mips" ]
then
echo "mipsel"
fi
}
@@ -0,0 +1,149 @@
#! /bin/bash
#
# Required Options:
# -a : arch (mips or arm)
# -q : qemu path to setup binfmt
# -b : build webkit
# -t : test webkit (assumes --build)
# -f <regexp>: filter to be passed to the run-javascript-tests script
# Optional Options:
# -i : enters a terminal
# Arguments:
# <path>: path to a local webkit checkout to use in building and testing
set -Eeuo pipefail

# Command line argument handling

INTERACTIVE=0
BUILD=0
TEST=0
FILTER=""
ARCH=
QEMU=
while getopts ":a:q:ibtf:" opt; do
case ${opt} in
a)
ARCH="$OPTARG"
;;
q)
QEMU="$OPTARG"
;;
i)
INTERACTIVE=1
;;
b)
BUILD=1
;;
t)
TEST=1
BUILD=1
;;
f)
TEST=1
BUILD=1
FILTER="$OPTARG"
;;
\?)
echo "Invalid option: $OPTARG" 1>&2
exit
;;
:)
echo "Invalid option: $OPTARG requires an argument" 1>&2
exit
;;
esac
done
shift $((OPTIND - 1))

if [[ $ARCH == '' ]]; then
echo "Please specify -a <arch>"
exit
fi

if [[ $# != 1 ]]; then
echo "Not enough arguments"
exit
fi
WEBKIT=$1

if [ ! -d $WEBKIT ]; then
echo "Directory $WEBKIT does not exist"
exit
fi

# Deal with exit
BINFMT_NAME=
function finish {
if [[ $BINFMT_NAME != '' ]]; then
echo "Cleaning up binfmt support"
echo -1 > /proc/sys/fs/binfmt_misc/$BINFMT_NAME
fi
}
trap finish EXIT SIGINT

# Output plan

echo "Using WebKit path $WEBKIT"
echo "Plan:"
if [[ $BUILD == 1 ]]; then
echo " * Build;"
fi
if [[ $TEST == 1 ]]; then
echo " * Test;"
if [[ $FILTER != "" ]]; then
echo " Using filter: \"$FILTER\""
fi
fi
if [[ $INTERACTIVE == 1 ]]; then
echo " * Going into interactive mode;"
fi

read -n1 -r -p "Press any key to continue or Ctrl-C to exit..." keyign

if [[ $QEMU != '' ]]; then
echo "Setting up $ARCH binfmt to run interpreter in $QEMU"
BINFMT_NAME="reprojsc-qemu-$ARCH"
if [ -f "/proc/sys/fs/binfmt_misc/$BINFMT_NAME" ]; then
echo -1 > /proc/sys/fs/binfmt_misc/$BINFMT_NAME
fi

if [[ $ARCH == "mips" ]]; then
echo ":$BINFMT_NAME:M:0:\\x7f\\x45\\x4c\\x46\\x01\\x01\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x08\\x00:\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\x00\\xfe\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xfe\\xff\\xff\\xff:$QEMU:OC" > /proc/sys/fs/binfmt_misc/register
elif [[ $ARCH == "arm" ]]; then
echo ":$BINFMT_NAME:M:0:\\x7f\\x45\\x4c\\x46\\x01\\x01\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x28\\x00:\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\x00\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xfe\\xff\\xff\\xff:$QEMU:OCF" > /proc/sys/fs/binfmt_misc/register
fi
fi

out=$(docker run --rm pmatos/jsc32-base:$ARCH /bin/true 2>&1 || true)
if `echo $out | grep -q "exec format error"`; then
echo "binfmt/qemu are not setup for $ARCH"
echo "use -q <qemu-path> to setup qemu using binfmt (make sure qemu-path points to the qemu with the right arch)"
exit 1
fi

did=$(docker run --rm -di -v $WEBKIT:/WebKit pmatos/jsc32-base:$ARCH)

UNAME=
if [[ "$ARCH" == "arm" ]]; then
UNAME="armv7l"
elif [[ "$ARCH" == "mips" ]]; then
UNAME="mips"
fi


if [[ "$(docker exec $did uname -m)" != "$UNAME" ]]; then
echo "Something is wrong - incorrect container architecture"
docker stop $did
exit
fi


set +e

[[ $BUILD == 1 ]] && docker exec $did /bin/bash -c "cd /WebKit && Tools/Scripts/build-jsc --release --jsc-only --cmakeargs=\"-DENABLE_STATIC_JSC=ON -DCMAKE_VERBOSE_MAKEFILE=ON -DENABLE_JIT=OFF\" $BUILD_JSC_ARGS"
echo "Build returned $?"
[[ $TEST == 1 ]] && docker exec $did /bin/bash -c "cd /WebKit && Tools/Scripts/run-javascriptcore-tests --no-build --no-fail-fast --json-output=jsc_results.json --release --memory-limited --no-testmasm --no-testair --no-testb3 --no-testdfg --no-testapi --jsc-only --no-jit-stress-tests $TEST_JSC_ARGS"
echo "Test returned $?"
[[ $INTERACTIVE == 1 ]] && docker exec -ti $did /bin/bash
docker stop $did

0 comments on commit 94bf25e

Please sign in to comment.
You can’t perform that action at this time.