Skip to content

Commit 94bf25e

Browse files
author
Paulo Matos
committed
Scripts to create image and run reproducible builds for JSC32
This adds script to update container image and script to run container image, create execution plan and run it. Also adds initial CI workflow using GitHub Actions to create and push the image.
1 parent 3f162e5 commit 94bf25e

File tree

9 files changed

+377
-0
lines changed

9 files changed

+377
-0
lines changed

‎.github/workflows/webkitbuild-image-creation.yml‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
name: CI
2+
3+
on:
4+
push:
5+
paths:
6+
- 'containers/jsc32-base/**'
7+
- '.github/workflows/webkitbuild-image-creation.yml'
8+
9+
jobs:
10+
build:
11+
12+
runs-on: ubuntu-latest
13+
strategy:
14+
matrix:
15+
arch: [mips, arm]
16+
17+
steps:
18+
- uses: actions/checkout@v1
19+
- name: Login to docker hub
20+
run: docker login -u pmatos -p ${{ secrets.DOCKER_PASS }}
21+
- name: Build
22+
working-directory: containers/jsc32-base/
23+
run: ./bimage.sh ${{ matrix.arch }}
24+
25+
test-image:
26+
runs-on: ubuntu-latest
27+
needs: build
28+
container:
29+
image: debian:buster
30+
options: --privileged -v /var/run/docker.sock:/var/run/docker.sock
31+
32+
strategy:
33+
fail-fast: false
34+
matrix:
35+
arch: [mips, arm]
36+
orch: [podman, docker]
37+
38+
steps:
39+
- name: Install qemu and binfmt support
40+
run: |
41+
apt-get update
42+
apt-get install -y gpg software-properties-common qemu-user-static binfmt-support
43+
- name: Install podman if needed
44+
if: matrix.orch == 'podman'
45+
run: |
46+
add-apt-repository -y ppa:projectatomic/ppa
47+
apt-get update
48+
apt-get install -y podman
49+
- name: Install docker if needed
50+
if: matrix.orch == 'docker'
51+
run: |
52+
apt-get install -y apt-transport-https ca-certificates curl gnupg2 software-properties-common
53+
curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add -
54+
apt-key fingerprint 0EBFCD88
55+
add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable"
56+
apt-get update
57+
apt-get install -y docker-ce docker-ce-cli containerd.io
58+
- name: Test image architecture
59+
run: ${{ matrix.orch }} run pmatos/jsc32-base:${{ matrix.arch }} /bin/bash -c "[[ \"$(uname -m)\" == \"${{ matrix.arch }}\" ]] || exit 1"

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# Temp files
2+
*~

‎containers/README.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# WebKit Containers
2+
3+
This folder contains the definition of several images that ease the development and reproducibility of problems in native and cross compiled JSC binaries.
4+
5+
## Images
6+

‎containers/jsc32-base/bimage.sh‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
#! /bin/bash
2+
## $PROG 0.1 - Build WebKit-ready docker image for 32bits mips and arm
3+
##
4+
## Usage: $PROG [ARCH]
5+
## Commands:
6+
## -h, --help Displays this help and exists
7+
## -v, --version Displays output version and exists
8+
## Examples:
9+
## $PROG arm
10+
## $PROG mips
11+
set -eu
12+
13+
PROG=${0##*/}
14+
die() { echo $@ >&2; exit 2; }
15+
16+
help() {
17+
grep "^##" "$0" | sed -e "s/^...//" -e "s/\$PROG/$PROG/g"; exit 0
18+
}
19+
version() {
20+
help | head -1
21+
}
22+
23+
[ $# = 0 ] && help
24+
while [ $# -ne 1 ]; do
25+
help
26+
exit 1
27+
done
28+
29+
ARCH=$1
30+
31+
if [ "$ARCH" != "arm" ] && [ "$ARCH" != "mips" ]
32+
then
33+
echo "Invalid architecture: $ARCH"
34+
help
35+
exit 1
36+
fi
37+
38+
docker build -t buster-$ARCH --build-arg ARCH=$ARCH -f build-image.Dockerfile .
39+
docker run --rm --privileged -v `pwd`/artifacts:/artifacts buster-$ARCH
40+
docker import `pwd`/artifacts/buster-$ARCH.tar jsc32-base:$ARCH-raw
41+
docker build -t pmatos/jsc32-base:$ARCH -f jsc32-base.Dockerfile --build-arg ARCH=$ARCH .
42+
docker push pmatos/jsc32-base:$ARCH

‎containers/jsc32-base/build-image.Dockerfile‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
FROM debian:stable
2+
3+
ARG ARCH
4+
# Need to pass arch argument to script
5+
WORKDIR /
6+
ADD ./build-image.sh .
7+
ADD ./utils.sh .
8+
ENV ARCH=${ARCH}
9+
ENTRYPOINT /build-image.sh ${ARCH}

‎containers/jsc32-base/build-image.sh‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
#! /bin/bash
2+
## $PROG 0.1 - Build WebKit-ready docker image for mipsel and arm
3+
##
4+
## Usage: $PROG [ARCH]
5+
## Commands:
6+
## -h, --help Displays this help and exists
7+
## -v, --version Displays output version and exists
8+
## Examples:
9+
## $PROG arm
10+
## $PROG mips
11+
set -Eeuo pipefail
12+
13+
source ./utils.sh
14+
15+
PROG=${0##*/}
16+
die() { echo $@ >&2; exit 2; }
17+
18+
help() {
19+
grep "^##" "$0" | sed -e "s/^...//" -e "s/\$PROG/$PROG/g"; exit 0
20+
}
21+
version() {
22+
help | head -1
23+
}
24+
25+
[ $# = 0 ] && help
26+
while [ $# -ne 1 ]; do
27+
help
28+
exit 1
29+
done
30+
31+
ARCH=$1
32+
33+
if [ "$ARCH" != "arm" ] && [ "$ARCH" != "mips" ]
34+
then
35+
echo "Invalid architecture: $ARCH"
36+
help
37+
exit 1
38+
fi
39+
40+
ROOTFS=`mktemp -d`
41+
QEMUARCH=$(qemuarch $ARCH)
42+
DEBIANARCH=$(debianarch $ARCH)
43+
44+
# Host dependencies
45+
apt-get update
46+
apt-get install -y qemu-user-static debootstrap binfmt-support
47+
debootstrap --foreign --no-check-gpg --arch=$DEBIANARCH buster $ROOTFS http://httpredir.debian.org/debian/
48+
49+
QEMU=`which qemu-$QEMUARCH-static`
50+
QEMUBASE=$(dirname $QEMU)
51+
mkdir -p $ROOTFS$QEMUBASE
52+
cp -v $QEMU $ROOTFS$QEMUBASE
53+
INTR=$(cat /proc/sys/fs/binfmt_misc/qemu-$QEMUARCH | grep interpreter | cut -d ' ' -f 2)
54+
if [[ "$INTR" != "$QEMU" ]]; then
55+
echo -1 > /proc/sys/fs/binfmt_misc/qemu-$QEMUARCH
56+
if [ "$ARCH" == "mips" ]
57+
then
58+
echo ":qemu-$QEMUARCH:M:0:\\x7f\\x45\\x4c\\x46\\x01\\x01\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x08\\x00:\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\x00\\xfe\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xfe\\xff\\xff\\xff:$QEMU:OC" > /proc/sys/fs/binfmt_misc/register
59+
elif [ "$ARCH" == "arm" ]
60+
then
61+
echo ":qemu-$QEMUARCH:M:0:\\x7f\\x45\\x4c\\x46\\x01\\x01\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x28\\x00:\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\x00\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xfe\\xff\\xff\\xff:$QEMU:OCF" > /proc/sys/fs/binfmt_misc/register
62+
fi
63+
fi
64+
# todo check that the interpreter exists in right place
65+
chroot $ROOTFS ./debootstrap/debootstrap --second-stage --verbose
66+
67+
mount -t devpts devpts $ROOTFS/dev/pts
68+
mount -t proc proc $ROOTFS/proc
69+
mount -t sysfs sysfs $ROOTFS/sys
70+
71+
chroot $ROOTFS apt-get update
72+
chroot $ROOTFS apt-get -y upgrade
73+
chroot $ROOTFS apt-get install -y g++ cmake libicu-dev git ruby-highline ruby-json python
74+
chroot $ROOTFS apt-get -y autoremove
75+
chroot $ROOTFS apt-get clean
76+
chroot $ROOTFS find /var/lib/apt/lists -type f -delete
77+
umount $ROOTFS/dev/pts
78+
umount $ROOTFS/proc
79+
umount $ROOTFS/sys
80+
81+
cd /artifacts
82+
tar --numeric-owner -cvf buster-$ARCH.tar -C $ROOTFS .

‎containers/jsc32-base/jsc32-base.Dockerfile‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
ARG ARCH
2+
FROM jsc32-base:$ARCH-raw
3+
4+
LABEL description="Minimal Debian image to reproduce JSC dev"
5+
LABEL maintainer="Paulo Matos <pmatos@igalia.com>"
6+
7+
CMD ["/bin/bash"]

‎containers/jsc32-base/utils.sh‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
#! /bin/bash
2+
3+
qemuarch() {
4+
if [ "$1" == "arm" ]
5+
then
6+
echo "arm"
7+
elif [ "$1" == "mips" ]
8+
then
9+
echo "mipsel"
10+
fi
11+
}
12+
13+
debianarch() {
14+
if [ "$1" == "arm" ]
15+
then
16+
echo "armhf"
17+
elif [ "$1" == "mips" ]
18+
then
19+
echo "mipsel"
20+
fi
21+
}

‎containers/reprojsc.sh‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
#! /bin/bash
2+
#
3+
# Required Options:
4+
# -a : arch (mips or arm)
5+
# -q : qemu path to setup binfmt
6+
# -b : build webkit
7+
# -t : test webkit (assumes --build)
8+
# -f <regexp>: filter to be passed to the run-javascript-tests script
9+
# Optional Options:
10+
# -i : enters a terminal
11+
# Arguments:
12+
# <path>: path to a local webkit checkout to use in building and testing
13+
set -Eeuo pipefail
14+
15+
# Command line argument handling
16+
17+
INTERACTIVE=0
18+
BUILD=0
19+
TEST=0
20+
FILTER=""
21+
ARCH=
22+
QEMU=
23+
while getopts ":a:q:ibtf:" opt; do
24+
case ${opt} in
25+
a)
26+
ARCH="$OPTARG"
27+
;;
28+
q)
29+
QEMU="$OPTARG"
30+
;;
31+
i)
32+
INTERACTIVE=1
33+
;;
34+
b)
35+
BUILD=1
36+
;;
37+
t)
38+
TEST=1
39+
BUILD=1
40+
;;
41+
f)
42+
TEST=1
43+
BUILD=1
44+
FILTER="$OPTARG"
45+
;;
46+
\?)
47+
echo "Invalid option: $OPTARG" 1>&2
48+
exit
49+
;;
50+
:)
51+
echo "Invalid option: $OPTARG requires an argument" 1>&2
52+
exit
53+
;;
54+
esac
55+
done
56+
shift $((OPTIND - 1))
57+
58+
if [[ $ARCH == '' ]]; then
59+
echo "Please specify -a <arch>"
60+
exit
61+
fi
62+
63+
if [[ $# != 1 ]]; then
64+
echo "Not enough arguments"
65+
exit
66+
fi
67+
WEBKIT=$1
68+
69+
if [ ! -d $WEBKIT ]; then
70+
echo "Directory $WEBKIT does not exist"
71+
exit
72+
fi
73+
74+
# Deal with exit
75+
BINFMT_NAME=
76+
function finish {
77+
if [[ $BINFMT_NAME != '' ]]; then
78+
echo "Cleaning up binfmt support"
79+
echo -1 > /proc/sys/fs/binfmt_misc/$BINFMT_NAME
80+
fi
81+
}
82+
trap finish EXIT SIGINT
83+
84+
# Output plan
85+
86+
echo "Using WebKit path $WEBKIT"
87+
echo "Plan:"
88+
if [[ $BUILD == 1 ]]; then
89+
echo " * Build;"
90+
fi
91+
if [[ $TEST == 1 ]]; then
92+
echo " * Test;"
93+
if [[ $FILTER != "" ]]; then
94+
echo " Using filter: \"$FILTER\""
95+
fi
96+
fi
97+
if [[ $INTERACTIVE == 1 ]]; then
98+
echo " * Going into interactive mode;"
99+
fi
100+
101+
read -n1 -r -p "Press any key to continue or Ctrl-C to exit..." keyign
102+
103+
if [[ $QEMU != '' ]]; then
104+
echo "Setting up $ARCH binfmt to run interpreter in $QEMU"
105+
BINFMT_NAME="reprojsc-qemu-$ARCH"
106+
if [ -f "/proc/sys/fs/binfmt_misc/$BINFMT_NAME" ]; then
107+
echo -1 > /proc/sys/fs/binfmt_misc/$BINFMT_NAME
108+
fi
109+
110+
if [[ $ARCH == "mips" ]]; then
111+
echo ":$BINFMT_NAME:M:0:\\x7f\\x45\\x4c\\x46\\x01\\x01\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x08\\x00:\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\x00\\xfe\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xfe\\xff\\xff\\xff:$QEMU:OC" > /proc/sys/fs/binfmt_misc/register
112+
elif [[ $ARCH == "arm" ]]; then
113+
echo ":$BINFMT_NAME:M:0:\\x7f\\x45\\x4c\\x46\\x01\\x01\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x28\\x00:\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\x00\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xfe\\xff\\xff\\xff:$QEMU:OCF" > /proc/sys/fs/binfmt_misc/register
114+
fi
115+
fi
116+
117+
out=$(docker run --rm pmatos/jsc32-base:$ARCH /bin/true 2>&1 || true)
118+
if `echo $out | grep -q "exec format error"`; then
119+
echo "binfmt/qemu are not setup for $ARCH"
120+
echo "use -q <qemu-path> to setup qemu using binfmt (make sure qemu-path points to the qemu with the right arch)"
121+
exit 1
122+
fi
123+
124+
did=$(docker run --rm -di -v $WEBKIT:/WebKit pmatos/jsc32-base:$ARCH)
125+
126+
UNAME=
127+
if [[ "$ARCH" == "arm" ]]; then
128+
UNAME="armv7l"
129+
elif [[ "$ARCH" == "mips" ]]; then
130+
UNAME="mips"
131+
fi
132+
133+
134+
if [[ "$(docker exec $did uname -m)" != "$UNAME" ]]; then
135+
echo "Something is wrong - incorrect container architecture"
136+
docker stop $did
137+
exit
138+
fi
139+
140+
141+
set +e
142+
143+
[[ $BUILD == 1 ]] && docker exec $did /bin/bash -c "cd /WebKit && Tools/Scripts/build-jsc --release --jsc-only --cmakeargs=\"-DENABLE_STATIC_JSC=ON -DCMAKE_VERBOSE_MAKEFILE=ON -DENABLE_JIT=OFF\" $BUILD_JSC_ARGS"
144+
echo "Build returned $?"
145+
[[ $TEST == 1 ]] && docker exec $did /bin/bash -c "cd /WebKit && Tools/Scripts/run-javascriptcore-tests --no-build --no-fail-fast --json-output=jsc_results.json --release --memory-limited --no-testmasm --no-testair --no-testb3 --no-testdfg --no-testapi --jsc-only --no-jit-stress-tests $TEST_JSC_ARGS"
146+
echo "Test returned $?"
147+
[[ $INTERACTIVE == 1 ]] && docker exec -ti $did /bin/bash
148+
docker stop $did
149+

0 commit comments

Comments
 (0)