diff --git a/.after_failure.sh b/.after_failure.sh new file mode 100755 index 0000000..b9283b2 --- /dev/null +++ b/.after_failure.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -x + +if [[ -e /tmp/base.bash.log ]]; then + cp /tmp/base.bash.log /tmp/$VARIANT-base.bash.log + ./publish.sh /tmp/$VARIANT-base.bash.log + tail -n 200 /tmp/base.bash.log +fi + +if [[ -e /tmp/build.bash.log ]]; then + cp /tmp/build.bash.log /tmp/$VARIANT-build.bash.log + ./publish.sh /tmp/$VARIANT-build.bash.log + tail -n 200 /tmp/build.bash.log; +fi + +if [[ -e /tmp/build_example.bash.log ]]; then + cp /tmp/build_example.bash.log /tmp/$VARIANT-build_example.bash.log + ./publish.sh /tmp/$VARIANT-build_example.bash.log + tail -n 200 /tmp/build_example.bash.log; +fi diff --git a/.travis.sh b/.travis.sh index e08696b..6c20a9e 100755 --- a/.travis.sh +++ b/.travis.sh @@ -27,7 +27,7 @@ fi done )& TICKER_PID=$! -trap 'kill ${TICKER_PID:-} ${BUILD_PID:-}' EXIT +trap 'kill ${TICKER_PID:-} ${BUILD_PID:-} &>/dev/null || :' EXIT ### base.bash @@ -41,9 +41,9 @@ fi ### build.bash -echo 'Running build of llvm-obfuscator... ' +echo "Running build of llvm-$VARIANT... " -make "ARCH=$ARCH" NO_TTY=y build &>/tmp/build.bash.log & +make NO_TTY=y "ARCH=$ARCH" "VARIANT=$VARIANT" build &>/tmp/build.bash.log & BUILD_PID=$! wait $BUILD_PID @@ -52,7 +52,7 @@ echo 'DONE.' ### build_example.bash -if [[ $ARCH == *arm* ]]; then +if [[ $ARCH == *arm* ]] && [[ $VARIANT == obfuscator ]]; then echo -n 'Building example project... ' make NO_TTY=y build-example &>/tmp/build_example.bash.log diff --git a/.travis.yml b/.travis.yml index 7a3cac3..bc6319d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,25 +10,39 @@ branches: services: - docker -before_install: - - pip install --user --upgrade awscli - - sudo apt-get update - - sudo apt-get install -y jq - -script: - - ./.travis.sh - -after_success: - - travis_wait tar -C output -cJf "llvm-obfuscator-$ARCH.txz" . - - ./publish.sh llvm-obfuscator-$ARCH.txz - -after_failure: - - if [[ -e /tmp/base.bash.log ]]; then echo "*** base.bash.log ***"; fi - - if [[ -e /tmp/base.bash.log ]]; then tail -n 500 /tmp/base.bash.log; fi - - if [[ -e /tmp/build.bash.log ]]; then echo "*** build.bash.log ***"; fi - - if [[ -e /tmp/build.bash.log ]]; then tail -n 500 /tmp/build.bash.log; fi - - if [[ -e /tmp/build_example.bash.log ]]; then echo "*** build_example.bash.log ***"; fi - - if [[ -e /tmp/build_example.bash.log ]]; then tail -n 500 /tmp/build_example.bash.log; fi +matrix: + include: + - env: + - VARIANT=obfuscator + - ARCH=arm,x86 + before_install: + - pip install --user --upgrade awscli + - sudo apt-get update + - sudo apt-get install -y jq + script: + - ./pull_ccache.bash --variant=$VARIANT --arch=$ARCH + - VARIANT=$VARIANT ./.travis.sh + after_success: + - travis_wait tar -C output -cJf "llvm-$VARIANT-$ARCH.txz" . + - ./publish.sh llvm-$VARIANT-$ARCH.txz + after_failure: + - ./.after_failure.sh + + - env: + - VARIANT=vanilla + - ARCH=arm,x86 + before_install: + - pip install --user --upgrade awscli + - sudo apt-get update + - sudo apt-get install -y jq + script: + - ./pull_ccache.bash --variant=$VARIANT --arch=$ARCH + - VARIANT=$VARIANT ./.travis.sh + after_success: + - travis_wait tar -C output -cJf "llvm-$VARIANT-$ARCH.txz" . + - ./publish.sh llvm-$VARIANT-$ARCH.txz + after_failure: + - ./.after_failure.sh env: global: @@ -44,5 +58,3 @@ env: - secure: "T1wlLXyhfGnMNMPtHv2FspxsWG09hVZgaucSJ6IxN+PZBbgjLb/TMSlqqLXfIzx6s0SuQIzX1xZE8lb/uOzfLtjmxBdc4dGYLW+qR4AwkghQl2DGMK+hzGAi9ZbeCGKWtX7cN4WxLN1tpcTyQBCC8LcnP0xVxd+7Znytdezmlr86ce2d8dmUo+kyLdN+vhEUnch9zhMpuZI+TxV2HQqCS/6e31myP+3Xzw3B6o6uF8dYZ0QlWlNU4KjUNNAD7cLqkxNd38E7wFrQJdyrk03/hZanatl2mcL7Te5Uxo5sWN5WteTNwq8dCKXiEXpkEqP1++4wWTQDeQWoPjHlhHzO4MxGAdEUnlFWafsV78OJvr/gwV0UxDFmfadPwGG/xPPJMyNJjXwDyeztswknbVaNZdbRXp3m7Iri+bWCvOZE9N7rGde/5q0JOz9rwJXRDMPnp7+lyupHQQIdswNpkFraHz3eZ/h4n+Dkr3RtkW0M6ZpPR+/r18uD7jwm/Ye73E9wzq1b6sL0knX/OEwBv3FVvaBaA4SWzb1glDlru8BKnZfVIZ/UxYas6FqGN6H9ROyCHtva86ffv7HvvnB/gUqcOfY5si7WvtPVApIc9kfILcSWEchRWTHQIeUOdEUBfoq8/Km0BL6gguOO9Ge9zzOrdo0NThh8x7NoE0DhHdQctJY=" # DOCKER_PASS - secure: "qEmUT+mBsHjLkrBYMZWiLl8+ZXq9UAv+DyMDww+N74hPtGiZxMqLuVIFq94qR/WhiI9zd7k+tjhy6TcgD8rpKoQnQSGXcotFGq6typ3UWEV0ekt40IMV5nBqopqGGIBVIHNfXjq0OTeYTF2eGncpNnznJHAQi8Dvj7vD99y5GA5RoZGSdsiqTyxBGMvZ0cE0QApkb1MH3lBVRzBuh6dotpa1RBlGR+i2xbcH3nAqy5RB38wZgMedAPnlUfg+uB+IQiF8+hbuSmHzADiabNLBkkB/UCOz59k/JkaeN+2gnyP6/so+SpOe1BKfpxKBcN74O4n056ZLDBRjTod2YSc49dKqxhZhQijwNJfFi3Tt5iNrFYdU3KPeqqfNLoQKmbBTKjGXbxHHXm3lnUuCTZH0Yd9QGKSAwn5kxDRy6DuyNMUGwFtu8YxzirwTJiQR6ayodMCMWjZCXuxUc0I5I2Y64CsofRWkuMbznjE4wJwptVPzTwIxkZnMt+aWVoQTXBtZVpiHf4CTJb65dyLDXeMOwhYnKzwLUrvvHHpOm7GygvAbbeDAzHumTDvt7s5nl2VwHNLmTK3TizvvpQCucIjG8M8dEd7iu4VRZh/rxGxV9MJ2qUNETa6hnfX+Z2xaLTcgYAG5z7p60sThPfFfYLopdJL1dO5eA/EncPzuQuh087w=" - matrix: - - ARCH=x86,arm diff --git a/Dockerfile b/Dockerfile.base similarity index 81% rename from Dockerfile rename to Dockerfile.base index d24e6cc..c78f88e 100644 --- a/Dockerfile +++ b/Dockerfile.base @@ -23,19 +23,28 @@ RUN apt-get update \ && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - \ && echo "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-4.0 main" \ >/etc/apt/sources.list.d/llvm40.list \ + && echo "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-6.0 main" \ + >/etc/apt/sources.list.d/llvm60.list \ && apt-get update \ - && apt-get upgrade \ + && apt-get upgrade -y \ && apt-get install -y libcurl4-openssl-dev \ build-essential \ bison \ flex \ ninja-build \ - llvm-4.0 \ - llvm-4.0-dev \ - clang-4.0 \ git \ + m4 \ + gperf \ + gawk \ + ncurses-dev \ + texinfo \ + help2man \ binutils-dev \ + libpthread-stubs0-dev \ + libtinfo-dev \ python \ + python-dev \ + ccache \ binutils-multiarch-dev \ g++-4.8-arm-linux-gnueabihf \ gcc-4.8-arm-linux-gnueabihf \ @@ -57,10 +66,9 @@ RUN apt-get update \ && tar -xzf cmake-3.10.1.tar.gz \ && cd cmake-3.10.1 \ && ./configure \ - && make -j4 \ - && make install \ + && make -j8 \ + && make -j8 install \ && cd .. && rm -rf cmake-* \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* + && apt-get clean # EOF diff --git a/Dockerfile.obfuscator b/Dockerfile.obfuscator new file mode 100644 index 0000000..9383c1e --- /dev/null +++ b/Dockerfile.obfuscator @@ -0,0 +1,14 @@ +# Copyright (C) 2017-2018 Swift Navigation Inc. +# Contact: Swift Navigation +# +# This source is subject to the license found in the file 'LICENSE' which must +# be be distributed together with this source. All other rights reserved. +# +# THIS CODE AND INFORMATION IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND, +# EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR PURPOSE. + +ARG DOCKER_NAMETAG +FROM $DOCKER_NAMETAG + +RUN apt-get install -y llvm-4.0 llvm-4.0-dev clang-4.0 libclang-4.0-dev diff --git a/Dockerfile.vanilla b/Dockerfile.vanilla new file mode 100644 index 0000000..da0acd9 --- /dev/null +++ b/Dockerfile.vanilla @@ -0,0 +1,14 @@ +# Copyright (C) 2017-2018 Swift Navigation Inc. +# Contact: Swift Navigation +# +# This source is subject to the license found in the file 'LICENSE' which must +# be be distributed together with this source. All other rights reserved. +# +# THIS CODE AND INFORMATION IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND, +# EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR PURPOSE. + +ARG DOCKER_NAMETAG +FROM $DOCKER_NAMETAG + +RUN apt-get install -y llvm-6.0 llvm-6.0-dev clang-6.0 libclang-6.0-dev diff --git a/Makefile b/Makefile index d672ceb..043ae39 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,10 @@ SCRIPTS := .travis.sh \ publish.sh \ run_build_shell.bash \ stage_sysroot.bash \ + do_clang_build.bash \ + push_ccache.bash \ + pull_ccache.bash \ + s3_download.bash \ all: check base build @@ -27,7 +31,7 @@ base: check-base $(CURDIR)/base.bash $(NO_TTY_ARG) build: check-build - $(CURDIR)/build.bash $(NO_TTY_ARG) --arch=$(ARCH) + $(CURDIR)/build.bash $(NO_TTY_ARG) --arch=$(ARCH) --variant=$(VARIANT) stage: check-stage_sysroot $(CURDIR)/stage_sysroot.bash $(NO_TTY_ARG) @@ -38,7 +42,25 @@ build-example: check-build_example run: check-run_build_shell $(CURDIR)/run_build_shell.bash $(NO_TTY_ARG) -clean: +clean-vanilla-build: + docker volume rm vanilla-llvm-build || : + docker volume rm vanilla-llvm-ccache || : + +clean-vanilla-src: + docker volume rm vanilla-llvm || : + +clean-vanilla: clean-vanilla-build clean-vanilla-src + +clean-obfuscator-build: docker volume rm obfuscator-llvm-build || : + docker volume rm obfuscator-llvm-ccache || : + +clean-obfuscator-src: docker volume rm obfuscator-llvm || : + +clean-obfuscator: clean-obfuscator-build clean-obfuscator-src + +clean-build: clean-vanilla clean-obfuscator + +clean: clean-build sudo rm -rf output/* diff --git a/base.bash b/base.bash index d7cf8ef..8e5fd8d 100755 --- a/base.bash +++ b/base.bash @@ -56,11 +56,16 @@ fi docker build \ --force-rm --no-cache \ - -f Dockerfile -t "$DOCKER_NAMETAG" . + -f Dockerfile.base -t "$DOCKER_NAMETAG" . -if [[ -n "${DOCKER_USER:-}" ]] && [[ -n "${DOCKER_PASS:-}" ]]; then - echo "$DOCKER_PASS" | docker login --username="$DOCKER_USER" --password-stdin - docker push "$DOCKER_NAMETAG" -else - echo "WARNING: not pushing new image to Docker Hub..." >&2 -fi +docker build \ + --force-rm --no-cache \ + --build-arg "DOCKER_NAMETAG=$DOCKER_NAMETAG" \ + -f Dockerfile.vanilla -t "$DOCKER_NAMETAG-vanilla" . + +docker build \ + --force-rm --no-cache \ + --build-arg "DOCKER_NAMETAG=$DOCKER_NAMETAG" \ + -f Dockerfile.obfuscator -t "$DOCKER_NAMETAG-obfuscator" . + +./push.bash diff --git a/build.bash b/build.bash index b397469..485646c 100755 --- a/build.bash +++ b/build.bash @@ -23,42 +23,70 @@ NO_TTY= while [[ $# -gt 0 ]]; do case $1 in - --arch=x86) ARCH="X86"; shift ;; - --arch=arm) ARCH="ARM"; shift ;; - --arch=arm,x86) ARCH="ARM\\;X86"; shift ;; - --arch=x86,arm) ARCH="X86\\;ARM"; shift ;; + --arch=x86) ARCH="X86"; shift ;; + --arch=arm) ARCH="ARM"; shift ;; + --arch=arm,x86) ARCH="ARM\\;X86"; shift ;; + --arch=x86,arm) ARCH="ARM\\;X86"; shift ;; - --verbose) VERBOSE="-v"; shift ;; - --no-tty) NO_TTY=--no-tty; shift ;; + --variant=vanilla) VARIANT="vanilla"; shift ;; + --variant=obfuscator) VARIANT="obfuscator"; shift ;; - *) shift ;; + --verbose) VERBOSE="-v"; shift ;; + --no-tty) NO_TTY=--no-tty; shift ;; + + *) shift ;; esac done if [[ -z "${ARCH:-}" ]]; then - echo "Error: must specify --arch=" + echo "Error: must specify --arch=" exit 1 fi -set -x -# -DLLVM_BUILD_TOOLS:BOOL=FALSE \ +if [[ -z "${VARIANT:-}" ]]; then + echo "Error: must a variant to build --variant=" + exit 1 +fi + +CXX_FLAGS="-L/toolchain/x86/x86_64-buildroot-linux-gnu/sysroot/lib \ + -L/toolchain/x86/x86_64-buildroot-linux-gnu/sysroot/usr/lib \ + -I/toolchain/x86/lib/gcc/x86_64-buildroot-linux-gnu/6.4.0/plugin/include" + +if [[ "$VARIANT" == "obfuscator" ]]; then + LLVM_REPO="https://github.com/obfuscator-llvm/obfuscator.git" + LLVM_BRANCH="llvm-4.0" + CLANG_REPO="" + CLANG_TOOLS_EXTRA_REPO="" + CPP_WRAPPER_DEFINE="-DCMAKE_CXX_COMPILER=/bin/cpp_wrapper" + PATCH_COMMAND="{ git apply /patches/*.patch || : ; }" + COMPILE_CPP_WRAPPER="cp -v /this_dir/cpp_wrapper.c /work/cpp_wrapper.c \ + && gcc -std=c99 -O3 -Wall /work/cpp_wrapper.c -o /bin/cpp_wrapper" +else + LLVM_REPO="https://github.com/llvm-mirror/llvm.git" + CLANG_REPO="https://github.com/llvm-mirror/clang.git" + CLANG_TOOLS_EXTRA_REPO="https://github.com/llvm-mirror/clang-tools-extra.git" + LLVM_BRANCH="release_60" + CPP_WRAPPER_DEFINE="-DCMAKE_CXX_COMPILER=/toolchain/x86/bin/x86_64-linux-g++" + PATCH_COMMAND="true" + COMPILE_CPP_WRAPPER="true" + CXX_FLAGS+=" -I/work/$VARIANT-llvm/tools/clang/include" + CXX_FLAGS+=" -I/work/build/tools/clang/include" +fi CMAKE_COMMAND="\ cmake -G Ninja \ - /work/obfuscator-llvm \ - -DCMAKE_INSTALL_PREFIX=/opt/llvm-obfuscator \ + /work/$VARIANT-llvm \ + -DCMAKE_INSTALL_PREFIX=/opt/llvm-$VARIANT \ -DLLVM_TARGETS_TO_BUILD=$ARCH \ - -DCMAKE_CXX_FLAGS='-DENDIAN_LITTLE=1 -I/toolchain/x86/lib/gcc/x86_64-buildroot-linux-gnu/6.4.0/plugin/include' \ + -DCMAKE_CXX_FLAGS='-DENDIAN_LITTLE=1 $CXX_FLAGS' \ + -DLLVM_CCACHE_BUILD=ON \ + $CPP_WRAPPER_DEFINE \ -DCMAKE_C_COMPILER=/toolchain/x86/bin/x86_64-linux-gcc \ - -DCMAKE_CXX_COMPILER=/bin/cpp_wrapper \ -DCMAKE_BUILD_TYPE=Release \ -DLLVM_BINUTILS_INCDIR=/usr/include \ - -DLLDB_DISABLE_CURSES:BOOL=TRUE \ -DLLVM_ENABLE_TERMINFO=0 \ -DLLVM_INCLUDE_TESTS=OFF" -PATCH_COMMAND="{ git apply /patches/*.patch || : ; }" - if [[ -z "$NO_TTY" ]]; then INTERACTIVE=("-i" "-t") else @@ -70,23 +98,21 @@ docker run ${INTERACTIVE[@]:-} --rm \ -v "$PWD/output/opt:/opt" \ -v "$PWD/patches:/patches" \ -v "$PWD:/this_dir" \ - -v obfuscator-llvm:/work/obfuscator-llvm \ - -v obfuscator-llvm-build:/work/build \ - "$DOCKER_NAMETAG" \ - /bin/bash -c "if [ ! -d /work/obfuscator-llvm/.git ]; then \ - git clone --depth=1 --single-branch -b llvm-4.0 \ - https://github.com/obfuscator-llvm/obfuscator.git \ - obfuscator-llvm; - else \ - (cd /work/obfuscator-llvm && git pull); \ - fi \ - && cp -v /this_dir/cpp_wrapper.c /work/cpp_wrapper.c \ - && gcc -std=c99 -O3 -Wall /work/cpp_wrapper.c -o /bin/cpp_wrapper \ - && cd /work/obfuscator-llvm \ - && $PATCH_COMMAND \ - && cd /work/build \ - && $CMAKE_COMMAND \ - && ninja $VERBOSE \ - && ninja $VERBOSE install" - -./stage_sysroot.bash $NO_TTY + -v $VARIANT-llvm-ccache:/work/ccache \ + -v $VARIANT-llvm:/work/$VARIANT-llvm \ + -v $VARIANT-llvm-build:/work/build \ + -e VARIANT=$VARIANT -e ARCH=$ARCH \ + -e VERBOSE=$VERBOSE -e NO_TTY=$NO_TTY \ + -e CPP_WRAPPER_DEFINE=$CPP_WRAPPER_DEFINE \ + -e CMAKE_COMMAND="$CMAKE_COMMAND" \ + -e LLVM_REPO=$LLVM_REPO \ + -e LLVM_BRANCH=$LLVM_BRANCH \ + -e CLANG_REPO=$CLANG_REPO \ + -e CLANG_TOOLS_EXTRA_REPO=$CLANG_TOOLS_EXTRA_REPO \ + -e PATCH_COMMAND=$PATCH_COMMAND \ + -e COMPILE_CPP_WRAPPER=$COMPILE_CPP_WRAPPER \ + -e CCACHE_DIR=/work/ccache \ + "$DOCKER_NAMETAG-$VARIANT" \ + /bin/bash -c "/this_dir/do_clang_build.bash" + +./stage_sysroot.bash $NO_TTY "--variant=$VARIANT" diff --git a/build_tag b/build_tag new file mode 100644 index 0000000..703a257 --- /dev/null +++ b/build_tag @@ -0,0 +1 @@ +v17 diff --git a/do_clang_build.bash b/do_clang_build.bash new file mode 100755 index 0000000..953e142 --- /dev/null +++ b/do_clang_build.bash @@ -0,0 +1,76 @@ +#!/usr/bin/env bash + +# Copyright (C) 2018 Swift Navigation Inc. +# Contact: Swift Navigation +# +# This source is subject to the license found in the file 'LICENSE' which must +# be be distributed together with this source. All other rights reserved. +# +# THIS CODE AND INFORMATION IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND, +# EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR PURPOSE. + +set -euo pipefail +IFS=$'\n\t' + +die_non_empty() + { echo "*** ERROR: variable $1 must be defined and non-empty" >&2; exit 1; } + +die_not_defined() + { echo "*** ERROR: variable $1 must be defined (can be empty)" >&2; exit 1; } + +[[ -n "${VARIANT:-}" ]] || die_non_empty + +[[ "${VARIANT}" == "obfuscator" ]] || [[ "${VARIANT}" == "vanilla" ]] || \ + { echo "*** ERROR: invalid variant (must be one of 'vanilla', or 'obfuscator')" >&2; exit 1; } + +[[ -n "${LLVM_REPO:-}" ]] || die_non_empty LLVM_REPO +[[ -n "${LLVM_BRANCH:-}" ]] || die_non_empty LLVM_BRANCH +[[ -n "${CMAKE_COMMAND:-}" ]] || die_non_empty CMAKE_COMMAND +[[ -n "${CCACHE_DIR:-}" ]] || die_non_empty CCACHE_DIR + +[[ -n "${CLANG_REPO+x}" ]] || die_not_defined CLANG_REPO +[[ -n "${CLANG_TOOLS_EXTRA_REPO+x}" ]] || die_not_defined CLANG_TOOLS_EXTRA_REPO + +[[ -n "${COMPILE_CPP_WRAPPER+x}" ]] || die_not_defined COMPILE_CPP_WRAPPER +[[ -n "${PATCH_COMMAND+x}" ]] || die_not_defined PATCH_COMMAND + +if [ ! -d "/work/$VARIANT-llvm/.git" ]; then + git clone --depth=1 --single-branch -b "$LLVM_BRANCH" \ + "$LLVM_REPO" "$VARIANT-llvm"; +else + (cd "/work/$VARIANT-llvm" && git pull); +fi + +if [ -n "${CLANG_REPO:-}" ]; then + if [ ! -d "/work/$VARIANT-llvm/tools/clang/.git" ]; then + git clone --depth=1 --single-branch -b "$LLVM_BRANCH" \ + "$CLANG_REPO" "$VARIANT-llvm/tools/clang"; + else + (cd "/work/$VARIANT-llvm/tools/clang" && git pull); + fi +fi + +if [ -n "${CLANG_TOOLS_EXTRA_REPO:-}" ]; then \ + if [ ! -d "/work/$VARIANT-llvm/tools/clang-tools-extra/.git" ]; then + git clone --depth=1 --single-branch -b "$LLVM_BRANCH" \ + "$CLANG_TOOLS_EXTRA_REPO" "$VARIANT-llvm/tools/clang-tools-extra"; + else + (cd "/work/$VARIANT-llvm/tools/clang-tools-extra" && git pull); + fi +fi + +export CCACHE_DIR=$CCACHE_DIR +echo "CCACHE_DIR: $CCACHE_DIR" + +eval "$COMPILE_CPP_WRAPPER" + +cd "/work/$VARIANT-llvm" +eval "$PATCH_COMMAND" + +cd /work/build + +eval "$CMAKE_COMMAND" + +eval ninja "$VERBOSE" +eval ninja "$VERBOSE" install diff --git a/docker_nametag b/docker_nametag index 2f65926..aae6e89 100644 --- a/docker_nametag +++ b/docker_nametag @@ -1 +1 @@ -swiftnav/arm-llvm-obf:4.0-ubuntu1604-2018.05.01 +swiftnav/arm-llvm-obf:2018.06.10 diff --git a/publish.sh b/publish.sh index b8971f5..e69dfc9 100755 --- a/publish.sh +++ b/publish.sh @@ -12,38 +12,40 @@ # # Script for publishing built binaries to S3. -set -e - -if [ "$TRAVIS_OS_NAME" != "linux" ]; then - exit -fi +set -xe REPO="${PWD##*/}" BUCKET="${BUCKET:-llvm-obfuscator-arm}" -BUILD_VERSION="$(git describe --tags --dirty --always)" +if [[ -z "$BUILD_VERSION" ]]; then + BUILD_VERSION="$(git describe --tags --always --dirty)" +fi + BUILD_PATH="$REPO/$BUILD_VERSION" + if [[ ! -z "$PRODUCT_VERSION" ]]; then BUILD_PATH="$BUILD_PATH/$PRODUCT_VERSION" fi + if [[ ! -z "$PRODUCT_REV" ]]; then BUILD_PATH="$BUILD_PATH/$PRODUCT_REV" fi + if [[ ! -z "$PRODUCT_TYPE" ]]; then BUILD_PATH="$BUILD_PATH/$PRODUCT_TYPE" fi echo "Uploading $* to $BUILD_PATH" -echo "Publish PULL_REQUEST ($TRAVIS_PULL_REQUEST)" -echo "Publish BRANCH ($TRAVIS_BRANCH)" -echo "Publish TAG ($TRAVIS_TAG)" + +[[ -n "$TRAVIS_PULL_REQUEST" ]] || echo "Publish PULL_REQUEST ($TRAVIS_PULL_REQUEST)" +[[ -n "$TRAVIS_BRANCH" ]] || echo "Publish BRANCH ($TRAVIS_BRANCH)" +[[ -n "$TRAVIS_TAG" ]] || echo "Publish TAG ($TRAVIS_TAG)" for file in "$@"; do KEY="$BUILD_PATH/$(basename "$file")" OBJECT="s3://$BUCKET/$KEY" if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then if [[ "$TRAVIS_BRANCH" == master || "$TRAVIS_TAG" == v* || "$TRAVIS_BRANCH" == v*-release ]]; then - OBJECT="s3://$BUCKET/$KEY" aws s3 cp "$file" "$OBJECT" fi else diff --git a/pull_ccache.bash b/pull_ccache.bash new file mode 100755 index 0000000..8ea1e52 --- /dev/null +++ b/pull_ccache.bash @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +# Copyright (C) 2018 Swift Navigation Inc. +# Contact: Swift Navigation +# +# This source is subject to the license found in the file 'LICENSE' which must +# be be distributed together with this source. All other rights reserved. +# +# THIS CODE AND INFORMATION IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND, +# EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR PURPOSE. + +set -euo pipefail +IFS=$'\n\t' + +DOCKER_NAMETAG=$(cat docker_nametag) + +while [[ $# -gt 0 ]]; do + case $1 in + + --arch=x86) ARCH="X86"; shift ;; + --arch=arm) ARCH="ARM"; shift ;; + --arch=arm,x86) ARCH="ARM\\;X86"; shift ;; + --arch=x86,arm) ARCH="ARM\\;X86"; shift ;; + + --variant=vanilla) VARIANT="vanilla"; shift ;; + --variant=obfuscator) VARIANT="obfuscator"; shift ;; + + *) shift ;; + esac +done + +if [[ -z "${ARCH:-}" ]]; then + echo "Error: must specify --arch=" + exit 1 +fi + +if [[ -z "${VARIANT:-}" ]]; then + echo "Error: must a variant to --variant=" + exit 1 +fi + +BUILD_VERSION="$(cat build_tag)" +ARCH="${ARCH//\\;/-}" + +CCACHE_ARCHIVE="ccache-${VARIANT}-${ARCH}-${BUILD_VERSION}.tbz2" + +./s3_download.bash "${CCACHE_ARCHIVE}" + +docker run --rm \ + -v "$PWD:/this_dir" \ + -v $VARIANT-llvm-ccache:/work/ccache \ + "$DOCKER_NAMETAG-$VARIANT" \ + /bin/bash -c "tar -xjf /this_dir/${CCACHE_ARCHIVE} -C /work/ccache ." diff --git a/push.bash b/push.bash new file mode 100755 index 0000000..981c506 --- /dev/null +++ b/push.bash @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +# Copyright (C) 2017 Swift Navigation Inc. +# Contact: Swift Navigation +# +# This source is subject to the license found in the file 'LICENSE' which must +# be be distributed together with this source. All other rights reserved. +# +# THIS CODE AND INFORMATION IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND, +# EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR PURPOSE. + +set -euo pipefail +IFS=$'\n\t' + +[[ -z "${DEBUG:-}" ]] || set -x + +DOCKER_NAMETAG=$(cat docker_nametag) + +if [[ -n "${DOCKER_USER:-}" ]] && [[ -n "${DOCKER_PASS:-}" ]]; then + echo "$DOCKER_PASS" | docker login --username="$DOCKER_USER" --password-stdin + docker push "$DOCKER_NAMETAG" + docker push "$DOCKER_NAMETAG-vanilla" + docker push "$DOCKER_NAMETAG-obfuscator" +else + echo "WARNING: not pushing new image to Docker Hub..." >&2 +fi diff --git a/push_ccache.bash b/push_ccache.bash new file mode 100755 index 0000000..26ec494 --- /dev/null +++ b/push_ccache.bash @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +# Copyright (C) 2018 Swift Navigation Inc. +# Contact: Swift Navigation +# +# This source is subject to the license found in the file 'LICENSE' which must +# be be distributed together with this source. All other rights reserved. +# +# THIS CODE AND INFORMATION IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND, +# EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR PURPOSE. + +set -euo pipefail +IFS=$'\n\t' + +[[ -z "${DEBUG:-}" ]] || set -x + +DOCKER_NAMETAG=$(cat docker_nametag) + +while [[ $# -gt 0 ]]; do + case $1 in + + --arch=x86) ARCH="X86"; shift ;; + --arch=arm) ARCH="ARM"; shift ;; + --arch=arm,x86) ARCH="ARM\\;X86"; shift ;; + --arch=x86,arm) ARCH="ARM\\;X86"; shift ;; + + --variant=vanilla) VARIANT="vanilla"; shift ;; + --variant=obfuscator) VARIANT="obfuscator"; shift ;; + + *) shift ;; + esac +done + +if [[ -z "${ARCH:-}" ]]; then + echo "Error: must specify --arch=" + exit 1 +fi + +if [[ -z "${VARIANT:-}" ]]; then + echo "Error: must a variant to build --variant=" + exit 1 +fi + +BUILD_VERSION="$(cat build_tag)" +ARCH="${ARCH//\\;/-}" + +CCACHE_ARCHIVE="ccache-${VARIANT}-${ARCH}-${BUILD_VERSION}.tbz2" + +docker run -it --rm \ + -v "$PWD:/this_dir" \ + -v $VARIANT-llvm-ccache:/work/ccache \ + "$DOCKER_NAMETAG-$VARIANT" \ + /bin/bash -c "tar -cjf /this_dir/${CCACHE_ARCHIVE} -C /work/ccache ." + +BUILD_VERSION="${BUILD_VERSION}" ./publish.sh "${CCACHE_ARCHIVE}" diff --git a/run_build_shell.bash b/run_build_shell.bash index acd6f67..c7fba89 100755 --- a/run_build_shell.bash +++ b/run_build_shell.bash @@ -13,6 +13,19 @@ set -euo pipefail IFS=$'\n\t' +while [[ $# -gt 0 ]]; do + case $1 in + --variant=vanilla) VARIANT="vanilla"; shift ;; + --variant=obfuscator)VARIANT="obfuscator"; shift ;; + *) shift ;; + esac +done + +if [[ -z "${VARIANT:-}" ]]; then + echo "Error: must a variant --variant=" + exit 1 +fi + DOCKER_NAMETAG=$(cat docker_nametag) BR2_TOOLCHAIN_LD_LIBRARY_PATH=/toolchain/x86/x86_64-buildroot-linux-gnu/lib64 @@ -24,10 +37,12 @@ docker run -i -t --rm \ -v "$PWD/bin:/wrapper-bin" \ -v "$PWD/patches:/patches" \ -v "$PWD:/this_dir" \ - -v obfuscator-llvm:/work/obfuscator-llvm \ - -v obfuscator-llvm-build:/work/build \ - "$DOCKER_NAMETAG" \ - /bin/bash -c "export PATH=/opt/llvm-obfuscator/bin:/opt/llvm-obfuscator/wrappers/bin:\$PATH; \ + -v $VARIANT-llvm-ccache:/work/ccache \ + -v $VARIANT-llvm:/work/$VARIANT-llvm \ + -v $VARIANT-llvm-build:/work/build \ + -e CCACHE_DIR=/work/ccache \ + "$DOCKER_NAMETAG-$VARIANT" \ + /bin/bash -c "export PATH=/opt/llvm-$VARIANT/bin:/opt/llvm-$VARIANT/wrappers/bin:\$PATH; \ cp -v /this_dir/cpp_wrapper.c /work/cpp_wrapper.c \ && gcc -std=c99 -O3 -Wall /work/cpp_wrapper.c -o /bin/cpp_wrapper; \ export BR2_TOOLCHAIN_PATH=/toolchain/arm; \ diff --git a/s3_download.bash b/s3_download.bash new file mode 100755 index 0000000..43670d5 --- /dev/null +++ b/s3_download.bash @@ -0,0 +1,39 @@ +#!/bin/bash + +# Copyright (C) 2016 Swift Navigation Inc. +# Contact: Fergus Noble +# +# This source is subject to the license found in the file 'LICENSE' which must +# be be distributed together with this source. All other rights reserved. +# +# THIS CODE AND INFORMATION IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND, +# EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR PURPOSE. +# +# Script for downloading firmware and NAP binaries from S3 to be incorporated +# into the Linux image. + +set -xe + +REPO="${PWD##*/}" +BUCKET="${BUCKET:-llvm-obfuscator-arm}" + +BUILD_VERSION="$(cat build_tag)" +BUILD_PATH="$REPO/$BUILD_VERSION" +if [[ ! -z "$PRODUCT_VERSION" ]]; then + BUILD_PATH="$BUILD_PATH/$PRODUCT_VERSION" +fi +if [[ ! -z "$PRODUCT_REV" ]]; then + BUILD_PATH="$BUILD_PATH/$PRODUCT_REV" +fi +if [[ ! -z "$PRODUCT_TYPE" ]]; then + BUILD_PATH="$BUILD_PATH/$PRODUCT_TYPE" +fi + +echo "Downloading $* to $BUILD_PATH" + +for file in "$@"; do + KEY="$BUILD_PATH/$(basename "$file")" + OBJECT="s3://$BUCKET/$KEY" + aws s3 cp "$OBJECT" "$file" +done diff --git a/stage_sysroot.bash b/stage_sysroot.bash index 867319e..2f205a0 100755 --- a/stage_sysroot.bash +++ b/stage_sysroot.bash @@ -28,32 +28,37 @@ if [[ -z "${DOCKERCEPTION:-}" ]]; then while [[ $# -gt 0 ]]; do case $1 in - --no-tty) - export INTERACTIVE=() - shift - ;; + --no-tty) export INTERACTIVE=(); shift ;; + --variant=vanilla) VARIANT="vanilla"; shift ;; + --variant=obfuscator)VARIANT="obfuscator"; shift ;; *) shift ;; esac done fi +if [[ -z "${VARIANT:-}" ]]; then + echo "Error: must a variant --variant=" + exit 1 +fi + stage_sysroot() { - OUT=/opt/llvm-obfuscator/sysroot + OUT=/opt/llvm-$VARIANT/sysroot rm -rf "$OUT" mkdir -p "$OUT" - WRAPPERS_BIN=/opt/llvm-obfuscator/wrappers/bin + WRAPPERS_BIN=/opt/llvm-$VARIANT/wrappers/bin mkdir -p $WRAPPERS_BIN rsync -asv '--exclude=.*.sw?' /this_dir/bin/ $WRAPPERS_BIN/ - LICENSE=/opt/llvm-obfuscator/ + LICENSE=/opt/llvm-$VARIANT/ cp -v /this_dir/LICENSE $LICENSE mkdir -p "${OUT}/buildroot" - cp -v /this_dir/toolchainfile.cmake "${OUT}/buildroot" + m4 "-DM4_VARIANT=${VARIANT}" /this_dir/toolchainfile.cmake.m4 \ + >"${OUT}/buildroot/toolchainfile.cmake" } run() { @@ -65,10 +70,11 @@ run() { -v "$PWD/example:/work/example" \ -v "$PWD/output/opt:/opt" \ -v "$PWD:/this_dir" \ - -v obfuscator-llvm:/work/obfuscator-llvm \ - -v obfuscator-llvm-build:/work/build \ + -v $VARIANT-llvm:/work/$VARIANT-llvm \ + -v $VARIANT-llvm-build:/work/build \ -e DOCKERCEPTION=1 \ - "$DOCKER_NAMETAG" \ + -e "VARIANT=$VARIANT" \ + "$DOCKER_NAMETAG-$VARIANT" \ /bin/bash -c ". /this_dir/stage_sysroot.bash; stage_sysroot" } diff --git a/toolchainfile.cmake b/toolchainfile.cmake.m4 similarity index 85% rename from toolchainfile.cmake rename to toolchainfile.cmake.m4 index 685173d..43ded08 100644 --- a/toolchainfile.cmake +++ b/toolchainfile.cmake.m4 @@ -8,7 +8,7 @@ # and store it in RELOCATED_HOST_DIR. # All the other variables that need to refer to HOST_DIR will use the # RELOCATED_HOST_DIR variable. -string(REPLACE "/opt/llvm-obfuscator/buildroot" "" RELOCATED_HOST_DIR ${CMAKE_CURRENT_LIST_DIR}) +string(REPLACE "/opt/llvm-M4_VARIANT/buildroot" "" RELOCATED_HOST_DIR ${CMAKE_CURRENT_LIST_DIR}) set(CMAKE_SYSTEM_NAME Linux) set(CMAKE_SYSTEM_PROCESSOR armv7l) @@ -44,18 +44,18 @@ set(CMAKE_EXE_LINKER_FLAGS "" CACHE STRING "Buildroot LDFLAGS for executables") set(CMAKE_INSTALL_SO_NO_EXE 0) -set(CMAKE_PROGRAM_PATH "${RELOCATED_HOST_DIR}/opt/llvm-obfuscator/bin") -set(CMAKE_SYSROOT "${RELOCATED_HOST_DIR}/opt/llvm-obfuscator/sysroot") -set(CMAKE_FIND_ROOT_PATH "${RELOCATED_HOST_DIR}/opt/llvm-obfuscator/sysroot") +set(CMAKE_PROGRAM_PATH "${RELOCATED_HOST_DIR}/opt/llvm-M4_VARIANT/bin") +set(CMAKE_SYSROOT "${RELOCATED_HOST_DIR}/opt/llvm-M4_VARIANT/sysroot") +set(CMAKE_FIND_ROOT_PATH "${RELOCATED_HOST_DIR}/opt/llvm-M4_VARIANT/sysroot") set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) -set(ENV{PKG_CONFIG_SYSROOT_DIR} "${RELOCATED_HOST_DIR}/opt/llvm-obfuscator/sysroot") +set(ENV{PKG_CONFIG_SYSROOT_DIR} "${RELOCATED_HOST_DIR}/opt/llvm-M4_VARIANT/sysroot") # This toolchain file can be used both inside and outside Buildroot. -set(CMAKE_C_COMPILER "${RELOCATED_HOST_DIR}/opt/llvm-obfuscator/wrappers/bin/arm-linux-gnueabihf-clang") -set(CMAKE_CXX_COMPILER "${RELOCATED_HOST_DIR}/opt/llvm-obfuscator/wrappers/bin/arm-linux-gnueabihf-clang++") +set(CMAKE_C_COMPILER "${RELOCATED_HOST_DIR}/opt/llvm-M4_VARIANT/wrappers/bin/arm-linux-gnueabihf-clang") +set(CMAKE_CXX_COMPILER "${RELOCATED_HOST_DIR}/opt/llvm-M4_VARIANT/wrappers/bin/arm-linux-gnueabihf-clang++") #if(1) # set(CMAKE_Fortran_FLAGS_DEBUG "" CACHE STRING "Debug Fortran FLAGS") # set(CMAKE_Fortran_FLAGS_RELEASE " -DNDEBUG" CACHE STRING "Release Fortran FLAGS")