From b6fac4c15d3c6330bf10144740beb236506be3cb Mon Sep 17 00:00:00 2001 From: Jason Mobarak Date: Sat, 9 Jun 2018 23:48:25 -0700 Subject: [PATCH 01/15] Support for building vanilla clang 6 cross compiler --- .travis.sh | 13 ++++++- .travis.yml | 10 +++--- Dockerfile | 11 +++++- Makefile | 13 +++++-- build.bash | 85 +++++++++++++++++++++++++++----------------- do_clang_build.bash | 67 ++++++++++++++++++++++++++++++++++ docker_nametag | 2 +- run_build_shell.bash | 19 ++++++++-- stage_sysroot.bash | 22 +++++++----- 9 files changed, 186 insertions(+), 56 deletions(-) create mode 100755 do_clang_build.bash diff --git a/.travis.sh b/.travis.sh index e08696b..65de7aa 100755 --- a/.travis.sh +++ b/.travis.sh @@ -43,7 +43,7 @@ fi echo 'Running build of llvm-obfuscator... ' -make "ARCH=$ARCH" NO_TTY=y build &>/tmp/build.bash.log & +make "ARCH=$ARCH" NO_TTY=y VARIANT=obfuscator build &>/tmp/build.obfuscator.log & BUILD_PID=$! wait $BUILD_PID @@ -59,3 +59,14 @@ if [[ $ARCH == *arm* ]]; then echo 'DONE.' fi + +make clean-obfuscator + +echo 'Running build of llvm-vanilla... ' + +make "ARCH=$ARCH" NO_TTY=y VARIANT=vanilla build &>/tmp/build.vanilla.log & +BUILD_PID=$! + +wait $BUILD_PID + +echo 'DONE.' diff --git a/.travis.yml b/.travis.yml index 7a3cac3..fd20605 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,15 +19,13 @@ script: - ./.travis.sh after_success: - - travis_wait tar -C output -cJf "llvm-obfuscator-$ARCH.txz" . - - ./publish.sh llvm-obfuscator-$ARCH.txz + - travis_wait tar -C output -cJf "llvm-buildroot-$ARCH.txz" . + - ./publish.sh llvm-buildroot-$ARCH.txz after_failure: - - if [[ -e /tmp/base.bash.log ]]; then echo "*** base.bash.log ***"; fi - if [[ -e /tmp/base.bash.log ]]; then tail -n 500 /tmp/base.bash.log; fi - - if [[ -e /tmp/build.bash.log ]]; then echo "*** build.bash.log ***"; fi - - if [[ -e /tmp/build.bash.log ]]; then tail -n 500 /tmp/build.bash.log; fi - - if [[ -e /tmp/build_example.bash.log ]]; then echo "*** build_example.bash.log ***"; fi + - if [[ -e /tmp/build.obfuscator.log ]]; then tail -n 500 /tmp/build.obfuscator.log; fi + - if [[ -e /tmp/build.vanilla.log ]]; then tail -n 500 /tmp/build.vanilla.log; fi - if [[ -e /tmp/build_example.bash.log ]]; then tail -n 500 /tmp/build_example.bash.log; fi env: diff --git a/Dockerfile b/Dockerfile index d24e6cc..d17e8e9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,7 +24,7 @@ RUN apt-get update \ && echo "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-4.0 main" \ >/etc/apt/sources.list.d/llvm40.list \ && apt-get update \ - && apt-get upgrade \ + && apt-get upgrade -y \ && apt-get install -y libcurl4-openssl-dev \ build-essential \ bison \ @@ -34,8 +34,17 @@ RUN apt-get update \ llvm-4.0-dev \ clang-4.0 \ git \ + m4 \ + gperf \ + gawk \ + ncurses-dev \ + texinfo \ + help2man \ binutils-dev \ + libpthread-stubs0-dev \ + libtinfo-dev \ python \ + python-dev \ binutils-multiarch-dev \ g++-4.8-arm-linux-gnueabihf \ gcc-4.8-arm-linux-gnueabihf \ diff --git a/Makefile b/Makefile index d672ceb..74422dd 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,7 @@ SCRIPTS := .travis.sh \ publish.sh \ run_build_shell.bash \ stage_sysroot.bash \ + do_clang_build.bash \ all: check base build @@ -27,7 +28,7 @@ base: check-base $(CURDIR)/base.bash $(NO_TTY_ARG) build: check-build - $(CURDIR)/build.bash $(NO_TTY_ARG) --arch=$(ARCH) + $(CURDIR)/build.bash $(NO_TTY_ARG) --arch=$(ARCH) --variant=$(VARIANT) stage: check-stage_sysroot $(CURDIR)/stage_sysroot.bash $(NO_TTY_ARG) @@ -38,7 +39,15 @@ build-example: check-build_example run: check-run_build_shell $(CURDIR)/run_build_shell.bash $(NO_TTY_ARG) -clean: +clean-vanilla: + docker volume rm vanilla-llvm-build || : + docker volume rm vanilla-llvm || : + +clean-obfuscator: docker volume rm obfuscator-llvm-build || : docker volume rm obfuscator-llvm || : + +clean-build: clean-vanilla clean-obfuscator + +clean: clean-build sudo rm -rf output/* diff --git a/build.bash b/build.bash index b397469..cc0ba1c 100755 --- a/build.bash +++ b/build.bash @@ -23,42 +23,65 @@ NO_TTY= while [[ $# -gt 0 ]]; do case $1 in - --arch=x86) ARCH="X86"; shift ;; - --arch=arm) ARCH="ARM"; shift ;; - --arch=arm,x86) ARCH="ARM\\;X86"; shift ;; - --arch=x86,arm) ARCH="X86\\;ARM"; shift ;; + --arch=x86) ARCH="X86"; shift ;; + --arch=arm) ARCH="ARM"; shift ;; + --arch=arm,x86) ARCH="ARM\\;X86"; shift ;; + --arch=x86,arm) ARCH="X86\\;ARM"; shift ;; - --verbose) VERBOSE="-v"; shift ;; - --no-tty) NO_TTY=--no-tty; shift ;; + --variant=vanilla) VARIANT="vanilla"; shift ;; + --variant=obfuscator) VARIANT="obfuscator"; shift ;; - *) shift ;; + --verbose) VERBOSE="-v"; shift ;; + --no-tty) NO_TTY=--no-tty; shift ;; + + *) shift ;; esac done if [[ -z "${ARCH:-}" ]]; then - echo "Error: must specify --arch=" + echo "Error: must specify --arch=" + exit 1 +fi + +if [[ -z "${VARIANT:-}" ]]; then + echo "Error: must a variant to build --variant=" exit 1 fi set -x -# -DLLVM_BUILD_TOOLS:BOOL=FALSE \ + +if [[ "$VARIANT" == "obfuscator" ]]; then + LLVM_REPO="https://github.com/obfuscator-llvm/obfuscator.git" + LLVM_BRANCH="llvm-4.0" + CLANG_REPO="" + CLANG_TOOLS_EXTRA_REPO="" + CPP_WRAPPER_DEFINE="-DCMAKE_CXX_COMPILER=/bin/cpp_wrapper" + PATCH_COMMAND="{ git apply /patches/*.patch || : ; }" + COMPILE_CPP_WRAPPER="cp -v /this_dir/cpp_wrapper.c /work/cpp_wrapper.c \ + && gcc -std=c99 -O3 -Wall /work/cpp_wrapper.c -o /bin/cpp_wrapper" +else + LLVM_REPO="https://github.com/llvm-mirror/llvm.git" + CLANG_REPO="https://github.com/llvm-mirror/clang.git" + CLANG_TOOLS_EXTRA_REPO="https://github.com/llvm-mirror/clang-tools-extra.git" + LLVM_BRANCH="release_60" + CPP_WRAPPER_DEFINE="-DCMAKE_CXX_COMPILER=/toolchain/x86/bin/x86_64-linux-g++" + PATCH_COMMAND="true" + COMPILE_CPP_WRAPPER="true" +fi CMAKE_COMMAND="\ cmake -G Ninja \ - /work/obfuscator-llvm \ - -DCMAKE_INSTALL_PREFIX=/opt/llvm-obfuscator \ + /work/$VARIANT-llvm \ + -DCMAKE_INSTALL_PREFIX=/opt/llvm-$VARIANT \ -DLLVM_TARGETS_TO_BUILD=$ARCH \ - -DCMAKE_CXX_FLAGS='-DENDIAN_LITTLE=1 -I/toolchain/x86/lib/gcc/x86_64-buildroot-linux-gnu/6.4.0/plugin/include' \ + -DCMAKE_CXX_FLAGS='-DENDIAN_LITTLE=1 -L/toolchain/x86/x86_64-buildroot-linux-gnu/sysroot/lib -L/toolchain/x86/x86_64-buildroot-linux-gnu/sysroot/usr/lib -I/toolchain/x86/lib/gcc/x86_64-buildroot-linux-gnu/6.4.0/plugin/include' \ + $CPP_WRAPPER_DEFINE \ -DCMAKE_C_COMPILER=/toolchain/x86/bin/x86_64-linux-gcc \ - -DCMAKE_CXX_COMPILER=/bin/cpp_wrapper \ -DCMAKE_BUILD_TYPE=Release \ -DLLVM_BINUTILS_INCDIR=/usr/include \ - -DLLDB_DISABLE_CURSES:BOOL=TRUE \ -DLLVM_ENABLE_TERMINFO=0 \ -DLLVM_INCLUDE_TESTS=OFF" -PATCH_COMMAND="{ git apply /patches/*.patch || : ; }" - if [[ -z "$NO_TTY" ]]; then INTERACTIVE=("-i" "-t") else @@ -70,23 +93,19 @@ docker run ${INTERACTIVE[@]:-} --rm \ -v "$PWD/output/opt:/opt" \ -v "$PWD/patches:/patches" \ -v "$PWD:/this_dir" \ - -v obfuscator-llvm:/work/obfuscator-llvm \ - -v obfuscator-llvm-build:/work/build \ + -v $VARIANT-llvm:/work/$VARIANT-llvm \ + -v $VARIANT-llvm-build:/work/build \ + -e VARIANT=$VARIANT -e ARCH=$ARCH \ + -e VERBOSE=$VERBOSE -e NO_TTY=$NO_TTY \ + -e CPP_WRAPPER_DEFINE=$CPP_WRAPPER_DEFINE \ + -e CMAKE_COMMAND="$CMAKE_COMMAND" \ + -e LLVM_REPO=$LLVM_REPO \ + -e LLVM_BRANCH=$LLVM_BRANCH \ + -e CLANG_REPO=$CLANG_REPO \ + -e CLANG_TOOLS_EXTRA_REPO=$CLANG_TOOLS_EXTRA_REPO \ + -e PATCH_COMMAND=$PATCH_COMMAND \ + -e COMPILE_CPP_WRAPPER=$COMPILE_CPP_WRAPPER \ "$DOCKER_NAMETAG" \ - /bin/bash -c "if [ ! -d /work/obfuscator-llvm/.git ]; then \ - git clone --depth=1 --single-branch -b llvm-4.0 \ - https://github.com/obfuscator-llvm/obfuscator.git \ - obfuscator-llvm; - else \ - (cd /work/obfuscator-llvm && git pull); \ - fi \ - && cp -v /this_dir/cpp_wrapper.c /work/cpp_wrapper.c \ - && gcc -std=c99 -O3 -Wall /work/cpp_wrapper.c -o /bin/cpp_wrapper \ - && cd /work/obfuscator-llvm \ - && $PATCH_COMMAND \ - && cd /work/build \ - && $CMAKE_COMMAND \ - && ninja $VERBOSE \ - && ninja $VERBOSE install" + /bin/bash -c "/this_dir/do_clang_build.bash" ./stage_sysroot.bash $NO_TTY diff --git a/do_clang_build.bash b/do_clang_build.bash new file mode 100755 index 0000000..695ba44 --- /dev/null +++ b/do_clang_build.bash @@ -0,0 +1,67 @@ +#!/usr/bin/env bash + +set -euo pipefail +IFS=$'\n\t' + +die_non_empty() + { echo "*** ERROR: variable $1 must be defined and non-empty" >&2; exit 1; } + +die_not_defined() + { echo "*** ERROR: variable $1 must be defined (can be empty)" >&2; exit 1; } + +[[ -n "${VARIANT}" ]] || die_not_defined + +[[ "${VARIANT}" == "obfuscator" ]] || [[ "${VARIANT}" == "vanilla" ]] || \ + { echo "*** ERROR: invalid variant (must be one of 'vanilla', or 'obfuscator')" >&2; exit 1; } + +[[ -n "${LLVM_REPO}" ]] || die_non_empty LLVM_REPO +[[ -n "${LLVM_BRANCH}" ]] || die_non_empty LLVM_BRANCH +[[ -n "${CMAKE_COMMAND}" ]] || die_non_empty CMAKE_COMMAND + +[[ -n "${CLANG_REPO+x}" ]] || die_not_defined CLANG_REPO +[[ -n "${CLANG_TOOLS_EXTRA_REPO+x}" ]] || die_not_defined CLANG_TOOLS_EXTRA_REPO + +[[ -n "${COMPILE_CPP_WRAPPER+x}" ]] || die_not_defined COMPILE_CPP_WRAPPER +[[ -n "${PATCH_COMMAND+x}" ]] || die_not_defined PATCH_COMMAND + +if [ ! -d "/work/$VARIANT-llvm/.git" ]; then + git clone --depth=1 --single-branch -b "$LLVM_BRANCH" \ + "$LLVM_REPO" "$VARIANT-llvm"; +else + (cd "/work/$VARIANT-llvm" && git pull); +fi + +if [ -n "${CLANG_REPO:-}" ]; then + if [ ! -d "/work/$VARIANT-llvm/tools/clang/.git" ]; then + git clone --depth=1 --single-branch -b "$LLVM_BRANCH" \ + "$CLANG_REPO" "$VARIANT-llvm/tools/clang"; + else + (cd "/work/$VARIANT-llvm/tools/clang" && git pull); + fi +fi + +if [ -n "${CLANG_TOOLS_EXTRA_REPO:-}" ]; then \ + if [ ! -d "/work/$VARIANT-llvm/tools/clang-tools-extra/.git" ]; then + git clone --depth=1 --single-branch -b "$LLVM_BRANCH" \ + "$CLANG_REPO" "$VARIANT-llvm/tools/clang-tools-extra"; + else + (cd "/work/$VARIANT-llvm/tools/clang-tools-extra" && git pull); + fi +fi + +eval "$COMPILE_CPP_WRAPPER" + +cd "/work/$VARIANT-llvm" +eval "$PATCH_COMMAND" + +cd /work/build + +eval "$CMAKE_COMMAND" + +if [[ -n "$VERBOSE" ]]; then + ninja -v + ninja -v install +else + ninja + ninja install +fi diff --git a/docker_nametag b/docker_nametag index 2f65926..271f7df 100644 --- a/docker_nametag +++ b/docker_nametag @@ -1 +1 @@ -swiftnav/arm-llvm-obf:4.0-ubuntu1604-2018.05.01 +swiftnav/arm-llvm-obf:4.0-ubuntu1604-2018.06.08 diff --git a/run_build_shell.bash b/run_build_shell.bash index acd6f67..8b29eb3 100755 --- a/run_build_shell.bash +++ b/run_build_shell.bash @@ -13,6 +13,19 @@ set -euo pipefail IFS=$'\n\t' +while [[ $# -gt 0 ]]; do + case $1 in + --variant=vanilla) VARIANT="vanilla"; shift ;; + --variant=obfuscator)VARIANT="obfuscator"; shift ;; + *) shift ;; + esac +done + +if [[ -z "${VARIANT:-}" ]]; then + echo "Error: must a variant --variant=" + exit 1 +fi + DOCKER_NAMETAG=$(cat docker_nametag) BR2_TOOLCHAIN_LD_LIBRARY_PATH=/toolchain/x86/x86_64-buildroot-linux-gnu/lib64 @@ -24,10 +37,10 @@ docker run -i -t --rm \ -v "$PWD/bin:/wrapper-bin" \ -v "$PWD/patches:/patches" \ -v "$PWD:/this_dir" \ - -v obfuscator-llvm:/work/obfuscator-llvm \ - -v obfuscator-llvm-build:/work/build \ + -v $VARIANT-llvm:/work/$VARIANT-llvm \ + -v $VARIANT-llvm-build:/work/build \ "$DOCKER_NAMETAG" \ - /bin/bash -c "export PATH=/opt/llvm-obfuscator/bin:/opt/llvm-obfuscator/wrappers/bin:\$PATH; \ + /bin/bash -c "export PATH=/opt/llvm-$VARIANT/bin:/opt/llvm-$VARIANT/wrappers/bin:\$PATH; \ cp -v /this_dir/cpp_wrapper.c /work/cpp_wrapper.c \ && gcc -std=c99 -O3 -Wall /work/cpp_wrapper.c -o /bin/cpp_wrapper; \ export BR2_TOOLCHAIN_PATH=/toolchain/arm; \ diff --git a/stage_sysroot.bash b/stage_sysroot.bash index 867319e..1dc44f8 100755 --- a/stage_sysroot.bash +++ b/stage_sysroot.bash @@ -28,28 +28,32 @@ if [[ -z "${DOCKERCEPTION:-}" ]]; then while [[ $# -gt 0 ]]; do case $1 in - --no-tty) - export INTERACTIVE=() - shift - ;; + --no-tty) export INTERACTIVE=(); shift ;; + --variant=vanilla) VARIANT="vanilla"; shift ;; + --variant=obfuscator)VARIANT="obfuscator"; shift ;; *) shift ;; esac done fi +if [[ -z "${VARIANT:-}" ]]; then + echo "Error: must a variant --variant=" + exit 1 +fi + stage_sysroot() { - OUT=/opt/llvm-obfuscator/sysroot + OUT=/opt/llvm-$VARIANT/sysroot rm -rf "$OUT" mkdir -p "$OUT" - WRAPPERS_BIN=/opt/llvm-obfuscator/wrappers/bin + WRAPPERS_BIN=/opt/llvm-$VARIANT/wrappers/bin mkdir -p $WRAPPERS_BIN rsync -asv '--exclude=.*.sw?' /this_dir/bin/ $WRAPPERS_BIN/ - LICENSE=/opt/llvm-obfuscator/ + LICENSE=/opt/llvm-$VARIANT/ cp -v /this_dir/LICENSE $LICENSE mkdir -p "${OUT}/buildroot" @@ -65,8 +69,8 @@ run() { -v "$PWD/example:/work/example" \ -v "$PWD/output/opt:/opt" \ -v "$PWD:/this_dir" \ - -v obfuscator-llvm:/work/obfuscator-llvm \ - -v obfuscator-llvm-build:/work/build \ + -v $VARIANT-llvm:/work/$VARIANT-llvm \ + -v $VARIANT-llvm-build:/work/build \ -e DOCKERCEPTION=1 \ "$DOCKER_NAMETAG" \ /bin/bash -c ". /this_dir/stage_sysroot.bash; stage_sysroot" From c913f3f3be1fa890ae228cd3857b516cd43386e6 Mon Sep 17 00:00:00 2001 From: Jason Mobarak Date: Sun, 10 Jun 2018 11:16:22 -0700 Subject: [PATCH 02/15] m4 toolchainfile --- stage_sysroot.bash | 3 ++- toolchainfile.cmake => toolchainfile.cmake.m4 | 14 +++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) rename toolchainfile.cmake => toolchainfile.cmake.m4 (85%) diff --git a/stage_sysroot.bash b/stage_sysroot.bash index 1dc44f8..1908900 100755 --- a/stage_sysroot.bash +++ b/stage_sysroot.bash @@ -57,7 +57,8 @@ stage_sysroot() { cp -v /this_dir/LICENSE $LICENSE mkdir -p "${OUT}/buildroot" - cp -v /this_dir/toolchainfile.cmake "${OUT}/buildroot" + m4 "-DM4_VARIANT=${VARIANT}" /this_dir/toolchainfile.cmake.m4 \ + >"${OUT}/buildroot/toolchainfile.cmake" } run() { diff --git a/toolchainfile.cmake b/toolchainfile.cmake.m4 similarity index 85% rename from toolchainfile.cmake rename to toolchainfile.cmake.m4 index 685173d..43ded08 100644 --- a/toolchainfile.cmake +++ b/toolchainfile.cmake.m4 @@ -8,7 +8,7 @@ # and store it in RELOCATED_HOST_DIR. # All the other variables that need to refer to HOST_DIR will use the # RELOCATED_HOST_DIR variable. -string(REPLACE "/opt/llvm-obfuscator/buildroot" "" RELOCATED_HOST_DIR ${CMAKE_CURRENT_LIST_DIR}) +string(REPLACE "/opt/llvm-M4_VARIANT/buildroot" "" RELOCATED_HOST_DIR ${CMAKE_CURRENT_LIST_DIR}) set(CMAKE_SYSTEM_NAME Linux) set(CMAKE_SYSTEM_PROCESSOR armv7l) @@ -44,18 +44,18 @@ set(CMAKE_EXE_LINKER_FLAGS "" CACHE STRING "Buildroot LDFLAGS for executables") set(CMAKE_INSTALL_SO_NO_EXE 0) -set(CMAKE_PROGRAM_PATH "${RELOCATED_HOST_DIR}/opt/llvm-obfuscator/bin") -set(CMAKE_SYSROOT "${RELOCATED_HOST_DIR}/opt/llvm-obfuscator/sysroot") -set(CMAKE_FIND_ROOT_PATH "${RELOCATED_HOST_DIR}/opt/llvm-obfuscator/sysroot") +set(CMAKE_PROGRAM_PATH "${RELOCATED_HOST_DIR}/opt/llvm-M4_VARIANT/bin") +set(CMAKE_SYSROOT "${RELOCATED_HOST_DIR}/opt/llvm-M4_VARIANT/sysroot") +set(CMAKE_FIND_ROOT_PATH "${RELOCATED_HOST_DIR}/opt/llvm-M4_VARIANT/sysroot") set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) -set(ENV{PKG_CONFIG_SYSROOT_DIR} "${RELOCATED_HOST_DIR}/opt/llvm-obfuscator/sysroot") +set(ENV{PKG_CONFIG_SYSROOT_DIR} "${RELOCATED_HOST_DIR}/opt/llvm-M4_VARIANT/sysroot") # This toolchain file can be used both inside and outside Buildroot. -set(CMAKE_C_COMPILER "${RELOCATED_HOST_DIR}/opt/llvm-obfuscator/wrappers/bin/arm-linux-gnueabihf-clang") -set(CMAKE_CXX_COMPILER "${RELOCATED_HOST_DIR}/opt/llvm-obfuscator/wrappers/bin/arm-linux-gnueabihf-clang++") +set(CMAKE_C_COMPILER "${RELOCATED_HOST_DIR}/opt/llvm-M4_VARIANT/wrappers/bin/arm-linux-gnueabihf-clang") +set(CMAKE_CXX_COMPILER "${RELOCATED_HOST_DIR}/opt/llvm-M4_VARIANT/wrappers/bin/arm-linux-gnueabihf-clang++") #if(1) # set(CMAKE_Fortran_FLAGS_DEBUG "" CACHE STRING "Debug Fortran FLAGS") # set(CMAKE_Fortran_FLAGS_RELEASE " -DNDEBUG" CACHE STRING "Release Fortran FLAGS") From 86b6d5a0935d0085e0d11e5bce19880b6efc9c67 Mon Sep 17 00:00:00 2001 From: Jason Mobarak Date: Sun, 10 Jun 2018 11:22:16 -0700 Subject: [PATCH 03/15] Add a build matrix --- .travis.sh | 17 +++-------------- .travis.yml | 49 ++++++++++++++++++++++++++++++++----------------- 2 files changed, 35 insertions(+), 31 deletions(-) diff --git a/.travis.sh b/.travis.sh index 65de7aa..6852554 100755 --- a/.travis.sh +++ b/.travis.sh @@ -41,9 +41,9 @@ fi ### build.bash -echo 'Running build of llvm-obfuscator... ' +echo "Running build of llvm-$VARIANT... " -make "ARCH=$ARCH" NO_TTY=y VARIANT=obfuscator build &>/tmp/build.obfuscator.log & +make "ARCH=$ARCH" NO_TTY=y VARIANT=$VARIANT build &>/tmp/build.bash.log & BUILD_PID=$! wait $BUILD_PID @@ -52,21 +52,10 @@ echo 'DONE.' ### build_example.bash -if [[ $ARCH == *arm* ]]; then +if [[ $ARCH == *arm* ]] && [[ $VARIANT == obfuscator ]]; then echo -n 'Building example project... ' make NO_TTY=y build-example &>/tmp/build_example.bash.log echo 'DONE.' fi - -make clean-obfuscator - -echo 'Running build of llvm-vanilla... ' - -make "ARCH=$ARCH" NO_TTY=y VARIANT=vanilla build &>/tmp/build.vanilla.log & -BUILD_PID=$! - -wait $BUILD_PID - -echo 'DONE.' diff --git a/.travis.yml b/.travis.yml index fd20605..f886aae 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,23 +10,38 @@ branches: services: - docker -before_install: - - pip install --user --upgrade awscli - - sudo apt-get update - - sudo apt-get install -y jq - -script: - - ./.travis.sh - -after_success: - - travis_wait tar -C output -cJf "llvm-buildroot-$ARCH.txz" . - - ./publish.sh llvm-buildroot-$ARCH.txz - -after_failure: - - if [[ -e /tmp/base.bash.log ]]; then tail -n 500 /tmp/base.bash.log; fi - - if [[ -e /tmp/build.obfuscator.log ]]; then tail -n 500 /tmp/build.obfuscator.log; fi - - if [[ -e /tmp/build.vanilla.log ]]; then tail -n 500 /tmp/build.vanilla.log; fi - - if [[ -e /tmp/build_example.bash.log ]]; then tail -n 500 /tmp/build_example.bash.log; fi +matrix: + include: + - env: + - VARIANT=obfuscator + before_install: + - pip install --user --upgrade awscli + - sudo apt-get update + - sudo apt-get install -y jq + script: + - VARIANT=$VARIANT ./.travis.sh + after_success: + - travis_wait tar -C output -cJf "llvm-$VARIANT-$ARCH.txz" . + - ./publish.sh llvm-$VARIANT-$ARCH.txz + after_failure: + - if [[ -e /tmp/base.bash.log ]]; then tail -n 500 /tmp/base.bash.log; fi + - if [[ -e /tmp/build.bash.log ]]; then tail -n 500 /tmp/build.bash.log; fi + - if [[ -e /tmp/build_example.bash.log ]]; then tail -n 500 /tmp/build_example.bash.log; fi + - env: + - VARIANT=vanilla + before_install: + - pip install --user --upgrade awscli + - sudo apt-get update + - sudo apt-get install -y jq + script: + - VARIANT=$VARIANT ./.travis.sh + after_success: + - travis_wait tar -C output -cJf "llvm-$VARIANT-$ARCH.txz" . + - ./publish.sh llvm-$VARIANT-$ARCH.txz + after_failure: + - if [[ -e /tmp/base.bash.log ]]; then tail -n 500 /tmp/base.bash.log; fi + - if [[ -e /tmp/build.bash.log ]]; then tail -n 500 /tmp/build.bash.log; fi + - if [[ -e /tmp/build_example.bash.log ]]; then tail -n 500 /tmp/build_example.bash.log; fi env: global: From 7a6ff20f2c21f1a14ccc20f24c1cb33246005526 Mon Sep 17 00:00:00 2001 From: Jason Mobarak Date: Sun, 10 Jun 2018 11:25:30 -0700 Subject: [PATCH 04/15] Set ARCH --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index f886aae..e9922ef 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,7 @@ matrix: include: - env: - VARIANT=obfuscator + - ARCH=x86,arm before_install: - pip install --user --upgrade awscli - sudo apt-get update @@ -29,6 +30,7 @@ matrix: - if [[ -e /tmp/build_example.bash.log ]]; then tail -n 500 /tmp/build_example.bash.log; fi - env: - VARIANT=vanilla + - ARCH=x86,arm before_install: - pip install --user --upgrade awscli - sudo apt-get update @@ -57,5 +59,3 @@ env: - secure: "T1wlLXyhfGnMNMPtHv2FspxsWG09hVZgaucSJ6IxN+PZBbgjLb/TMSlqqLXfIzx6s0SuQIzX1xZE8lb/uOzfLtjmxBdc4dGYLW+qR4AwkghQl2DGMK+hzGAi9ZbeCGKWtX7cN4WxLN1tpcTyQBCC8LcnP0xVxd+7Znytdezmlr86ce2d8dmUo+kyLdN+vhEUnch9zhMpuZI+TxV2HQqCS/6e31myP+3Xzw3B6o6uF8dYZ0QlWlNU4KjUNNAD7cLqkxNd38E7wFrQJdyrk03/hZanatl2mcL7Te5Uxo5sWN5WteTNwq8dCKXiEXpkEqP1++4wWTQDeQWoPjHlhHzO4MxGAdEUnlFWafsV78OJvr/gwV0UxDFmfadPwGG/xPPJMyNJjXwDyeztswknbVaNZdbRXp3m7Iri+bWCvOZE9N7rGde/5q0JOz9rwJXRDMPnp7+lyupHQQIdswNpkFraHz3eZ/h4n+Dkr3RtkW0M6ZpPR+/r18uD7jwm/Ye73E9wzq1b6sL0knX/OEwBv3FVvaBaA4SWzb1glDlru8BKnZfVIZ/UxYas6FqGN6H9ROyCHtva86ffv7HvvnB/gUqcOfY5si7WvtPVApIc9kfILcSWEchRWTHQIeUOdEUBfoq8/Km0BL6gguOO9Ge9zzOrdo0NThh8x7NoE0DhHdQctJY=" # DOCKER_PASS - secure: "qEmUT+mBsHjLkrBYMZWiLl8+ZXq9UAv+DyMDww+N74hPtGiZxMqLuVIFq94qR/WhiI9zd7k+tjhy6TcgD8rpKoQnQSGXcotFGq6typ3UWEV0ekt40IMV5nBqopqGGIBVIHNfXjq0OTeYTF2eGncpNnznJHAQi8Dvj7vD99y5GA5RoZGSdsiqTyxBGMvZ0cE0QApkb1MH3lBVRzBuh6dotpa1RBlGR+i2xbcH3nAqy5RB38wZgMedAPnlUfg+uB+IQiF8+hbuSmHzADiabNLBkkB/UCOz59k/JkaeN+2gnyP6/so+SpOe1BKfpxKBcN74O4n056ZLDBRjTod2YSc49dKqxhZhQijwNJfFi3Tt5iNrFYdU3KPeqqfNLoQKmbBTKjGXbxHHXm3lnUuCTZH0Yd9QGKSAwn5kxDRy6DuyNMUGwFtu8YxzirwTJiQR6ayodMCMWjZCXuxUc0I5I2Y64CsofRWkuMbznjE4wJwptVPzTwIxkZnMt+aWVoQTXBtZVpiHf4CTJb65dyLDXeMOwhYnKzwLUrvvHHpOm7GygvAbbeDAzHumTDvt7s5nl2VwHNLmTK3TizvvpQCucIjG8M8dEd7iu4VRZh/rxGxV9MJ2qUNETa6hnfX+Z2xaLTcgYAG5z7p60sThPfFfYLopdJL1dO5eA/EncPzuQuh087w=" - matrix: - - ARCH=x86,arm From 7c307a05c9a5f5abe67b80310c65b30c2a5b9b14 Mon Sep 17 00:00:00 2001 From: Jason Mobarak Date: Sun, 10 Jun 2018 11:30:18 -0700 Subject: [PATCH 05/15] Fix repo path --- .travis.sh | 2 +- do_clang_build.bash | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.sh b/.travis.sh index 6852554..eee7873 100755 --- a/.travis.sh +++ b/.travis.sh @@ -27,7 +27,7 @@ fi done )& TICKER_PID=$! -trap 'kill ${TICKER_PID:-} ${BUILD_PID:-}' EXIT +trap 'kill ${TICKER_PID:-} ${BUILD_PID:-} &>/dev/null' EXIT ### base.bash diff --git a/do_clang_build.bash b/do_clang_build.bash index 695ba44..0cb5dd0 100755 --- a/do_clang_build.bash +++ b/do_clang_build.bash @@ -43,7 +43,7 @@ fi if [ -n "${CLANG_TOOLS_EXTRA_REPO:-}" ]; then \ if [ ! -d "/work/$VARIANT-llvm/tools/clang-tools-extra/.git" ]; then git clone --depth=1 --single-branch -b "$LLVM_BRANCH" \ - "$CLANG_REPO" "$VARIANT-llvm/tools/clang-tools-extra"; + "$CLANG_TOOLS_EXTRA_REPO" "$VARIANT-llvm/tools/clang-tools-extra"; else (cd "/work/$VARIANT-llvm/tools/clang-tools-extra" && git pull); fi From b84a98f9ca044310f534160e0aed8b4c5651b0db Mon Sep 17 00:00:00 2001 From: Jason Mobarak Date: Sun, 10 Jun 2018 13:34:55 -0700 Subject: [PATCH 06/15] Make sure VARIANT is passed down --- .travis.sh | 2 +- build.bash | 2 +- stage_sysroot.bash | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.sh b/.travis.sh index eee7873..649b83b 100755 --- a/.travis.sh +++ b/.travis.sh @@ -43,7 +43,7 @@ fi echo "Running build of llvm-$VARIANT... " -make "ARCH=$ARCH" NO_TTY=y VARIANT=$VARIANT build &>/tmp/build.bash.log & +make NO_TTY=y "ARCH=$ARCH" "VARIANT=$VARIANT" build &>/tmp/build.bash.log & BUILD_PID=$! wait $BUILD_PID diff --git a/build.bash b/build.bash index cc0ba1c..04b6b9e 100755 --- a/build.bash +++ b/build.bash @@ -108,4 +108,4 @@ docker run ${INTERACTIVE[@]:-} --rm \ "$DOCKER_NAMETAG" \ /bin/bash -c "/this_dir/do_clang_build.bash" -./stage_sysroot.bash $NO_TTY +./stage_sysroot.bash $NO_TTY "--variant=$VARIANT" diff --git a/stage_sysroot.bash b/stage_sysroot.bash index 1908900..70dadac 100755 --- a/stage_sysroot.bash +++ b/stage_sysroot.bash @@ -73,6 +73,7 @@ run() { -v $VARIANT-llvm:/work/$VARIANT-llvm \ -v $VARIANT-llvm-build:/work/build \ -e DOCKERCEPTION=1 \ + -e "VARIANT=$VARIANT" \ "$DOCKER_NAMETAG" \ /bin/bash -c ". /this_dir/stage_sysroot.bash; stage_sysroot" } From e56f08449c0a99bbc1f0d700e8c6f306d5521770 Mon Sep 17 00:00:00 2001 From: Jason Mobarak Date: Sun, 10 Jun 2018 15:59:04 -0700 Subject: [PATCH 07/15] Debug travis failure --- .after_failure.sh | 21 +++++++++++++++++++++ .travis.sh | 2 +- .travis.yml | 9 +++------ 3 files changed, 25 insertions(+), 7 deletions(-) create mode 100755 .after_failure.sh diff --git a/.after_failure.sh b/.after_failure.sh new file mode 100755 index 0000000..b9283b2 --- /dev/null +++ b/.after_failure.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -x + +if [[ -e /tmp/base.bash.log ]]; then + cp /tmp/base.bash.log /tmp/$VARIANT-base.bash.log + ./publish.sh /tmp/$VARIANT-base.bash.log + tail -n 200 /tmp/base.bash.log +fi + +if [[ -e /tmp/build.bash.log ]]; then + cp /tmp/build.bash.log /tmp/$VARIANT-build.bash.log + ./publish.sh /tmp/$VARIANT-build.bash.log + tail -n 200 /tmp/build.bash.log; +fi + +if [[ -e /tmp/build_example.bash.log ]]; then + cp /tmp/build_example.bash.log /tmp/$VARIANT-build_example.bash.log + ./publish.sh /tmp/$VARIANT-build_example.bash.log + tail -n 200 /tmp/build_example.bash.log; +fi diff --git a/.travis.sh b/.travis.sh index 649b83b..6c20a9e 100755 --- a/.travis.sh +++ b/.travis.sh @@ -27,7 +27,7 @@ fi done )& TICKER_PID=$! -trap 'kill ${TICKER_PID:-} ${BUILD_PID:-} &>/dev/null' EXIT +trap 'kill ${TICKER_PID:-} ${BUILD_PID:-} &>/dev/null || :' EXIT ### base.bash diff --git a/.travis.yml b/.travis.yml index e9922ef..8d441ec 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,9 +25,8 @@ matrix: - travis_wait tar -C output -cJf "llvm-$VARIANT-$ARCH.txz" . - ./publish.sh llvm-$VARIANT-$ARCH.txz after_failure: - - if [[ -e /tmp/base.bash.log ]]; then tail -n 500 /tmp/base.bash.log; fi - - if [[ -e /tmp/build.bash.log ]]; then tail -n 500 /tmp/build.bash.log; fi - - if [[ -e /tmp/build_example.bash.log ]]; then tail -n 500 /tmp/build_example.bash.log; fi + - ./.after_failure.sh + - env: - VARIANT=vanilla - ARCH=x86,arm @@ -41,9 +40,7 @@ matrix: - travis_wait tar -C output -cJf "llvm-$VARIANT-$ARCH.txz" . - ./publish.sh llvm-$VARIANT-$ARCH.txz after_failure: - - if [[ -e /tmp/base.bash.log ]]; then tail -n 500 /tmp/base.bash.log; fi - - if [[ -e /tmp/build.bash.log ]]; then tail -n 500 /tmp/build.bash.log; fi - - if [[ -e /tmp/build_example.bash.log ]]; then tail -n 500 /tmp/build_example.bash.log; fi + - ./.after_failure.sh env: global: From 2fe5fd2a42d8dad8310a3d7205674552bc49cbb2 Mon Sep 17 00:00:00 2001 From: Jason Mobarak Date: Sun, 10 Jun 2018 21:59:33 -0700 Subject: [PATCH 08/15] Attempt to use ccache, stage correct llvm packages --- .travis.yml | 4 ++++ Dockerfile => Dockerfile.base | 13 ++++++------- Dockerfile.obfuscator | 14 ++++++++++++++ Dockerfile.vanilla | 14 ++++++++++++++ base.bash | 19 ++++++++++++------- build.bash | 16 +++++++++++++--- do_clang_build.bash | 31 ++++++++++++++++++++----------- docker_nametag | 2 +- push.bash | 27 +++++++++++++++++++++++++++ run_build_shell.bash | 2 +- stage_sysroot.bash | 2 +- 11 files changed, 113 insertions(+), 31 deletions(-) rename Dockerfile => Dockerfile.base (92%) create mode 100644 Dockerfile.obfuscator create mode 100644 Dockerfile.vanilla create mode 100755 push.bash diff --git a/.travis.yml b/.travis.yml index 8d441ec..f59dbd9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,10 @@ branches: services: - docker +cache: + directories: + - output/ccache + matrix: include: - env: diff --git a/Dockerfile b/Dockerfile.base similarity index 92% rename from Dockerfile rename to Dockerfile.base index d17e8e9..c78f88e 100644 --- a/Dockerfile +++ b/Dockerfile.base @@ -23,6 +23,8 @@ RUN apt-get update \ && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - \ && echo "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-4.0 main" \ >/etc/apt/sources.list.d/llvm40.list \ + && echo "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-6.0 main" \ + >/etc/apt/sources.list.d/llvm60.list \ && apt-get update \ && apt-get upgrade -y \ && apt-get install -y libcurl4-openssl-dev \ @@ -30,9 +32,6 @@ RUN apt-get update \ bison \ flex \ ninja-build \ - llvm-4.0 \ - llvm-4.0-dev \ - clang-4.0 \ git \ m4 \ gperf \ @@ -45,6 +44,7 @@ RUN apt-get update \ libtinfo-dev \ python \ python-dev \ + ccache \ binutils-multiarch-dev \ g++-4.8-arm-linux-gnueabihf \ gcc-4.8-arm-linux-gnueabihf \ @@ -66,10 +66,9 @@ RUN apt-get update \ && tar -xzf cmake-3.10.1.tar.gz \ && cd cmake-3.10.1 \ && ./configure \ - && make -j4 \ - && make install \ + && make -j8 \ + && make -j8 install \ && cd .. && rm -rf cmake-* \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* + && apt-get clean # EOF diff --git a/Dockerfile.obfuscator b/Dockerfile.obfuscator new file mode 100644 index 0000000..9383c1e --- /dev/null +++ b/Dockerfile.obfuscator @@ -0,0 +1,14 @@ +# Copyright (C) 2017-2018 Swift Navigation Inc. +# Contact: Swift Navigation +# +# This source is subject to the license found in the file 'LICENSE' which must +# be be distributed together with this source. All other rights reserved. +# +# THIS CODE AND INFORMATION IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND, +# EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR PURPOSE. + +ARG DOCKER_NAMETAG +FROM $DOCKER_NAMETAG + +RUN apt-get install -y llvm-4.0 llvm-4.0-dev clang-4.0 libclang-4.0-dev diff --git a/Dockerfile.vanilla b/Dockerfile.vanilla new file mode 100644 index 0000000..da0acd9 --- /dev/null +++ b/Dockerfile.vanilla @@ -0,0 +1,14 @@ +# Copyright (C) 2017-2018 Swift Navigation Inc. +# Contact: Swift Navigation +# +# This source is subject to the license found in the file 'LICENSE' which must +# be be distributed together with this source. All other rights reserved. +# +# THIS CODE AND INFORMATION IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND, +# EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR PURPOSE. + +ARG DOCKER_NAMETAG +FROM $DOCKER_NAMETAG + +RUN apt-get install -y llvm-6.0 llvm-6.0-dev clang-6.0 libclang-6.0-dev diff --git a/base.bash b/base.bash index d7cf8ef..8e5fd8d 100755 --- a/base.bash +++ b/base.bash @@ -56,11 +56,16 @@ fi docker build \ --force-rm --no-cache \ - -f Dockerfile -t "$DOCKER_NAMETAG" . + -f Dockerfile.base -t "$DOCKER_NAMETAG" . -if [[ -n "${DOCKER_USER:-}" ]] && [[ -n "${DOCKER_PASS:-}" ]]; then - echo "$DOCKER_PASS" | docker login --username="$DOCKER_USER" --password-stdin - docker push "$DOCKER_NAMETAG" -else - echo "WARNING: not pushing new image to Docker Hub..." >&2 -fi +docker build \ + --force-rm --no-cache \ + --build-arg "DOCKER_NAMETAG=$DOCKER_NAMETAG" \ + -f Dockerfile.vanilla -t "$DOCKER_NAMETAG-vanilla" . + +docker build \ + --force-rm --no-cache \ + --build-arg "DOCKER_NAMETAG=$DOCKER_NAMETAG" \ + -f Dockerfile.obfuscator -t "$DOCKER_NAMETAG-obfuscator" . + +./push.bash diff --git a/build.bash b/build.bash index 04b6b9e..9e9705d 100755 --- a/build.bash +++ b/build.bash @@ -16,6 +16,7 @@ IFS=$'\n\t' DOCKER_NAMETAG=$(cat docker_nametag) mkdir -p output/opt +mkdir -p output/ccache VERBOSE= NO_TTY= @@ -48,7 +49,9 @@ if [[ -z "${VARIANT:-}" ]]; then exit 1 fi -set -x +CXX_FLAGS="-L/toolchain/x86/x86_64-buildroot-linux-gnu/sysroot/lib \ + -L/toolchain/x86/x86_64-buildroot-linux-gnu/sysroot/usr/lib \ + -I/toolchain/x86/lib/gcc/x86_64-buildroot-linux-gnu/6.4.0/plugin/include" if [[ "$VARIANT" == "obfuscator" ]]; then LLVM_REPO="https://github.com/obfuscator-llvm/obfuscator.git" @@ -67,6 +70,8 @@ else CPP_WRAPPER_DEFINE="-DCMAKE_CXX_COMPILER=/toolchain/x86/bin/x86_64-linux-g++" PATCH_COMMAND="true" COMPILE_CPP_WRAPPER="true" + CXX_FLAGS+=" -I/work/$VARIANT-llvm/tools/clang/include" + CXX_FLAGS+=" -I/work/build/tools/clang/include" fi CMAKE_COMMAND="\ @@ -74,7 +79,8 @@ CMAKE_COMMAND="\ /work/$VARIANT-llvm \ -DCMAKE_INSTALL_PREFIX=/opt/llvm-$VARIANT \ -DLLVM_TARGETS_TO_BUILD=$ARCH \ - -DCMAKE_CXX_FLAGS='-DENDIAN_LITTLE=1 -L/toolchain/x86/x86_64-buildroot-linux-gnu/sysroot/lib -L/toolchain/x86/x86_64-buildroot-linux-gnu/sysroot/usr/lib -I/toolchain/x86/lib/gcc/x86_64-buildroot-linux-gnu/6.4.0/plugin/include' \ + -DCMAKE_CXX_FLAGS='-DENDIAN_LITTLE=1 $CXX_FLAGS' \ + -DLLVM_CCACHE_BUILD=ON \ $CPP_WRAPPER_DEFINE \ -DCMAKE_C_COMPILER=/toolchain/x86/bin/x86_64-linux-gcc \ -DCMAKE_BUILD_TYPE=Release \ @@ -88,11 +94,14 @@ else INTERACTIVE=() fi +# -v $VARIANT-llvm-ccache:/work/ccache \ + # shellcheck disable=SC2068 docker run ${INTERACTIVE[@]:-} --rm \ -v "$PWD/output/opt:/opt" \ -v "$PWD/patches:/patches" \ -v "$PWD:/this_dir" \ + -v "$PWD/output/ccache:/work/ccache" \ -v $VARIANT-llvm:/work/$VARIANT-llvm \ -v $VARIANT-llvm-build:/work/build \ -e VARIANT=$VARIANT -e ARCH=$ARCH \ @@ -105,7 +114,8 @@ docker run ${INTERACTIVE[@]:-} --rm \ -e CLANG_TOOLS_EXTRA_REPO=$CLANG_TOOLS_EXTRA_REPO \ -e PATCH_COMMAND=$PATCH_COMMAND \ -e COMPILE_CPP_WRAPPER=$COMPILE_CPP_WRAPPER \ - "$DOCKER_NAMETAG" \ + -e CCACHE_DIR=/work/ccache \ + "$DOCKER_NAMETAG-$VARIANT" \ /bin/bash -c "/this_dir/do_clang_build.bash" ./stage_sysroot.bash $NO_TTY "--variant=$VARIANT" diff --git a/do_clang_build.bash b/do_clang_build.bash index 0cb5dd0..953e142 100755 --- a/do_clang_build.bash +++ b/do_clang_build.bash @@ -1,5 +1,15 @@ #!/usr/bin/env bash +# Copyright (C) 2018 Swift Navigation Inc. +# Contact: Swift Navigation +# +# This source is subject to the license found in the file 'LICENSE' which must +# be be distributed together with this source. All other rights reserved. +# +# THIS CODE AND INFORMATION IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND, +# EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR PURPOSE. + set -euo pipefail IFS=$'\n\t' @@ -9,14 +19,15 @@ die_non_empty() die_not_defined() { echo "*** ERROR: variable $1 must be defined (can be empty)" >&2; exit 1; } -[[ -n "${VARIANT}" ]] || die_not_defined +[[ -n "${VARIANT:-}" ]] || die_non_empty [[ "${VARIANT}" == "obfuscator" ]] || [[ "${VARIANT}" == "vanilla" ]] || \ { echo "*** ERROR: invalid variant (must be one of 'vanilla', or 'obfuscator')" >&2; exit 1; } -[[ -n "${LLVM_REPO}" ]] || die_non_empty LLVM_REPO -[[ -n "${LLVM_BRANCH}" ]] || die_non_empty LLVM_BRANCH -[[ -n "${CMAKE_COMMAND}" ]] || die_non_empty CMAKE_COMMAND +[[ -n "${LLVM_REPO:-}" ]] || die_non_empty LLVM_REPO +[[ -n "${LLVM_BRANCH:-}" ]] || die_non_empty LLVM_BRANCH +[[ -n "${CMAKE_COMMAND:-}" ]] || die_non_empty CMAKE_COMMAND +[[ -n "${CCACHE_DIR:-}" ]] || die_non_empty CCACHE_DIR [[ -n "${CLANG_REPO+x}" ]] || die_not_defined CLANG_REPO [[ -n "${CLANG_TOOLS_EXTRA_REPO+x}" ]] || die_not_defined CLANG_TOOLS_EXTRA_REPO @@ -49,6 +60,9 @@ if [ -n "${CLANG_TOOLS_EXTRA_REPO:-}" ]; then \ fi fi +export CCACHE_DIR=$CCACHE_DIR +echo "CCACHE_DIR: $CCACHE_DIR" + eval "$COMPILE_CPP_WRAPPER" cd "/work/$VARIANT-llvm" @@ -58,10 +72,5 @@ cd /work/build eval "$CMAKE_COMMAND" -if [[ -n "$VERBOSE" ]]; then - ninja -v - ninja -v install -else - ninja - ninja install -fi +eval ninja "$VERBOSE" +eval ninja "$VERBOSE" install diff --git a/docker_nametag b/docker_nametag index 271f7df..aae6e89 100644 --- a/docker_nametag +++ b/docker_nametag @@ -1 +1 @@ -swiftnav/arm-llvm-obf:4.0-ubuntu1604-2018.06.08 +swiftnav/arm-llvm-obf:2018.06.10 diff --git a/push.bash b/push.bash new file mode 100755 index 0000000..981c506 --- /dev/null +++ b/push.bash @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +# Copyright (C) 2017 Swift Navigation Inc. +# Contact: Swift Navigation +# +# This source is subject to the license found in the file 'LICENSE' which must +# be be distributed together with this source. All other rights reserved. +# +# THIS CODE AND INFORMATION IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND, +# EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR PURPOSE. + +set -euo pipefail +IFS=$'\n\t' + +[[ -z "${DEBUG:-}" ]] || set -x + +DOCKER_NAMETAG=$(cat docker_nametag) + +if [[ -n "${DOCKER_USER:-}" ]] && [[ -n "${DOCKER_PASS:-}" ]]; then + echo "$DOCKER_PASS" | docker login --username="$DOCKER_USER" --password-stdin + docker push "$DOCKER_NAMETAG" + docker push "$DOCKER_NAMETAG-vanilla" + docker push "$DOCKER_NAMETAG-obfuscator" +else + echo "WARNING: not pushing new image to Docker Hub..." >&2 +fi diff --git a/run_build_shell.bash b/run_build_shell.bash index 8b29eb3..8cdc600 100755 --- a/run_build_shell.bash +++ b/run_build_shell.bash @@ -39,7 +39,7 @@ docker run -i -t --rm \ -v "$PWD:/this_dir" \ -v $VARIANT-llvm:/work/$VARIANT-llvm \ -v $VARIANT-llvm-build:/work/build \ - "$DOCKER_NAMETAG" \ + "$DOCKER_NAMETAG-$VARIANT" \ /bin/bash -c "export PATH=/opt/llvm-$VARIANT/bin:/opt/llvm-$VARIANT/wrappers/bin:\$PATH; \ cp -v /this_dir/cpp_wrapper.c /work/cpp_wrapper.c \ && gcc -std=c99 -O3 -Wall /work/cpp_wrapper.c -o /bin/cpp_wrapper; \ diff --git a/stage_sysroot.bash b/stage_sysroot.bash index 70dadac..2f205a0 100755 --- a/stage_sysroot.bash +++ b/stage_sysroot.bash @@ -74,7 +74,7 @@ run() { -v $VARIANT-llvm-build:/work/build \ -e DOCKERCEPTION=1 \ -e "VARIANT=$VARIANT" \ - "$DOCKER_NAMETAG" \ + "$DOCKER_NAMETAG-$VARIANT" \ /bin/bash -c ". /this_dir/stage_sysroot.bash; stage_sysroot" } From af610b3b98a4d22bcddf7028ad653b134d087abc Mon Sep 17 00:00:00 2001 From: Jason Mobarak Date: Mon, 11 Jun 2018 14:01:12 -0700 Subject: [PATCH 09/15] Pull ccache from S3 --- .travis.yml | 5 ++-- Makefile | 16 ++++++++++-- build.bash | 7 ++---- describe_repo.bash | 2 ++ publish.sh | 3 +-- pull_ccache.bash | 55 +++++++++++++++++++++++++++++++++++++++++ push_ccache.bash | 59 ++++++++++++++++++++++++++++++++++++++++++++ run_build_shell.bash | 2 ++ s3_download.bash | 39 +++++++++++++++++++++++++++++ 9 files changed, 177 insertions(+), 11 deletions(-) create mode 100755 describe_repo.bash create mode 100755 pull_ccache.bash create mode 100755 push_ccache.bash create mode 100755 s3_download.bash diff --git a/.travis.yml b/.travis.yml index f59dbd9..592770b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ matrix: include: - env: - VARIANT=obfuscator - - ARCH=x86,arm + - ARCH=arm,x86 before_install: - pip install --user --upgrade awscli - sudo apt-get update @@ -33,12 +33,13 @@ matrix: - env: - VARIANT=vanilla - - ARCH=x86,arm + - ARCH=arm,x86 before_install: - pip install --user --upgrade awscli - sudo apt-get update - sudo apt-get install -y jq script: + - ./pull_ccache.bash --variant=$VARIANT --arch=$ARCH - VARIANT=$VARIANT ./.travis.sh after_success: - travis_wait tar -C output -cJf "llvm-$VARIANT-$ARCH.txz" . diff --git a/Makefile b/Makefile index 74422dd..2da33ad 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,8 @@ SCRIPTS := .travis.sh \ run_build_shell.bash \ stage_sysroot.bash \ do_clang_build.bash \ + push_ccache.bash \ + pull_ccache.bash \ all: check base build @@ -39,14 +41,24 @@ build-example: check-build_example run: check-run_build_shell $(CURDIR)/run_build_shell.bash $(NO_TTY_ARG) -clean-vanilla: +clean-vanilla-build: docker volume rm vanilla-llvm-build || : + docker volume rm vanilla-llvm-ccache || : + +clean-vanilla-src: docker volume rm vanilla-llvm || : -clean-obfuscator: +clean-vanilla: clean-vanilla-build clean-vanilla-src + +clean-obfuscator-build: docker volume rm obfuscator-llvm-build || : + docker volume rm obfuscator-llvm-ccache || : + +clean-obfuscator-src: docker volume rm obfuscator-llvm || : +clean-obfuscator: clean-obfuscator-build clean-obfuscator-src + clean-build: clean-vanilla clean-obfuscator clean: clean-build diff --git a/build.bash b/build.bash index 9e9705d..485646c 100755 --- a/build.bash +++ b/build.bash @@ -16,7 +16,6 @@ IFS=$'\n\t' DOCKER_NAMETAG=$(cat docker_nametag) mkdir -p output/opt -mkdir -p output/ccache VERBOSE= NO_TTY= @@ -27,7 +26,7 @@ while [[ $# -gt 0 ]]; do --arch=x86) ARCH="X86"; shift ;; --arch=arm) ARCH="ARM"; shift ;; --arch=arm,x86) ARCH="ARM\\;X86"; shift ;; - --arch=x86,arm) ARCH="X86\\;ARM"; shift ;; + --arch=x86,arm) ARCH="ARM\\;X86"; shift ;; --variant=vanilla) VARIANT="vanilla"; shift ;; --variant=obfuscator) VARIANT="obfuscator"; shift ;; @@ -94,14 +93,12 @@ else INTERACTIVE=() fi -# -v $VARIANT-llvm-ccache:/work/ccache \ - # shellcheck disable=SC2068 docker run ${INTERACTIVE[@]:-} --rm \ -v "$PWD/output/opt:/opt" \ -v "$PWD/patches:/patches" \ -v "$PWD:/this_dir" \ - -v "$PWD/output/ccache:/work/ccache" \ + -v $VARIANT-llvm-ccache:/work/ccache \ -v $VARIANT-llvm:/work/$VARIANT-llvm \ -v $VARIANT-llvm-build:/work/build \ -e VARIANT=$VARIANT -e ARCH=$ARCH \ diff --git a/describe_repo.bash b/describe_repo.bash new file mode 100755 index 0000000..328cd4b --- /dev/null +++ b/describe_repo.bash @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +git describe --tags --always --dirty diff --git a/publish.sh b/publish.sh index b8971f5..9affda5 100755 --- a/publish.sh +++ b/publish.sh @@ -21,7 +21,7 @@ fi REPO="${PWD##*/}" BUCKET="${BUCKET:-llvm-obfuscator-arm}" -BUILD_VERSION="$(git describe --tags --dirty --always)" +BUILD_VERSION="$(./describe_repo.bash)" BUILD_PATH="$REPO/$BUILD_VERSION" if [[ ! -z "$PRODUCT_VERSION" ]]; then BUILD_PATH="$BUILD_PATH/$PRODUCT_VERSION" @@ -43,7 +43,6 @@ for file in "$@"; do OBJECT="s3://$BUCKET/$KEY" if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then if [[ "$TRAVIS_BRANCH" == master || "$TRAVIS_TAG" == v* || "$TRAVIS_BRANCH" == v*-release ]]; then - OBJECT="s3://$BUCKET/$KEY" aws s3 cp "$file" "$OBJECT" fi else diff --git a/pull_ccache.bash b/pull_ccache.bash new file mode 100755 index 0000000..3d2ba48 --- /dev/null +++ b/pull_ccache.bash @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +# Copyright (C) 2018 Swift Navigation Inc. +# Contact: Swift Navigation +# +# This source is subject to the license found in the file 'LICENSE' which must +# be be distributed together with this source. All other rights reserved. +# +# THIS CODE AND INFORMATION IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND, +# EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR PURPOSE. + +set -euo pipefail +IFS=$'\n\t' + +while [[ $# -gt 0 ]]; do + case $1 in + + --arch=x86) ARCH="X86"; shift ;; + --arch=arm) ARCH="ARM"; shift ;; + --arch=arm,x86) ARCH="ARM\\;X86"; shift ;; + --arch=x86,arm) ARCH="ARM\\;X86"; shift ;; + + --variant=vanilla) VARIANT="vanilla"; shift ;; + --variant=obfuscator) VARIANT="obfuscator"; shift ;; + + --verbose) VERBOSE="-v"; shift ;; + --no-tty) NO_TTY=--no-tty; shift ;; + + *) shift ;; + esac +done + +if [[ -z "${ARCH:-}" ]]; then + echo "Error: must specify --arch=" + exit 1 +fi + +if [[ -z "${VARIANT:-}" ]]; then + echo "Error: must a variant to --variant=" + exit 1 +fi + +BUILD_VERSION="$(./describe_repo.bash)" +ARCH="$(echo ${ARCH} | sed 's@\\;@-@g')" + +CCACHE_ARCHIVE="ccache-${VARIANT}-${ARCH}-${BUILD_VERSION}.tbz2" + +./s3_download.bash ${CCACHE_ARCHIVE} + +docker run ${INTERACTIVE[@]:-} --rm \ + -v "$PWD:/this_dir" \ + -v $VARIANT-llvm-ccache:/work/ccache \ + "$DOCKER_NAMETAG-$VARIANT" \ + /bin/bash -c "tar -xjf ${CCACHE_ARCHIVE} -C /work/ccache ." diff --git a/push_ccache.bash b/push_ccache.bash new file mode 100755 index 0000000..dd5d7e6 --- /dev/null +++ b/push_ccache.bash @@ -0,0 +1,59 @@ +#!/usr/bin/env bash + +# Copyright (C) 2018 Swift Navigation Inc. +# Contact: Swift Navigation +# +# This source is subject to the license found in the file 'LICENSE' which must +# be be distributed together with this source. All other rights reserved. +# +# THIS CODE AND INFORMATION IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND, +# EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR PURPOSE. + +set -euo pipefail +IFS=$'\n\t' + +[[ -z "${DEBUG:-}" ]] || set -x + +DOCKER_NAMETAG=$(cat docker_nametag) + +while [[ $# -gt 0 ]]; do + case $1 in + + --arch=x86) ARCH="X86"; shift ;; + --arch=arm) ARCH="ARM"; shift ;; + --arch=arm,x86) ARCH="ARM\\;X86"; shift ;; + --arch=x86,arm) ARCH="ARM\\;X86"; shift ;; + + --variant=vanilla) VARIANT="vanilla"; shift ;; + --variant=obfuscator) VARIANT="obfuscator"; shift ;; + + --verbose) VERBOSE="-v"; shift ;; + --no-tty) NO_TTY=--no-tty; shift ;; + + *) shift ;; + esac +done + +if [[ -z "${ARCH:-}" ]]; then + echo "Error: must specify --arch=" + exit 1 +fi + +if [[ -z "${VARIANT:-}" ]]; then + echo "Error: must a variant to build --variant=" + exit 1 +fi + +BUILD_VERSION="$(./describe_repo.bash)" +ARCH="$(echo ${ARCH} | sed 's@\\;@-@g')" + +CCACHE_ARCHIVE="ccache-${VARIANT}-${ARCH}-${BUILD_VERSION}.tbz2" + +docker run -it --rm \ + -v "$PWD:/this_dir" \ + -v $VARIANT-llvm-ccache:/work/ccache \ + "$DOCKER_NAMETAG-$VARIANT" \ + /bin/bash -c "tar -cjf ${CCACHE_ARCHIVE} -C /work/ccache ." + +./publish.sh "${CCACHE_ARCHIVE}" diff --git a/run_build_shell.bash b/run_build_shell.bash index 8cdc600..c7fba89 100755 --- a/run_build_shell.bash +++ b/run_build_shell.bash @@ -37,8 +37,10 @@ docker run -i -t --rm \ -v "$PWD/bin:/wrapper-bin" \ -v "$PWD/patches:/patches" \ -v "$PWD:/this_dir" \ + -v $VARIANT-llvm-ccache:/work/ccache \ -v $VARIANT-llvm:/work/$VARIANT-llvm \ -v $VARIANT-llvm-build:/work/build \ + -e CCACHE_DIR=/work/ccache \ "$DOCKER_NAMETAG-$VARIANT" \ /bin/bash -c "export PATH=/opt/llvm-$VARIANT/bin:/opt/llvm-$VARIANT/wrappers/bin:\$PATH; \ cp -v /this_dir/cpp_wrapper.c /work/cpp_wrapper.c \ diff --git a/s3_download.bash b/s3_download.bash new file mode 100755 index 0000000..7814ff2 --- /dev/null +++ b/s3_download.bash @@ -0,0 +1,39 @@ +#!/bin/bash + +# Copyright (C) 2016 Swift Navigation Inc. +# Contact: Fergus Noble +# +# This source is subject to the license found in the file 'LICENSE' which must +# be be distributed together with this source. All other rights reserved. +# +# THIS CODE AND INFORMATION IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND, +# EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR PURPOSE. +# +# Script for downloading firmware and NAP binaries from S3 to be incorporated +# into the Linux image. + +set -xe + +REPO="${PWD##*/}" +BUCKET="${BUCKET:-llvm-obfuscator-arm}" + +BUILD_VERSION="$(describe_repo.bash)" +BUILD_PATH="$REPO/$BUILD_VERSION" +if [[ ! -z "$PRODUCT_VERSION" ]]; then + BUILD_PATH="$BUILD_PATH/$PRODUCT_VERSION" +fi +if [[ ! -z "$PRODUCT_REV" ]]; then + BUILD_PATH="$BUILD_PATH/$PRODUCT_REV" +fi +if [[ ! -z "$PRODUCT_TYPE" ]]; then + BUILD_PATH="$BUILD_PATH/$PRODUCT_TYPE" +fi + +echo "Downloading $* to $BUILD_PATH" + +for file in "$@"; do + KEY="$BUILD_PATH/$(basename "$file")" + OBJECT="s3://$BUCKET/$KEY" + aws s3 cp "$OBJECT" "$file" +done From ee5a1e9c7e0ea5f43f85a5c39c82c13371553c86 Mon Sep 17 00:00:00 2001 From: Jason Mobarak Date: Mon, 11 Jun 2018 14:39:36 -0700 Subject: [PATCH 10/15] Remove travis check --- publish.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/publish.sh b/publish.sh index 9affda5..c34c8af 100755 --- a/publish.sh +++ b/publish.sh @@ -12,11 +12,7 @@ # # Script for publishing built binaries to S3. -set -e - -if [ "$TRAVIS_OS_NAME" != "linux" ]; then - exit -fi +set -xe REPO="${PWD##*/}" BUCKET="${BUCKET:-llvm-obfuscator-arm}" From bc1a3bffd3f143543d40d28729866f2d1181f71c Mon Sep 17 00:00:00 2001 From: Jason Mobarak Date: Mon, 11 Jun 2018 15:06:02 -0700 Subject: [PATCH 11/15] Reference this_dir --- pull_ccache.bash | 2 +- push_ccache.bash | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pull_ccache.bash b/pull_ccache.bash index 3d2ba48..db15f96 100755 --- a/pull_ccache.bash +++ b/pull_ccache.bash @@ -52,4 +52,4 @@ docker run ${INTERACTIVE[@]:-} --rm \ -v "$PWD:/this_dir" \ -v $VARIANT-llvm-ccache:/work/ccache \ "$DOCKER_NAMETAG-$VARIANT" \ - /bin/bash -c "tar -xjf ${CCACHE_ARCHIVE} -C /work/ccache ." + /bin/bash -c "tar -xjf /this_dir/${CCACHE_ARCHIVE} -C /work/ccache ." diff --git a/push_ccache.bash b/push_ccache.bash index dd5d7e6..0fb68ed 100755 --- a/push_ccache.bash +++ b/push_ccache.bash @@ -54,6 +54,6 @@ docker run -it --rm \ -v "$PWD:/this_dir" \ -v $VARIANT-llvm-ccache:/work/ccache \ "$DOCKER_NAMETAG-$VARIANT" \ - /bin/bash -c "tar -cjf ${CCACHE_ARCHIVE} -C /work/ccache ." + /bin/bash -c "tar -cjf /this_dir/${CCACHE_ARCHIVE} -C /work/ccache ." ./publish.sh "${CCACHE_ARCHIVE}" From dd45e0db92e9fc60895014af403cb67bc0417ddf Mon Sep 17 00:00:00 2001 From: Jason Mobarak Date: Mon, 11 Jun 2018 16:49:45 -0700 Subject: [PATCH 12/15] Bugs --- pull_ccache.bash | 2 ++ s3_download.bash | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pull_ccache.bash b/pull_ccache.bash index db15f96..875d072 100755 --- a/pull_ccache.bash +++ b/pull_ccache.bash @@ -13,6 +13,8 @@ set -euo pipefail IFS=$'\n\t' +DOCKER_NAMETAG=$(cat docker_nametag) + while [[ $# -gt 0 ]]; do case $1 in diff --git a/s3_download.bash b/s3_download.bash index 7814ff2..f60773b 100755 --- a/s3_download.bash +++ b/s3_download.bash @@ -18,7 +18,7 @@ set -xe REPO="${PWD##*/}" BUCKET="${BUCKET:-llvm-obfuscator-arm}" -BUILD_VERSION="$(describe_repo.bash)" +BUILD_VERSION="$(./describe_repo.bash)" BUILD_PATH="$REPO/$BUILD_VERSION" if [[ ! -z "$PRODUCT_VERSION" ]]; then BUILD_PATH="$BUILD_PATH/$PRODUCT_VERSION" From d0c3e41ea27b00176ad07720c305870a5dfc1691 Mon Sep 17 00:00:00 2001 From: Jason Mobarak Date: Mon, 11 Jun 2018 18:28:02 -0700 Subject: [PATCH 13/15] Use a tag to store the ccache blob --- Makefile | 2 ++ describe_repo.bash | 2 -- most_recent_tag.bash | 2 ++ publish.sh | 15 +++++++++++---- pull_ccache.bash | 11 ++++------- push_ccache.bash | 9 +++------ s3_download.bash | 2 +- 7 files changed, 23 insertions(+), 20 deletions(-) delete mode 100755 describe_repo.bash create mode 100755 most_recent_tag.bash diff --git a/Makefile b/Makefile index 2da33ad..4fa44ba 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,8 @@ SCRIPTS := .travis.sh \ do_clang_build.bash \ push_ccache.bash \ pull_ccache.bash \ + most_recent_tag.bash \ + s3_download.bash \ all: check base build diff --git a/describe_repo.bash b/describe_repo.bash deleted file mode 100755 index 328cd4b..0000000 --- a/describe_repo.bash +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env bash -git describe --tags --always --dirty diff --git a/most_recent_tag.bash b/most_recent_tag.bash new file mode 100755 index 0000000..2e2205f --- /dev/null +++ b/most_recent_tag.bash @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +git describe --tags --abbrev=0 diff --git a/publish.sh b/publish.sh index c34c8af..e69dfc9 100755 --- a/publish.sh +++ b/publish.sh @@ -17,22 +17,29 @@ set -xe REPO="${PWD##*/}" BUCKET="${BUCKET:-llvm-obfuscator-arm}" -BUILD_VERSION="$(./describe_repo.bash)" +if [[ -z "$BUILD_VERSION" ]]; then + BUILD_VERSION="$(git describe --tags --always --dirty)" +fi + BUILD_PATH="$REPO/$BUILD_VERSION" + if [[ ! -z "$PRODUCT_VERSION" ]]; then BUILD_PATH="$BUILD_PATH/$PRODUCT_VERSION" fi + if [[ ! -z "$PRODUCT_REV" ]]; then BUILD_PATH="$BUILD_PATH/$PRODUCT_REV" fi + if [[ ! -z "$PRODUCT_TYPE" ]]; then BUILD_PATH="$BUILD_PATH/$PRODUCT_TYPE" fi echo "Uploading $* to $BUILD_PATH" -echo "Publish PULL_REQUEST ($TRAVIS_PULL_REQUEST)" -echo "Publish BRANCH ($TRAVIS_BRANCH)" -echo "Publish TAG ($TRAVIS_TAG)" + +[[ -n "$TRAVIS_PULL_REQUEST" ]] || echo "Publish PULL_REQUEST ($TRAVIS_PULL_REQUEST)" +[[ -n "$TRAVIS_BRANCH" ]] || echo "Publish BRANCH ($TRAVIS_BRANCH)" +[[ -n "$TRAVIS_TAG" ]] || echo "Publish TAG ($TRAVIS_TAG)" for file in "$@"; do KEY="$BUILD_PATH/$(basename "$file")" diff --git a/pull_ccache.bash b/pull_ccache.bash index 875d072..3111349 100755 --- a/pull_ccache.bash +++ b/pull_ccache.bash @@ -26,9 +26,6 @@ while [[ $# -gt 0 ]]; do --variant=vanilla) VARIANT="vanilla"; shift ;; --variant=obfuscator) VARIANT="obfuscator"; shift ;; - --verbose) VERBOSE="-v"; shift ;; - --no-tty) NO_TTY=--no-tty; shift ;; - *) shift ;; esac done @@ -43,14 +40,14 @@ if [[ -z "${VARIANT:-}" ]]; then exit 1 fi -BUILD_VERSION="$(./describe_repo.bash)" -ARCH="$(echo ${ARCH} | sed 's@\\;@-@g')" +BUILD_VERSION="$(./most_recent_tag.bash)" +ARCH="${ARCH//\\;/-}" CCACHE_ARCHIVE="ccache-${VARIANT}-${ARCH}-${BUILD_VERSION}.tbz2" -./s3_download.bash ${CCACHE_ARCHIVE} +./s3_download.bash "${CCACHE_ARCHIVE}" -docker run ${INTERACTIVE[@]:-} --rm \ +docker run --rm \ -v "$PWD:/this_dir" \ -v $VARIANT-llvm-ccache:/work/ccache \ "$DOCKER_NAMETAG-$VARIANT" \ diff --git a/push_ccache.bash b/push_ccache.bash index 0fb68ed..6145a10 100755 --- a/push_ccache.bash +++ b/push_ccache.bash @@ -28,9 +28,6 @@ while [[ $# -gt 0 ]]; do --variant=vanilla) VARIANT="vanilla"; shift ;; --variant=obfuscator) VARIANT="obfuscator"; shift ;; - --verbose) VERBOSE="-v"; shift ;; - --no-tty) NO_TTY=--no-tty; shift ;; - *) shift ;; esac done @@ -45,8 +42,8 @@ if [[ -z "${VARIANT:-}" ]]; then exit 1 fi -BUILD_VERSION="$(./describe_repo.bash)" -ARCH="$(echo ${ARCH} | sed 's@\\;@-@g')" +BUILD_VERSION="$(./most_recent_tag.bash)" +ARCH="${ARCH//\\;/-}" CCACHE_ARCHIVE="ccache-${VARIANT}-${ARCH}-${BUILD_VERSION}.tbz2" @@ -56,4 +53,4 @@ docker run -it --rm \ "$DOCKER_NAMETAG-$VARIANT" \ /bin/bash -c "tar -cjf /this_dir/${CCACHE_ARCHIVE} -C /work/ccache ." -./publish.sh "${CCACHE_ARCHIVE}" +BUILD_VERSION="${BUILD_VERSION}" ./publish.sh "${CCACHE_ARCHIVE}" diff --git a/s3_download.bash b/s3_download.bash index f60773b..c09dbd4 100755 --- a/s3_download.bash +++ b/s3_download.bash @@ -18,7 +18,7 @@ set -xe REPO="${PWD##*/}" BUCKET="${BUCKET:-llvm-obfuscator-arm}" -BUILD_VERSION="$(./describe_repo.bash)" +BUILD_VERSION="$(./most_recent_tag.bash)" BUILD_PATH="$REPO/$BUILD_VERSION" if [[ ! -z "$PRODUCT_VERSION" ]]; then BUILD_PATH="$BUILD_PATH/$PRODUCT_VERSION" From abd42151d325ec2b2f846cb5c42e56bec8009ffd Mon Sep 17 00:00:00 2001 From: Jason Mobarak Date: Mon, 11 Jun 2018 21:41:24 -0700 Subject: [PATCH 14/15] Stop fighting with travis/github --- Makefile | 1 - build_tag | 1 + most_recent_tag.bash | 2 -- pull_ccache.bash | 2 +- push_ccache.bash | 2 +- s3_download.bash | 2 +- 6 files changed, 4 insertions(+), 6 deletions(-) create mode 100644 build_tag delete mode 100755 most_recent_tag.bash diff --git a/Makefile b/Makefile index 4fa44ba..043ae39 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,6 @@ SCRIPTS := .travis.sh \ do_clang_build.bash \ push_ccache.bash \ pull_ccache.bash \ - most_recent_tag.bash \ s3_download.bash \ all: check base build diff --git a/build_tag b/build_tag new file mode 100644 index 0000000..703a257 --- /dev/null +++ b/build_tag @@ -0,0 +1 @@ +v17 diff --git a/most_recent_tag.bash b/most_recent_tag.bash deleted file mode 100755 index 2e2205f..0000000 --- a/most_recent_tag.bash +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env bash -git describe --tags --abbrev=0 diff --git a/pull_ccache.bash b/pull_ccache.bash index 3111349..8ea1e52 100755 --- a/pull_ccache.bash +++ b/pull_ccache.bash @@ -40,7 +40,7 @@ if [[ -z "${VARIANT:-}" ]]; then exit 1 fi -BUILD_VERSION="$(./most_recent_tag.bash)" +BUILD_VERSION="$(cat build_tag)" ARCH="${ARCH//\\;/-}" CCACHE_ARCHIVE="ccache-${VARIANT}-${ARCH}-${BUILD_VERSION}.tbz2" diff --git a/push_ccache.bash b/push_ccache.bash index 6145a10..26ec494 100755 --- a/push_ccache.bash +++ b/push_ccache.bash @@ -42,7 +42,7 @@ if [[ -z "${VARIANT:-}" ]]; then exit 1 fi -BUILD_VERSION="$(./most_recent_tag.bash)" +BUILD_VERSION="$(cat build_tag)" ARCH="${ARCH//\\;/-}" CCACHE_ARCHIVE="ccache-${VARIANT}-${ARCH}-${BUILD_VERSION}.tbz2" diff --git a/s3_download.bash b/s3_download.bash index c09dbd4..43670d5 100755 --- a/s3_download.bash +++ b/s3_download.bash @@ -18,7 +18,7 @@ set -xe REPO="${PWD##*/}" BUCKET="${BUCKET:-llvm-obfuscator-arm}" -BUILD_VERSION="$(./most_recent_tag.bash)" +BUILD_VERSION="$(cat build_tag)" BUILD_PATH="$REPO/$BUILD_VERSION" if [[ ! -z "$PRODUCT_VERSION" ]]; then BUILD_PATH="$BUILD_PATH/$PRODUCT_VERSION" From d63df28459f4e697912a479b9836e83a3b44c7df Mon Sep 17 00:00:00 2001 From: Jason Mobarak Date: Mon, 11 Jun 2018 22:43:36 -0700 Subject: [PATCH 15/15] Use ccache to build obfuscator too --- .travis.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 592770b..bc6319d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,10 +10,6 @@ branches: services: - docker -cache: - directories: - - output/ccache - matrix: include: - env: @@ -24,6 +20,7 @@ matrix: - sudo apt-get update - sudo apt-get install -y jq script: + - ./pull_ccache.bash --variant=$VARIANT --arch=$ARCH - VARIANT=$VARIANT ./.travis.sh after_success: - travis_wait tar -C output -cJf "llvm-$VARIANT-$ARCH.txz" .