From 0b3506608ad794d807d9c91d5c2dfb1dee2a9b06 Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Thu, 8 Sep 2016 11:36:25 -0700 Subject: [PATCH 1/6] fix caching for language:generic --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index bb6b781..6ac9c33 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,8 +11,8 @@ branches: - /^v[0-9]/ cache: - ccache: true - apt: true + directories: + - ${HOME}/.ccache notifications: email: false From 886908a6c4ac7451afe5b16bd5112d6793b477f9 Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Thu, 8 Sep 2016 11:59:49 -0700 Subject: [PATCH 2/6] fix coverage logic so it truly only runs when COVERAGE=true [skip ci] --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6ac9c33..9343b85 100644 --- a/.travis.yml +++ b/.travis.yml @@ -100,8 +100,7 @@ script: ./scripts/publish.sh; fi; - | - if [ -n "${COVERAGE}" ]; then - which llvm-cov + if [ "${COVERAGE:-false}" == true ]; then curl -S -f https://codecov.io/bash -o codecov chmod +x codecov ./codecov -x "llvm-cov gcov" -Z From 6e9278c57c0bd55932fefb43b718ddddd8daa27b Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Thu, 8 Sep 2016 14:04:56 -0700 Subject: [PATCH 3/6] avoid installing massive (> 300MB) documentation package unless explicitly building docs [skip ci] --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 2b5503f..c481296 100644 --- a/package.json +++ b/package.json @@ -48,6 +48,6 @@ "prepublish": "npm ls", "install": "node-pre-gyp install --fallback-to-build", "test": "node test/index.js", - "docs": "documentation build src/node_osrm.cpp --polyglot -f md -o docs/api.md" + "docs": "npm install documentation@^4.0.0-beta2 && documentation build src/node_osrm.cpp --polyglot -f md -o docs/api.md" } } From dc1ce792a4735b2911d85ac9ee529cd35a9d956a Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Thu, 8 Sep 2016 14:05:44 -0700 Subject: [PATCH 4/6] stop bundling node-pre-gyp which is no longer necessary --- package.json | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index c481296..9e22d0d 100644 --- a/package.json +++ b/package.json @@ -31,11 +31,8 @@ }, "dependencies": { "nan": "^2.1.0", - "node-pre-gyp": "~0.6.7" + "node-pre-gyp": "~0.6.30" }, - "bundledDependencies": [ - "node-pre-gyp" - ], "devDependencies": { "aws-sdk": "~2.0.31", "documentation": "^4.0.0-beta2", @@ -45,7 +42,7 @@ "node": ">=4" }, "scripts": { - "prepublish": "npm ls", + "preinstall": "npm install node-pre-gyp", "install": "node-pre-gyp install --fallback-to-build", "test": "node test/index.js", "docs": "npm install documentation@^4.0.0-beta2 && documentation build src/node_osrm.cpp --polyglot -f md -o docs/api.md" From 347fe5f7f790125fbe4b10f28419b9d98228069e Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Thu, 8 Sep 2016 10:53:26 -0700 Subject: [PATCH 5/6] refactor travis scripts --- .travis.yml | 111 ++++++++++++++------------------------ scripts/build.sh | 33 +++++------- scripts/publish.sh | 8 ++- scripts/setup_compiler.sh | 10 ++++ scripts/setup_mason.sh | 6 +-- scripts/travis_helper.sh | 71 ++++++++++++++++++++++++ scripts/validate_tag.sh | 24 --------- 7 files changed, 143 insertions(+), 120 deletions(-) create mode 100644 scripts/setup_compiler.sh create mode 100644 scripts/travis_helper.sh delete mode 100755 scripts/validate_tag.sh diff --git a/.travis.yml b/.travis.yml index 9343b85..94d47af 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,85 +23,54 @@ notifications: sudo: false +env: + global: + - CCACHE_TEMPDIR=/tmp/.ccache-temp + - CCACHE_COMPRESS=1 + - CASHER_TIME_OUT=1000 + - CC="clang" + - CXX="clang++" + - secure: KitzGZjoDblX/3heajcvssGz0JnJ/k02dr2tu03ksUV+6MogC3RSQudqyKY57+f8VyZrcllN/UOlJ0Q/3iG38Oz8DljC+7RZxtkVmE1SFBoOezKCdhcvWM12G3uqPs7hhrRxuUgIh0C//YXEkulUrqa2H1Aj2xeen4E3FAqEoy0= + - secure: WLGmxl6VTVWhXGm6X83GYNYzPNsvTD+9usJOKM5YBLAdG7cnOBQBNiCCUKc9OZMMZVUr3ec2/iigakH5Y8Yc+U6AlWKzlORyqWLuk4nFuoedu62x6ocQkTkuOc7mHiYhKd21xTGMYauaZRS6kugv4xkpGES2UjI2T8cjZ+LN2jU= + +addons_default: &apt_packages + apt: + sources: [ 'ubuntu-toolchain-r-test' ] + # note: libstdc++-5-dev is required by clang-3.8 for its headers (clang-3.8 is installed by mason on linux) + packages: [ 'libstdc++-5-dev', 'apport', 'gdb' ] + +script_default: &build_and_publish + - ./scripts/build.sh + - ./scripts/publish.sh + matrix: include: # Linux - os: linux - compiler: ": clang38-release-node-v4" - env: NODE="4" TARGET=Release CC="clang-3.8" CXX="clang++-3.8" PUBLISHABLE=true - # note: libstdc++-5-dev is required by clang-3.8 for its headers (clang-3.8 is installed by mason below) - addons: - apt: - sources: [ 'ubuntu-toolchain-r-test' ] - packages: [ 'libstdc++-5-dev', 'apport', 'gdb' ] + env: NODE="4" TARGET=Release + addons: *apt_packages + script: *build_and_publish - os: linux - compiler: ": clang38-debug-node-v4" - env: NODE="4" TARGET=Debug CC="clang-3.8" CXX="clang++-3.8" PUBLISHABLE=true - # note: libstdc++-5-dev is required by clang-3.8 for its headers (clang-3.8 is installed by mason below) - addons: - apt: - sources: [ 'ubuntu-toolchain-r-test' ] - packages: [ 'libstdc++-5-dev', 'apport', 'gdb' ] + env: NODE="4" TARGET=Debug NPM_FLAGS="--debug" + addons: *apt_packages + script: *build_and_publish # OS X - os: osx # https://docs.travis-ci.com/user/languages/objective-c/#Supported-OS-X-iOS-SDK-versions - osx_image: xcode8 # upgrades clang from 6 -> 7 - compiler: clang - env: NODE="4" TARGET=Release PUBLISHABLE=true + osx_image: xcode8 + env: NODE="4" TARGET=Release + script: *build_and_publish - os: osx - osx_image: xcode8 # upgrades clang from 6 -> 7 - compiler: clang - env: NODE="4" TARGET=Debug PUBLISHABLE=true + # https://docs.travis-ci.com/user/languages/objective-c/#Supported-OS-X-iOS-SDK-versions + osx_image: xcode8 + env: NODE="4" TARGET=Debug NPM_FLAGS="--debug" + script: *build_and_publish # Linux coverage build - os: linux - compiler: ": clang38-debug-coverage-node-v4" - env: NODE="4" TARGET=Debug CC="clang-3.8" CXX="clang++-3.8" COVERAGE=true PUBLISHABLE=false - addons: - apt: - sources: [ 'ubuntu-toolchain-r-test' ] - packages: [ 'libstdc++-5-dev', 'apport', 'gdb' ] - -env: - global: - - CCACHE_TEMPDIR=/tmp/.ccache-temp - - CCACHE_COMPRESS=1 - - CASHER_TIME_OUT=1000 - - secure: KitzGZjoDblX/3heajcvssGz0JnJ/k02dr2tu03ksUV+6MogC3RSQudqyKY57+f8VyZrcllN/UOlJ0Q/3iG38Oz8DljC+7RZxtkVmE1SFBoOezKCdhcvWM12G3uqPs7hhrRxuUgIh0C//YXEkulUrqa2H1Aj2xeen4E3FAqEoy0= - - secure: WLGmxl6VTVWhXGm6X83GYNYzPNsvTD+9usJOKM5YBLAdG7cnOBQBNiCCUKc9OZMMZVUr3ec2/iigakH5Y8Yc+U6AlWKzlORyqWLuk4nFuoedu62x6ocQkTkuOc7mHiYhKd21xTGMYauaZRS6kugv4xkpGES2UjI2T8cjZ+LN2jU= - -before_install: -# workaround travis rvm bug -# http://superuser.com/questions/1044130/why-am-i-having-how-can-i-fix-this-error-shell-session-update-command-not-f -- | - if [[ "${TRAVIS_OS_NAME}" == "osx" ]]; then - rvm get head || true - fi -- source scripts/setup_mason.sh -- scripts/validate_tag.sh -- | - if [[ $(uname -s) == 'Linux' ]]; then - mason install clang 3.8.0 - export PATH=$(mason prefix clang 3.8.0)/bin:${PATH} - which clang++ - fi - -install: -- | - if [[ "$(uname -s)" == "Linux" ]]; then - export JOBS=2 - source ./scripts/build.sh - else - export JOBS=4 - source ./scripts/build.sh - fi - -script: -- if [[ ${PUBLISHABLE} == true ]]; then - ./scripts/publish.sh; - fi; -- | - if [ "${COVERAGE:-false}" == true ]; then - curl -S -f https://codecov.io/bash -o codecov - chmod +x codecov - ./codecov -x "llvm-cov gcov" -Z - fi + env: NODE="4" TARGET=Debug COVERAGE=true NPM_FLAGS="--debug" + addons: *apt_packages + script: + - ./scripts/build.sh + - curl -S -f https://codecov.io/bash -o codecov + - chmod +x codecov + - ./codecov -x "llvm-cov gcov" -Z diff --git a/scripts/build.sh b/scripts/build.sh index 8e12d6e..d2885b3 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -11,29 +11,24 @@ export NODE=${NODE:-4} export CURRENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source ${CURRENT_DIR}/travis_helper.sh + +mapbox_time "setup-mason" source ${CURRENT_DIR}/setup_mason.sh +mapbox_time "setup-compiler" source ${CURRENT_DIR}/setup_compiler.sh + # ensure we start inside the root directory (one level up) cd ${CURRENT_DIR}/../ -if [[ $(which wget) ]]; then - echo "Success: Found wget"; -else +if [[ ! $(which wget) ]]; then echo "echo wget must be installed"; exit 1; fi; -if [[ $(which pkg-config) ]]; then - echo "Success: Found pkg-config"; -else +if [[ ! $(which pkg-config) ]]; then echo "echo pkg-config must be installed"; exit 1; fi; -if [[ ${TARGET} == 'Debug' ]]; then - export NPM_FLAGS="--debug" -else - export NPM_FLAGS="" -fi - if [[ "$(uname -s)" == "Darwin" ]]; then if [[ -f /etc/sysctl.conf ]] && [[ $(grep shmmax /etc/sysctl.conf) ]]; then echo "Note: found shmmax setting in /etc/sysctl.conf, not modifying" @@ -42,17 +37,17 @@ if [[ "$(uname -s)" == "Darwin" ]]; then sudo sysctl -w kern.sysv.shmmax=4294967296 sudo sysctl -w kern.sysv.shmall=1048576 sudo sysctl -w kern.sysv.shmseg=128 - brew install md5sha1sum + mapbox_time "brew" brew install md5sha1sum fi fi # install consistent node version -source ./scripts/install_node.sh ${NODE} +mapbox_time "install_node" source ./scripts/install_node.sh ${NODE} if [[ ${TARGET} == 'Debug' ]]; then - export BUILD_TYPE=Debug && source ./bootstrap.sh + mapbox_time "bootstrap" export BUILD_TYPE=Debug && source ./bootstrap.sh else - source ./bootstrap.sh + mapbox_time "bootstrap" source ./bootstrap.sh fi # only set coverage flags for node-osrm to avoid @@ -61,20 +56,20 @@ if [[ ${COVERAGE} == true ]]; then export LDFLAGS="${LDFLAGS:-} --coverage" && export CXXFLAGS="${CXXFLAGS:-} --coverage" fi -npm install --build-from-source ${NPM_FLAGS} --clang=1 +mapbox_time "npm-install" npm install --build-from-source ${NPM_FLAGS} --clang=1 # run tests, with backtrace support if [[ "$(uname -s)" == "Linux" ]]; then ulimit -c unlimited -S RESULT=0 - make test || RESULT=$? + mapbox_time "make-test" make test || RESULT=$? for i in $(find ./ -maxdepth 1 -name 'core*' -print); do gdb $(which node) $i -ex "thread apply all bt" -ex "set pagination 0" -batch; done; if [[ ${RESULT} != 0 ]]; then exit $RESULT; fi else # todo: coredump support on OS X - make test + mapbox_time "make-test" make test fi diff --git a/scripts/publish.sh b/scripts/publish.sh index a615e2c..a719526 100755 --- a/scripts/publish.sh +++ b/scripts/publish.sh @@ -1,5 +1,11 @@ #!/bin/bash +set -eu +set -o pipefail + +# should be set for debug builds +export NPM_FLAGS=${NPM_FLAGS:-} + echo "dumping binary meta..." ./node_modules/.bin/node-pre-gyp reveal ${NPM_FLAGS} @@ -25,7 +31,7 @@ else echo "This is a push commit, continuing to package..." ./node_modules/.bin/node-pre-gyp package ${NPM_FLAGS} - COMMIT_MESSAGE=$(git log --format=%B --no-merges | head -n 1 | tr -d '\n') + export COMMIT_MESSAGE=$(git log --format=%B --no-merges | head -n 1 | tr -d '\n') echo "Commit message: ${COMMIT_MESSAGE}" if [[ ${COMMIT_MESSAGE} =~ "[publish binary]" ]]; then diff --git a/scripts/setup_compiler.sh b/scripts/setup_compiler.sh new file mode 100644 index 0000000..433aee5 --- /dev/null +++ b/scripts/setup_compiler.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -eu +set -o pipefail + +if [[ $(uname -s) == 'Linux' ]]; then + mason install clang 3.8.0 + export PATH=$(mason prefix clang 3.8.0)/bin:${PATH} + which clang++ +fi \ No newline at end of file diff --git a/scripts/setup_mason.sh b/scripts/setup_mason.sh index 60d6c2b..f0e5afe 100755 --- a/scripts/setup_mason.sh +++ b/scripts/setup_mason.sh @@ -20,8 +20,4 @@ function setup_mason() { export CC=${CC:-clang} } - -setup_mason - -set +eu -set +o pipefail \ No newline at end of file +setup_mason \ No newline at end of file diff --git a/scripts/travis_helper.sh b/scripts/travis_helper.sh new file mode 100644 index 0000000..c7b98ec --- /dev/null +++ b/scripts/travis_helper.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +# This script is sourced, so do not set -e or -o pipefail here. Doing so would +# bleed into Travis' wrapper script, which messes with their workflow, e.g. +# preventing after_failure scripts to be triggered. + +case `uname -s` in + 'Darwin') JOBS=$((`sysctl -n hw.ncpu` + 2)) ;; + 'Linux') JOBS=$((`nproc` + 2)) ;; + *) JOBS=2 ;; +esac + +function mapbox_time_start { + local name=$1 + mapbox_timer_name=$name + + mapbox_fold start $name + + mapbox_timer_id=$(printf %08x $(( RANDOM * RANDOM ))) + eval "mapbox_start_time_$mapbox_timer_id=$(mapbox_nanoseconds)" + echo -en "travis_time:start:$mapbox_timer_id\n" +} + +function mapbox_time_finish { + local name=${1:-$mapbox_timer_name} + local timer_id=${2:-$mapbox_timer_id} + local timer_start="mapbox_start_time_$timer_id" + eval local start_time=\${$timer_start} + local end_time=$(mapbox_nanoseconds) + local duration=$(($end_time-$start_time)) + echo -en "travis_time:end:$timer_id:start=$start_time,finish=$end_time,duration=$duration\n" + + mapbox_fold end $name +} + +function mapbox_time { + local name=$1 ; shift + mapbox_time_start $name + local timer_id=$mapbox_timer_id + echo "\$ $@" + $@ + mapbox_time_finish $name $timer_id +} + +function mapbox_fold { + local action=$1 + local name=$2 + local ANSI_CLEAR="\e[0m" + echo -en "travis_fold:${action}:${name}\r${ANSI_CLEAR}" +} + +function mapbox_nanoseconds { + local cmd="date" + local format="+%s%N" + local os=$(uname -s) + + if hash gdate > /dev/null 2>&1; then + cmd="gdate" # use gdate if available + elif [[ "$os" = Darwin ]]; then + format="+%s000000000" # fallback to second precision on darwin (does not support %N) + fi + + $cmd -u $format +} + +export JOBS +export -f mapbox_fold +export -f mapbox_nanoseconds +export -f mapbox_time +export -f mapbox_time_start +export -f mapbox_time_finish diff --git a/scripts/validate_tag.sh b/scripts/validate_tag.sh deleted file mode 100755 index 67d039b..0000000 --- a/scripts/validate_tag.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -set -u - -# let's catch the case where we tag but -# forget to increment the package.json version - -# check if we are on a tag -if [ `git describe --tags --always HEAD` ]; then - echo 'looks like we are on a tag' - if [[ $TRAVIS_BRANCH == `git describe --tags --always HEAD` ]]; then - echo 'git reports the same tag as travis' - # now check to make sure package.json `version` matches - MODULE_VERSION=$(node -e "console.log(require('./package.json').version)") - if [[ $MODULE_VERSION != $TRAVIS_BRANCH ]] && [[ v$MODULE_VERSION != $TRAVIS_BRANCH ]]; then - echo "package.json version ($MODULE_VERSION) does not match tag ($TRAVIS_BRANCH)" - exit 1 - else - echo "Validation success: package.json ($MODULE_VERSION) matches tag ($TRAVIS_BRANCH)" - fi - else - echo "warning: travis thinks the tag ($TRAVIS_BRANCH) differs from git (`git describe --tags --always HEAD`)" - fi -fi From 1ef41cf3a924ad49aa9b04bedd833972f5c4137c Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Thu, 8 Sep 2016 15:43:57 -0700 Subject: [PATCH 6/6] Also build for node v6 (upcoming LTS) --- .travis.yml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 94d47af..2ac99df 100644 --- a/.travis.yml +++ b/.travis.yml @@ -54,6 +54,14 @@ matrix: env: NODE="4" TARGET=Debug NPM_FLAGS="--debug" addons: *apt_packages script: *build_and_publish + - os: linux + env: NODE="6" TARGET=Release + addons: *apt_packages + script: *build_and_publish + - os: linux + env: NODE="6" TARGET=Debug NPM_FLAGS="--debug" + addons: *apt_packages + script: *build_and_publish # OS X - os: osx # https://docs.travis-ci.com/user/languages/objective-c/#Supported-OS-X-iOS-SDK-versions @@ -61,10 +69,18 @@ matrix: env: NODE="4" TARGET=Release script: *build_and_publish - os: osx - # https://docs.travis-ci.com/user/languages/objective-c/#Supported-OS-X-iOS-SDK-versions osx_image: xcode8 env: NODE="4" TARGET=Debug NPM_FLAGS="--debug" script: *build_and_publish + - os: osx + # https://docs.travis-ci.com/user/languages/objective-c/#Supported-OS-X-iOS-SDK-versions + osx_image: xcode8 + env: NODE="6" TARGET=Release + script: *build_and_publish + - os: osx + osx_image: xcode8 + env: NODE="6" TARGET=Debug NPM_FLAGS="--debug" + script: *build_and_publish # Linux coverage build - os: linux env: NODE="4" TARGET=Debug COVERAGE=true NPM_FLAGS="--debug"