Skip to content
Permalink
Browse files

split travis build to two jobs and use cache storing dependency

Summary:
mcrouter oos travis build could still timeout because total build time is longer than 50mins. This diff split travis build into two staged jobs: build_deps_and_cache and mcrouter.
Job build_deps_and_cache builds fb dependencies and stores in travis cache, took ~32mins
Job mcrouter fetches the dependency file from cache and continue mcrouter build, took ~34mins

Although each travis job has 50mins limit, there is no limit on total build time. Using cache so that jobs could share progress  between each other. This way also make sure we have enough room for future build time increase.

Reviewed By: edenzik

Differential Revision: D18372976

fbshipit-source-id: a7bf45432e5603d3c2a93528772e54770b0efb9f
  • Loading branch information
disylh authored and facebook-github-bot committed Nov 7, 2019
1 parent 16416eb commit 951b33b4a5f9a31273d561178d21752457d42228
@@ -6,7 +6,21 @@ language: cpp
compiler:
- gcc

script: ./mcrouter/scripts/install_ubuntu_18.04.sh "$(pwd)"/mcrouter-install
cache:
directories:
- $HOME/travis_cache

jobs:
include:
- stage: build_deps_and_cache
script: >
./mcrouter/scripts/install_ubuntu_18.04.sh "$(pwd)"/mcrouter-install deps
&& tar -zcvf $HOME/travis_cache/install_deps_ubuntu_18.04.tar.gz -C "$(pwd)"/mcrouter-install/install .
- stage: mcrouter
script: >
mkdir -p "$(pwd)"/mcrouter-install/install
&& tar -xzvf $HOME/travis_cache/install_deps_ubuntu_18.04.tar.gz -C "$(pwd)"/mcrouter-install/install
&& ./mcrouter/scripts/install_ubuntu_18.04.sh "$(pwd)"/mcrouter-install mcrouter
notifications:
email:
@@ -4,6 +4,13 @@ all: .folly-done .fizz-done .wangle-done .rsocket-done .fmt-done .fbthrift-done
${RECIPES_DIR}/mcrouter.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR)
touch $@

mcrouter:
${RECIPES_DIR}/mcrouter.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR)
touch $@

deps: .folly-done .fizz-done .wangle-done .rsocket-done .fmt-done .fbthrift-done
touch $@

.folly-done:
${RECIPES_DIR}/folly.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR)
touch $@
@@ -7,11 +7,13 @@
set -ex

MAKE_FILE="$1"
PKG_DIR="${2%/}"/pkgs
INSTALL_DIR="${2%/}"/install
INSTALL_AUX_DIR="${2%/}"/install/aux
TARGET="$2"
PKG_DIR="${3%/}"/pkgs
INSTALL_DIR="${3%/}"/install
INSTALL_AUX_DIR="${3%/}"/install/aux

shift 2
[ -n "$MAKE_FILE" ] || ( echo "Make file missing"; exit 1 )
[ -n "$TARGET" ] || ( echo "Target missing"; exit 1 )

mkdir -p "$PKG_DIR" "$INSTALL_DIR" "$INSTALL_AUX_DIR"

@@ -22,6 +24,6 @@ export REPO_BASE_DIR

export LDFLAGS="-ljemalloc $LDFLAGS"

make all -j3 -f "$MAKE_FILE" PKG_DIR="$PKG_DIR" INSTALL_DIR="$INSTALL_DIR" INSTALL_AUX_DIR="$INSTALL_AUX_DIR"
make "$TARGET" -j3 -f "$MAKE_FILE" PKG_DIR="$PKG_DIR" INSTALL_DIR="$INSTALL_DIR" INSTALL_AUX_DIR="$INSTALL_AUX_DIR"

printf "%s\n" "Mcrouter installed in $INSTALL_DIR/bin/mcrouter"
printf "%s\n" "make $TARGET for $MAKE_FILE done"
@@ -6,7 +6,10 @@

set -ex

[ -n "$1" ] || ( echo "Install dir missing"; exit 1 )
BASE_DIR="$1"
TARGET="${2:-all}"

[ -n "$BASE_DIR" ] || ( echo "Base dir missing"; exit 1 )

sudo apt-get update

@@ -42,4 +45,4 @@ sudo apt-get install -y \

cd "$(dirname "$0")" || ( echo "cd fail"; exit 1 )

./get_and_build_everything_by_make.sh "Makefile_ubuntu-18.04" "$@"
./get_and_build_by_make.sh "Makefile_ubuntu-18.04" "$TARGET" "$BASE_DIR"

0 comments on commit 951b33b

Please sign in to comment.
You can’t perform that action at this time.