executable file 347 lines (277 sloc) 10.8 KB
# shellcheck disable=SC2164
# Copyright 2017 Google Inc.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# See the License for the specific language governing permissions and
# limitations under the License.
# Outline of this file.
# 0. Initialization and helper methods.
# 1. Installation of dependencies.
# 2. Installation of Go tools and vendored Go dependencies.
# 3. Detection of installed MySQL and setting MYSQL_FLAVOR.
# 4. Installation of development related steps e.g. creating Git hooks.
# 0. Initialization and helper methods.
# Run parallel make, based on number of cores available.
case $(uname) in
Linux) NB_CORES=$(grep -c '^processor' /proc/cpuinfo);;
Darwin) NB_CORES=$(sysctl hw.ncpu | awk '{ print $2 }');;
if [ -n "$NB_CORES" ]; then
export MAKEFLAGS="-j$((NB_CORES+1)) -l${NB_CORES}"
function fail() {
echo "ERROR: $1"
exit 1
[[ "$(dirname "$0")" = "." ]] || fail " must be run from its current directory"
go version &>/dev/null || fail "Go is not installed or is not on \$PATH"
# Set up the proper GOPATH for go get below.
source ./dev.env
# Create main directories.
mkdir -p "$VTROOT/dist"
mkdir -p "$VTROOT/bin"
mkdir -p "$VTROOT/lib"
mkdir -p "$VTROOT/vthook"
# Set up required soft links.
# TODO(mberlin): Which of these can be deleted?
ln -snf "$VTTOP/config" "$VTROOT/config"
ln -snf "$VTTOP/data" "$VTROOT/data"
ln -snf "$VTTOP/py" "$VTROOT/py-vtdb"
ln -snf "$VTTOP/go/vt/zkctl/" "$VTROOT/bin/"
ln -snf "$VTTOP/test/" "$VTROOT/vthook/"
ln -snf "$VTTOP/test/vthook-test_backup_error" "$VTROOT/vthook/test_backup_error"
ln -snf "$VTTOP/test/vthook-test_backup_transform" "$VTROOT/vthook/test_backup_transform"
# install_dep is a helper function to generalize the download and installation of dependencies.
# If the installation is successful, it puts the installed version string into
# the $dist/.installed_version file. If the version has not changed, bootstrap
# will skip future installations.
function install_dep() {
if [[ $# != 4 ]]; then
fail "install_dep function requires exactly 4 parameters (and not $#). Parameters: $*"
local name="$1"
local version="$2"
local dist="$3"
local install_func="$4"
if [[ -f "$version_file" && "$(cat "$version_file")" == "$version" ]]; then
echo "skipping $name install. remove $dist to force re-install."
echo "installing $name $version"
# shellcheck disable=SC2064
trap "fail '$name build failed'; exit 1" ERR
# Cleanup any existing data and re-create the directory.
rm -rf "$dist"
mkdir -p "$dist"
# Change $CWD to $dist before calling "install_func".
pushd "$dist" >/dev/null
# -E (same as "set -o errtrace") makes sure that "install_func" inherits the
# trap. If here's an error, the trap will be called which will exit this
# script.
set -E
$install_func "$version" "$dist"
set +E
popd >/dev/null
trap - ERR
echo "$version" > "$version_file"
# 1. Installation of dependencies.
# Install the gRPC Python library (grpcio) and the protobuf gRPC Python plugin (grpcio-tools) from PyPI.
# Dependencies like the Python protobuf package will be installed automatically.
function install_grpc() {
local version="$1"
local dist="$2"
# Python requires a very recent version of virtualenv.
# We also require a recent version of pip, as we use it to
# upgrade the other tools.
# For instance, setuptools doesn't work with pip 6.0:
# (and setuptools is used by grpc install).
$VIRTUALENV -v "$grpc_virtualenv"
$PIP install --upgrade pip
$PIP install --upgrade --ignore-installed virtualenv
$PIP install --upgrade grpcio=="$grpcio_ver" grpcio-tools=="$grpcio_ver"
install_dep "gRPC" "1.10.0" "$VTROOT/dist/grpc" install_grpc
# Install protoc.
function install_protoc() {
local version="$1"
local dist="$2"
case $(uname) in
Linux) local platform=linux;;
Darwin) local platform=osx;;
wget "$version/protoc-$version-$"
unzip "protoc-$version-$"
ln -snf "$dist/bin/protoc" "$VTROOT/bin/protoc"
install_dep "protoc" "$protoc_ver" "$VTROOT/dist/vt-protoc-$protoc_ver" install_protoc
# Install Zookeeper.
function install_zookeeper() {
local version="$1"
local dist="$2"
wget "$zk/$zk.tar.gz"
tar -xzf "$zk.tar.gz"
mkdir -p lib
cp "$zk/contrib/fatjar/$zk-fatjar.jar" lib
# TODO(sougou): when version changes, see if we can drop the 'zip -d' hack to get the fatjars working.
# If yes, also delete "zip" from the Dockerfile files and the manual build instructions again.
# 3.4.10 workaround: Delete META-INF files which should not be in there.
zip -d "lib/$zk-fatjar.jar" 'META-INF/*.SF' 'META-INF/*.RSA' 'META-INF/*SF'
rm -rf "$zk" "$zk.tar.gz"
install_dep "Zookeeper" "$zk_ver" "$VTROOT/dist/vt-zookeeper-$zk_ver" install_zookeeper
# Download and install etcd, link etcd binary into our root.
function install_etcd() {
local version="$1"
local dist="$2"
wget "$download_url/$version/$tar_file"
tar xzf "$tar_file"
rm "$tar_file"
ln -snf "$dist/etcd-${version}-linux-amd64/etcd" "$VTROOT/bin/etcd"
install_dep "etcd" "v3.1.0-rc.1" "$VTROOT/dist/etcd" install_etcd
# Download and install consul, link consul binary into our root.
function install_consul() {
local version="$1"
local dist="$2"
wget "${download_url}/${version}/consul_${version}"
unzip "consul_${version}"
ln -snf "$dist/consul" "$VTROOT/bin/consul"
install_dep "Consul" "1.0.6" "$VTROOT/dist/consul" install_consul
# Install py-mock.
function install_pymock() {
local version="$1"
local dist="$2"
# For some reason, it seems like setuptools won't create directories even with the --prefix argument
mkdir -p lib/python2.7/site-packages
PYTHONPATH=$(prepend_path "$PYTHONPATH" "$dist/lib/python2.7/site-packages")
pushd "$VTTOP/third_party/py" >/dev/null
tar -xzf "mock-$version.tar.gz"
cd "mock-$version"
$PYTHON ./ install --prefix="$dist"
cd ..
rm -r "mock-$version"
popd >/dev/null
install_dep "py-mock" "$pymock_version" "$VTROOT/dist/py-mock-$pymock_version" install_pymock
# Download Selenium (necessary to run test/
function install_selenium() {
local version="$1"
local dist="$2"
# PYTHONPATH is removed for `pip install` because otherwise it can pick up go/dist/grpc/usr/local/lib/python2.7/site-packages
# instead of go/dist/selenium/lib/python3.5/site-packages and then can't find module 'pip._vendor.requests'
PYTHONPATH='' $PIP install selenium
install_dep "Selenium" "latest" "$VTROOT/dist/selenium" install_selenium
# Download chromedriver (necessary to run test/
function install_chromedriver() {
local version="$1"
local dist="$2"
curl -sL "$version/" >
unzip -o -q -d "$dist"
install_dep "chromedriver" "2.40" "$VTROOT/dist/chromedriver" install_chromedriver
# 2. Installation of Go tools and vendored Go dependencies.
# Install third-party Go tools used as part of the development workflow.
# DO NOT ADD LIBRARY DEPENDENCIES HERE. Instead use govendor as described below.
# Note: We explicitly do not vendor the tools below because a) we want to stay
# on their latest version and b) it's easier to "go install" them this way.
gotools=" \ \ \ \ \ \ \ \
echo "Installing dev tools with 'go get'..."
# shellcheck disable=SC2086
go get -u $gotools || fail "Failed to download some Go tools with 'go get'. Please re-run in case of transient errors."
# Download dependencies that are version-pinned via govendor.
# To add a new dependency, run:
# govendor fetch <package_path>
# Existing dependencies can be updated to the latest version with 'fetch' as well.
# Then:
# git add vendor/vendor.json
# git commit
# See for more options.
echo "Updating govendor dependencies..."
govendor sync || fail "Failed to download/update dependencies with govendor. Please re-run in case of transient errors."
# 3. Detection of installed MySQL and setting MYSQL_FLAVOR.
# find mysql and prepare to use libmysqlclient
if [ -z "$MYSQL_FLAVOR" ]; then
echo "MYSQL_FLAVOR environment variable not set. Using default: $MYSQL_FLAVOR"
case "$MYSQL_FLAVOR" in
myversion="$("$VT_MYSQL_ROOT/bin/mysql" --version)"
[[ "$myversion" =~ Distrib\ 5\.[67] ]] || fail "Couldn't find MySQL 5.6+ in $VT_MYSQL_ROOT. Set VT_MYSQL_ROOT to override search location."
echo "Found MySQL 5.6+ installation in $VT_MYSQL_ROOT."
myversion="$("$VT_MYSQL_ROOT/bin/mysql" --version)"
[[ "$myversion" =~ MariaDB ]] || fail "Couldn't find MariaDB in $VT_MYSQL_ROOT. Set VT_MYSQL_ROOT to override search location."
echo "Found MariaDB installation in $VT_MYSQL_ROOT."
# save the flavor that was used in bootstrap, so it can be restored
# every time dev.env is sourced.
# 4. Installation of development related steps e.g. creating Git hooks.
# Create the Git hooks.
echo "creating git hooks"
mkdir -p "$VTTOP/.git/hooks"
ln -sf "$VTTOP/misc/git/pre-commit" "$VTTOP/.git/hooks/pre-commit"
ln -sf "$VTTOP/misc/git/prepare-commit-msg.bugnumber" "$VTTOP/.git/hooks/prepare-commit-msg"
ln -sf "$VTTOP/misc/git/commit-msg" "$VTTOP/.git/hooks/commit-msg"
(cd "$VTTOP" && git config core.hooksPath "$VTTOP/.git/hooks")
echo "bootstrap finished - run 'source dev.env' in your shell before building."