build_mesos

#!/bin/bash
set -o errexit -o nounset -o pipefail
export LC_ALL=C
function -h {
cat <<USAGE
 USAGE: build_mesos (--repo <git URL>)?
                    (--ref <version>)?
                    (--build-version <debian_revision or rpm release>)?

  Performs a build in ./mesos-repo after checking out a recent copy of Mesos.
  The default is to checkout:

    $repo

  You can specify a different Mesos Git URL with \`--repo'. Note that it does
  not work to use an \`='; the \`--repo' option must be separated from the Git
  URL with a space.

  Nominal version is the Mesos major.minor.patch version and is autodetected
  from the repository checkout. The detected version can be overridden with
  --ref.

  Build version is an additional version string used for package releases. For
  debian packages this is known as the 'debian revision' and for rpm packages
  it is known as the 'rpm release version'. By default this is set to a
  snapshot timestamp schema of the form: 0.1.%Y%m%d%H%M%S. Here are some
  examples of what this might look like:

    * Snapshot build:                   0.1.20140809173810
    * Mesos release candidate 1:        0.2.rc1
    * Mesos release candidate 2:        0.3.rc2
    * Mesos release:                    1.0
    * Mesos release with packaging fix: 1.1

  The repo can be given as ...?ref=prod7 or even ...?prod7 to select a
  particular branch to build.

  Rather than have this script perform all the build steps, you can point it at
  all or part of an exsting build and have it go from there. Use \`--src-dir' to
  specify the code checkout to use. \`--build-dir' to specify where 'make'
  should be run from. If you have already run built mesos in that directory,
  then you can use \`--prebuilt' to tell the script to just run make install
  inside the build directory to make the package.

USAGE
}; function --help { -h ;}

function globals {
  this="$(cd "$(dirname "$0")" && pwd -P)"
  name=mesos
  build_version="0.1.$(date -u +'%Y%m%d%H%M%S')"
  repo=https://git-wip-us.apache.org/repos/asf/mesos.git
  use_git_version=false
  prebuilt=false
  with_python=true
  start_with="system"
  enable_gpu=false
}; globals

function as_absolute {
  if [[ "$1" == /* ]]; then
    echo "$1"
  else
    echo "$start_dir/$1"
  fi
}

function main {
  while [[ $# -gt 0 ]]
  do
    case "$1" in                                      # Munging globals, beware
      --src-dir)                src_dir="$2"       ; shift 2 ;;
      --build-dir)              build_dir="$2"     ; shift 2 ;;
      --prebuilt)               prebuilt=true      ; shift 1 ;;
      --without-python)         with_python=false  ; shift 1 ;;
      --repo)                   repo="$2"          ; shift 2 ;;
      --branch)                 branch="$2"        ; shift 2 ;;
      --ref)                    ref="$2"           ; shift 2 ;;
      --build-version)          build_version="$2" ; shift 2 ;;
      --enable-gpu)             enable_gpu=true    ; shift 1 ;;
      *)                        err 'Argument error. Please see help.' ;;
    esac
  done

  : "${src_dir:=mesos-repo}"
  : "${build_dir:=$src_dir/build}"

  src_dir=$(as_absolute "$src_dir")
  build_dir=$(as_absolute "$build_dir")

  # Use the more specific dir, e.g. centos/6 will be used over centos if present
  if [[ -d "$this/$linux" ]]
  then asset_dir="$linux"
  elif [[ -d "${linux%%/*}" ]]
  then asset_dir="${linux%%/*}"
  else err "Unable to determine asset_dir for $linux"; exit 1
  fi

  if ! $prebuilt; then
    checkout
  fi
  version=${version:-"$(mesos_version)"}      # Set from checkout if no version
  ( cd "$src_dir" && go )
}

function go {
  clean
  if ! $prebuilt; then
    build
  fi
  create_installation
  create_lib_symlinks
  pkg
  if [[ $with_python ]];then
    save_python_egg
  fi
}

function mesos_version {
  local configure_ac="$src_dir"/configure.ac
  if [[ -f "$configure_ac" ]]
  then
    # Error if AC_INIT is not found or extracted version does not match pattern
    local ac_init=$(grep '^AC_INIT(\[mesos\], \[' "$configure_ac")
    if [[ ! "$ac_init" =~ .*([0-9]+\.[0-9]+\.[0-9]+).* ]]
    then
      err "Unable to extract Mesos version from: ${configure_ac}"
      exit 1
    fi
    out "${BASH_REMATCH[1]}"
  else
    err "Unable to find ${configure_ac}; checkout required"
    exit 1
  fi
}

function maybe_append_git_hash {
  if $use_git_version && git rev-parse --git-dir &>/dev/null
  then out "$1-g$(git log -n1 --format=%h)"
  else out "$1"
  fi
}

repo=https://git-wip-us.apache.org/repos/asf/mesos.git
function checkout {
  local url=( $(url_split "$repo") )
  local repository="${url[0]}"
  local query="${url[1]:-}"

  if [[ -z "${ref:-}" ]]; then
    if [[ ${url[2]:-} ]]
    then err "Setting fragment (#) does nothing. Try query (?) instead."
    fi
    case "$query" in
      ref=*|h=*|branch=*|tag=*) local ref="${query#*=}" ;;
      *)                        local ref="$query" ;;
    esac
  fi
  if [[ -d "$src_dir" ]]
  then msg "Found directory \`mesos-repo'; skipping checkout."
  else msg "Cloning: $repository at $ref" && git clone "$repository" mesos-repo
  fi
  repodir="${this}/mesos-repo"
  msg "git checkout -f $ref"
  ( cd $src_dir && ( [[ ! ${ref:-} ]] || git fetch --tags && git checkout -f "$ref" ) && "$@" )
}

function build {(
  autoreconf -f -i -Wall,no-obsolete
  ./bootstrap
  mkdir -p "$build_dir"
  cd "$build_dir"
  "$src_dir"/configure $(configure_opts)
  # optimal number of build jobs is 2x num_cores
  make -j $(($(num_cores)*2))
)}

function os_release {
  msg "Trying /etc/os-release..."
  if [[ -f /etc/os-release ]]
  then
    ( source /etc/os-release && display_version "$ID" "$VERSION_ID" )
    return 0
  fi
  msg "Trying /etc/redhat-release..."
  if [[ -f /etc/redhat-release ]]
  then
    # Seems to be formatted as: <distro> release <version> (<remark>)
    #                           CentOS release 6.3 (Final)
    if [[ $(cat /etc/redhat-release) =~ \
          ^(.+)' '+release' '+([^ ]+)' '+'('[^')']+')'$ ]]
    then
      local os
      case "${BASH_REMATCH[1]}" in
        'Red Hat '*) os=RedHat ;;
        *)           os="${BASH_REMATCH[1]}" ;;
      esac
      display_version "$os" "${BASH_REMATCH[2]}"
      return 0
    else
      err "/etc/redhat-release not like: <distro> release <version> (<remark>)"
    fi
  fi
  if which sw_vers &> /dev/null
  then
    local product="$(sw_vers -productName)"
    case "$product" in
      'Mac OS X') display_version MacOSX "$(sw_vers -productVersion)" ;;
      *) err "Expecting productName to be 'Mac OS X', not '$product'!";;
    esac
    return 0
  fi
  err "Could not determine OS version!"
}

function display_version {
  local os="$( tr A-Z a-z <<<"$1" )" version="$( tr A-Z a-z <<<"$2" )"
  case "$os" in
    redhat|centos|debian|fedora) out "$os/${version%%.*}" ;;   # Ignore minor versions
    macosx)               out "$os/${version%.*}" ;;  # Ignore bug fix releases
    *)                    out "$os/$version" ;;
  esac
}

function create_installation {(
  local pwd="$(pwd -P)"
  mkdir -p toor
  mkdir -p inst
  local inst_dir="$pwd"/inst
  ( cd "$build_dir" && make install DESTDIR=$inst_dir )
  cd toor
  mkdir -p usr/share/doc/mesos etc/default etc/mesos var/log/mesos
  mkdir -p etc/mesos-master etc/mesos-slave var/lib/mesos
  cp ../CHANGELOG                usr/share/doc/mesos/
  cp "$this"/default/mesos*      etc/default/
  echo zk://localhost:2181/mesos > etc/mesos/zk
  if [[ $(vercomp "$version" 0.19.0) == '>' ]] ||
     [[ $(vercomp "$version" 0.19.0) == '=' ]]
  then
    echo /var/lib/mesos          > etc/mesos-master/work_dir
    echo 1                       > etc/mesos-master/quorum
  fi
  init_scripts "$linux"
  if ! $prebuilt; then
    jars
  fi
  mkdir -p usr/bin
  mkdir -p usr/lib
  mkdir -p usr/libexec/mesos/
  mkdir -p usr/share
  mkdir -p usr/sbin
  cp $inst_dir/usr/bin/mesos* usr/bin
  cp $inst_dir/usr/lib/*.la usr/lib
  cp $inst_dir/usr/lib/libmesos-*.so usr/lib
  cp $inst_dir/usr/libexec/mesos/mesos* usr/libexec/mesos
  cp -r $inst_dir/usr/share usr/
  cp $inst_dir/usr/sbin/mesos-master usr/sbin
  cp $inst_dir/usr/sbin/mesos-slave usr/sbin
)}

function create_lib_symlinks {(
  cd "$src_dir/toor"
  # symlink current version to libmesos.so
  if [ ! -L usr/lib/libmesos.so ]; then
    ( cd usr/lib && ln -s "$(ls -1 libmesos-*.so)" "libmesos.so" )
  fi
  if [[ ! -d usr/local/lib ]]
  then
    msg "Create symlinks for backwards compatibility (e.g. Marathon currently"
    msg "expects libmesos.so to exist in /usr/local/lib)."
    mkdir -p usr/local/lib
    # ensure symlinks are relative so they work as expected in the final env
    ( cd usr/local/lib && cp -s ../../lib/lib*.so . )
  fi
)}

function init_scripts {
  mkdir -p usr/bin
  cp -p "$this"/mesos-init-wrapper usr/bin
  case "$1" in
    debian/*)
      mkdir -p etc/init.d
      cp -p "$this"/init/master.init etc/init.d/mesos-master
      cp -p "$this"/init/slave.init etc/init.d/mesos-slave ;;
    ubuntu/12.*|ubuntu/13.*|ubuntu/14.*|ubuntu/15.*|redhat/6|redhat/6.*|centos/6|centos/6.*)
      mkdir -p etc/init
      cp "$this"/upstart/master.upstart etc/init/mesos-master.conf
      cp "$this"/upstart/slave.upstart etc/init/mesos-slave.conf ;;
    fedora/*|redhat/7|redhat/7.*|centos/7|centos/7.*|ubuntu/16*)
      mkdir -p usr/lib/systemd/system
      cp "$this"/systemd/master.systemd usr/lib/systemd/system/mesos-master.service
      cp "$this"/systemd/slave.systemd usr/lib/systemd/system/mesos-slave.service ;;
    *) err "Not sure how to make init scripts for: $1" ;;
  esac
}

function runit {
  mkdir -p etc/service
  mkdir -p etc/sv/mesos-master
  mkdir -p etc/sv/mesos-slave
  cp -r "$this"/runit/master/* etc/sv/mesos-master
  cp -r "$this"/runit/slave/* etc/sv/mesos-slave
}

function jars {
  mkdir -p usr/share/java/
  if [[ -d "$build_dir"/src/java/target ]]
  then mv "$build_dir"/src/java/target/mesos-*.jar usr/share/java # Mesos >= 0.18.1
  else mv "$build_dir"/src/mesos-*.jar usr/share/java             # Mesos <  0.18.1
  fi
}

function pkg {
  case "$linux" in
    ubuntu/*|debian/*) deb_ ;;
    centos/*|redhat/*|fedora/*) rpm_ ;;
    *)                 err "Not sure how to package for: $linux" ;;
  esac
}

function architecture {
  case "$linux" in
    ubuntu/*|debian/*) dpkg-architecture -qDEB_BUILD_ARCH ;;
    centos/*|redhat/*|fedora/*) arch ;;
    *)                 err "Not sure how to determine arch for: $linux" ;;
  esac
}

function find_gem_bin {
  gem env | sed -n '/^ *- EXECUTABLE DIRECTORY: */ { s/// ; p }'
}

function deb_ {
  local scripts="${linux%%/*}"
  local opts=( -t deb
               -d 'java-runtime-headless'
               -d "libcurl3"
               --after-install "$this/$asset_dir/mesos.postinst"
               --after-remove "$this/$asset_dir/mesos.postrm"
               --config-files "/etc/default/mesos"
               --config-files "/etc/default/mesos-master"
               --config-files "/etc/default/mesos-slave"
               --config-files "/etc/mesos/zk"
               --license "Apache 2.0"
                )
  if [[ $start_with == "runit" ]];then
    opts+=( -d 'runit' )
  fi

  if [[ $(vercomp "$version" 0.20.1) == '>' ]]; then
    opts+=( -d "libsvn1" )
  fi

  if [[ $(check_pkg 'libcurl4-nss-dev') -eq 1 ]];then
    opts+=( -d 'libcurl4-nss-dev' )
  elif [[ $(check_pkg 'libcurl4-openssl-dev') -eq 1 ]];then
    opts+=( -d 'libcurl4-openssl-dev' )
  elif [[ $(check_pkg 'libcurl4-gnutls-dev') -eq 1 ]];then
    opts+=( -d 'libcurl4-gnutls-dev' )
  else
    err "missing libcurl-dev package"
    exit 1
  fi

  fpm_ "${opts[@]}" -p "$this"/pkg.deb
}

function check_pkg {
  local ret=-1
  case "$linux" in
    ubuntu/*|debian/*)
     ret=$(dpkg --get-selections $1 | grep "install" | wc -l)
    ;;
    centos/*|redhat/*)
      err "not implemented yet"
    ;;
    *) err "Not sure how to get installed packages for: $linux" ;;
  esac
  echo $ret
}


function rpm_ {
  local opts=( -t rpm
               -d libcurl
               --after-install "$this/$asset_dir/mesos.postinst"
               --after-remove "$this/$asset_dir/mesos.postrm" )
  fpm_ "${opts[@]}" -p "$this"/pkg.rpm
}

# Doesn't actually work the same as the others...
function osx_ {(
  arch=x86_64
  gem_bin=/usr/bin
  fpm_ -t osxpkg --osxpkg-identifier-prefix org.apache
)}

function fpm_ {
  local version="$(maybe_append_git_hash "$version")"
  local opts=( -s dir
               -n "$name"
               -v "$version"
               --iteration "$build_version"
               --description
"Cluster resource manager with efficient resource isolation
Apache Mesos is a cluster manager that offers efficient resource isolation
and sharing across distributed applications, or frameworks. It can run
Hadoop, MPI, Hypertable, Spark (a new framework for low-latency interactive
and iterative jobs), and other applications."
               --url=http://mesos.apache.org/
               -a "$arch"
               --category misc
               --vendor ""
               -m mesos-dev@incubator.apache.org
               --prefix=/ )
  ( cd toor && "$gem_bin"/fpm "${opts[@]}" "$@" -- . )
}

function save_python_egg {
  local python_dist="$build_dir"/src/python/dist
  if ls -d "$build_dir"/src/python/native/dist/*.egg &>/dev/null
  then
    # Eggs were found in the old location, use that instead
    python_dist="$build_dir"/src/python/native/dist
  fi
  local eggs=( "$python_dist"/*.egg )
  cp "${eggs[@]}" "$this"/
  if [[ $(vercomp "$version" 0.20.0) == '<' ]]
  then
    # Old way to create the distribution egg
    cat "${eggs[@]}" > "$this"/mesos.egg
  else
    # Distribute mesos.native (mesos.interface can be found on pypi)
    cp "$this"/mesos.native*.egg "$this"/mesos.egg
  fi
}

function python_pkg_ {
  # Takes an egg and turns it into a deb.
  # Takes a egg filename as the first argument, and any extra fpm args after that.
  local egg_path=$1
  shift
  # For fpm to work on python stuff, the package name must be in a directory
  # local to the current path. In our build tree the python modules are here
  cd "$this/mesos-repo/build/src/python"

  # component here is the namespaced python module. i.e. mesos.native
  local basename=$(basename ${egg_path})
  local component=${basename%%-*}

  # Note: Version must stay with the stock version (no extra) for fpm to handle
  # The dependencies correctly.
  local version=${version%%-*}

  local opts=( -s python
               -n "python-$component"
               -v "$version"
               --description
"Cluster resouce manager with efficient resource isolation
Apache Mesos is a cluster manager that offers efficient resource isolation
and sharing across distributed applications, or frameworks. It can run
Hadoop, MPI, Hypertable, Spark (a new framework for low-latency interactive
and iterative jobs), and other applications."
               --url=http://incubator.apache.org/mesos/
               -a "$arch"
               --category misc
               --vendor "Apache Mesos"
               -m mesos-dev@incubator.apache.org
               )
  if [[ $component =~ ^.*\..*$ ]]; then
    # Components with a . in the name need to be moved first so that fpm can
    # pick up on the right folder and find the setup.py
    local second_part=${component##*\.}
    mv "$second_part" "$component"
  fi
  fpm "${opts[@]}" "$@" -- "$component"

}

function upload {
  local pkg="$name"_"$version"_"$arch".deb
  local url="${1%/}"/"$linux"/"$pkg"
  curl -X PUT "$url" --data-binary @"$2" >/dev/null
  out "$url"
}

function get_system_info {
  linux="$(os_release)"                 # <distro>/<version>, like ubuntu/12.10
  arch="$(architecture)"          # In the format used to label distro packages
  gem_bin="$(find_gem_bin)"                          # Might not be on the PATH
  start_dir="$PWD"
  PATH="$PATH${gem_bin:+:$gem_bin}"
}

function url_fragment {
  local step1="${1%#}"#       # Ensure URL ends in #, even if it has a fragment
  local step2="${step1#*#}"                                # Clip up to first #
  out "${step2%#}"                    # Remove trailing #, guaranteed by step 1
}

# Split URL in to resource, query and fragment.
function url_split {
  local fragment= query=
  local sans_fragment="${1%%#*}"
  local sans_query="${sans_fragment%%'?'*}"
  [[ $1             = $sans_fragment ]] || fragment="${1#*#}"
  [[ $sans_fragment = $sans_query    ]] || query="${sans_fragment#*'?'}"
  out "$sans_query"
  out "$query"
  out "$fragment"
}


function clean {
  build_dir="$(pwd -P)/build"
  if [[ ! $prebuilt && -d $build_dir ]]; then
    msg "removing build directory..."
    rm -rf $build_dir
  fi
  inst_dir="$(pwd -P)/inst"
  if [[ ! $prebuilt && -d $inst_dir ]]; then
    msg "removing inst directory..."
    rm -rf $inst_dir
  fi
 for package in "$this/pkg.deb" "$this/pkg.rpm" "$this/python-mesos.deb"; do
    if [[ -f $package ]]; then
      msg "removing old package $package"
      rm $package
    fi
  done
  local num_eggs=`ls $this/*.egg -1 | wc -l`
  if [[ $num_eggs -gt 0 ]];then
    msg "removing old python egg"
    rm -f "$this"/*.egg
  fi
}


# Print the number of cores detected. If we are unable to determine the number
# of cores, print a warning and assume "1" core.
function num_cores {
  local cores=
  if hash nproc &>/dev/null
  then cores="$(nproc)"                                   # Linux based systems
  else                                                            # OSX and BSD
    if cores="$(sysctl -n hw.ncpu)"
    then : # Do nothing, success
    else
      msg "Could not find nproc and sysctl failed; defaulting to 1 core."
      cores=1
    fi
  fi
  out "$cores"
}

# Return Mesos configuration options
function configure_opts {
  local options="--prefix=/usr"
  if [[ "$version" == 0.18.0-rc4 ]] || [[ "$repo" =~ 0\.18\.0-rc4$ ]]
  then options+=" --without-cxx11"                # See: MESOS-750 and MESOS-1095
  fi
  if [[ $(vercomp "$version" 0.21.0) == '>' ]] ||
     [[ $(vercomp "$version" 0.21.0) == '=' ]]
  then options+=" --enable-optimize"
  fi

  if ${enable_gpu} 
  then  options+=" --enable-nvidia-gpu-support --with-nvml-include=/opt/nvidia-gdk/usr/include --with-nvml-lib='/opt/nvidia-gdk/usr/src/gdk/nvml/lib'"
  fi 

  out "$options"
}

# Compares version strings $1 with $2 and prints '=', '>', or '<'
# Only works if compared strings have the same number of positions, for example:
#   vercomp 0.19    0.2   # good
#   vercomp 0.19.0  0.2.0 # good
#   vercomp 0.19.0  0.2   # bad
# Adapted from: http://stackoverflow.com/a/4025065/3389824
function vercomp {
  if [[ $1 == $2 ]]
  then
    out '='
    return
  fi
  local IFS=.
  local i ver1=($1) ver2=($2)
  # fill empty fields in ver1 with zeros
  for ((i=${#ver1[@]}; i<${#ver2[@]}; i++))
  do
    ver1[i]=0
  done
  for ((i=0; i<${#ver1[@]}; i++))
  do
    if [[ -z ${ver2[i]} ]]
    then
      # fill empty fields in ver2 with zeros
      ver2[i]=0
    fi
    if ((10#${ver1[i]} > 10#${ver2[i]}))
    then
      out '>'
      return
    fi
    if ((10#${ver1[i]} < 10#${ver2[i]}))
    then
      out '<'
      return
    fi
  done
  out =
}

function msg { out "$*" >&2 ;}
function err { local x=$? ; msg "$*" ; return $(( $x == 0 ? 1 : $x )) ;}
function out { printf '%s\n' "$*" ;}

if [[ ${1:-} ]] && declare -F | cut -d' ' -f3 | fgrep -qx -- "${1:-}"
then
  case "$1" in
    -h|--help|go|url_split|create_installation|checkout|build|osx_) : ;;
    *) get_system_info ;;
  esac
  "$@"
else
  get_system_info
  main "$@"
fi