Deploy

#!/bin/sh

##H Deploy [OPTIONS] ROOT [SERVICE...]
##H
##H Deploy one or more SERVICEs into the ROOT directory.
##H
##H Possible options:
##H  -A ARCH        Install platform ARCH instead of the default one.
##H  -R RELEASE     Take package versions from RELEASE meta-package.
##H  -M             Activate $MIGRATION commands.
##H  -a             Activate production installation under multiple accounts.
##H  -r A=B         Use repository B instead of A (e.g. comp=comp.pre).
##H  -t VERSION     Install software as VERSION.
##H  -p AUTH-DIR    Use private secrets from AUTH-DIR (default: use dummies).
##H  -pl AUTH-DIR   Create a link to secrets on AUTH-DIR (overrides -p).
##H  -s STAGES      Install only STAGES parts of recipes (prep sw post).
##H  -H HOST        Install as if for HOST (default: hostname).
##H  -w HOST        Use the following host instead of cmsrep.cern.ch.
##H  -h             Display this help.
##H
##H Remaining parameters are installation root directory ROOT and list
##H of service names to install. Per-service deployment recipes and
##H configuration are taken from the same directory as this script.
##H
##H Example to install service 'foo' into /data as version hg1103-backend
##H from pre-release repository, using official secrets from 'secrets'.
##H
##H   scp -rp cmsweb@lxplus.cern.ch:private/conf secrets
##H   svn co svn+ssh://svn.cern.ch/reps/Infrastructure/trunk/Deployment d
##H   d/Deploy -a -r comp=comp.pre -V hg1103a-backend -A secrets /data foo

MIGRATION=

repoedit=n
now=$(date +%Y%m%d%H%M%S)
host=$(hostname -s)
rpmhost="cmsrep.cern.ch"
stages="prep sw"
nogroups=true
servicedir="$(cd $(dirname $0) && pwd)"
cfgversion=
privauthdir=
authlink=false
release=
svcs=

case $(uname) in
  Darwin ) arch=osx108_amd64_gcc481 xargsr="xargs" ;;
  Linux  ) linuxmajor=$(egrep -o 'release [0-9]+{1}\.' /etc/redhat-release|cut -d" " -f2|tr -d .)
           case $linuxmajor in
             5 ) arch=slc5_amd64_gcc461 ;;
             6 ) arch=slc6_amd64_gcc493 ;;
             7 ) arch=slc7_amd64_gcc630 ;;
           esac
           xargsr="xargs -r"
           ;;
  *      ) echo "unsupported architecture" 1>&2; exit 1 ;;
esac

while [ $# -ge 1 ]; do
  case $1 in
    -A ) arch="$2"; shift; shift ;;
    -R ) release="cms+$(echo $2 | tr @ +)"; shift; shift ;;
    -M ) MIGRATION=true; shift ;;
    -a ) nogroups=false; shift ;;
    -r ) repoedit="s/${2%=*}/${2#*=}/; $repoedit"; shift; shift ;;
    -s ) stages="$2"; shift; shift ;;
    -t ) cfgversion="$2"; shift; shift ;;
    -p ) privauthdir="$2"; shift; shift ;;
    -pl) privauthdir="$2"; authlink=true; shift; shift ;;
    -H ) host="$2"; shift; shift ;;
    -w ) rpmhost="$2"; shift; shift ;;
    -h ) perl -ne '/^##H/ && do { s/^##H ?//; print }' < $0 1>&2; exit 1 ;;
    -* ) echo "$0: unrecognised option $1, use -h for help" 1>&2; exit 1 ;;
    *  ) break ;;
  esac
done

chgrp=true chown=true chmod=true
$nogroups || chgrp=chgrp chown=chown chmod=chmod

if [ $# -lt 1 ]; then
  echo "Usage: $0 [OPTIONS] ROOT [SERVICE...] (use -h for help)" 1>&2
  exit 1
fi

if [ X"$cfgversion" = X ]; then
  echo "$0: missing software tag version (-t option)" 1>&2
  exit 1
fi

case $cfgversion in */* | . | .. | .* )
  echo "$0: invalid configuration version name '$cfgversion'" 1>&2
  exit 1 ;;
esac

if ! $authlink && [ X"$privauthdir" != X ] && [ ! -d "$privauthdir" ]; then
  echo "$privauthdir: not a directory" 1>&2
  exit 1
fi

# Set SCRAM_ARCH. Lots of things won't work without it being set.
export SCRAM_ARCH=$arch

# Make sure the installation directory exists.
root="$1"; shift
[ -d $root ] || mkdir -p $root || exit $?

# Create directory for deployment logs. Must be writeable by _sw.
[ -d $root/.deploy ] || {
  mkdir -p $root/.deploy || exit $?
  $chmod ug+rw,o-w $root/.deploy || exit $?
  $chgrp _sw $root/.deploy || exit $?
}


# Create directory for the configuration version we are about to create.
grepo=$(echo "comp" | sed -e "$repoedit")
glabel=$(echo $grepo | sed 's/^comp//; s/^\([^.]\)/.\1/; s/^/sw/; n')
for dir in $root/$cfgversion{,/bin,/apps.$glabel,/config}; do
  [ -d $dir ] || {
    mkdir -p $dir || exit $?
    $chmod ug+rw,o-w $dir || exit $?
    $chgrp _sw $dir || exit $?
  }
done

for dir in $root/{logs,state,enabled}; do
  [ -d $dir ] || mkdir -p $dir || exit $?
done

[ -d $root/auth ] || {
  mkdir -p $root/auth || exit $?
  $chmod u=rwx,g=rx,o-rwx $root/auth || exit $?
  $chgrp _config $root/auth || exit $?
}

# Get list of services to deploy
for svc; do svcs="$svcs $svc"; done

######################################################################
# Installation routines

# Set group if we are running in multi-account mode.
setgroup()
{
  local opts=
  local priv=
  local group=

  while [ $# -ge 1 ]; do
    case $1 in -*) opts="$opts $1"; shift ;; *) break;; esac
  done

  local priv=$1
  local group=$2
  shift; shift

  if [ $# -ge 1 ]; then
    $nogroups || chgrp $opts $group ${1+"$@"}
    chmod $opts $priv ${1+"$@"}
  fi
}

# Produce crontab entry for sysboot rule
sysboot()
{
  if [ ! -f $root/enabled/$project ]; then :; else
    if $nogroups; then
      echo "@reboot $project_config/manage ${1-sysboot}"
    else
      echo "@reboot sudo -H -u _$project bashs -l -c '$project_config/manage ${1-sysboot}'"
    fi
  fi
}

# Clean up crontab entries for the project
mkcrontab()
{
  crontab -l | { fgrep -v -e $project_config/ || true; }
}

# Create application proxy
mkproxy()
{
  local path=$PWD/proxy/proxy.cert
  local link=$(echo $path | md5sum | awk '{print $1}' | cut -b 1-8)
  local pxfile=$root/$cfgversion/auth/proxy/PX$link
  local pxdir=$root/$cfgversion/auth/proxy
  if $nogroups; then local acct=$USER; else local acct=_$project; fi

  mkdir -p $pxdir
  setgroup u+rwx,g+rx,o-rwx _config $pxdir

  echo "$acct $path" > $pxfile
  setgroup u+rw,go-rwx _config $pxfile

  mkdir -p $PWD/proxy
  setgroup ug+rwx,o-rwx $acct $(dirname $path)
}

# Create project directory, with some additional directories.
# With groups enabled the project directory is writable by _sw.
mkproj()
{
  local nolog=false
  local nostate=false
  while [ $# -ge 1 ]; do
    case $1 in
      -s ) nostate=true; shift ;;
      -l ) nolog=true; shift ;;
      -* ) echo "$0: mkproj: unexpected option $1" 1>&2; exit 1;;
      *  ) break ;;
    esac
  done

  mkdir -p $root/$cfgversion/config/$project
  $chgrp _sw $root/$cfgversion/config/$project
  $chmod ug+rw,o-w $root/$cfgversion/config/$project

  if $nolog; then :; else
    mkdir -p $project_logs
    setgroup ug+rw,o-w _$project $project_logs
  fi

  if $nostate; then :; else
    mkdir -p $project_state
    setgroup ug+rw,o-w _$project $project_state
    cd $project_state
    for dir; do
      mkdir -p $dir
      setgroup ug+rw,o-w _$project $dir
    done
  fi
}

# Source software management environment for a repository.
# Automatically bootstrap the requested area if necessary.
inrepo()
{
  local repo=$1
  local swarea=$2
  if [ ! -f $swarea/.bootstrapped ]; then
    note "INFO: bootstrapping $repo software area in $swarea"
    (set -e
     rm -fr $swarea
     mkdir -p $swarea
     cd $swarea
     curl -sO http://$rpmhost/cmssw/bootstrap.sh
     sh -x ./bootstrap.sh -architecture $arch -path $PWD -repository $repo \
       -server $rpmhost setup > $PWD/bootstrap-$arch.log 2>&1
     touch .bootstrapped)
    if [ $? = 0 ]; then
      note "INFO: bootstrap successful"
    else
      note "ERROR: bootstrap failed"
      exit 3
    fi
  fi
}

# Deploy stage of some component.
deploy()
{
  local stage=$1; shift
  local pkg version variant
  case $1 in
    *@* ) pkg=${1%@*} version=${1#*@} ;;
    *   ) pkg=$1 version= ;;
  esac
  local var=${2-default}
  local name=$(echo $pkg | tr - _)
  local varname=$(echo $var | tr - _)
  eval local did="\$${name}_${varname}_deployed"
  if [ X$did = X ]; then
    if [ -f $servicedir/$pkg/deploy ]; then
      eval "deploy_${name}_variants=default"
      eval "deploy_${name}_deps() { true; }"
      eval "deploy_${name}_prep() { true; }"
      eval "deploy_${name}_sw() { true; }"
      eval "deploy_${name}_post() { enable; }"
      eval "deploy_${name}_auth() { true; }"
      . $servicedir/$pkg/deploy
    else
      note "ERROR: cannot deploy $pkg, no $servicedir/$pkg/deploy"
      exit 4
    fi

    eval local variants="\$deploy_${name}_variants"
    case " $variants " in *" $var "* ) ;; * )
      note "ERROR: $pkg does not have variant '$var'"
      exit 5 ;;
    esac

    local setup=:
    setup="$setup; project='$pkg'"
    setup="$setup; project_version_override='$version'"
    setup="$setup; project_config_src='$servicedir/$pkg'"
    setup="$setup; project_config='$root/current/config/$pkg'"
    setup="$setup; project_auth='$root/$cfgversion/auth/$pkg'"
    setup="$setup; project_state='$root/state/$pkg'"
    setup="$setup; project_logs='$root/logs/$pkg'"
    setup="$setup; variant='$var'"
    setup="$setup; cd \$root; [ ! -d \$project_state ] || cd \$project_state"

    eval "$setup; deploy_${name}_deps"
    note "INFO: deploying $pkg - variant: $var, version: ${version:-default}"
    eval "$setup; deploy_${name}_$stage"
    eval "${name}_${varname}_deployed=1"
  else
    true
  fi
}

deploy_pkg()
{
  local authfiles=
  local authfile=
  local link=
  while [ $# -ge 1 ]; do
    case $1 in
      -a ) authfiles="$authfiles $2"; shift; shift ;;
      -l ) link="$2"; shift; shift ;;
      -* ) echo "$0: deploy_pkg: unexpected option $1" 1>&2; exit 1;;
      *  ) break ;;
    esac
  done

  if [ $# = 0 ]; then :; else
    local repo=$(echo $1 | sed -e "$repoedit")
    local label=$(echo $repo | sed 's/^comp//; s/^\([^.]\)/.\1/; s/^/sw/; n')
    local swarea=$root/$cfgversion/$label
    local ver=${project_version_override:-${3:-auto}}
    local pkg=$2

    case $ver:$pkg in auto:${release%+*} )
      ver=${release##*+} ;;
    esac

    case $ver:$release in auto: )
      note "ERROR: deploying '$pkg' version 'auto' requires -R META-RELEASE"
      exit 4 ;;
    esac

    (set -e
     inrepo $repo $swarea
     [ X"$ver" != Xauto ] ||
       ver=$($swarea/common/cmspkg -a $arch depends "$release" |
             fgrep "Depends: $pkg+" | awk -F+ '{print $NF}')
     $swarea/common/cmspkg -a $arch update
     $swarea/common/cmspkg -a $arch -y install $($swarea/common/cmspkg -a $arch search fake | awk -F' ' '{print $1}')
     $swarea/common/cmspkg -a $arch -y install $pkg+$ver

     # Create application link.
     [ X"$link" != X ] || link=${pkg#*+}
     rm -f $root/$cfgversion/apps.$label/$link
     ln -s ../$label/$arch/$(echo $pkg+$ver | tr + /) $root/$cfgversion/apps.$label/$link)
    [ $? = 0 ]

    (set -e
     inrepo $repo $swarea
     $swarea/common/cmspkg -a $arch clean)
    [ $? = 0 ]

    # Fix privs dropped by cmspkg/rpm
    setgroup ug+rw,o-w _sw $swarea
    if $nogroups; then :; else
      BADFILES=$(find $swarea/$arch -type f ! -group _sw -print)
      [ X"$BADFILES" = X ] || note "WARNING: removing $BADFILES"
      find $swarea/$arch -type f ! -group _sw -print0 | $xargsr -t -0 rm -f
      find $swarea/$arch ! -type l -perm -2 -print0 | $xargsr -t -0 chmod o-w
      find $swarea/$arch -type d ! -perm -660 -print0 | $xargsr -t -0 chmod ug+rw
      find $swarea/$arch -type f ! -group _sw -print0 | $xargsr -t -0 chgrp _sw
    fi
  fi

  # Install configuration.
  if $nogroups; then :; else
    BADFILES=$(find $root/$cfgversion/config/$project -type f ! -group _config -print)
    [ X"$BADFILES" = X ] || note "WARNING: removing $BADFILES"
    find $root/$cfgversion/config/$project -type f ! -group _config -print0 | $xargsr -t -0 rm -f
  fi

  (cd $project_config_src; find . -type f -print0 |
   tar --null -T /dev/stdin -cf -) | tar -C $root/$cfgversion/config/$project -xvvf - \
     --exclude CVS --exclude .svn --exclude .git \
     --exclude '*~' --exclude '.#*' --exclude '*.swp'
  setgroup ug+r,go-w _config $(find $root/$cfgversion/config/$project -type f)

  # Install authentication file if any.
  $authlink || for authfile in $authfiles; do
    local authname=$(echo $project | tr - _)
    local authsrc=
    local authdest=
    case $authfile in
      *:* ) authsrc=${authfile#*:} authdest=${authfile%:*} ;;
      *   ) authsrc=$authfile authdest=$(basename $authfile) ;;
    esac

    mkdir -p $project_auth
    if [ X"$privauthdir" != X ] && [ -f $privauthdir/$authsrc ]; then
      rm -f $project_auth/$authdest
      cp $privauthdir/$authsrc $project_auth/$authdest
    elif [ -f $project_auth/$authdest ]; then
      true
    else
      eval "deploy_${authname}_auth $authsrc" > $project_auth/$authdest
    fi

    setgroup u=rwx,g=rx,o-rwx _config $(find $project_auth -type d)
    setgroup ug=r,o-rwx _config $(find $project_auth -type f)
  done
}

# Enable or disable server on a particular node.
enable() { touch $root/enabled/$project; }
disable() { rm -f $root/enabled/$project; }

note() { echo "$*" >> $errlog; echo "$*" 1>&11; }

######################################################################
# Start error log and make sure it gets wiped on exit.
errlog=$root/.deploy/$(date +%Y%m%d-%H%M%S)-$$-$(echo $stages | tr -d ' ').log
trap "(exit 77); exit 77" 1 2 15
trap 'err=$?; note "ERROR: installation failed with exit code $err."; exit $err' ERR
: > $errlog
exec 11>&1 22>&2
exec 1>>$errlog 2>&1
set -eEx

# Record beginning of installation.
note "INFO: $now: starting deployment of: $*"
note "INFO: installation log can be found in $errlog"

# Make sure auth dir is ready before running sw
if $authlink; then
  note "INFO: Updating $root/$cfgversion/auth to $privauthdir"
  rm -f $root/$cfgversion/auth
  ln -s $privauthdir $root/$cfgversion/auth
else
  [ -d $root/$cfgversion/auth ] || {
    mkdir -p $root/$cfgversion/auth
    $chmod ug+rw,o-w $root/$cfgversion/auth
    $chgrp _sw $root/$cfgversion/auth
  }
fi

for stage in $stages; do
  # Ensure links are pointing to the right release before running post
  if [ X$stage = Xpost ]; then
    if [ -e $root/$cfgversion/apps ] && [ X"$(readlink $root/$cfgversion/apps)" = X"apps.$glabel" ]; then :; else
      note "INFO: Updating $root/$cfgversion/apps to apps.$glabel"
      rm -f $root/$cfgversion/apps
      ln -s apps.$glabel $root/$cfgversion/apps
    fi
    if [ -e $root/current ] && [ X"$(readlink $root/current)" = X"$cfgversion" ]; then :; else
      note "INFO: Updating $root/current to $cfgversion"
      rm -f $root/current
      ln -s $cfgversion $root/current
    fi
  fi
  
  # Loop over all requested services and install them one at a time.
  (for svc in $svcs; do deploy $stage $(echo $svc | tr / ' '); done)
done

# Resume normal output.
exit=$?
set +eEx
exec 1>&11 2>&22

# Report program result
case $exit in
  0 ) note "INFO: installation completed sucessfully" ;;
  * ) note "ERROR: installation failed with exit code $exit" ;;
esac

exit $exit