cassandra-common

#!/bin/bash
# vim: set ts=4 sw=4 et si:

. ./scripts/volume-common.sh
ETC_CASSANDRA="${ETC_CASSANDRA:-/etc/cassandra}"
CASSANDRA_ENV="${ETC_CASSANDRA}/cassandra-env.sh"
CASSANDRA_YML="${ETC_CASSANDRA}/cassandra.yaml"
CASSANDRA_ACCESS="${ETC_CASSANDRA}/access.properties"
CASSANDRA_PASSWD="${ETC_CASSANDRA}/passwd.properties"
CASSANDRA_RACKDC="${ETC_CASSANDRA}/cassandra-rackdc.properties"
CASSANDRA_TOPOLOGY="${ETC_CASSANDRA}/cassandra-topology.properties"
CASSANDRA_USER="cassandra"
CASSANDRA_GROUP="cassandra"
NAGIOS_EXPORT_DIR="/var/lib/nagios/export"

set -ex

export LANG=en_US.UTF-8

# Comms-less leader determination
# Picks the unit with the lowest id.
am_i_the_leader () {
  # FWIW this is race-prone (as not all units have the same 'relation-list'
  # view at the same point in time), so am_i_the_leader it's kinda ~best-effort.
  # Until juju-core provides facilities for master election, keep this
  # nasty workaround.
  # Leader is the lowest unit number and won't be in the
  # list of relation peers
  if [ -z "${JUJU_RELATION_ID}" ]; then
    juju-log "${FUNCNAME[1]}: ${FUNCNAME[0]}: not running under relation context"
    return 2
  fi
  local units=$(relation-list)
  juju-log "related units seen: ${units}"
  local unit_no=${JUJU_UNIT_NAME##*/}
  local unit
  for unit in ${units}; do
    peer_unit_no=${unit##*/}
    if [ "${peer_unit_no}" -lt "${unit_no}" ]; then
        juju-log "I am not the leader"
        return 1
    fi
  done
  juju-log "I am the leader"
  return 0
}

bzr_ci () {
  local msg=${FUNCNAME[1]}
  test -n "$1" && msg="$1"
  bzr st ${ETC_CASSANDRA}|egrep -q '^modified|added' || return 1
  bzr ci -m "[$(date "+%Y-%m-%d %H:%M:%S")] ${JUJU_UNIT_NAME}: ${msg} auto-commit" ${ETC_CASSANDRA} || return 1
  return 0
}


migrate_cassandra_data() {
  local from=${1?} to=${2?}
  [[ $from = $to ]] && return 0
  /etc/init.d/cassandra stop
  rsync -vaP "$from/" "$to/"
  mv "$from" "$from.migrated_${JUJU_UNIT_NAME//\//-}@$(date +%s)"
}

# When the volume is mounted by another (fresh) unit, getent passwd
# entry *can* get a different UID (depending e.g. on user order
# creation and/or other role accounts present), thus the need
# to fix permissions (ownership) over cassandra data dir.
fix_cassandra_perms() {
    local data_dir=${1?}
    chown -R cassandra:cassandra ${data_dir}
}

# Install cassandra source and install packages
install_cassandra () {
  juju-log "Installing Cassandra"
  # Install the repository
  APT_REPO_SPEC="$(config-get apt-repo-spec)"
  APT_REPO_KEY=$(config-get apt-repo-key)
  # Check for configured extra packages to install from config
  EXTRA_PACKAGES="$(config-get extra_packages)"
  # Get the debian package in the files directory.
  DEBIAN_PACKAGE_FILE=($CHARM_DIR/files/cassandra_*.deb)
  # Does the debain cassandra package exist in the charm directory?
  if [[ -f ${DEBIAN_PACKAGE_FILE} ]]; then
    # The apt-get update command is necessary to install java.
    apt-get update -qq
    # Install the prerequisites for cassandra.
    DEBIAN_FRONTEND=noninteractive apt-get -qq install -y openjdk-7-jre-headless libjna-java python-support python-cheetah dnsutils bzr ${EXTRA_PACKAGES}
    # Use the debian package manager to install the cassandra package.
    dpkg -i ${DEBIAN_PACKAGE_FILE}
  else
    # Add the key
    if [[ -n "${APT_REPO_KEY}" ]]; then
      # Use port 80 to get the keyserver key.
      apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys ${APT_REPO_KEY}
    fi
    if [[ -n "${APT_REPO_SPEC}" ]]; then
      echo "${APT_REPO_SPEC}" > /etc/apt/sources.list.d/cassandra.list
    fi
    # Update the repositories

    apt-get update
    # Ensure that cassandra does not startup before we have configured it
    disable_cassandra_start
    # Install the package
    DEBIAN_FRONTEND=noninteractive apt-get -qq install -y cassandra python-cheetah python-pip dnsutils bzr ${EXTRA_PACKAGES}

    # Install charm-helpers for benchmarking support
    pip install charmhelpers
  fi
  bzr_ci || :
}

# Update the cassandra environment with the appropriate JMX port
configure_jmx_port () {
  check_units_to_update || return 0
  juju-log "Configuring JMX port"
  JMX_PORT=$(config-get jmx-port)
  test -n "${JMX_PORT}" || return 1
  test ${JMX_PORT} -gt 0 -a ${JMX_PORT} -lt 65535 || return 1
  sed -i -e "s/^JMX_PORT=.*/JMX_PORT=\"${JMX_PORT}\"/" ${CASSANDRA_ENV}
  # Open port ready for expose
  open-port ${JMX_PORT}/TCP
}

get_private_ip () {
  # Make sure that we use the resolvable private address
  # some dig versions dealxs with both hostnames and ip
  # addresses, but don't depend on that.
  local address=$(unit-get private-address)
  if [[ ${address} =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]]; then
      # already an IP
      echo "${address}"
  else
      # not an IP (some providers return hostnames), resolve it
      dig +short "${address}"
  fi
}

relation-get-all() {
    # reltype is e.g. 'cluster'
    # relkey is e.g. 'seed-nodes'
    # default separator is space
    local reltype=${1:?} relkey=${2:?} separator="${3:- }"
    local rid unit value=
    # aggregate all values from every related unit
    value=$(
        for rid in $(relation-ids ${reltype});do
            for unit in $(relation-list -r ${rid});do
                relation-get -r ${rid} ${relkey} ${unit}
            done
        done
    )
    echo "${value}" | sort -u | paste -d "${separator}" -s
}

write_seed_nodes() {
    local seeds="${1?}"
    # write to config if non-null seeds, or seed value coming from
    if [ -n "${seeds}" ] || [ "$(config-get force-seed-nodes)" != "" ] || \
        [ "$(config-get allow-single-node)" = "True" ]; then
        local seeds_comma=$(echo "$seeds" | sed -r 's/ +/,/g')
        juju-log "Setting seeds node to be '${seeds_comma}'"
        sed -i -e "s/\- seeds:.*/\- seeds: \"${seeds_comma}\"/" ${CASSANDRA_YML}
    else
        # This happens when e.g. the unit relates to a non-leader
        juju-log "Skipping seeds node setting (no seeds)"
    fi
}

get_seed_nodes() {
    local force_seed_nodes="$(config-get force-seed-nodes)"
    local allow_single_node="$(config-get allow-single-node)"
    local seeds

    if [ -n "${force_seed_nodes}" ]; then
        seeds="${force_seed_nodes}"
    elif [ "${allow_single_node}" == "True" ]; then
        seeds="$(get_private_ip)"
        # If running under relation context ...
    elif [ -n "${JUJU_RELATION_ID}" ]; then
        if am_i_the_leader; then
            seeds="$(running_nodes)"
            # Too early, maybe bootstraping the whole cluster ...
            if [ -z "${seeds}" ]; then
                seeds=$(get_private_ip)
            fi
        else
            seeds=$(relation-get-all cluster seed-nodes)
        fi
    fi
    juju-log "${FUNCNAME[1]}: ${FUNCNAME[0]}: seeds='$seeds'"
    echo "${seeds}"
}

srv_root_get() {
  sed -r -n 's,commitlog_directory: *([^ ]+)/commitlog.*,\1,p' ${CASSANDRA_YML}
}

srv_root_from_volid() {
  local volid=${1?missing volid}
  local mntpoint
  if mntpoint=$(volume_mount_point_from_volid "${volid}"); then
    echo "${mntpoint}/cassandra"
  else
    echo "/var/lib/cassandra"
  fi
}
srv_root_save() {
  local srv_root=${1:?missing srv_root}
  sed -i -r -e $'/data_file_directories:/{\nN;s|- /.*|- '"${srv_root}/data"'|}' \
         -e "s|^(commitlog_directory):.*|\1: ${srv_root}/commitlog|" \
         -e "s|^(saved_caches_directory):.*|\1: ${srv_root}/savedcache_dir|" \
         ${CASSANDRA_YML}
  test -d "${srv_root}" || {
    mkdir -p "${srv_root}"
    chown -R ${CASSANDRA_USER}:${CASSANDRA_GROUP} "${srv_root}"
  }
}
disable_cassandra_start() {
  # Ensure that cassandra does not startup before we have configured it
  if [ -e /usr/sbin/policy-rc.d ]; then
      mv /usr/sbin/policy-rc.d /usr/sbin/policy-rc.d.orig
  fi
  install -m 755 files/policy-rc.d /usr/sbin/policy-rc.d
}

enable_cassandra_start() {
  if [ -e /usr/sbin/policy-rc.d.orig ]; then
      mv /usr/sbin/policy-rc.d.orig /usr/sbin/policy-rc.d
  else
      rm -rf /usr/sbin/policy-rc.d.orig
  fi
}

get_initial_token() {
    token_map_unitname=$(config-get token-map-by-unitname)
    token_map_volid=$(config-get token-map-by-volid)
    initial_token=""
    if [[ -z ${token_map_unitname} ]] && [[ -z ${token_map_volid} ]]; then
        echo $initial_token
        return
    elif [[ -n ${token_map_unitname} ]] && [[ -n ${token_map_volid} ]]; then
        juju-log "ERROR: cannot specify token-map-by-unitname and token-map-by-volid"
        exit 1
    elif [[ -n ${token_map_unitname} ]]; then
        token_map=$token_map_unitname
        token_key=${JUJU_UNIT_NAME}
    elif [[ -n ${token_map_volid} ]]; then
        token_map=$token_map_volid
        token_key=$(volume_get_volume_id)
    fi
    python="import sys;from yaml import load;token_map = load(sys.stdin);print token_map.get(\"${token_key}\")"
    initial_token=$(echo $token_map | python -c "${python}")
    echo ${initial_token/None/}
}

# Construct the cassandra.yaml file from the appropriate information above
configure_cassandra () {
  check_units_to_update || return 0
  juju-log "Configuring Cassandra"
  local LISTEN_ADDRESS=$(get_private_ip)
  local CLUSTER_PORT=$(config-get cluster-port)
  local CLIENT_PORT=$(config-get client-port)
  local CLUSTER_NAME=$(config-get cluster-name)
  local AUTO_MEMORY=$(config-get auto-memory)
  local HEAP_SIZE=$(config-get heap-size)
  local NEW_GEN_SIZE=$(config-get new-gen-size)
  local SRV_ROOT=$(config-get srv-root)
  local COMPACTION_THROUGHPUT=$(config-get compaction-throughput)
  local STREAM_THROUGHPUT=$(config-get stream-throughput)
  local NUM_TOKENS=$(config-get num-tokens)
  local INITIAL_TOKEN=$(get_initial_token)
  local PARTITIONER=$(config-get partitioner)
  local ENDPOINT_SNITCH=$(config-get endpoint_snitch)
  local DATACENTER=$(config-get datacenter)
  local RACK=$(config-get rack)
  local DC_SUFFIX=$(config-get dc_suffix)
  local PREFER_LOCAL=$(config-get prefer_local | tr '[:upper:]' '[:lower:]')
  # This needs to 'detect' whether its running in ec2
  # and use the ec2Snitch instead if it is - or we could
  # specify this as a configuration parameter
  # non-trivial sed to edit data_file_directories (has its value in the line following it)
  sed -i -r -e "s/^cluster_name:.*/cluster_name: \'${CLUSTER_NAME}\'/" \
         -e "s/^(storage_port):.*/\1: ${CLUSTER_PORT}/" \
         -e "s/^(listen_address):.*/\1: ${LISTEN_ADDRESS}/" \
         -e "s/^(rpc_address):.*/\1: ${LISTEN_ADDRESS}/" \
         -e "s/^(rpc_port):.*/\1: ${CLIENT_PORT}/" \
         -e "s/^(compaction_throughput_mb_per_sec):.*/\1: ${COMPACTION_THROUGHPUT}/" \
         -e "s/^(# )?(stream_throughput_outbound_megabits_per_sec):.*/\2: ${STREAM_THROUGHPUT}/" \
         -e "s/^(partitioner):.*/\1: ${PARTITIONER}/" \
         -e "s/^(endpoint_snitch):.*/\1: ${ENDPOINT_SNITCH}/" \
         ${CASSANDRA_YML}

  ## Number of tokens or initial token
  if [ $NUM_TOKENS -eq 0 ] && [ -n "$INITIAL_TOKEN" ]; then
      sed -i -r -e "s/^(# )?(num_tokens):.*/#\2: ${NUM_TOKENS}/" \
         -e "s/^(# )?(initial_token):.*/\2: ${INITIAL_TOKEN}/" \
         ${CASSANDRA_YML}
  elif [ $NUM_TOKENS -gt 0 ] && [ -z "$INITIAL_TOKEN" ]; then
      sed -i -r -e "s/^(# )?(num_tokens):.*/\2: ${NUM_TOKENS}/" \
         -e "s/^(# )?(initial_token):.*/# \2: ${INITIAL_TOKEN}/" \
         ${CASSANDRA_YML}
  else
        juju-log "ERROR: cannot specify num_tokens and initial_token"
        exit 1
  fi

  ## Enpoint Snitch configuration
  # XXX Handle more endpoint snitch options
  if [[ "$ENDPOINT_SNITCH" =~ .*SimpleSnitch ]]; then
    rm -f ${CASSANDRA_RACKDC} ${CASSANDRA_TOPOLOGY}
  elif [[ "$ENDPOINT_SNITCH" =~ .*GossipingPropertyFileSnitch ]]; then
    rm -f ${CASSANDRA_TOPOLOGY}
    (
        echo "dc=${DATACENTER}"
        echo "rack=${RACK}"
        if [ -n "$DC_SUFFIX" ]; then
          echo "dc_suffix=${DC_SUFFIX}"
        fi
        if [ "$PREFER_LOCAL" = "true" ]; then
          echo "prefer_local=true"
        fi
    ) > ${CASSANDRA_RACKDC}
  else
    rm -f ${CASSANDRA_RACKDC}
  fi

  ## Storage configuration
  local volid
  local IO_SCHEDULER=$(config-get io-scheduler)
  if volid=$(volume_get_volume_id);then
    juju-log "INFO: volid=${volid}"
  else
    disable_cassandra_start
    juju-log "ERROR: invalid storage_config at ${FUNCNAME[0]}"
    return 1
  fi
  local srv_root_curr=$(srv_root_get)
  (
  set -e
  [[ -z "${volid}" ]] && exit 0
  if volume_is_permanent "${volid}";then
    local srv_root_new=$(srv_root_from_volid ${volid})
    if [[ ${srv_root_new} != ${srv_root_curr} ]];then
      volume_init_and_mount ${volid}
      if [[ -d ${srv_root_curr}/data ]];then
        juju-log "WARNING: found already existing cassandra data, migrating ${srv_root_curr} -> ${srv_root_new}"
        migrate_cassandra_data ${srv_root_curr} ${srv_root_new}
      fi
      fix_cassandra_perms ${srv_root_new}
      srv_root_save ${srv_root_new}
    fi
    # Set the kernel block scheduler
    if [[ -n "$IO_SCHEDULER" ]]; then
        DEV=$(df $srv_root_new | awk '{print $1}' |tail -n 1)
        DEV=${DEV#\/dev\/}
        DEV=${DEV%[0-9]}
        echo "$IO_SCHEDULER" > /sys/block/${DEV}/queue/scheduler
    fi
  fi
  ) && enable_cassandra_start || disable_cassandra_start

  # Open port ready for expose
  open-port ${CLIENT_PORT}/TCP

  # Get the architecture from the uname command.
  ARCHITECTURE=`uname -m`
  if [ "$ARCHITECTURE" == "ppc64le" ]; then
    # On the ppc64le architecture requires a higher value for stack size.
    sed -i -e "s/-Xss[0-9]*k/-Xss1664k/" ${CASSANDRA_ENV}
  else
    # Default cassandra-env.sh for some cassandra 1.0.x specifies -Xss128k
    # while 160k min is required
    sed -i -e "s/-Xss[0-9]*k/-Xss256k/" ${CASSANDRA_ENV}
  fi
  # OpenJDK does not handle the UseCondCardMark flag.
  sed -i -e "s/-XX:+UseCondCardMark//" ${CASSANDRA_ENV}

  # Configure memory settings as specified in configuration
  if [ "$AUTO_MEMORY" = "False" ]; then
    juju-log "Configuring Manual Memory Setttings"
    sed -i -e "s/^[#]MAX_HEAP_SIZE=.*/MAX_HEAP_SIZE=\"${HEAP_SIZE}\"/" \
           -e "s/^[#]HEAP_NEWSIZE=.*/HEAP_NEWSIZE=\"${NEW_GEN_SIZE}\"/" ${CASSANDRA_ENV}
  else
    sed -i -e "s/^[#]MAX_HEAP_SIZE=.*/#MAX_HEAP_SIZE=\"${HEAP_SIZE}\"/" \
           -e "s/^[#]HEAP_NEWSIZE=.*/#HEAP_NEWSIZE=\"${NEW_GEN_SIZE}\"/" ${CASSANDRA_ENV}
  fi
  local use_simpleauth=$(config-get use-simpleauth)
  if [[ ${use_simpleauth} == True ]];then
      local auth_opts
      install -o ${CASSANDRA_USER} -m 600 /dev/null ${CASSANDRA_PASSWD}
      config-get auth-passwd64|base64 -d > ${CASSANDRA_PASSWD}
      config-get auth-access64|base64 -d > ${CASSANDRA_ACCESS}
      auth_opts="-Dpasswd.properties=${CASSANDRA_PASSWD} -Daccess.properties=${CASSANDRA_ACCESS}"
      add_JVM_OPTS_line "Daccess.properties" "$auth_opts"
      replace_YAML_key authenticator org.apache.cassandra.auth.SimpleAuthenticator
  else
      rm -f ${CASSANDRA_PASSWD} ${CASSANDRA_ACCESS}
      del_JVM_OPTS_line "Daccess.properties"
      replace_YAML_key authenticator org.apache.cassandra.auth.AllowAllAuthenticator
  fi
  local extra_jvm_opts=$(config-get extra-jvm-opts)
  if [[ ${extra_jvm_opts} != "" ]];then
      add_JVM_OPTS_line "JUJU_extra-jvm-opts" "$extra_jvm_opts"
  else
      del_JVM_OPTS_line "JUJU_extra-jvm-opts"
  fi
  reconfigure_cluster_seeds
}

# Remove JVM_OPTS line by regex
del_JVM_OPTS_line() {
  local regex="${1?}"
  juju-log "Deleting from JVM_OPTS: key=${regex}"
  sed -ri "\!^JVM_OPTS=.*${regex}!d" ${CASSANDRA_ENV} || true
}

# Add JVM_OPTS line, use regex remove it before, if already present
add_JVM_OPTS_line() {
  local regex="${1?}" line="${2?}"
  del_JVM_OPTS_line "${regex}"
  juju-log "Adding to JVM_OPTS: ${line} (key=${regex})"
  echo "JVM_OPTS=\"\$JVM_OPTS ${line}\" ## ${regex} " >> ${CASSANDRA_ENV}
}

# Replace an *existent* YAML key: value
# Note that key will be treated as a regex
replace_YAML_key() {
  local key="${1?}" value="${2?}"
  juju-log "Replacing YAML: key=${key} value=${value}"
  sed -ri -e "s!^(${key}): .*!\1: ${value}!" ${CASSANDRA_YML}
}

# Service Control Commands
cassandra_is_running() {
    juju-log "${FUNCNAME[1]}: ${FUNCNAME[0]} $@"
    service cassandra status
}

# wait until "$func $*" OK, loop $times sleeping $secs (default: 1)
wait_until() {
    local secs=${1:?} times=${2:?} func=${3:?}
    shift 3
    juju-log "${FUNCNAME[1]}: ${FUNCNAME[0]} $@"
    while let times=times-1; do
        if ${func} ${*}; then
            juju-log "${FUNCNAME[0]} $func $*: OK (times=$times left)"
            return 0
        fi
        juju-log "${FUNCNAME[0]} $func $*: sleep ${secs}, times=$times left"
        sleep ${secs}
    done
    return 1
}

# ditto wait_until, with opposite logic
wait_until_not() {
    local secs=${1:?} times=${2:?} func=${3:?}
    juju-log "${FUNCNAME[1]}: ${FUNCNAME[0]} $@"
    while let times=times-1; do
        if ${func} ${*}; then
            :
        else
            juju-log "${FUNCNAME[0]} $func $*: OK (times=$times left)"
            return 0
        fi
        juju-log "${FUNCNAME[0]} $func $*: sleep ${secs}, times=$times left"
        sleep ${secs}
    done
    return 1
}

restart_cassandra () {
  juju-log "Restarting Cassandra"
  service cassandra restart
  wait_until 1 60 cassandra_is_running
}

start_cassandra() {
  juju-log "Starting Cassandra"
  service cassandra status || service cassandra start
  wait_until 1 60 cassandra_is_running
}

stop_cassandra() {
  juju-log "Stopping Cassandra"
  service cassandra stop || :
  wait_until_not 1 60 cassandra_is_running
}

setup_database_interface () {
  juju-log "Setup Cassandra database interface"
  relation-set port="$(config-get client-port)"
}

setup_jmx_interface () {
  juju-log "Setup Cassandra JMX interface"
  relation-set port="$(config-get jmx-port)"
}

reconfigure_cluster_seeds() {
    juju-log "Reconfiguring cluster seeds"
    local seeds=$(get_seed_nodes)
    write_seed_nodes "${seeds}"
    # If under relation context, and I'm leader => propagate seed-nodes to other units.
    if [ -n "${seeds}" -a -n "${JUJU_RELATION_ID}" ] && am_i_the_leader; then
        juju-log "Setting relation seed-nodes to ${seeds}"
        relation-set seed-nodes="${seeds}"
    fi
}

nodestatus_is() {
    local ip="${1:?}" wanted_status="${2:?}"
    local nodestatus="$(nodestatus ${ip})"
    juju-log "nodestatus='${nodestatus}' wanted_status='${wanted_status}'"
    [[ ${nodestatus} =~ ${wanted_status} ]]
    return $?
}

nodestatus() {
    local node_ip=$1
    local VER=$(dpkg-query --show --showformat='${Version}' cassandra)
    local MAJ=${VER%%.*}
    local MIN_REL=${VER#*.}
    local MIN=${MIN_REL%%.*}
    local RING_COLS='$3$4'
    if [ $MAJ == "0" ] || ([ $MAJ == "1" ] && [ $MIN == "0" ]); then
        # nodetool ring columns are different in 1.0 and lower
        RING_COLS='$4$5'
    fi
    nodetool -h $node_ip ring 2>/dev/null | \
        awk "/^$node_ip / {print $RING_COLS}" | \
        tail -n1
}

running_nodes() {
    local node_ip=$(get_private_ip)
    nodetool -h $node_ip ring 2>/dev/null | \
        awk '/Up *Normal/ {print $1}' | \
        sort -u | \
        paste -d " " -s
}

echo_yml_expression() {
    python -c "import yaml,sys;print yaml.load(sys.stdin)$1" < ${CASSANDRA_YML}
}

current_file_seeds() {
    echo_yml_expression '["seed_provider"][0]["parameters"][0]["seeds"]'
}

bootstrap () {
    local IP
    local count status
    if am_i_the_leader; then
        :
    else
        local file_seeds=$(current_file_seeds)
        case "${file_seeds}" in
            ""|127.*)
                juju-log "Returning soft-fail on not-yet received external seeds"
                return 0;;
        esac
    fi
    juju-log "Waiting for this node to join the cluster"
    # wait until cassandra is UP ok, 3 retries
    # start_cassandra will no-op if already running
    count=3
    while let count=count-1; do
        start_cassandra
        if wait_until 60 2 nodestatus_is $(get_private_ip) "Up(Joining|Normal)"; then
            return 0
        fi
    done
    juju-log "Timed out waiting for Up Normal/Joining"
    exit 2
}

remove_down_nodes () {
    reconfigure_cluster_seeds
}

decommission_node () {
  juju-log "Decommissioning node - this may take some time..."
  nodetool -h $(get_private_ip) decommission
}
run_exec_d_hooks() {
  local exec_hook=${1:?}
  if [[ -d exec.d ]]; then
    shopt -s nullglob
    for f in exec.d/*/charm-${exec_hook}; do
    [[ -x "$f" ]] || continue
    ${SHELL} -c "$f"|| {
        ## bail out if anyone fails
        juju-log -l ERROR "$f: returned exit_status=$? "
        exit 1
    }
    done
  fi
}

update_nagios_checks() {
  if [ -d "/usr/lib/nagios/plugins" ] && [ -d "/etc/nagios/nrpe.d" ]; then
    IP=`get_private_ip`
    install --owner=root --group=root --mode=0555 files/check_cassandra_heap.sh /usr/lib/nagios/plugins/check_cassandra_heap.sh
    export NAGIOS_CONTEXT="$(config-get nagios_context)"
    export NAGIOS_HOSTNAME="${NAGIOS_CONTEXT}-${JUJU_UNIT_NAME//\//-}"
    export NAGIOS_SERVICEGROUP=${NAGIOS_CONTEXT}

    export WARN_PCT="$(config-get nagios_heapchk_warn_pct)"
    export CRIT_PCT="$(config-get nagios_heapchk_crit_pct)"
    export SERVICE_DESCRIPTION="Cassandra server heap usage"
    export NRPE_CMD_NAME="check_${NAGIOS_CONTEXT}_cassandra_heap"
    export NRPE_CMD="/usr/lib/nagios/plugins/check_cassandra_heap.sh ${IP} ${WARN_PCT} ${CRIT_PCT}"
    if [ ! -d "${NAGIOS_EXPORT_DIR}" ]; then
        mkdir -p "${NAGIOS_EXPORT_DIR}"
    fi
    cheetah fill --env -p templates/nrpe_cmd_file.tmpl > /etc/nagios/nrpe.d/${NRPE_CMD_NAME}.cfg
    cheetah fill --env -p templates/nrpe_service_file.tmpl > ${NAGIOS_EXPORT_DIR}/service__${NAGIOS_HOSTNAME}_${NRPE_CMD_NAME}.cfg
  fi
}

# Check if units-to-update comma separated list contains my number, or "all",
# returns 0 if this unit should be updated
check_units_to_update() {
  local unit
  local units_to_update=$(config-get units-to-update)
  local unitnum=${JUJU_UNIT_NAME##*/}
  [[ $units_to_update == "all" ]] && return 0
  (
    OLDIFS="$IFS"
    IFS=" ,"
    for unit in ${units_to_update}; do
      if [[ $unit == $unitnum ]];then
        juju-log "check_units_to_update: \"${unitnum}\" in \"${units_to_update}\", updating."
        exit 0
      fi
    done
    juju-log "check_units_to_update: \"${unitnum}\" not in \"${units_to_update}\", skipping."
    exit 1
  )
  return $?
}

hook_main() {
local COMMAND="$1"
export BZR_HOME=/root
case $COMMAND in
  install)
    run_exec_d_hooks "pre-install"
    # Install cassandra first run only
    [[ -d /usr/share/cassandra ]] || install_cassandra
    # Initialize /etc/cassandra bzr
    bzr whoami "${JUJU_UNIT_NAME} <juju@${HOSTNAME}>"
    test -d ${ETC_CASSANDRA}/.bzr || bzr init ${ETC_CASSANDRA} && bzr add ${ETC_CASSANDRA}
    bzr_ci "main:${COMMAND}" ${ETC_CASSANDRA} || :
    # Update the cassandra environment with the appropriate JMX port
    configure_jmx_port
    bzr_ci || :
    # Construct the cassandra.yaml file from the appropriate information above
    configure_cassandra
    bzr_ci || :
    run_exec_d_hooks "post-install"
    ;;
  config-changed)
    needs_restart=
    # Update the cassandra environment with the appropriate JMX port
    configure_jmx_port
    bzr_ci && needs_restart=:
    # Construct the cassandra.yaml file from the appropriate information above
    configure_cassandra
    bzr_ci && needs_restart=:
    # be sure to kick a cassandra restart if not
    # running yet
    cassandra_is_running || needs_restart=:
    if [ $needs_restart ]; then
      # Restart as required
      restart_cassandra
    fi
    # Fill in nagios checks
    update_nagios_checks
    ;;
  upgrade-charm)
    check_units_to_update || exit 0
    juju-log "Updating this charm - currently no-op"
    ;;
  stop)
    stop_cassandra
    ;;
  start)
    if [ "$(config-get allow-single-node)" == "True" ]; then
        start_cassandra
    fi
    ;;
  database-relation-joined)
    setup_database_interface
    ;;
  cluster-relation-joined)
    juju-log "Joining cassandra cluster..."
    am_i_the_leader || sleep $(( 30 + RANDOM%15 ))
    ;;
  cluster-relation-changed)
    reconfigure_cluster_seeds
    # if there were config changes, restart cassandra
    bzr_ci && restart_cassandra
    bootstrap
    ;;
  cluster-relation-departed)
    reconfigure_cluster_seeds
    # if there were config changes, restart cassandra
    bzr_ci && restart_cassandra
    remove_down_nodes
    ;;
  jmx-relation-joined)
    setup_jmx_interface
    ;;
  nrpe-external-master-relation-changed)
    update_nagios_checks
    ;;
  *)
    juju-log "Command not recognised: $COMMAND"
    ;;
esac
}
# Actual processing, if not unittesting
test -n "$CASSANDRA_TESTING" || hook_main ${0##*/}