manage

#!/bin/bash
export MSYS_NO_PATHCONV=1
export DOCKERHOST=${APPLICATION_URL-$(docker run --rm --net=host eclipse/che-ip)}

SCRIPT_HOME="$( cd "$( dirname "$0" )" && pwd )"
export COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-von}"
export DEFAULT_CLI_SCRIPT_DIR='./cli-scripts'

export LEDGER_TIMEOUT="${LEDGER_TIMEOUT:-60}"
export LEDGER_URL_CONFIG="${LEDGER_URL_CONFIG}"

export ROOT_BACKUP_DIR=backup
export SHELL_CMD='bash'

# Running on Windows?
if [[ "$OSTYPE" == "msys" ]]; then
  # Prefix interactive terminal commands ...
  terminalEmu="winpty"
fi

# ========================================================================================================
# Dynamically detect the version of docker compose and adjust the '--log-level' syntax appropriately.
# --------------------------------------------------------------------------------------------------------
# Default to using the existing syntax
dockerCompose="docker-compose --log-level ERROR"
dockerComposeVersion=$(docker-compose version --short | sed 's~v~~;s~-.*~~')
dockerComposeVersion=${dockerComposeVersion%.*}
if [[ $(awk "BEGIN {print (${dockerComposeVersion} >= 2.0) ? \"true\" : \"false\" }") == "true" ]]; then
  # Use the new syntax when version 2.0.0 or greater is detected.
  dockerCompose="docker --log-level error compose"
fi
# ========================================================================================================

# =================================================================================================================
# Usage:
# -----------------------------------------------------------------------------------------------------------------
usage () {
  cat <<-EOF

  Usage: $0 [command] [--logs] [options]

  Commands:

  build - Build the docker images for the project.
          You need to do this first.

  start | up - Starts all containers
       You can include a '--wait' parameter which will wait until the ledger is active
       You can include a '--taa-sample' parameter which will install the sample Transaction Authorisation Agreement
        files into the ledger. Alternatively you can create your own config/aml.json and config/taa.json files which
        will get installed into the ledger at start (see config/sample_aml.json and config/sample_taa.json for format)
       When using the '--logs' option, use ctrl-c to exit logging. Use "down" or "stop" to stop the run.
        Examples:
        $0 start
        $0 start --logs
        $0 start <ip_proxy_1>,<ip_proxy_2>,<ip_proxy_3>,<ip_proxy_4> &
        $0 start --wait --logs
        $0 start --taa-sample

  start-web - Start the web server to monitor an existing ledger, requires GENESIS_URL and LEDGER_SEED params
        Example:
        $0 start-web GENESIS_URL=http://foo.bar/genesis.txt LEDGER_SEED=00000000000000000000000000000012

  logs - To tail the logs of running containers (ctrl-c to exit).
         Use the '--no-tail' option to only print log without tailing.
          Examples:
          $0 logs
          $0 logs --no-tail

  down | rm - Brings down the services and removes the volumes (storage) and containers.

  stop - Stops the services.  This is a non-destructive process.  The volumes and containers
         are not deleted so they will be reused the next time you run start.

  rebuild - Rebuild the docker images.

  dockerhost - Print the ip address of the Docker Host Adapter as it is seen by containers running in docker.

  generateSecrets - Generate a random set of secrets using openssl; a Seed and a Key.

  indy-cli - Run Indy-Cli commands in a Indy-Cli container environment.

        $0 indy-cli -h
          - Display specific help documentation.

  cli - Run a command in an Indy-Cli container.

        $0 cli -h
          - Display specific help documentation.

  backup [description] - Backup the current von-network environment.
          Creates a set of tar.gz archives of each of the environment's volumes.
          Backup sets are stored in a ./backup/date/time folder structure.
          Examples:
          $0 backup
          $0 backup "The description of my environment's current state."
            - Make a backup and include the description in a ReadMe.txt file.

  restore [filter] - Restore a given backup set.  Defaults to restoring the most recent backup.
          Examples:
          $0 restore
            - Restore the most recent backup.
          $0 restore 13-15-37
            - Restore the backup from 13-15-37 today.
          $0 restore von_client-data_2021-08-23_08-21-08.tar
            - Restore the backup set containing the von_client-data_2021-08-23_08-21-08.tar archive.
          $0 restore 2021-08-23
            - Restore the most recent backup set from 2021-08-23.

  restoreArchive <archive> <volume> [tarOptions]- Restore a tar.gz archive to a named volume.
          Useful for restoring an archive for inspection and debugging purposes.
          Examples:
          $0 restoreArchive ./backup/Node3-ledger-backup.tar.gz node3-bcovrin-tes
          $0 restoreArchive ./backup/Node3-ledger-backup.tar.gz node3-bcovrin-test --strip=1
            - Restore the archive to the named volume, stripping the first level directory from the archive.
            - Useful in the scenario where the archive contains additional directory levels that aren’t needed in the restored copy.

  debugVolume <volume> [volumeMountFolder] - Mount a named volume into a 'debug' instance of the 'von-network-base' image with an interactive shell.
          Provides a containerized environment to perform analysis on the ledger databases and files.
          Starting with 'bcgovimages/von-image:node-1.12-4' the base image for von-network conatins the RocksDB sst_dump tool that can be used to verify
          and inspect the RocksDB database files; '*.sst' files.
          For example the command 'find /debug_volume/ -name "*.sst" | xargs -I {} sst_dump --file={} --command=verify' can be used to do a quick
          verification on all the database files once the container starts.
          Usage information for sst_dump can be found here; https://github.com/facebook/rocksdb/wiki/Administration-and-Data-Access-Tool
          Examples:
          $0 debugVolume node3-bcovrin-test
          $0 debugVolume node1-bcovrin-test /home/indy/ledger
            - Mount the named volume to '/home/indy/ledger'
EOF
exit 1
}

indyCliUsage () {
  cat <<-EOF

  Usage:
    $0 [options] indy-cli [-h] [command] [parameters]

    Run Indy-Cli commands in a Indy-Cli container environment.
      - Refer to the cli-scripts directory for available scripts and their parameters.
      - Refer to './docs/Writing Transactions to a Ledger for an Un-privileged Author.md' for
        additional examples.

    Options:
      -v <FullyQualifiedPathToScripts/>
        - Mount a script volume to the container.  By default the 'cli-scripts' directory is mounted to the container.

    Examples:

    $0 indy-cli
      - Start an interactive indy-cli session in your Indy-Cli Container.

    $0 indy-cli --help
      - Get usage information for the indy-cli.
EOF
exit 1
}

cliUsage () {
  cat <<-EOF

  Usage:
    $0 [options] cli [-h] [command]

    Run a command in an Indy-Cli container.

    Options:
      -v <FullyQualifiedPathToScripts/>
        - Mount a script volume to the container.  By default the 'cli-scripts' directory is mounted to the container.

    Examples:

    $0 cli reset
      - Reset your Indy-CLI container's environment

    $0 cli init-pool localpool http://192.168.65.3:9000/genesis
    $0 cli init-pool MainNet https://raw.githubusercontent.com/sovrin-foundation/sovrin/stable/sovrin/pool_transactions_live_genesis
      - Initialize the pool for your Indy-CLI container's environment.
EOF
exit 1
}

# -----------------------------------------------------------------------------------------------------------------
# Initialization:
# -----------------------------------------------------------------------------------------------------------------
while getopts v:h FLAG; do
  case $FLAG in
    v ) VOLUMES=$OPTARG ;;
    h ) usage ;;
    \? ) #unrecognized option - show help
      echo -e \\n"Invalid script option: -${OPTARG}"\\n
      usage
      ;;
  esac
done
shift $((OPTIND-1))

# -----------------------------------------------------------------------------------------------------------------
# Functions:
# -----------------------------------------------------------------------------------------------------------------
function toLower() {
  echo $(echo ${@} | tr '[:upper:]' '[:lower:]')
}

function initDockerBuildArgs() {
  dockerBuildArgs=""

  # HTTP proxy, prefer lower case
  if [[ "${http_proxy}" ]]; then
    dockerBuildArgs=" ${dockerBuildArgs} --build-arg http_proxy=${http_proxy}"
  else
    if [[ "${HTTP_PROXY}" ]]; then
      dockerBuildArgs=" ${dockerBuildArgs} --build-arg http_proxy=${HTTP_PROXY}"
    fi
  fi

  # HTTPS proxy, prefer lower case
  if [[ "${https_proxy}" ]]; then
    dockerBuildArgs=" ${dockerBuildArgs} --build-arg https_proxy=${https_proxy}"
  else
    if [[ "${HTTPS_PROXY}" ]]; then
      dockerBuildArgs=" ${dockerBuildArgs} --build-arg https_proxy=${HTTPS_PROXY}"
    fi
  fi

  echo ${dockerBuildArgs}
}

function initEnv() {

  if [ -f .env ]; then
    while read line; do
      if [[ ! "$line" =~ ^\# ]] && [[ "$line" =~ .*= ]]; then
        export ${line//[$'\r\n']}
      fi
    done <.env
  fi

  for arg in "$@"; do
    # Remove recognized arguments from the list after processing.
    shift
    case "$arg" in
      *=*)
        export "${arg}"
        ;;
      --logs)
        TAIL_LOGS=1
        ;;
      --wait)
        WAIT_FOR_LEDGER=1
        ;;
      --taa-sample)
        USE_SAMPLE_TAA=1
        ;;
      *)
        # If not recognized, save it for later procesing ...
        set -- "$@" "$arg"
        ;;
    esac
  done

  IP=""
  IPS=""
  if [ ! -z $(echo ${1} | grep '[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}') ]; then
    if [[ $1 == *","* ]]; then
      IPS="$1"
    else
      IP="$1"
    fi
  fi
  export IP="$IP" IPS="$IPS"

  export LOG_LEVEL=${LOG_LEVEL:-info}
  export RUST_LOG=${RUST_LOG:-warning}
}

function runCliCommand() {

  unset displayCliUsage
  for arg in "$@"; do
    # Remove recognized arguments from the list after processing.
    shift
    case "$arg" in
      -h)
        displayCliUsage=1
        ;;
      *)
        # If not recognized, save it for later procesing ...
        set -- "$@" "$arg"
        ;;
    esac
  done

  initEnv "$@"
  cliCmd="${1}"
  shift || cliCmd=""

  if [ ! -z "${displayCliUsage}" ] && [[ "${cliCmd}" == "indy-cli" ]]; then
    indyCliUsage
  elif [ ! -z "${displayCliUsage}" ] && [[ -z "${cliCmd}" ]]; then
    cliUsage
  fi

  cmd="${terminalEmu} ${dockerCompose} \
    run "

  if [ -z "${VOLUMES}" ] && [ -d "${DEFAULT_CLI_SCRIPT_DIR}" ] ; then
    VOLUMES=$(realpath ${DEFAULT_CLI_SCRIPT_DIR})
  fi

  if [ ! -z "${VOLUMES}" ]; then
    shopt -s extglob
    paths=$(echo "${VOLUMES}" | sed -n 1'p' | tr ',' '\n')
    for path in ${paths}; do
      path=${path%%+(/)}
      mountPoint=${path##*/}
      if [[ "$OSTYPE" == "msys" ]]; then
        # When running on Windows, you need to prefix the path with an extra '/'
        path="/${path}"
      fi
      cmd+=" -v '${path}:/home/indy/${mountPoint}:Z'"
    done
  fi

  # Need to escape quotes and commas so they don't get removed along the way ...
  escapedArgs=$(echo $@ | sed "s~'~\\\'~g" | sed 's~\"~\\"~g')

  # Quote the escaped args so docker compose does not try to perform any processing on them ...
  # Separate the command and the args so they don't get treated as one argument by the scripts in the container ...
  cmd+="
    --rm client \
    ./scripts/manage ${cliCmd} \"${escapedArgs}\""

  eval ${cmd}
}

function logs() {
  (
    local OPTIND
    local unset _force
    local unset no_tail
    while getopts ":f-:" FLAG; do
      case $FLAG in
        f ) local _force=1 ;;
        - )
            case ${OPTARG} in
                "no-tail"*) no_tail=1
              ;;
            esac
      esac
    done
    shift $((OPTIND-1))

    log_args=()
    (( no_tail != 1 )) && log_args+=( '-f' )
    if [ ! -z "${TAIL_LOGS}" ] || [ ! -z "${_force}" ]; then
      ${dockerCompose} \
        logs \
        "${log_args[@]}" "$@"
    fi
  )
}

pingLedger(){
  ledger_url=${1}

  # ping ledger web browser for genesis txns
  local rtnCd=$(curl -s --write-out '%{http_code}' --output /dev/null ${ledger_url}/genesis)
  if (( ${rtnCd} == 200 )); then
    return 0
  else
    return 1
  fi
}

function wait_for_ledger() {
  (
    # if flag is set, wait for ledger to activate before continuing
    local rtnCd=0
    if [ ! -z "${WAIT_FOR_LEDGER}" ]; then
      # Wait for ledger server to start ...
      local startTime=${SECONDS}
      # use global LEDGER_URL
      local LEDGER_URL="${LEDGER_URL_CONFIG:-http://localhost:9000}"
      printf "waiting for ledger to start"
      while ! pingLedger "$LEDGER_URL"; do
        printf "."
        local duration=$(($SECONDS - $startTime))
        if (( ${duration} >= ${LEDGER_TIMEOUT} )); then
          echoRed "\nThe Indy Ledger failed to start within ${duration} seconds.\n"
          rtnCd=1
          break
        fi
        sleep 1
      done
    fi
    return ${rtnCd}
  )
}

function install_taa() {
  (
    # if flag is set, copy the sample config/sample_aml.json and config/sample_taa.json
    # to config/aml.json and config/taa.json. Also create a marker file so that we
    # clean up on shutdown and still support backward compatibility where people previously
    # their own custom versions and don't want them removed.
    local rtnCd=0
    if [ ! -z "${USE_SAMPLE_TAA}" ]; then
      rtnCd=$(cp -f ./config/sample_aml.json ./config/aml.json) && $(cp -f ./config/sample_taa.json ./config/taa.json)
      touch ./config/.samples.used
    fi
    return ${rtnCd}
  )
}

function remove_taa() {
  (
    # if the marker exists indicating we created the aml and taa files, make sure we remove them to clean up.
    if [ -f "./config/.samples.used" ]; then
      rm -f ./config/aml.json ./config/taa.json ./config/.samples.used
    fi
  )
}

function generateKey(){
  (
    _length=${1:-48}
    # Format can be `-base64` or `-hex`
    _format=${2:--base64}
    echo $(openssl rand ${_format} ${_length})
  )
}

function generateSeed(){
  (
    _prefix=${1}
    _seed=$(echo "${_prefix}$(generateKey 32)" | fold -w 32 | head -n 1 )
    _seed=$(echo -n "${_seed}")
    echo ${_seed}
  )
}

function generateSecrets() {
  echo
  echo "Seed: $(generateSeed)"
  echo "Key: $(generateKey)"
  echo
}

function backup() {
  (
    _msg=$@
    volumes=$(${dockerCompose} config --volumes)
    timeStamp=`date +\%Y-\%m-\%d_%H-%M-%S`
    datePart=${timeStamp%%_*}
    timePart=${timeStamp#*_}
    backupDir=${ROOT_BACKUP_DIR}/${datePart}/${timePart}
    backupVolumeMount=$(getVolumeMount ./${ROOT_BACKUP_DIR})/
    mkdir -p ./${backupDir}
    if [ ! -z "${_msg}" ]; then
      echo "${_msg}" > ./${backupDir}/ReadMe.txt
    fi

    for volume in ${volumes}; do
      volume=$(echo ${volume} |sed 's~\r$~~')
      sourceVolume=${COMPOSE_PROJECT_NAME}_${volume}
      archiveName=${sourceVolume}_${timeStamp}.tar.gz
      archivePath="/${backupDir}/${archiveName}"

      echoYellow \\n"Backing up ${sourceVolume} to ${archivePath} ..."
      docker run \
        --rm \
        --name von-network-backup \
        -v ${backupVolumeMount}:/${ROOT_BACKUP_DIR} \
        -v ${sourceVolume}:/source_volume von-network-base \
        tar -czvf ${archivePath} -C /source_volume/ .
    done
  )
}

function restore() {
  (
    _fileName=${1}
    archivePath=$(findBackup ${1})
    archiveDirectory=${archivePath%/*}
    datePart=$(echo ${archivePath} | awk -F_ '{print $3}')
    timePart=$(echo ${archivePath} | awk -F_ '{print $4}')
    archiveSuffix="${datePart}_${timePart}"

    if promptForConfirmation "You are about to restore from the '${archiveDirectory}' backup set.\nYour existing data will be lost if not backed up first."; then
      volumes=$(${dockerCompose} config --volumes)
      for volume in ${volumes}; do
        volume=$(echo ${volume} |sed 's~\r$~~')
        targetVolume=${COMPOSE_PROJECT_NAME}_${volume}
        archiveName=${targetVolume}_${archiveSuffix}
        archivePath="${archiveDirectory}/${archiveName}"

        restoreArchive -q "${archivePath}" "${targetVolume}"
      done
    else
      echo -e \\n"Restore aborted."
    fi
  )
}

function restoreArchive()
{
  (
    local OPTIND
    local quiet
    unset quiet
    while getopts q FLAG; do
      case $FLAG in
        q ) quiet=1 ;;
      esac
    done
    shift $((OPTIND-1))

    archive=${1}
    volume=${2}
    tarOptions=${3} # Example "--strip=1", to remove the first directory level.
    if [ -z ${archive} ] || [ -z ${volume} ]; then
      echoYellow "You must supply the path to the archive and the name of the volume to which the archive will be restored."
      exit 1
    fi

    archiveFolder=${archive%/*}
    archiveName=${archive##*/}
    archiveToRestore=/${ROOT_BACKUP_DIR}/${archiveName}
    archiveVolumeMount=$(getVolumeMount ${archiveFolder})

    if [ ! -z "${quiet}" ] || promptForConfirmation "You are about to restore '${archive}' to ${volume}.\nYour existing data will be lost if not backed up first." ; then
      deleteVolume ${volume}
      echoYellow \\n"Restoring ${volume} from ${archive} ..."
      docker run \
        --rm \
        --name von-network-restore \
        --user root \
        -v ${archiveVolumeMount}:/${ROOT_BACKUP_DIR} \
        -v ${volume}:/target_volume von-network-base \
        tar --same-owner -xzvpf ${archiveToRestore} -C /target_volume/ ${tarOptions}
    else
      echo -e \\n"Restore aborted."
    fi
  )
}

function debugVolume()
{
  (
    volume=${1}
    volumeMountFolder=${2:-/debug_volume}
    if [ -z ${volume} ]; then
      echoYellow "You must supply the name of the volume to attach to the debug session."
      exit 1
    fi

    backupVolumeMount=$(getVolumeMount ./${ROOT_BACKUP_DIR})/

    echo -e "\nOpening a debug session with the followig volume mounts:\n  - '${volume}':'${volumeMountFolder}'\n  - '${backupVolumeMount}':'/${ROOT_BACKUP_DIR}'\n"
    docker run \
      --rm \
      -it \
      --network="host" \
      --user root \
      -v ${backupVolumeMount}:/${ROOT_BACKUP_DIR} \
      -v ${volume}:${volumeMountFolder} \
      --entrypoint ${SHELL_CMD} \
      von-network-base
  )
}

function findBackup(){
  (
    _fileName=${1}

    # If no backup file was specified, find the most recent set.
    # Otherwise treat the value provided as a filter to find the most recent backup set matching the filter.
    if [ -z "${_fileName}" ]; then
      _fileName=$(find ${ROOT_BACKUP_DIR}* -type f -printf '%T@ %p\n' | grep .tar.gz | sort | tail -n 1 | sed 's~^.* \(.*$\)~\1~')
    else
      _fileName=$(find ${ROOT_BACKUP_DIR}* -type f -printf '%T@ %p\n' | grep .tar.gz | grep ${_fileName} | sort | tail -n 1 | sed 's~^.* \(.*$\)~\1~')
    fi

    echo "${_fileName}"
  )
}

function echoYellow (){
  (
  _msg=${1}
  _yellow='\e[33m'
  _nc='\e[0m' # No Color
  echo -e "${_yellow}${_msg}${_nc}" >&2
  )
}

function promptForConfirmation(){
  (
    _msg=${@}
    echoYellow "\n${_msg}"
    read -n1 -s -r -p $'\e[33mWould you like to continue?\e[0m  Press \'Y\' to continue, or any other key to exit ...\n' key
    if [[ $(toLower ${key}) == 'y' ]]; then
      return 0
    else
      return 1
    fi
  )
}

function deleteVolume() {
  (
    volume=${1}

    echoYellow \\n"Deleting volume '${volume}' ..."
    containerId=$(docker volume rm ${volume} 2>&1 >/dev/null | sed -e 's~.*\[\(.*\)\]~\1~' | grep -v ${volume})
    if [ ! -z "${containerId}" ]; then
      # The volume is in use by a container.  Remove the container before deleting the volume.
      docker stop ${containerId} > /dev/null 2>&1
      docker rm ${containerId} > /dev/null 2>&1
      docker volume rm ${volume} > /dev/null 2>&1
    fi
  )
}

function deleteVolumes() {
  (
    _projectName=${COMPOSE_PROJECT_NAME:-docker}

    echoYellow \\n"Stopping and removing any running containers ..."
    ${dockerCompose} down -v

    _pattern="^${_projectName}_\|^docker_"
    _volumes=$(docker volume ls -q | grep ${_pattern})
    if [ ! -z "${_volumes}" ]; then
      echoYellow "Removing project volumes ..."
      echo ${_volumes} | xargs docker volume rm
    fi
  )
}

function getVolumeMount() {
  path=${1}
  path=$(realpath ${path})
  path=${path%%+(/)}
  if [[ "$OSTYPE" == "msys" ]]; then
    # When running on Windows, you need to prefix the path with an extra '/'
    path="/${path}"
  fi
  echo ${path}
}
# =================================================================================================================

pushd "${SCRIPT_HOME}" >/dev/null
COMMAND=$(toLower ${1})
shift || COMMAND=usage

case "${COMMAND}" in
  start|up)
      initEnv "$@"
      install_taa
      ${dockerCompose} \
        up \
        -d webserver node1 node2 node3 node4
      wait_for_ledger
      logs
      echo 'Want to see the scrolling container logs? Run "./manage logs"'
    ;;
  start-combined)
      initEnv "$@"
      install_taa
      ${dockerCompose} \
        up \
        -d webserver nodes
      wait_for_ledger
      logs
    ;;
  start-web)
      initEnv "$@"
      if [ -z "$LEDGER_SEED" ]; then
        export ANONYMOUS=1
      fi
      ${dockerCompose} \
        up \
        -d webserver
      wait_for_ledger
      logs webserver
    ;;
  synctest)
      initEnv "$@"
      ${dockerCompose} \
        up \
        -d synctest node1 node2 node3 node4
      logs -f synctest
    ;;
  cli)
      runCliCommand $@
    ;;
  indy-cli)
      runCliCommand indy-cli $@
    ;;
  logs)
      initEnv "$@"
      logs -f "$@"
    ;;
  stop)
      initEnv "$@"
      ${dockerCompose} \
        stop
      remove_taa
    ;;
  down|rm)
      initEnv "$@"
      deleteVolumes
      remove_taa
    ;;
  build)
      docker build $(initDockerBuildArgs) -t von-network-base .
    ;;
  rebuild)
      docker build --no-cache $(initDockerBuildArgs) -t von-network-base .
    ;;
  dockerhost)
      echo -e \\n"DockerHost: ${DOCKERHOST}"\\n
    ;;
  generatesecrets)
      generateSecrets
    ;;

  backup)
      backup "$@"
    ;;
  restore)
      restore $@
    ;;
  restorearchive)
      archive=${1}
      volume=${2}
      tarOptions=${3}
      restoreArchive ${archive} ${volume} ${tarOptions}
    ;;
  debugvolume)
      volume=${1}
      volumeMountFolder=${2}
      debugVolume ${volume} ${volumeMountFolder}
    ;;
  *)
      usage;;
esac

popd >/dev/null