dcache

#
# chkconfig: 2345 92 8
# description: dCache init script

set -e

# Solaris doesn't have a POSIX compliant shell as /bin/sh. We
# try to find one and execute it.
if [ "$1" = "%" ]; then
    shift
elif [ "`uname`" = "SunOS" ]; then
    if [ -x /usr/xpg4/bin/sh ]; then
        exec /usr/xpg4/bin/sh $0 % "$@"
    elif [ -x /bin/bash ]; then
        exec /bin/bash $0 % "$@"
    else
        echo "Cannot find POSIX compliant shell. This script will"
        echo "probably break, but we attempt to execute it anyway."
    fi
fi

# Prints help screen and exits with error status 2
usage()
{
    echo "Usage: $(basename $0) [OPTION]... COMMAND"
    echo
    echo "Valid commands are:"
    echo "   alarm list (predefined types)"
    echo "   alarm send [-h=<src host>] [-d=<src domain>] [-s=<src service/cell] [-t=<alarm type>] <message>"
    echo "   alarm add    (interactive)"
    echo "   alarm modify (interactive)"
    echo "   alarm remove (interactive)"
    echo "   billing [--format=raw|files|json|yaml] [--since=DATE] [--until=DATE] [-f=<file>] [<path>|<pnfsid>|<dn>]..."
    echo "   check-config"
    echo "   condrestart [<domain>]..."
    echo "   database ls"
    echo "   database update [<cell>@<domain>]..."
    echo "   database showUpdateSQL [<cell>@<domain>]..."
    echo "   database tag <tag> [<cell>@<domain>]..."
    echo "   database rollback <tag> [<cell>@<domain>]..."
    echo "   database rollbackToDate <date/time> [<cell>@<domain>]..."
    echo "   database listLocks [<cell>@<domain>]..."
    echo "   database releaseLocks [<cell>@<domain>]..."
    echo "   database doc <cell>@<domain> <out-dir>"
    echo "   dump heap [--force] <domain> <file>"
    echo "   dump threads [<domain>...]"
    echo "   kpwd <command> [-debug] [<command argument>]..."
    echo "   ports"
    echo "   pool convert <name> <target-type>"
    echo "   pool create [--meta=file|db] [--size=<bytes>]"
    echo "               [--lfs=none|precious|volatile|transient]"
    echo "               <directory> <name> <domain>"
    echo "   pool ls"
    echo "   pool reconstruct <directory> <target dir>"
    echo "   pool yaml <name>"
    echo "   property <property-name> [<domain-name> [<cell-name>]]"
    echo "   restart [<domain>]..."
    echo "   services"
    echo "   start [<domain>]..."
    echo "   status"
    echo "   stop [<domain>]..."
    echo "   version"
    echo
    echo "Size is specified in bytes, or optionally followed by K, M, G or T"
    echo "for powers of 1024. Size is rounded down to the nearest integer"
    echo "number of GiB."
    exit 2
} 1>&2

# Get the canonical path of $1. Only returns a truly canonical path
# if readlink is available. Otherwise an absolute path which does not
# end in a symlink is returned.
getCanonicalPath() # in $1 = path, out $2 = canonical path
{
    local link
    local ret
    link="$1"
    if readlink -f . > /dev/null 2>&1; then
        ret="$(readlink -f $link)"
    else
        ret="$(cd $(dirname $link); pwd)/$(basename $link)"
        while [ -h "$ret" ]; do
            link="$(ls -ld $ret | sed 's/.*-> //')"
            if [ -z "${link##/*}" ]; then
                ret="${link}"
            else
                link=$(dirname $ret)/${link}
                ret="$(cd $(dirname $link); pwd)/$(basename $link)"
            fi
        done
    fi
    eval $2=\"$ret\"
}

# Returns true if $1 is contained as a word in $2.
contains() # in $1 = word, in $2+ = list
{
    local word
    word=$1
    shift
    for i in "$@"; do
        if [ "$word" = "$i" ]; then
            return 0
        fi
    done
    return 1
}

# Generic option parser. Both single and multi character options are
# supported. Single character options start with a single dash and
# multi character options with a double dash. Single character options
# can be combined, e.g. rather than -a -b -c one can use -abc.
#
# The first argument is a list of valid options. Remaining arguments
# are the options to be parsed. When finding an option not in the list
# of valid options, the usage() is called.
#
# Parsing stops when no arguments are left or a non-option argument
# is found.
#
# Options can have an optional value.
#
# For each option found the variable opt_X, where X is the
# option, is defined. If a value is provided for the option, then
# opt_X is set to that value, otherwise to 1.
#
# The return value is the number of words of $1 that were processed.
#
parseOptions() # $1 = list of valid options
{
    local valid
    local count
    local name
    local value
    local rest
    local option

    valid=$1
    count=0

    shift
    while [ $# -gt 0 ]; do
        option=$1
        case $option in
            --*=*)
                option=${option#--}    # Strip leading double dash
                name=${option%%=*}
                value=${option#*=}
                ;;

            -?=*)
                option=${option#-}     # Strip leading dash
                name=${option%%=*}
                value=${option#*=}
                ;;

            --?*)
                name=${option#--}      # Strip leading double dash
                value=1
                ;;

            -?*)
                option=${option#-}     # Strip leading dash
                while [ -n "$option" ]; do
                    rest=${option#?}       # Strip leading character
                    name=${option%${rest}} # Strip the rest to get name
                    if ! contains $name $valid; then
                        usage
                    fi

                    option=${rest}

                    eval "opt_${name}=1"
                done
                count=$((${count}+1))
                shift
                continue
                ;;

            *)
                break
                ;;
        esac

        if ! contains $name $valid; then
            usage
        fi

        eval "opt_${name}=\"${value}\""

        shift
        count=$((${count}+1))
    done

    return $count
}

# Dumps the heap. Terminates the script in case of failure.
dumpHeap() # $1=force, $2=live, $3=file, $4=pid, $5=error
{
    if ! $jmap ${1:+-F} -dump:${2:+live,}format=b,file=$3 $4; then
        fail 1 "$5"
    fi

    if [ ! -f "$3" ]; then
        fail 1 "$5"
    fi
}

# display dCache package version
showVersion()
{
    CLASSPATH="$(getProperty dcache.paths.classpath)" quickJava org.dcache.util.Version
}

#  print either the user a PID is running as (if $1 is non-empty)
#  or the user the domain is configured to run as otherwise.
userForProcessOrDomain() # $1=pid, $2=domain
{
    if [ -n "$1" ]; then
        processUser $1
    else
        user=$(getProperty dcache.user "$domain")
        if [ -z "$user" ]; then
            user="[whoever runs \"dcache start\"]"
        fi
        echo "$user"
    fi
}



# Checks for the existing of OOM heap dump files and generates
# a warning for each file.
checkForHeapDumpFiles()
{
    local file
    local domain
    for domain in $(getProperty dcache.domains); do
        file=$(getProperty dcache.java.oom.file "$domain")
        if [ -e "$file" ]; then
            printp "A heap dump file, $file, was found for domain
                    $domain. The file was generated
     $(if type stat > /dev/null 2>&1; then echo at $(stat -c '%x' "$file"); fi)
                    as a result of an out of memory failure in the domain." \
                    "The dump contains debugging information that may
                    help a developer determine the cause of high memory
                    usage. Please note that the dump may contain
                    confidential information." \
                    "As long as the file exists no other dumps
                    will be generated on out of memory failures. Please
                    move or delete $file. Consider increasing the
                    dcache.java.memory.heap property. The current value
                    is $(getProperty dcache.java.memory.heap "$domain")."
            echo
        fi
    done
}

poolConvertMeta() # $1 = domain, $2 = cell, $3 = type
{
    classpath=$(getProperty dcache.paths.classpath "$1" "$2")
    path=$(getProperty pool.path "$1" "$2")
    src=$(getProperty pool.plugins.meta "$1" "$2")
    name=$(getProperty pool.name "$1" "$2")

    if [ "$(printSimpleDomainStatus "$1")" != "stopped" ]; then
        fail 1 "Domain '$1' has to be stopped before pool '$name'
                can be converted."
    fi

    if [ "$src" = "$3" ]; then
        fail 2 "Cannot convert pool '$name', as it is already of type $src."
    fi

    CLASSPATH="$classpath" quickJava org.dcache.pool.repository.MetaDataStoreCopyTool "$path" "$src" "$3"

    printp ""\
           "The pool meta data database of '$name' was converted from
            type $src to type $3. Note that to use the new meta data
            store, the pool configuration must be updated by adjusting
            the metaDataRepository property, eg, in the layout file:" \
           "metaDataRepository=$3"

    exit 0
}

poolDumpYaml() # $1 = domain, $2 = cell
{
    classpath=$(getProperty dcache.paths.classpath "$1" "$2")
    path=$(getProperty pool.path "$1" "$2")
    type=$(getProperty pool.plugins.meta "$1" "$2")
    CLASSPATH="$classpath" quickJava org.dcache.pool.repository.MetaDataStoreYamlTool "$path" "$type"
}

if [ $# -eq 0 ]; then
    usage
fi

@DCACHE_LOAD_CONFIG@

lib="$(getProperty dcache.paths.share.lib)"

case "$1" in
    start)
        shift
        . ${lib}/utils.sh
        . ${lib}/services.sh
        checkForHeapDumpFiles
        domains=$(printDomains "$@")
        for domain in $domains; do
            domainStart $domain || :
        done

        lock=$(getProperty dcache.paths.lock.file)
        if [ -n "$lock" ]; then
            touch "$lock" 2> /dev/null || :
        fi
        ;;

    stop)
        shift

        . ${lib}/utils.sh
        . ${lib}/services.sh
        domains=$(printDomains "$@")
        reverse domains_backward $domains
        for domain in $domains_backward; do
            domainStop $domain || :
        done

        lock=$(getProperty dcache.paths.lock.file)
        if [ -n "$lock" ]; then
            rm -f "$lock"
        fi

        checkForHeapDumpFiles
        ;;

    restart)
        shift
        . ${lib}/utils.sh
        . ${lib}/services.sh
        checkForHeapDumpFiles
        domains=$(printDomains "$@")

        reverse domains_backward $domains
        for domain in $domains_backward; do
            domainStop "$domain" || :
        done

        for domain in $domains; do
            domainStart "$domain" || :
        done
        ;;

    condrestart)
        shift
        . ${lib}/utils.sh
        . ${lib}/services.sh
        checkForHeapDumpFiles
        domains=$(printDomains "$@")

        reverse domains_backward $domains
        for domain in $domains_backward; do
            if [ "$(printSimpleDomainStatus "$domain")" = "stopped" ]; then
                running_domains="$running_domains $domain"
                domainStop "$domain"
            fi
        done

        reverse domains_to_start $running_domains
        for domain in $domains_to_start; do
            domainStart "$domain"
        done
        ;;

    status)
        shift
        . ${lib}/utils.sh
        . ${lib}/services.sh
        message=$(
            rc_0=0
            rc_1=0
            rc_3=0
            printf "DOMAIN\tSTATUS\tPID\tUSER\tLOG\n"
                for domain in $(getProperty dcache.domains); do
                    count=$((${count}+1))
                    rc=0
                    status=$(printDetailedDomainStatus "$domain") || rc=$?
                    eval rc_$rc=$((rc_$rc+1))
                    pid=$(printJavaPid "$domain") || :
                    user=$(userForProcessOrDomain "$pid" "$domain") || :
                    log=$(getProperty dcache.log.file "$domain")
                    printf "${domain}\t${status}\t${pid}\t${user}\t${log}\n"
                done

                # Derive common exit status
                if [ "$count" = "$((rc_3+rc_1))" -a -n "$rc_1" ]; then
                    exit 1     # program is dead and /var/run pid file exists
                elif [ "$count" = "$rc_3" ]; then
                    if [ -f "$(getProperty dcache.paths.lock.file)" ]; then
                        exit 2 # program is dead and /var/run file exists
                    else
                        exit 3 # program is not running
                    fi
                elif [ "$count" != "$rc_0" ]; then
                    exit 4     # program of service status is unknown
                fi
            ) || rc=$?

        echo "$message" | column
        checkForHeapDumpFiles

        [ -z "$rc" ] || exit $rc
        ;;

    services)
        shift
        . ${lib}/utils.sh
        (
            printf "DOMAIN\tSERVICE\tCELL\tREPLICABLE\n"
            for domain in $(getProperty dcache.domains); do
                for cell in $(getProperty dcache.domain.cells "$domain"); do
                    service=$(getProperty dcache.domain.service "$domain" "$cell")
                    if [ "$(getScopedProperty cell.replicable "$domain" "$cell")" = "true" ]; then
                        replicable=Yes
                    else
                        replicable=No
                    fi
                    printf "${domain}\t${service}\t${cell}\t${replicable:-true}\n"
                done
            done
        ) | column
        ;;

    version)
        showVersion
        ;;

    pool)
        shift

        if [ $# -eq 0 ]; then
            usage
        fi

        . ${lib}/utils.sh
        . ${lib}/pool.sh
        . ${lib}/services.sh

        command=$1
        shift

        case "$command" in
            create)
                parseOptions "meta size lfs" "$@" || shift $?

                [ $# -ne 3 ] && usage

                path="$1"
                name="$2"
                domain="$3"

                createPool "${path}" "${name}" "$domain" "${opt_size}" "$opt_meta" "$opt_lfs"
                ;;

            convert)
                [ $# -ne 2 ] && usage
                name="$1"
                case "$2" in
                    db)
                        type=org.dcache.pool.repository.meta.db.BerkeleyDBMetaDataRepository
                        ;;
                    file)
                        type=org.dcache.pool.repository.meta.file.FileMetaDataRepository
                        ;;
                    *)
                        type="$2"
                        ;;
                esac

                doForPoolOrFail "$name" poolConvertMeta "$type"
                ;;

            yaml)
                [ $# -ne 1 ] && usage
                doForPoolOrFail "$1" poolDumpYaml
                ;;

            reconstruct)
                [ $# -ne 2 ] && usage
                src="$1"
                dst="$2"

                # Check that we have a meta directory
                if [ ! -d "$src/meta" ]; then
                    fail 2 "The pool appears not to have a Berkeley DB holding
                            the meta data, as there is no $src/meta directory."
                fi

                # Make sure the destination does not exist
                if [ -e "$dst" ]; then
                    fail 2 "$dst already exists. The target directory must
                            not exist prior to recovering a pool."
                fi

                # Reconstruct the DB
                mkdir -p "$dst" || fail 1 "Failed to create $dst"
                reconstructMeta "${src}/meta" "${dst}" || fail 1 "Operation aborted"

                printp "The pool meta data database of $src was reconstructed
                        and stored in $dst. You have to manually replace
                        $src/meta with the content of $dst."
                ;;

            ls)
                (
                    printf "POOL\tDOMAIN\tMETA\tSIZE\tFREE\tPATH\n"
                    for domain in $(getProperty dcache.domains); do
                        for cell in $(getProperty dcache.domain.cells "$domain"); do
                            service=$(getProperty dcache.domain.service "$domain" "$cell")
                            if [ "$service" = "pool" ]; then
                                name=$(getProperty pool.name "$domain" "$cell")
                                path=$(getProperty pool.path "$domain" "$cell")
                                meta=$(getProperty pool.plugins.meta "$domain" "$cell")
                                case "$meta" in
                                    org.dcache.pool.repository.meta.db.BerkeleyDBMetaDataRepository)
                                        meta=db
                                        ;;
                                    org.dcache.pool.repository.meta.file.FileMetaDataRepository)
                                        meta=file
                                        ;;
                                    *)
                                        meta=other
                                        ;;
                                esac

                                printf "${name}\t${domain}\t${meta}\t$(getSizeOfPool "$path")\t$(getFreeSpace ${path}/data)G\t${path}\n"
                            fi
                        done
                    done
                ) | column
                ;;

            *)
                usage
                ;;
        esac
        ;;

    dump)
        shift

        if [ $# -eq 0 ]; then
            usage
        fi

        . ${lib}/utils.sh
        . ${lib}/services.sh

        command=$1
        shift

        case "$command" in
            heap)
                parseOptions "force" "$@" || shift $?

                [ $# -ne 2 ] && usage

                domain="$1"
                file="$2"

                findJavaTool jmap ||
                fail 1 "Could not find the jmap command, part of the Java 6
                        JDK. This command is required for producing a heap
                        dump. Please ensure that either jmap is in the path
                        or update JAVA_HOME."

                if [ -f ${file} ]; then
                    fail 1 "${file} already exists. Heap not dumped."
                fi

                if ! pid=$(printJavaPid "$domain"); then
                    fail 1 "Domain ${domain} is not running."
                fi

                user=$(processUser $pid)
                whoami=$(id|sed 's/.*uid=[0-9]*(\([^)]*\)).*/\1/')
                if [ "$user" != "$whoami" ]; then
                    if [ "$whoami" = "root" ]; then
                        exec su "$user" -c "\"$0\" dump heap \"$domain\" \"$file\""
                    else
                        fail 1 "Permission denied. Only $user and root can dump the heap of $domain."
                    fi
                fi

                if [ -z "$opt_force" ]; then
                    dumpHeap "" "live" "$file" "$pid" \
                        "Failed to dump the heap; please consult
                         the previous error message for possible
                         reasons. The dump might succeed when using
                         the --force option."
                else
                    dumpHeap "force" "" "$file" "$pid" \
                        "Failed to dump the heap; please consult
                         the previous error message for possible
                         reasons."
                fi

                printp "The heap of domain ${domain} has been written to
                        ${file}. Notice that the file might contain
                        confidential information."
                ;;

            threads)
                for domain in $(printDomains "$@"); do
                    if pid=$(printJavaPid "$domain"); then
                        if ! kill -s QUIT "${pid}"; then
                            fail 1 "Failed to dump stack traces. Likely
                                    the current user does not have the
                                    proper permissions."
                        fi
                        LOG_FILE="$(getProperty dcache.log.file "$domain")"
                        printp "Stack traces for $domain have been written to $LOG_FILE."
                    fi
                done
                ;;

            *)
                usage
                ;;
        esac
        ;;

    kpwd)
        shift

        . ${lib}/utils.sh

        if [ $# -eq 0 ]; then
            CLASSPATH="$(getProperty dcache.paths.classpath)" quickJava org.dcache.auth.KAuthFile
        else
            command="$1"
            shift

            kpwdFile="$(getProperty gplazma.kpwd.file)"
            if [ ! -e "$kpwdFile" ]; then
                touch "$kpwdFile"
            fi

            CLASSPATH="$(getProperty dcache.paths.classpath)" quickJava org.dcache.auth.KAuthFile "$command" "$(getProperty gplazma.kpwd.file)" "$@"
        fi
        ;;

    ports)
        . ${lib}/utils.sh
        broker_domain=$(getProperty dcache.broker.domain)
        (
            printf "DOMAIN\tCELL\tSERVICE\tPROTO\tPORT\n"
            for domain in $(getProperty dcache.domains); do
                if [ "$domain" = "$broker_domain" ]; then
                    prop_prefix=dcache.broker.net.ports
                else
                    prop_prefix=dcache.non-broker.net.ports
                fi

                for port in $(getProperty "${prop_prefix}.tcp" "$domain"); do
                    printf "${domain}\t-\t-\tTCP\t${port}\n"
                done

                for port in $(getProperty "${prop_prefix}.udp" "$domain"); do
                    printf "${domain}\t-\t-\tUDP\t${port}\n"
                done

                for cell in $(getProperty dcache.domain.cells "$domain"); do
                    service=$(getProperty dcache.domain.service "$domain" "$cell")

                    for port in $(getScopedProperty net.ports.tcp "$domain" "$cell"); do
                        printf "${domain}\t${cell}\t${service}\tTCP\t$(echo $port | tr : -)\n"
                    done

                    for port in $(getScopedProperty net.ports.udp "$domain" "$cell"); do
                        printf "${domain}\t${cell}\t${service}\tUDP\t$(echo $port | tr : -)\n"
                    done
                done
            done
        ) | column

        for domain in $(getProperty dcache.domains); do
            if [ "$domain" = "$broker_domain" ]; then
                prop_prefix=dcache.broker.net.ports
            else
                prop_prefix=dcache.non-broker.net.ports
            fi

            for port in $(getProperty "${prop_prefix}.tcp" "$domain") \
                        $(getProperty "${prop_prefix}.udp" "$domain"); do
                 has_broker_ports=true

                 if [ "$port" = "0" ]; then
                     has_zero_ports=true
                     break
                 fi
            done

            for cell in $(getProperty dcache.domain.cells "$domain"); do
                for port in $(getScopedProperty net.ports.tcp "$domain" "$cell") \
                    $(getScopedProperty net.ports.udp "$domain" "$cell"); do
                    if [ "$port" = "0" ]; then
                        has_zero_ports=true
                        break 2
                    fi
                done
            done
        done

        if [ "$has_broker_ports" = "true" ]; then
            printp "" "Ports with '-' under the CELL and SERVICE columns provide
                    inter-domain communication for dCache.  They are
                    established independently of any service in the layouts
                    file and are configured by the broker.* family of
                    properties."
        fi

        if [ "$has_zero_ports" = "true" ]; then
            printp "" "Entries where the port number is zero indicates that a random
                       port number is chosen.  The chosen port is guaranteed not to
                       conflict with already open ports."
        fi
        ;;

    alarm)
        shift

        if [ $# -eq 0 ]; then
            usage
        fi

        command=$1
        shift

        . ${lib}/utils.sh
        . ${lib}/alarm.sh

        case "$command" in
            list)
                list_alarms
                ;;

            send)
                send_alarm $@
                ;;

            add)
                handle_alarm_definition $command
                ;;

            modify)
                handle_alarm_definition $command
                ;;

            remove)
                handle_alarm_definition $command
                ;;

            *)
                usage
                ;;
        esac
        ;;

    database)
        shift

        if [ $# -eq 0 ]; then
            usage
        fi

        . ${lib}/utils.sh
        . ${lib}/database.sh

        command=$1
        shift

        case "$command" in
            ls)
                (
                    printf "DOMAIN\tCELL\tDATABASE\tHOST\tUSER\tMIN-\tMAX-CONNS\tMANAGEABLE\tAUTO\n"
                    totalMinConnections=0
                    totalMaxConnections=0
                    for domain in $(getProperty dcache.domains); do
                        for cell in $(getProperty dcache.domain.cells "$domain"); do
                            if hasDatabase "$domain" "$cell"; then
                                host=$(getScopedProperty db.host "$domain" "$cell")
                                user=$(getScopedProperty db.user "$domain" "$cell")
                                changelog=$(getScopedProperty db.schema.changelog "$domain" "$cell")
                                database=$(getScopedProperty db.name "$domain" "$cell")
                                maxConnections=$(getScopedProperty db.connections.max "$domain" "$cell")
                                minConnections=$(getScopedProperty db.connections.min "$domain" "$cell")
                                if [ -n "$maxConnections" ]; then
                                    totalMaxConnections=$(( $totalMaxConnections + $maxConnections ))
                                fi
                                if [ -n "$minConnections" ]; then
                                    totalMinConnections=$(( $totalMinConnections + $minConnections ))
                                fi

                                if hasManagedDatabase "$domain" "$cell"; then
                                    manageable=Yes
                                else
                                    manageable=No
                                fi
                                if [ "$(getScopedProperty db.schema.auto "$domain" "$cell")" = "true" ]; then
                                    auto=Yes
                                else
                                    auto=No
                                fi
                                printf "${domain}\t${cell}\t${database}\t${host}\t${user}\t${minConnections}\t${maxConnections}\t${manageable}\t${auto}\n"
                            fi
                        done
                    done
                    printf "TOTAL\t\t\t\t\t${totalMinConnections}\t${totalMaxConnections}\t\t\n"
                ) | column
                ;;

            update)
                for domain in $(getProperty dcache.domains); do
                    for cell in $(getProperty dcache.domain.cells "$domain"); do
                        if [ $# -eq 0 ] || matchesAny "$cell@$domain" "$@"; then
                            if hasManagedDatabase "$domain" "$cell"; then
                                printf "%s: \n" "$cell@$domain"
                                liquibase "$domain" "$cell" update
                            fi
                        fi
                    done
                done
                ;;

            showUpdateSQL)
                for domain in $(getProperty dcache.domains); do
                    for cell in $(getProperty dcache.domain.cells "$domain"); do
                        if [ $# -eq 0 ] || matchesAny "$cell@$domain" "$@"; then
                            if hasManagedDatabase "$domain" "$cell"; then
                                printf -- "-- %s: \n" "$cell@$domain"
                                liquibase "$domain" "$cell" updateSQL
                            fi
                        fi
                    done
                done
                ;;

            tag)
                if [ $# -lt 1 ]; then
                    usage
                fi

                tag="$1"
                shift

                for domain in $(getProperty dcache.domains); do
                    for cell in $(getProperty dcache.domain.cells "$domain"); do
                        if [ $# -eq 0 ] || matchesAny "$cell@$domain" "$@"; then
                            if hasManagedDatabase "$domain" "$cell"; then
                                printf "%s: " "$cell@$domain"
                                liquibase "$domain" "$cell" tag "$tag"
                            fi
                        fi
                    done
                done
                ;;

            rollback)
                if [ $# -lt 1 ]; then
                    usage
                fi

                tag="$1"
                shift

                for domain in $(getProperty dcache.domains); do
                    for cell in $(getProperty dcache.domain.cells "$domain"); do
                        if [ $# -eq 0 ] || matchesAny "$cell@$domain" "$@"; then
                            if hasManagedDatabase "$domain" "$cell"; then
                                printf "%s: " "$cell@$domain"
                                liquibase "$domain" "$cell" rollback "$tag"
                            fi
                        fi
                    done
                done
                ;;

            rollbackToDate)
                if [ $# -lt 1 ]; then
                    usage
                fi

                date="$1"
                shift

                for domain in $(getProperty dcache.domains); do
                    for cell in $(getProperty dcache.domain.cells "$domain"); do
                        if [ $# -eq 0 ] || matchesAny "$cell@$domain" "$@"; then
                            if hasManagedDatabase "$domain" "$cell"; then
                                printf "%s: " "$cell@$domain"
                                liquibase "$domain" "$cell" rollbackToDate "$date"
                            fi
                        fi
                    done
                done
                ;;

            listLocks)
                for domain in $(getProperty dcache.domains); do
                    for cell in $(getProperty dcache.domain.cells "$domain"); do
                        if [ $# -eq 0 ] || matchesAny "$cell@$domain" "$@"; then
                            if hasManagedDatabase "$domain" "$cell"; then
                                printf "%s: " "$cell@$domain"
                                liquibase "$domain" "$cell" listLocks
                            fi
                        fi
                    done
                done
                ;;

            releaseLocks)
                for domain in $(getProperty dcache.domains); do
                    for cell in $(getProperty dcache.domain.cells "$domain"); do
                        if [ $# -eq 0 ] || matchesAny "$cell@$domain" "$@"; then
                            if hasManagedDatabase "$domain" "$cell"; then
                                printf "%s: " "$cell@$domain"
                                liquibase "$domain" "$cell" releaseLocks
                            fi
                        fi
                    done
                done
                ;;

            doc)
                if [ $# -ne 2 ]; then
                    usage
                fi
                liquibase "${1##*@}" "${1%@*}" dbDoc "$2"
                ;;

            *)
                usage
                ;;
        esac
        ;;

    billing)
        shift
        . ${lib}/billing.sh

        parseOptions "format since until f" "$@" || shift $?

        case "${opt_format}" in
            "")
                billing_find ${opt_f:+-f="${opt_f}"} ${opt_since:+-since="${opt_since}"} ${opt_until:+-until="${opt_until}"} "$@"
                ;;
            raw)
                billing_find ${opt_f:+-f="${opt_f}"} ${opt_since:+-since="${opt_since}"} ${opt_until:+-until="${opt_until}"} "$@"
                ;;
            files)
                billing_find -files ${opt_f:+-f="${opt_f}"} ${opt_since:+-since="${opt_since}"} ${opt_until:+-until="${opt_until}"} "$@"
                ;;
            json)
                billing_find -json ${opt_f:+-f="${opt_f}"} ${opt_since:+-since="${opt_since}"} ${opt_until:+-until="${opt_until}"} "$@"
                ;;
            yaml)
                billing_find -yaml ${opt_f:+-f="${opt_f}"} ${opt_since:+-since="${opt_since}"} ${opt_until:+-until="${opt_until}"} "$@"
                ;;
        esac
        ;;

    loader)
        shift
        bootLoader "$@"
        ;;

    check-config)
        shift
        . ${lib}/utils.sh
        bootLoader check-config | while read line; do
            printpi "$line" "^[^:]*:[^:]*:"
        done
        ;;

    property)
        shift
        getProperty "$@"
        ;;

    *)
        usage
        ;;
esac