Update a couple of scripts

Mostly style. Author: Franck Cuny <fcuny@apache.org> Author: Leigh Stewart <lstewart@twitter.com> Author: Khurrum Nasim <khurrumnasimm@gmail.com> Reviewers: Sijie Guo <sijie@apache.org> Closes #46 from franckcuny/fcuny/update-scripts and squashes the following commits: 50c16ec [Franck Cuny] Rename a variable in one of the shell script. 924ba7e [Franck Cuny] Run `git rev-parse` only once when building the bundle. f1d3b3f [Franck Cuny] Remove more duplicated shell scripts. 90ce66f [Franck Cuny] Do not duplicate code for the 'bundle' script. 13987ea [Franck Cuny] Build cobertura report only on master for successful builds 525ec9a [Khurrum Nasim] Fix deadlock on BKSyncLogReaderDLSN 23c9190 [Leigh Stewart] Add documentation for distributedlog-benchmark 85aa8a7 [Franck Cuny] Make the shell style more consistent. 4f81ba2 [Franck Cuny] Rely on git to find the root directory.
twitter-archive · Dec 3, 2016 · a546e62 · a546e62
1 parent 2a677d1
commit a546e62
Show file tree

Hide file tree

Showing 9 changed files with 305 additions and 539 deletions.
diff --git a/distributedlog-benchmark/bin/bundle b/distributedlog-benchmark/bin/bundle
@@ -18,19 +18,8 @@
 # * limitations under the License.
 # */
 
-BASEDIR=$(dirname "$0")
-DISTRIBUTEDLOG_ROOT="${BASEDIR}/../.."
-
 set -e
-shopt -s dotglob
 
-cd ${DISTRIBUTEDLOG_ROOT}
-rm -rf distributedlog-benchmark/lib
-rm -rf dist/distributedlog-benchmark*
-mkdir -p dist/distributedlog-benchmark/{conf,lib}
-mvn clean install -Ptwitter-ostrich-provider -pl distributedlog-benchmark -am -DskipTests
-cp -rf distributedlog-benchmark/conf/* dist/distributedlog-benchmark/conf
-cp -rf distributedlog-benchmark/lib/* dist/distributedlog-benchmark/lib
-cp -rf distributedlog-benchmark/target/distributedlog-benchmark-*.jar dist/distributedlog-benchmark
-cd dist
-zip -r distributedlog-benchmark.zip distributedlog-benchmark >/dev/null
+DLOG_ROOT=$(git rev-parse --show-toplevel)
+
+bash "${DLOG_ROOT}"/scripts/bundle benchmark
diff --git a/distributedlog-benchmark/bin/dbench b/distributedlog-benchmark/bin/dbench
@@ -18,59 +18,13 @@
 # * limitations under the License.
 # */
 
-# check if net.ipv6.bindv6only is set to 1
-bindv6only=$(/sbin/sysctl -n net.ipv6.bindv6only 2> /dev/null)
-if [ -n "$bindv6only" ] && [ "$bindv6only" -eq "1" ]
-then
-  echo "Error: \"net.ipv6.bindv6only\" is set to 1 - Java networking could be broken"
-  echo "For more info (the following page also applies to dlog): http://wiki.apache.org/hadoop/HadoopIPv6"
-  exit 1
-fi
+set -e
 
-# See the following page for extensive details on setting
-# up the JVM to accept JMX remote management:
-# http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html
-# by default we allow local JMX connections
-if [ "x$JMXLOCALONLY" = "x" ]
-then
-    JMXLOCALONLY=false
-fi
+DLOG_ROOT=$(git rev-parse --show-toplevel)
 
-if [ "x$JMXDISABLE" = "x" ]
-then
-    echo "JMX enabled by default" >&2
-    # for some reason these two options are necessary on jdk6 on Ubuntu
-    #   accord to the docs they are not necessary, but otw jconsole cannot
-    #   do a local attach
-    JMX_ARGS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.local.only=$JMXLOCALONLY"
-else
-    echo "JMX disabled by user request" >&2
-fi
+DLOG_HOME="${DLOG_ROOT}/distributedlog-benchmark"
 
-BINDIR=`dirname "$0"`
-DLOG_HOME=`cd $BINDIR/.. > /dev/null;pwd`
-
-DEFAULT_LOG_CONF=$DLOG_HOME/conf/log4j.properties
-
-source $DLOG_HOME/conf/dlogenv.sh
-
-# exclude tests jar
-RELEASE_JAR=`ls $DLOG_HOME/distributedlog-*.jar 2> /dev/null | egrep -v 'tests|javadoc|sources' | tail -1`
-if [ $? == 0 ]; then
-    DLOG_JAR=$RELEASE_JAR
-fi
-
-# exclude tests jar
-BUILT_JAR=`ls $DLOG_HOME/target/distributedlog-*.jar 2> /dev/null | egrep -v 'tests|javadoc|sources' | tail -1`
-if [ $? != 0 ] && [ ! -e "$DLOG_JAR" ]; then
-    echo "\nCouldn't find dlog jar.";
-    echo "Make sure you've run 'mvn package'\n";
-    exit 1;
-elif [ -e "$BUILT_JAR" ]; then
-    DLOG_JAR=$BUILT_JAR
-fi
-
-dbench_help() {
+usage() {
     cat <<EOF
 Usage: dlog <command>
 where command is one of:
@@ -89,67 +43,13 @@ These variable can also be set in conf/dlogenv.sh
 EOF
 }
 
-add_maven_deps_to_classpath() {
-    MVN="mvn"
-    if [ "$MAVEN_HOME" != "" ]; then
-	MVN=${MAVEN_HOME}/bin/mvn
-    fi
-
-    # Need to generate classpath from maven pom. This is costly so generate it
-    # and cache it. Save the file into our target dir so a mvn clean will get
-    # clean it up and force us create a new one.
-    f="${DLOG_HOME}/target/cached_classpath.txt"
-    if [ ! -f "${f}" ]
-    then
-	${MVN} -f "${DLOG_HOME}/pom.xml" dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null
-    fi
-    DLOG_CLASSPATH=${CLASSPATH}:`cat "${f}"`
-}
-
-if [ -d "$DLOG_HOME/lib" ]; then
-    for i in $DLOG_HOME/lib/*.jar; do
-	DLOG_CLASSPATH=$DLOG_CLASSPATH:$i
-    done
-else
-    add_maven_deps_to_classpath
-fi
-
-# if no args specified, show usage
-if [ $# = 0 ]; then
-    dbench_help;
-    exit 1;
-fi
+cd "${DLOG_ROOT}"
+source ./scripts/common.sh
 
 # get arguments
 COMMAND=$1
 shift
 
-if [ -z "$DLOG_LOG_CONF" ]; then
-    DLOG_LOG_CONF=$DEFAULT_LOG_CONF
-fi
-
-DLOG_CLASSPATH="$DLOG_JAR:$DLOG_CLASSPATH:$DLOG_EXTRA_CLASSPATH"
-if [ "$DLOG_LOG_CONF" != "" ]; then
-    DLOG_CLASSPATH="`dirname $DLOG_LOG_CONF`:$DLOG_CLASSPATH"
-    OPTS="$OPTS -Dlog4j.configuration=`basename $DLOG_LOG_CONF`"
-fi
-OPTS="-cp $DLOG_CLASSPATH $OPTS $DLOG_EXTRA_OPTS"
-
-OPTS="$OPTS $DLOG_EXTRA_OPTS"
-
-# Disable ipv6 as it can cause issues
-OPTS="$OPTS -Djava.net.preferIPv4Stack=true"
-
-# log directory & file
-DLOG_ROOT_LOGGER=${DLOG_ROOT_LOGGER:-"INFO,R"}
-DLOG_LOG_DIR=${DLOG_LOG_DIR:-"$DLOG_HOME/logs"}
-DLOG_LOG_FILE=${DLOG_LOG_FILE:-"dbench.log"}
-
-#Configure log configuration system properties
-OPTS="$OPTS -Ddlog.root.logger=$DLOG_ROOT_LOGGER"
-OPTS="$OPTS -Ddlog.log.dir=$DLOG_LOG_DIR"
-OPTS="$OPTS -Ddlog.log.file=$DLOG_LOG_FILE"
-
 BENCH_ARGS="""
   --provider ${STATS_PROVIDER} \\
   --conf ${BENCH_CONF_FILE:-"${DLOG_HOME}/conf/benchmark.conf"} \\
@@ -165,7 +65,9 @@ BENCH_ARGS="""
 
 #Change to DLOG_HOME to support relative paths
 cd "$DLOG_HOME"
-if [ $COMMAND == "bkwrite" ]; then
+
+case "${COMMAND}" in
+  bkwrite)
     BENCH_WRITE_ARGS="""
     --messagesize ${MSG_SIZE} \\
     --rate ${INITIAL_RATE} \\
@@ -175,7 +77,8 @@ if [ $COMMAND == "bkwrite" ]; then
     """
     BENCH_ARGS="${BENCH_ARGS} \\ ${BENCH_WRITE_ARGS} \\ --mode dlwrite \\"
     exec java $OPTS $JMX_ARGS com.twitter.distributedlog.benchmark.Benchmarker $BENCH_ARGS $@
-elif [ $COMMAND == "write" ]; then
+    ;;
+  write)
     BENCH_WRITE_ARGS="""
     --messagesize ${MSG_SIZE} \\
     --rate ${INITIAL_RATE} \\
@@ -185,18 +88,21 @@ elif [ $COMMAND == "write" ]; then
     """
     BENCH_ARGS="${BENCH_ARGS} \\ ${BENCH_WRITE_ARGS} \\ --mode write \\"
     exec java $OPTS $JMX_ARGS com.twitter.distributedlog.benchmark.Benchmarker $BENCH_ARGS $@
-elif [ $COMMAND == "read" ]; then
+    ;;
+  read)
     BENCH_READ_ARGS="""
     --readers-per-stream ${NUM_READERS_PER_STREAM} \\
     --max-stream-id ${MAX_STREAM_ID} \\
     --truncation-interval ${TRUNCATION_INTERVAL} \\
     """
     BENCH_ARGS="${BENCH_ARGS} \\ ${BENCH_READ_ARGS} \\ --mode read \\"
     exec java $OPTS $JMX_ARGS com.twitter.distributedlog.benchmark.Benchmarker $BENCH_ARGS $@
-elif [ $COMMAND == "help" ]; then
-    dbench_help;
-else
+    ;;
+  help)
+    usage
+    ;;
+  *)
     exec java $OPTS $COMMAND $@
-fi
-
+    ;;
+esac
 
diff --git a/distributedlog-core/bin/dlog b/distributedlog-core/bin/dlog
@@ -18,154 +18,58 @@
 # * limitations under the License.
 # */
 
-# check if net.ipv6.bindv6only is set to 1
-bindv6only=$(/sbin/sysctl -n net.ipv6.bindv6only 2> /dev/null)
-if [ -n "$bindv6only" ] && [ "$bindv6only" -eq "1" ]
-then
-  echo "Error: \"net.ipv6.bindv6only\" is set to 1 - Java networking could be broken"
-  echo "For more info (the following page also applies to dlog): http://wiki.apache.org/hadoop/HadoopIPv6"
-  exit 1
-fi
+set -e
 
-# See the following page for extensive details on setting
-# up the JVM to accept JMX remote management:
-# http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html
-# by default we allow local JMX connections
-if [ "x$JMXLOCALONLY" = "x" ]
-then
-    JMXLOCALONLY=false
-fi
+DLOG_ROOT=$(git rev-parse --show-toplevel)
+DLOG_HOME="${DLOG_ROOT}"
 
-if [ "x$JMXDISABLE" = "x" ]
-then
-    echo "JMX enabled by default" >&2
-    # for some reason these two options are necessary on jdk6 on Ubuntu
-    #   accord to the docs they are not necessary, but otw jconsole cannot
-    #   do a local attach
-    JMX_ARGS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.local.only=$JMXLOCALONLY"
-else
-    echo "JMX disabled by user request" >&2
-fi
-
-BINDIR=`dirname "$0"`
-DLOG_HOME=`cd $BINDIR/.. > /dev/null;pwd`
-
-DEFAULT_LOG_CONF=$DLOG_HOME/conf/log4j.properties
-
-source $DLOG_HOME/conf/dlogenv.sh
-
-# exclude tests jar
-RELEASE_JAR=`ls $DLOG_HOME/distributedlog-*.jar 2> /dev/null | egrep -v 'tests|javadoc|sources' | tail -1`
-if [ $? == 0 ]; then
-    DLOG_JAR=$RELEASE_JAR
-fi
-
-# exclude tests jar
-BUILT_JAR=`ls $DLOG_HOME/target/distributedlog-*.jar 2> /dev/null | egrep -v 'tests|javadoc|sources' | tail -1`
-if [ $? != 0 ] && [ ! -e "$DLOG_JAR" ]; then
-    echo "\nCouldn't find dlog jar.";
-    echo "Make sure you've run 'mvn package'\n";
-    exit 1;
-elif [ -e "$BUILT_JAR" ]; then
-    DLOG_JAR=$BUILT_JAR
-fi
-
-dlog_help() {
-    cat <<EOF
-Usage: dlog <command>
+usage() {
+  cat <<EOF
+Usage: runner <command>
 where command is one of:
-    local               Run distributedlog sandbox
-    example             Run distributedlog example
-    tool                Run distributedlog tool
-    proxy_tool          Run distributedlog proxy tool to interact with proxies
-    balancer            Run distributedlog balancer
-    admin               Run distributedlog admin tool
-    help                This help message
+  local               Run distributedlog sandbox
+  example             Run distributedlog example
+  tool                Run distributedlog tool
+  proxy_tool          Run distributedlog proxy tool to interact with proxies
+  balancer            Run distributedlog balancer
+  admin               Run distributedlog admin tool
+  help                This help message
 
 or command is the full name of a class with a defined main() method.
 
 Environment variables:
-   DLOG_LOG_CONF        Log4j configuration file (default $DEFAULT_LOG_CONF)
-   DLOG_EXTRA_OPTS      Extra options to be passed to the jvm
-   DLOG_EXTRA_CLASSPATH Add extra paths to the dlog classpath
+  DLOG_LOG_CONF        Log4j configuration file (default $DEFAULT_LOG_CONF)
+  DLOG_EXTRA_OPTS      Extra options to be passed to the jvm
+  DLOG_EXTRA_CLASSPATH Add extra paths to the dlog classpath
 
 These variable can also be set in conf/dlogenv.sh
 EOF
 }
 
-add_maven_deps_to_classpath() {
-    MVN="mvn"
-    if [ "$MAVEN_HOME" != "" ]; then
-	MVN=${MAVEN_HOME}/bin/mvn
-    fi
-
-    # Need to generate classpath from maven pom. This is costly so generate it
-    # and cache it. Save the file into our target dir so a mvn clean will get
-    # clean it up and force us create a new one.
-    f="${DLOG_HOME}/target/cached_classpath.txt"
-    if [ ! -f "${f}" ]
-    then
-	${MVN} -f "${DLOG_HOME}/pom.xml" dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null
-    fi
-    DLOG_CLASSPATH=${CLASSPATH}:`cat "${f}"`
-}
-
-if [ -d "$DLOG_HOME/lib" ]; then
-    for i in $DLOG_HOME/lib/*.jar; do
-	DLOG_CLASSPATH=$DLOG_CLASSPATH:$i
-    done
-else
-    add_maven_deps_to_classpath
-fi
-
-# if no args specified, show usage
-if [ $# = 0 ]; then
-    dlog_help;
-    exit 1;
-fi
+cd "${DLOG_ROOT}"
+source ./scripts/common.sh
 
 # get arguments
 COMMAND=$1
 shift
 
-if [ -z "$DLOG_LOG_CONF" ]; then
-    DLOG_LOG_CONF=$DEFAULT_LOG_CONF
-fi
-
-DLOG_CLASSPATH="$DLOG_JAR:$DLOG_CLASSPATH:$DLOG_EXTRA_CLASSPATH"
-if [ "$DLOG_LOG_CONF" != "" ]; then
-    DLOG_CLASSPATH="`dirname $DLOG_LOG_CONF`:$DLOG_CLASSPATH"
-    OPTS="$OPTS -Dlog4j.configuration=`basename $DLOG_LOG_CONF`"
-fi
-OPTS="-cp $DLOG_CLASSPATH $OPTS $DLOG_EXTRA_OPTS"
-
-OPTS="$OPTS $DLOG_EXTRA_OPTS"
-
-# Disable ipv6 as it can cause issues
-OPTS="$OPTS -Djava.net.preferIPv4Stack=true"
-
-# log directory & file
-DLOG_ROOT_LOGGER=${DLOG_ROOT_LOGGER:-"INFO,R"}
-DLOG_LOG_DIR=${DLOG_LOG_DIR:-"$DLOG_HOME/logs"}
-DLOG_LOG_FILE=${DLOG_LOG_FILE:-"dlog.log"}
-
-#Configure log configuration system properties
-OPTS="$OPTS -Ddlog.root.logger=$DLOG_ROOT_LOGGER"
-OPTS="$OPTS -Ddlog.log.dir=$DLOG_LOG_DIR"
-OPTS="$OPTS -Ddlog.log.file=$DLOG_LOG_FILE"
-
 #Change to DLOG_HOME to support relative paths
 cd "$DLOG_HOME"
-if [ $COMMAND == "local" ]; then
+
+case "${COMMAND}" in
+  local)
     exec java $OPTS $JMX_ARGS com.twitter.distributedlog.LocalDLMEmulator $@
-elif [ $COMMAND == "tool" ]; then
+    ;;
+  tool)
     exec java $OPTS com.twitter.distributedlog.tools.Tool com.twitter.distributedlog.tools.DistributedLogTool $@
-elif [ $COMMAND == "admin" ]; then
+    ;;
+  admin)
     exec java $OPTS com.twitter.distributedlog.tools.Tool com.twitter.distributedlog.admin.DistributedLogAdmin $@
-elif [ $COMMAND == "help" ]; then
-    dlog_help;
-else
+    ;;
+  help)
+    usage
+    ;;
+  *)
     exec java $OPTS $COMMAND $@
-fi
-
-
+    ;;
+esac