Skip to content
Permalink
Browse files
Merge pull request #241 from myui/dev/AddScriptForLaunchingMixServers
Dev/add script for launching mix servers
  • Loading branch information
myui committed Dec 2, 2015
2 parents 2147d17 + 59cd843 commit f15bc06a15e24026c89bf12b858d74c6f69ff281
Show file tree
Hide file tree
Showing 9 changed files with 281 additions and 16 deletions.
@@ -2,9 +2,7 @@
/.fatjar
/build
target
core/target/*
mixserv/target/*
nlp/target/*
logs
.git
.settings
*.swp
@@ -1,4 +1,15 @@
#!/bin/sh

cd ..
if [ "$HIVEMALL_HOME" == "" ]; then
if [ -e ../bin/${0##*/} ]; then
HIVEMALL_HOME=".."
elif [ -e ./bin/${0##*/} ]; then
HIVEMALL_HOME="."
else
echo "env HIVEMALL_HOME not defined"
exit 1
fi
fi

cd $HIVEMALL_HOME
mvn clean package -Dskiptests=true -Dmaven.test.skip=true
@@ -1,4 +1,15 @@
#!/bin/sh

cd ..
if [ "$HIVEMALL_HOME" == "" ]; then
if [ -e ../bin/${0##*/} ]; then
HIVEMALL_HOME=".."
elif [ -e ./bin/${0##*/} ]; then
HIVEMALL_HOME="."
else
echo "env HIVEMALL_HOME not defined"
exit 1
fi
fi

cd $HIVEMALL_HOME
mvn clean deploy -DperformRelease=true -Dskiptests=true -Dmaven.test.skip=true
@@ -17,23 +17,43 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# Start a MIX server instance on this machine
# Start MIX server instances on each machine specified
# in the conf/MIXSERV_LIST file

usage="Usage: mixserv_cluster.sh (start|stop|status)"

if [ "$HIVEMALL_HOME" == "" ]; then
echo env HIVEMALL_HOME not defined
echo "env HIVEMALL_HOME not defined"
exit 1
fi

# Load a version number from a VERSION file
VERSION=`cat "$HIVEMALL_HOME/VERSION"`
JARFILE="$HIVEMALL_HOME/target/hivemall-mixserv-$VERSION-fat.jar"
cmd=$1
case $cmd in
(start)
;;
(stop)
;;
(status)
;;
(*)
echo $usage
exit 1
;;
esac

MIXSERV_HOSTS="$HIVEMALL_HOME/conf/MIXSERV_LIST"
MIXSERV_SSH_OPTS="-o StrictHostKeyChecking=no"

if [ -f "$JARFILE" ]; then
JMXOPTS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9010 -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.authenticate=false"
VMOPTS="-Xmx4g -da -server -XX:+PrintGCDetails -XX:+UseNUMA -XX:+UseParallelGC $VMOPTS"
# VMOPTS="-Xmx4g -da -server -XX:+PrintGCDetails -XX:+UseNUMA -XX:+UseParallelGC $JMXOPTS $VMOPTS"
java ${VMOPTS} -jar ${JARFILE} $@
# Load host entries from the servers file
if [ -f "$MIXSERV_HOSTS" ]; then
HOSTLIST=`cat $MIXSERV_HOSTS`
else
echo executable jar ${JARFILE} not found
HOSTLIST=localhost
fi

for slave in `echo "$HOSTLIST" | sed "s/#.*$//;/^$/d"`; do
ssh $MIXSERV_SSH_OPTS "$slave" "$HIVEMALL_HOME/bin/mixserv_daemon.sh" $cmd 2>&1 | sed "s/^/$slave: /" &
done

wait

@@ -0,0 +1,155 @@
#!/bin/sh

# Hivemall: Hive scalable Machine Learning Library
#
# Copyright (C) 2015 Makoto YUI
# Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

usage="Usage: mixserv_daemon.sh (start|stop|status)"

# If no args specified, show usage
if [ $# -ne 1 ]; then
echo $usage
exit 1
fi

if [ "$HIVEMALL_HOME" == "" ]; then
if [ -e ../bin/${0##*/} ]; then
HIVEMALL_HOME=".."
elif [ -e ./bin/${0##*/} ]; then
HIVEMALL_HOME="."
else
echo "env HIVEMALL_HOME not defined"
exit 1
fi
fi

# Load global variables
. "$HIVEMALL_HOME/conf/mixserv_env.sh"

# Load a version number from a VERSION file
VERSION=`cat $HIVEMALL_HOME/VERSION`

MIXSERV_PID_DIR="/tmp"
MIXSERV_PID_FILE="$MIXSERV_PID_DIR/hivemall-mixserv-$VERSION-$USER.pid"
MIXSERV_LOG_DIR="$HIVEMALL_HOME/logs"
MIXSERV_LOG_FILE="$MIXSERV_LOG_DIR/hivemall-mixserv-$VERSION-$USER-$HOSTNAME.out"

rotate_mixserv_log() {
log=$1
num=5
if [ -n "$2" ]; then
num=$2
fi
if [ -f "$log" ]; then # rotate logs
while [ $num -gt 1 ]; do
prev=`expr $num - 1`
[ -f "$log.$prev" ] && mv "$log.$prev" "$log.$num"
num=$prev
done
mv "$log" "$log.$num";
fi
}

# Sanitize log directory
mkdir -p "$MIXSERV_LOG_DIR"
touch "$MIXSERV_LOG_DIR"/.hivemall_test > /dev/null 2>&1
TEST_LOG_DIR=$?
if [ "${TEST_LOG_DIR}" = "0" ]; then
rm -f "$MIXSERV_LOG_DIR"/.hivemall_test
else
chown "$USER" "$MIXSERV_LOG_DIR"
fi

# Set default scheduling priority
if [ "$MIXSERV_NICENESS" = "" ]; then
export MIXSERV_NICENESS="0"
fi

case $1 in

(start)

# Check if the MIX server has already run
if [ -f $MIXSERV_PID_FILE ]; then
TARGET_ID="$(cat $MIXSERV_PID_FILE)"
if [[ $(ps -p "$TARGET_ID" -o comm=) =~ "java" ]]; then
echo "MIX server has already run as process ${TARGET_ID}"
exit 0
fi
fi

JARFILE="$HIVEMALL_HOME/target/hivemall-mixserv-$VERSION-fat.jar"
if [ -f "$JARFILE" ]; then
# Launch a MIX server
rotate_mixserv_log "$MIXSERV_LOG_FILE"
echo "starting a MIX server, logging to ${MIXSERV_LOG_FILE}"

nohup nice -n "$MIXSERV_NICENESS" java ${MIXSERV_JMXOPTS} ${MIXSERV_VMOPTS} \
-jar "$JARFILE" ${MIXSERV_OPS} > "$MIXSERV_LOG_FILE" 2>&1 &

newpid="$!"
echo $newpid > "$MIXSERV_PID_FILE"
sleep 1

# Checks if the process has died
if [[ ! $(ps -p "$newpid" -o comm=) =~ "java" ]]; then
echo "failed to launch a MIX server"
fi
else
echo "executable jar ${JARFILE} not found"
exit 1
fi
;;

(stop)

if [ -f $MIXSERV_PID_FILE ]; then
TARGET_ID="$(cat $MIXSERV_PID_FILE)"
if [[ $(ps -p "$TARGET_ID" -o comm=) =~ "java" ]]; then
echo "stopping a MIX server (PID=${TARGET_ID})"
kill "$TARGET_ID" && rm -f "$MIXSERV_PID_FILE"
else
echo "no MIX server to stop"
fi
else
echo "no MIX server to stop"
fi
;;

(status)

if [ -f $MIXSERV_PID_FILE ]; then
TARGET_ID="$(cat $MIXSERV_PID_FILE)"
if [[ $(ps -p "$TARGET_ID" -o comm=) =~ "java" ]]; then
echo "MIX server is running (PID=${TARGET_ID})"
exit 0
else
echo "file ${MIXSERV_PID_FILE} is present but MIX server is not running"
exit 1
fi
else
echo "MIX server is not running"
exit 2
fi
;;

(*)
echo $usage
exit 1
;;

esac

@@ -0,0 +1,31 @@
#!/bin/sh

# Hivemall: Hive scalable Machine Learning Library
#
# Copyright (C) 2015 Makoto YUI
# Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

if [ "$HIVEMALL_HOME" == "" ]; then
if [ -e ../bin/${0##*/} ]; then
HIVEMALL_HOME=".."
elif [ -e ./bin/${0##*/} ]; then
HIVEMALL_HOME="."
else
echo "env HIVEMALL_HOME not defined"
exit 1
fi
fi

/bin/sh "$HIVEMALL_HOME/bin/mixserv_daemon.sh" start
@@ -0,0 +1,31 @@
#!/bin/sh

# Hivemall: Hive scalable Machine Learning Library
#
# Copyright (C) 2015 Makoto YUI
# Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

if [ "$HIVEMALL_HOME" == "" ]; then
if [ -e ../bin/${0##*/} ]; then
HIVEMALL_HOME=".."
elif [ -e ./bin/${0##*/} ]; then
HIVEMALL_HOME="."
else
echo "env HIVEMALL_HOME not defined"
exit 1
fi
fi

/bin/sh "$HIVEMALL_HOME/bin/mixserv_daemon.sh" stop
@@ -0,0 +1 @@
localhost
@@ -0,0 +1,7 @@
#!/bin/sh

MIXSERV_JMXOPTS+="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9010 -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.authenticate=false"
MIXSERV_VMOPTS+="-Xmx4g -da -server -XX:+PrintGCDetails -XX:+UseNUMA -XX:+UseParallelGC"
MIXSERV_OPS+="-sync 30"
MIXSERV_NICENESS="0"

0 comments on commit f15bc06

Please sign in to comment.