Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Add cluster control scripts. Patch on http://code.google.com/p/projec…

  • Loading branch information...
commit 4786eaef2fcfebc2c5ad284388f5dccea52a33fa 1 parent 35581b6
@jkreps jkreps authored
View
42 bin/generate_cluster_xml.py
@@ -0,0 +1,42 @@
+import sys
+import random
+
+if len(sys.argv) != 3:
+ print >> sys.stderr, "USAGE: python generate_partitions.py <nodes_file> <partitions_per_node>"
+ sys.exit()
+
+FORMAT_WIDTH = 10
+
+nodes = 0
+for line in open(sys.argv[1],'r'):
+ nodes+=1
+
+partitions = int(sys.argv[2])
+
+ids = range(nodes * partitions)
+
+# use known seed so this is repeatable
+random.seed(92873498274)
+random.shuffle(ids)
+
+print '<cluster>'
+print '<name>prodcluster</name>'
+id = 0
+for host in open(sys.argv[1],'r'):
+ print '<server>'
+ print " <id>%d</id>" % id
+ print " <host>%s</host>" % host.strip()
+ print ' <http-port>8081</http-port>'
+ print ' <socket-port>6666</socket-port>'
+ print ' <partitions>',
+ node_ids = sorted(ids[id*partitions:(id+1)*partitions])
+ for j in xrange(len(node_ids)):
+ print str(node_ids[j]) + ',',
+ if j % FORMAT_WIDTH == FORMAT_WIDTH - 1:
+ print ' ',
+ print ' </partitions>'
+ print '</server>'
+ id += 1
+print '</cluster>'
+
+
View
0  bin/run-wrapper.sh → bin/run-class.sh
File renamed without changes
View
41 bin/voldemort-config.sh
@@ -0,0 +1,41 @@
+
+#
+# Copyright 2008-2009 bebo, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Standard include defining ways to determine VOLDEMORT_HOME, etc.
+
+#export VOL_SSH_OPTS=""
+
+this="$0"
+while [ -h "$this" ]; do
+ ls=`ls -ld "$this"`
+ link=`expr "$ls" : '.*-> \(.*\)$'`
+ if expr "$link" : '.*/.*' > /dev/null; then
+ this="$link"
+ else
+ this=`dirname "$this"`/"$link"
+ fi
+done
+
+# convert relative path to absolute path
+bin=`dirname "$this"`
+script=`basename "$this"`
+bin=`cd "$bin"; pwd`
+this="$bin/$script"
+
+export VOL_HOME=`dirname "$this"`/..
+export VOL_CONF_DIR="$VOL_HOME"
+
View
81 bin/voldemort-destroy-all-data.sh
@@ -0,0 +1,81 @@
+#!/bin/bash
+
+#
+# Copyright 2008-2009 bebo, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Kills all the BDB data files after showing affected nodes and the
+# respective data directory path. User must enter 'Yes' before it proceeds.
+
+
+usage="Usage: voldemort-destroy-all-data.sh [--config <conf-dir>]"
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin/voldemort-config.sh"
+
+if [ "$#" != "2" ] && [ "$#" != "0" ]
+then
+ echo $usage
+ exit 1
+fi
+
+#check to see if the conf dir is given as an optional argument
+if [ $# -eq 2 ]
+then
+ if [ "--config" = "$1" ]
+ then
+ shift
+ confdir=$1
+ shift
+ VOL_CONF_DIR=$confdir
+ nodeid=$1
+ fi
+fi
+
+HOSTLIST="$VOL_CONF_DIR/config/nodes"
+datadir="$(cd $VOL_CONF_DIR && pwd)/data/bdb"
+
+# Make user confirm the selection
+echo ------------------------------------------------------------------
+echo ------------------------------------------------------------------
+echo
+echo You are about to delete all data on the following nodes:
+echo
+echo $(cat $HOSTLIST)
+echo
+echo in $datadir
+echo
+echo
+echo ------------------------------------------------------------------
+echo ------------------------------------------------------------------
+
+echo -n "Type 'Yes' if you would like to continue: "
+read a
+if [[ $a != "Yes" ]]
+then
+ echo "Didn't get a 'Yes' input. Quitting for now."
+ exit 1
+fi
+
+
+for node in `cat "$HOSTLIST"`; do
+ echo $node: Removing data directory $datadir
+ ssh $VOL_SSH_OPTS $node "rm -rf $datadir" &
+done
+wait
+echo
+echo "Done destroying the data directories."
View
57 bin/voldemort-node-id.sh
@@ -0,0 +1,57 @@
+#!/bin/bash
+
+#
+# Copyright 2008-2009 bebo, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Script to remove and change the node.id from local server.properties. Is
+# also used by the sync script across all nodes.
+
+usage="Usage: voldemort-node-id.sh [--config <conf-dir>] <node id>"
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin/voldemort-config.sh"
+
+if [ $# -eq 0 ] || [ $# -eq 2 ] || [ $# -gt 3 ]
+then
+ echo $usage
+ exit 1
+fi
+
+nodeid=$1
+
+#check to see if the conf dir is given as an optional argument
+if [ $# -eq 3 ]
+then
+ if [ "--config" = "$1" ]
+ then
+ shift
+ confdir=$1
+ shift
+ VOL_CONF_DIR=$confdir
+ nodeid=$1
+ fi
+fi
+
+SERVER_PROPERTIES="$VOL_CONF_DIR/config/server.properties"
+
+grep -v "node.id" $SERVER_PROPERTIES > /tmp/$$
+echo "node.id=$nodeid" >> /tmp/$$
+cp -f /tmp/$$ $SERVER_PROPERTIES
+rm /tmp/$$
+
+echo $(hostname): node.id=$nodeid
View
6 bin/voldemort-remote-test.sh
@@ -16,10 +16,10 @@
# limitations under the License.
#
-if [ $# -ne 3 ];
+if [ $# -ne 4 ];
then
- echo 'USAGE: bin/voldemort-remote-test.sh bootstrap-url num-requests start-num'
+ echo 'USAGE: bin/voldemort-remote-test.sh bootstrap-url num-requests value-size start-number'
exit 1
fi
-bin/run-wrapper.sh voldemort.performance.RemoteTest ${1} ${2} ${3}
+bin/run-wrapper.sh voldemort.performance.RemoteTest ${1} ${2} ${3} ${4}
View
56 bin/voldemort-start-all.sh
@@ -0,0 +1,56 @@
+#!/bin/bash
+
+#
+# Copyright 2008-2009 bebo, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Name says it all. Like all the other scripts the --config parameter allow
+# you to determine, which configuration to use. Log output goes into a logs
+# directory on each node (will be auto created if it doesn't exist).
+
+
+usage="Usage: voldemort-start-all.sh [--config <conf-dir>]"
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin/voldemort-config.sh"
+
+if [ "$#" != "0" ] && [ "$#" != "2" ]
+then
+ echo $usage
+ exit 1
+fi
+
+#check to see if the conf dir is given as an optional argument
+if [ $# -eq 2 ]
+then
+ if [ "--config" = "$1" ]
+ then
+ shift
+ confdir=$1
+ shift
+ VOL_CONF_DIR=$confdir
+ nodeid=$1
+ fi
+fi
+
+export HOSTLIST="$VOL_CONF_DIR/config/nodes"
+
+for node in `cat "$HOSTLIST"`; do
+ ssh $VOL_SSH_OPTS $node "mkdir $VOL_HOME/logs 2>/dev/null; nohup $bin/voldemort-server.sh `cd $VOL_CONF_DIR && pwd` >$VOL_HOME/logs/voldemort-server.log 2>$VOL_HOME/logs/voldemort-server.err </dev/null& " &
+done
+wait
+
View
58 bin/voldemort-stop-all.sh
@@ -0,0 +1,58 @@
+#!/bin/bash
+
+#
+# Copyright 2008-2009 bebo, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Stops the local node. Note: As a future enhancement the PID of the current
+# running voldemort server could be saved, so finding out, which Java process
+# to end would be more reliable in case there are multiple Voldemort
+# instances running on the same host under the same user. The PID would be
+# ideally saved on a per config basis.
+
+
+usage="Usage: voldemort-stop-all.sh [--config <conf-dir>]"
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin/voldemort-config.sh"
+
+if [ "$#" != "0" ] && [ "$#" != "2" ]
+then
+ echo $usage
+ exit 1
+fi
+
+#check to see if the conf dir is given as an optional argument
+if [ $# -eq 2 ]
+then
+ if [ "--config" = "$1" ]
+ then
+ shift
+ confdir=$1
+ shift
+ VOL_CONF_DIR=$confdir
+ nodeid=$1
+ fi
+fi
+
+export HOSTLIST="$VOL_CONF_DIR/config/nodes"
+
+for node in `cat "$HOSTLIST"`; do
+ ssh $VOL_SSH_OPTS $node "$bin/voldemort-stop.sh" &
+done
+wait
+
View
42 bin/voldemort-stop.sh
@@ -0,0 +1,42 @@
+#!/bin/bash
+
+#
+# Copyright 2008-2009 bebo, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+usage="Usage: voldemort-stop.sh"
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin/voldemort-config.sh"
+
+if [ "$#" != "0" ]
+then
+ echo $usage
+ exit 1
+fi
+
+pids=`ps xwww | grep voldemort.server.VoldemortServe[r] | awk '{print $1}'`
+
+if [ "$pids" != "" ]
+then
+ echo $(hostname)': Stopping Voldemort...'
+ kill $pids
+ exit 0
+fi
+
+echo $(hostname)': Voldemort Server not running!'
+exit 1
View
73 bin/voldemort-sync-config.sh
@@ -0,0 +1,73 @@
+#!/bin/bash
+
+#
+# Copyright 2008-2009 bebo, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Syncs the entire voldemort install directory over to the other nodes using rsync. The
+# script is smart enough to exclude the BDB data directories (if they are in
+# their standard locations) to avoid overwriting data on the nodes. Once
+# everything is synced over it prepares node.ids for each node and ships them
+# over. The script is also driven by the 'nodes' file.
+
+usage="Usage: voldemort-sync-config.sh [--config <conf-dir>]"
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin/voldemort-config.sh"
+
+#check to see if the conf dir is given as an optional argument
+if [ $# -eq 2 ]
+then
+ if [ "--config" = "$1" ]
+ then
+ shift
+ confdir=$1
+ shift
+ VOL_CONF_DIR=$confdir
+ fi
+fi
+
+export HOSTLIST="$VOL_CONF_DIR/config/nodes"
+
+echo -n "Syncing config changes... "
+
+# don't copy the local berkeley db files to all nodes
+excludefile="/tmp/rsync-excludes$$"
+echo "bdb" > $excludefile
+cd `cd $VOL_HOME && pwd`
+find config -name bdb -type d >> $excludefile
+cd -
+
+# Sync everything but BDB files over
+for node in `cat "$HOSTLIST"`; do
+ rsync $VOL_SSH_OPTS --exclude-from $excludefile --delete -ra `cd $VOL_HOME && pwd`/ $node:`cd $VOL_HOME && pwd` && echo -n "$node " &
+done
+# Wait until config is all pushed out.
+wait
+rm $excludefile
+echo "done!"
+
+echo "Reassigning node IDs..."
+nodeid=0
+abs_conf_dir=`cd $VOL_CONF_DIR && pwd`
+for node in `cat "$HOSTLIST"`; do
+ ssh $node $bin/voldemort-node-id.sh --config $abs_conf_dir $nodeid &
+ nodeid=$[$nodeid + 1]
+done
+wait
+echo "Done syncing Voldemort home directory and reassigning node IDs"
+exit 0
Please sign in to comment.
Something went wrong with that request. Please try again.