Skip to content
Browse files

Merge branch 'ec2-testing' of git://github.com/kirktrue/voldemort int…

…o rebalancing
  • Loading branch information...
2 parents efc6f4f + 27e1f6f commit 95aab9fdb6ddeb845f5fcf86417d919cb3d19c0e @bbansal bbansal committed Nov 5, 2009
Showing with 4,143 additions and 0 deletions.
  1. +4 −0 .classpath
  2. +3 −0 build.xml
  3. +47 −0 contrib/ec2-testing/bin/run-class.sh
  4. +18 −0 contrib/ec2-testing/bin/voldemort-clustercleaner.sh
  5. +18 −0 contrib/ec2-testing/bin/voldemort-clustergenerator.sh
  6. +18 −0 contrib/ec2-testing/bin/voldemort-clusterremotetest.sh
  7. +18 −0 contrib/ec2-testing/bin/voldemort-clusterstarter.sh
  8. +18 −0 contrib/ec2-testing/bin/voldemort-clusterstopper.sh
  9. +18 −0 contrib/ec2-testing/bin/voldemort-deployer.sh
  10. +18 −0 contrib/ec2-testing/bin/voldemort-ec2instancecreator.sh
  11. +18 −0 contrib/ec2-testing/bin/voldemort-ec2instanceterminator.sh
  12. +46 −0 contrib/ec2-testing/examples/remotetest/graph99percentile.groovy
  13. +41 −0 contrib/ec2-testing/examples/remotetest/graph99percentile.sh
  14. +51 −0 contrib/ec2-testing/examples/remotetest/graphAverageTime.groovy
  15. +39 −0 contrib/ec2-testing/examples/remotetest/graphAverageTime.sh
  16. +58 −0 contrib/ec2-testing/examples/remotetest/multiHostTest.groovy
  17. +75 −0 contrib/ec2-testing/examples/remotetest/multiHostTest.sh
  18. +131 −0 contrib/ec2-testing/examples/remotetest/remotetest.sh
  19. +62 −0 contrib/ec2-testing/examples/remotetest/remotetestparser.scala
  20. BIN contrib/ec2-testing/lib/typica.jar
  21. +46 −0 contrib/ec2-testing/resources/commands.properties
  22. +9 −0 contrib/ec2-testing/resources/log4j.properties
  23. +54 −0 contrib/ec2-testing/src/java/voldemort/utils/ClusterCleaner.java
  24. +178 −0 contrib/ec2-testing/src/java/voldemort/utils/ClusterGenerator.java
  25. +171 −0 contrib/ec2-testing/src/java/voldemort/utils/ClusterNodeDescriptor.java
  26. +49 −0 contrib/ec2-testing/src/java/voldemort/utils/ClusterStarter.java
  27. +52 −0 contrib/ec2-testing/src/java/voldemort/utils/ClusterStopper.java
  28. +48 −0 contrib/ec2-testing/src/java/voldemort/utils/Deployer.java
  29. +98 −0 contrib/ec2-testing/src/java/voldemort/utils/Ec2Connection.java
  30. +88 −0 contrib/ec2-testing/src/java/voldemort/utils/HostNamePair.java
  31. +44 −0 contrib/ec2-testing/src/java/voldemort/utils/RemoteOperation.java
  32. +37 −0 contrib/ec2-testing/src/java/voldemort/utils/RemoteOperationException.java
  33. +62 −0 contrib/ec2-testing/src/java/voldemort/utils/RemoteTest.java
  34. +206 −0 contrib/ec2-testing/src/java/voldemort/utils/app/VoldemortApp.java
  35. +71 −0 contrib/ec2-testing/src/java/voldemort/utils/app/VoldemortClusterCleanerApp.java
  36. +69 −0 contrib/ec2-testing/src/java/voldemort/utils/app/VoldemortClusterGeneratorApp.java
  37. +72 −0 contrib/ec2-testing/src/java/voldemort/utils/app/VoldemortClusterRemoteTestApp.java
  38. +169 −0 contrib/ec2-testing/src/java/voldemort/utils/app/VoldemortClusterStarterApp.java
  39. +71 −0 contrib/ec2-testing/src/java/voldemort/utils/app/VoldemortClusterStopperApp.java
  40. +72 −0 contrib/ec2-testing/src/java/voldemort/utils/app/VoldemortDeployerApp.java
  41. +92 −0 contrib/ec2-testing/src/java/voldemort/utils/app/VoldemortEc2InstanceCreatorApp.java
  42. +75 −0 contrib/ec2-testing/src/java/voldemort/utils/app/VoldemortEc2InstanceTerminatorApp.java
  43. +75 −0 contrib/ec2-testing/src/java/voldemort/utils/impl/CommandLineParameterizer.java
  44. +66 −0 contrib/ec2-testing/src/java/voldemort/utils/impl/CommandLineParser.java
  45. +155 −0 contrib/ec2-testing/src/java/voldemort/utils/impl/CommandLineRemoteOperation.java
  46. +48 −0 contrib/ec2-testing/src/java/voldemort/utils/impl/CommandOutputListener.java
  47. +48 −0 contrib/ec2-testing/src/java/voldemort/utils/impl/DelegatingCommandOutputListener.java
  48. +51 −0 contrib/ec2-testing/src/java/voldemort/utils/impl/ExitCodeCallable.java
  49. +67 −0 contrib/ec2-testing/src/java/voldemort/utils/impl/LoggingCommandOutputListener.java
  50. +130 −0 contrib/ec2-testing/src/java/voldemort/utils/impl/RsyncDeployer.java
  51. +98 −0 contrib/ec2-testing/src/java/voldemort/utils/impl/SshClusterCleaner.java
  52. +178 −0 contrib/ec2-testing/src/java/voldemort/utils/impl/SshClusterStarter.java
  53. +98 −0 contrib/ec2-testing/src/java/voldemort/utils/impl/SshClusterStopper.java
  54. +115 −0 contrib/ec2-testing/src/java/voldemort/utils/impl/SshRemoteTest.java
  55. +243 −0 contrib/ec2-testing/src/java/voldemort/utils/impl/TypicaEc2Connection.java
  56. +100 −0 contrib/ec2-testing/src/java/voldemort/utils/impl/UnixCommand.java
  57. +136 −0 contrib/ec2-testing/test/voldemort/utils/SmokeTest.java
  58. +1 −0 src/java/log4j.properties
View
4 .classpath
@@ -1,6 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" path="src/java"/>
+ <classpathentry kind="src" path="contrib/ec2-testing/resources"/>
+ <classpathentry kind="src" path="contrib/ec2-testing/src/java"/>
+ <classpathentry kind="src" path="contrib/ec2-testing/test"/>
<classpathentry kind="src" path="contrib/hadoop-store-builder/test"/>
<classpathentry kind="src" path="contrib/hadoop-store-builder/src/java"/>
<classpathentry kind="src" path="contrib/mongodb/example"/>
@@ -40,5 +43,6 @@
<classpathentry kind="lib" path="lib/commons-pool-1.5.2.jar"/>
<classpathentry kind="lib" path="lib/je-3.3.87.jar"/>
<classpathentry kind="lib" path="lib/protobuf-java-2.2.0.jar"/>
+ <classpathentry kind="lib" path="contrib/ec2-testing/lib/typica.jar"/>
<classpathentry kind="output" path="classes"/>
</classpath>
View
3 build.xml
@@ -184,6 +184,9 @@
<classpath refid="main-classpath" />
<classpath refid="contrib-classpath" />
</javac>
+ <copy todir="${contrib.classes.dir}">
+ <fileset dir="${contrib.root.dir}/ec2-testing/resources" />
+ </copy>
</target>
<target name="contrib-jar" depends="contrib-build" description="Build contrib jar file">
View
47 contrib/ec2-testing/bin/run-class.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+#
+# Copyright 2008-2009 LinkedIn, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+if [ $# -lt 1 ]; then
+ echo $0 java-class-name [options]
+ exit 1
+fi
+
+base_dir=$(dirname $0)/../../..
+
+for file in $base_dir/dist/*.jar;
+do
+ CLASSPATH=$CLASSPATH:$file
+done
+
+for file in $base_dir/lib/*.jar;
+do
+ CLASSPATH=$CLASSPATH:$file
+done
+
+for file in $base_dir/contrib/ec2-testing/lib/*.jar;
+do
+ CLASSPATH=$CLASSPATH:$file
+done
+
+CLASSPATH=$CLASSPATH:$base_dir/dist/resources
+
+if [ -z $VOLD_OPTS ]; then
+ VOLD_OPTS="-Xmx2G -server -Dcom.sun.management.jmxremote"
+fi
+
+export CLASSPATH
+java $VOLD_OPTS -cp $CLASSPATH $@
View
18 contrib/ec2-testing/bin/voldemort-clustercleaner.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+#
+# Copyright 2009 LinkedIn, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+$(dirname $0)/run-class.sh voldemort.utils.app.VoldemortClusterCleanerApp $@
View
18 contrib/ec2-testing/bin/voldemort-clustergenerator.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+#
+# Copyright 2009 LinkedIn, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+$(dirname $0)/run-class.sh voldemort.utils.app.VoldemortClusterGeneratorApp $@
View
18 contrib/ec2-testing/bin/voldemort-clusterremotetest.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+#
+# Copyright 2009 LinkedIn, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+$(dirname $0)/run-class.sh voldemort.utils.app.VoldemortClusterRemoteTestApp $@
View
18 contrib/ec2-testing/bin/voldemort-clusterstarter.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+#
+# Copyright 2009 LinkedIn, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+$(dirname $0)/run-class.sh voldemort.utils.app.VoldemortClusterStarterApp $@
View
18 contrib/ec2-testing/bin/voldemort-clusterstopper.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+#
+# Copyright 2009 LinkedIn, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+$(dirname $0)/run-class.sh voldemort.utils.app.VoldemortClusterStopperApp $@
View
18 contrib/ec2-testing/bin/voldemort-deployer.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+#
+# Copyright 2009 LinkedIn, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+$(dirname $0)/run-class.sh $CLASSPATH voldemort.utils.app.VoldemortDeployerApp $@
View
18 contrib/ec2-testing/bin/voldemort-ec2instancecreator.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+#
+# Copyright 2009 LinkedIn, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+$(dirname $0)/run-class.sh voldemort.utils.app.VoldemortEc2InstanceCreatorApp $@
View
18 contrib/ec2-testing/bin/voldemort-ec2instanceterminator.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+#
+# Copyright 2009 LinkedIn, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+$(dirname $0)/run-class.sh voldemort.utils.app.VoldemortEc2InstanceTerminatorApp $@
View
46 contrib/ec2-testing/examples/remotetest/graph99percentile.groovy
@@ -0,0 +1,46 @@
+// Written by Matthew Garcia
+/* Program takes the raw data files provided by arguments and
+ then parses each file adding the writes, reads, and deletes
+ transactions to seperate Lists. After this it sorts the lists
+ by value and gets the index that is 99% of the total size of the
+ list. It then prints each test number followed by the 99% highest
+ of read, write, and delete transactions(in a format that gnuplot can
+ use to create a graph based on the data). Output may be changed so
+ that this information is sent to a file. */
+
+public double listSorter(List thetimes) {
+ Collections.sort(thetimes)
+ int size = thetimes.size()
+ double percent = size * 0.99
+ BigDecimal bd = new BigDecimal(Double.toString(percent))
+ bd = bd.setScale(0, BigDecimal.ROUND_HALF_UP)
+ return thetimes.get(bd.intValue() - 1)
+}
+
+// Get raw data files from user.
+for (i in 0..args.length - 1) {
+ def inputFile = new java.io.File(args[i])
+
+ // Checks that the data files exist
+ if (!inputFile.exists()) {
+ println "File you entered - ${inputFile.getAbsolutePath()} - does not exist."
+ System.exit(1)
+ }
+
+ def writes = []
+ def deletes = []
+ def reads = []
+
+ inputFile.getText().eachLine {
+ def data = it.split(" ")
+
+ if (data.length >= 5) {
+ reads.add(data[2].toDouble())
+ writes.add(data[3].toDouble())
+ deletes.add(data[4].toDouble())
+ }
+ }
+
+ // Prints the data in a format that can be used by gnuplot.
+ println "${i + 1} ${listSorter(reads)} ${listSorter(writes)} ${listSorter(deletes)}"
+}
View
41 contrib/ec2-testing/examples/remotetest/graph99percentile.sh
@@ -0,0 +1,41 @@
+# Written by Matthew Garcia
+# Creates a .pg file based on the datafile given by the user.
+# The program then runs the .pg file and outputs the data to
+# a .png file, creating a graph for the user.
+# After the program is done with the data file and .pg file
+# it then deletes the files.
+
+DATE=`date +%C%y_%m_%d_%H_%M_%S`
+PG_FILE=${DATE}graph99percentile.pg
+IMG_FILE=graph99percentile.png
+DATA_FILE=${DATE}graph99percentile.dat
+
+if [ "$1" = "" ]
+ then
+ echo "Usage: $0 Takes raw data files as arguments."
+ exit 1
+fi
+
+groovy graph99percentile.groovy $@ > $DATA_FILE
+
+# Creates a .pg file by outputting the file strings to it.
+
+cat > $PG_FILE <<End-of-Message
+#!/usr/bin/gnuplot
+reset
+set terminal png
+set xlabel "Test Number"
+set ylabel "Transactions/Second"
+set yrange [0:10000]
+set title "99 percent of Transactions per second"
+set key reverse Left outside
+set grid
+set style data linespoints
+plot "${DATA_FILE}" using 1:2 title "Reads", \\
+"" using 1:3 title "Writes", \\
+"" using 1:4 title "Deletes"
+End-of-Message
+
+chmod +x $PG_FILE
+./$PG_FILE > $IMG_FILE
+rm -f $PG_FILE $DATA_FILE
View
51 contrib/ec2-testing/examples/remotetest/graphAverageTime.groovy
@@ -0,0 +1,51 @@
+def record = []
+// Written by Matthew Garcia
+/* Parses through multiple files containing raw data.
+ As it parses through it adds the total time for
+ writes, deletes, and reads, and then divides them
+ by the number of iterations to come up with the average.
+ The program then prints out a number defining the test,
+ followed by the average reads, writes, and deletes for
+ that test (in a way so that gnuplot can parse and graph it). */
+
+for (i in 0..args.length - 1) {
+ // Get raw data files from user.
+ def inputFile = new java.io.File(args[i])
+
+ //Checks that the raw data files exist.
+ if (!inputFile.exists()) {
+ println "File you entered - ${inputFile.getAbsolutePath()} - does not exist."
+ System.exit(1)
+ }
+
+ def writesTotal = 0.0
+ def writesCount = 0
+ def deletesTotal = 0.0
+ def deletesCount = 0
+ def readsTotal = 0.0
+ def readsCount = 0
+
+ inputFile.getText().eachLine {
+ def data = it.split(" ")
+
+ if (data.length >= 5) {
+ // Gets total seconds for each transaction, and the total iterations.
+ writesTotal += data[3].toDouble()
+ writesCount++
+ deletesTotal += data[4].toDouble()
+ deletesCount++
+ readsTotal += data[2].toDouble()
+ readsCount++
+ }
+ }
+
+ def writesAvg = writesTotal / writesCount
+ def deletesAvg = deletesTotal / deletesCount
+ def readsAvg = readsTotal / readsCount
+
+ // Adds the transactions to a List in a way that is parsable by gnuplot.
+ record.add("${i + 1} ${readsAvg} ${writesAvg} ${deletesAvg}")
+}
+
+for (i in 0..record.size - 1)
+ println record[i]
View
39 contrib/ec2-testing/examples/remotetest/graphAverageTime.sh
@@ -0,0 +1,39 @@
+#!/bin/sh
+# Written by Matthew Garcia
+# Runs a groovy script that outputs data to a .dat file.
+# Creates a .pg file based on the datafile given by the user.
+# The .pg file is run using the .dat file created, and output
+# directed to a .png file, creating a graph for the user.
+# After the program is done with the .dat file and .pg file
+# it then deletes the files.
+
+DATE=`date +%C%y_%m_%d_%H_%M_%S`
+PG_FILE=${DATE}graphAverageTime.pg
+IMG_FILE=graphAverageTime.png
+DATA_FILE=${DATE}graphAverageTime.dat
+
+if [ "$1" = "" ]
+ then
+ echo "Usage: $0 Takes raw data files as arguments."
+ exit 1
+fi
+
+groovy graphAverageTime.groovy $@ > $DATA_FILE
+cat > $PG_FILE <<End-of-Message
+#!/usr/bin/gnuplot
+reset
+set terminal png
+set xlabel "Test Number"
+set ylabel "Transactions/Second"
+set yrange [0:10000]
+set title "Average Transactions per second"
+set key reverse Left outside
+set grid
+set style data linespoints
+plot '$DATA_FILE' using 1:2 title "Reads", \\
+"" using 1:3 title "Writes", \\
+"" using 1:4 title "Deletes"
+End-of-Message
+chmod +x $PG_FILE
+./$PG_FILE > $IMG_FILE
+rm -f $PG_FILE $DATA_FILE
View
58 contrib/ec2-testing/examples/remotetest/multiHostTest.groovy
@@ -0,0 +1,58 @@
+// Written by Matthew Garcia
+/* Takes one raw data input file as an argument. Parses through the
+ file and seperates the write data output by hosts used in the users test.
+ It then puts the different hosts used into a file used to hold the hostnames.
+ The program prints out every tenth iteration from the test of each host in a
+ way that gnuplot can print out a graph from a .pg file */
+
+def inputFile = new java.io.File(args[0])
+Map hostNameWritesMap = [:]
+
+// Gets the raw data file given by arguments
+if (!inputFile.exists()) {
+ println "File you entered - ${inputFile.getAbsolutePath()} - does not exist."
+ System.exit(1)
+}
+
+def numIterations = 0
+
+// Parses each line in the files and splits them, placing the split data into a String array.
+inputFile.getText().eachLine {
+ def data = it.split(" ")
+
+ if (data.length >= 4) {
+ def host = data[0]
+ def writes = data[3]
+
+ // If the user is using an instance of ec2, this cuts off the tail end of the host name.
+ def hostName = host.replace(".compute-1.amazonaws.com", "")
+ def writesList = hostNameWritesMap[hostName]
+
+ if (writesList == null) {
+ writesList = []
+ hostNameWritesMap[hostName] = writesList
+ }
+
+ // Adds the write record to a list containing all writes in the file(mapped to a particular hostname).
+ writesList << writes
+ numIterations = Math.max(numIterations, writesList.size())
+ }
+}
+
+// Gets an int value of 10% of the total number of iterations.
+//def percent = numIterations / 10
+
+println numIterations
+
+for (i in 0..numIterations - 1) {
+ // Prints the data if the iteration number is divisible by 10% of the total number of iterations
+ //if ((i + 1).intValue() % percent == 0) {
+ print "${(i + 1)} "
+
+ // Prints the write record for each host. and each tenth iteration, then creates a new line.
+ for (hostName in hostNameWritesMap.keySet())
+ print "${hostNameWritesMap[hostName][i]} "
+
+ println ""
+ //}
+}
View
75 contrib/ec2-testing/examples/remotetest/multiHostTest.sh
@@ -0,0 +1,75 @@
+#!/bin/sh
+# Written by Matthew Garcia
+# Runs a groovy script that creates a text file containing the information
+# that will be parsed, as well as a text file containing the hostnames of
+# the machines used in the test. It uses awk to get the number of hosts
+# in the script. Afterwards it creates a .pg file using the hosts file
+# and the text file containing the information to be graphed. After
+# creating the .pg file it is run and output is directed to a .png file.
+# The .png file contains the data created from the groovy script in a graph.
+
+DATE=`date +%C%y_%m_%d_%H_%M_%S`
+PG_FILE=${DATE}multiHostTest.pg
+IMG_FILE=multiHostTest.png
+DATA_FILE=${DATE}multiHostTest.dat
+HOSTS_FILE=${DATE}multiHostTest-hosts.dat
+
+if [ "$1" = "" ]
+ then
+ echo "Usage: $0 Takes raw data files as arguments."
+ exit 1
+fi
+
+groovy multiHostTest.groovy $@ > $DATA_FILE
+
+cat $1 | cut -d' ' -f1 | uniq > $HOSTS_FILE
+intI=`awk 'END{print NR}' $HOSTS_FILE`
+
+# Creates the .pg file
+cat > $PG_FILE <<End-of-Message
+#!/usr/bin/gnuplot
+reset
+set terminal png
+set xlabel "Test Number"
+set ylabel "Transactions/Second"
+set yrange [0:8000]
+set title "Average Transactions per second"
+set key reverse Left outside
+set grid
+set style data linespoints
+End-of-Message
+
+# Loops through and appends to the .pg file for each host.
+for ((i = 1; $i <= $intI; i++)) {
+ hostName=$(awk -v i=$i 'NR==i {print $1}' $HOSTS_FILE)
+ usingLine="using 1:$(($i + 1)) title \"$hostName\""
+
+ if [ $i == 1 ]
+ then
+ # If the current host is the first host, it appends a
+ # plot String to the .pg file.
+ echo -n "plot \"$DATA_FILE\" $usingLine" >> $PG_FILE
+ else
+ # If neither of the above cases are true the file will
+ # append to the file the "" string with the ending
+ # having a , and \ (meaning that the file has more to
+ # plot.
+ echo -n "\"\" $usingLine" >> $PG_FILE
+ fi
+
+ if [ $i != $intI ]
+ then
+ # If the current host is NOT the last host, it appends
+ # a string starting with "" (which means using the same
+ # data stated already) and ends without a \
+ echo ", \\" >> $PG_FILE
+ else
+ # This is just to be nice formatting since we don't
+ # put out any newlines above
+ echo "" >> $PG_FILE
+ fi
+}
+
+chmod +x $PG_FILE
+./$PG_FILE > $IMG_FILE
+rm -f $DATA_FILE $HOSTS_FILE $PG_FILE
View
131 contrib/ec2-testing/examples/remotetest/remotetest.sh
@@ -0,0 +1,131 @@
+#!/bin/sh
+
+HOSTS_SERVERS=/tmp/hosts-servers
+HOSTS_CLIENTS=/tmp/hosts-clients
+LOG_SERVERS_FILE=/tmp/log-servers
+LOG_CLIENTS_FILE=/tmp/log-clients
+COMMANDS_FILE=/tmp/commands
+
+if [ $# -ne 10 ]
+ then
+ echo "Usage: $0 <access key ID> <secret key> <ami> <key pair ID> <# clients> <# servers> <partitions per server> <config dir> <# requests> <# iterations>"
+ exit 1
+fi
+
+accessKeyId=$1
+secretKey=$2
+ami=$3
+keypairId=$4
+numClients=$5
+numServers=$6
+partitions=$7
+configDir=$8
+numRequests=$9
+iterations=${10}
+
+rm -f $LOG_SERVERS_FILE $LOG_CLIENTS_FILE $COMMANDS_FILE
+
+# Create instances in EC2.
+if [ $numClients -gt 0 ]
+ then
+ ./contrib/ec2-testing/bin/voldemort-ec2instancecreator.sh --accessid $accessKeyId --secretkey $secretKey --ami $ami --instances $numClients --keypairid $keypairId > $HOSTS_CLIENTS 2>> $LOG_SERVERS_FILE
+ exitCode=$?
+
+ if [ $exitCode -ne 0 ]
+ then
+ echo "Exit code from voldemort-ec2instancecreator.sh for clients: $exitCode, please see logs for more details"
+ exit 1
+ fi
+fi
+
+if [ $numServers -gt 0 ]
+ then
+ ./contrib/ec2-testing/bin/voldemort-ec2instancecreator.sh --accessid $accessKeyId --secretkey $secretKey --ami $ami --instances $numServers --keypairid $keypairId > $HOSTS_SERVERS 2>> $LOG_SERVERS_FILE
+ exitCode=$?
+
+ if [ $exitCode -ne 0 ]
+ then
+ echo "Exit code from voldemort-ec2instancecreator.sh for servers: $exitCode, please see logs for more details"
+ exit 1
+ fi
+fi
+
+# Generate the cluster.xml file
+./contrib/ec2-testing/bin/voldemort-clustergenerator.sh --hostnames $HOSTS_SERVERS --partitions $partitions > $configDir/config/cluster.xml
+exitCode=$?
+
+if [ $exitCode -ne 0 ]
+ then
+ echo "Exit code from voldemort-clustergenerator.sh: $exitCode, please see logs for more details"
+ exit 1
+fi
+
+# Deploy our files to the remote host.
+./contrib/ec2-testing/bin/voldemort-deployer.sh --hostnames $HOSTS_SERVERS --source `pwd` --sshprivatekey $EC2_RSAKEYPAIR --parent "server" --logging debug 2>> $LOG_SERVERS_FILE
+exitCode=$?
+
+if [ $exitCode -ne 0 ]
+ then
+ echo "Exit code from voldemort-deployer.sh for deploying servers: $exitCode, please see logs for more details"
+ exit 1
+fi
+
+./contrib/ec2-testing/bin/voldemort-deployer.sh --hostnames $HOSTS_SERVERS --source $configDir/* --sshprivatekey $EC2_RSAKEYPAIR --parent "testconfig" --logging debug 2>> $LOG_SERVERS_FILE
+exitCode=$?
+
+if [ $exitCode -ne 0 ]
+ then
+ echo "Exit code from voldemort-deployer.sh for deploying server configuration: $exitCode, please see logs for more details"
+ exit 1
+fi
+
+./contrib/ec2-testing/bin/voldemort-deployer.sh --hostnames $HOSTS_CLIENTS --source `pwd` --sshprivatekey $EC2_RSAKEYPAIR --parent "client" --logging debug 2>> $LOG_SERVERS_FILE
+exitCode=$?
+
+if [ $exitCode -ne 0 ]
+ then
+ echo "Exit code from voldemort-deployer.sh for deploying clients: $exitCode, please see logs for more details"
+ exit 1
+fi
+
+# We can't easily check the exit code as we run this in the background...
+./contrib/ec2-testing/bin/voldemort-clusterstarter.sh --hostnames $HOSTS_SERVERS --sshprivatekey $EC2_RSAKEYPAIR --voldemortroot "server/voldemort" --voldemorthome "testconfig" --clusterxml $configDir/config/cluster.xml --logging debug 2>> $LOG_SERVERS_FILE &
+
+sleep 10
+
+bootstrapHost=`tail -n 1 $HOSTS_SERVERS | cut -d'=' -f2`
+
+counter=0
+sleep=30
+
+cat $HOSTS_CLIENTS |
+while read line
+do
+ rampSeconds=$(($sleep * $counter))
+ startKeyIndex=$(($numRequests * $counter))
+ counter=$(($counter + 1))
+ externalHost=`echo $line | cut -d'=' -f1`
+ echo "$externalHost=cd client/voldemort ; sleep $rampSeconds ; ./bin/voldemort-remote-test.sh -r -w -d --iterations $iterations --start-key-index $startKeyIndex tcp://${bootstrapHost}:6666 test $numRequests" >> $COMMANDS_FILE
+done
+
+./contrib/ec2-testing/bin/voldemort-clusterremotetest.sh --hostnames $HOSTS_CLIENTS --sshprivatekey $EC2_RSAKEYPAIR --commands $COMMANDS_FILE --logging debug 2>> $LOG_SERVERS_FILE > $LOG_CLIENTS_FILE
+exitCode=$?
+
+if [ $exitCode -ne 0 ]
+ then
+ echo "Exit code from voldemort-clusterremotetest.sh: $exitCode, please see logs for more details"
+ exit 1
+fi
+
+
+# Send SIGTERM
+pids=`ps xwww | grep voldemort.utils.app.VoldemortClusterStarterApp | grep -v "run-class.sh" | grep -v "grep" | awk '{print $1}'`
+
+if [ "$pids" != "" ]
+ then
+ kill -15 $pids
+fi
+
+./contrib/ec2-testing/examples/remotetest/remotetestparser.scala $LOG_CLIENTS_FILE
+
+rm -f $LOG_SERVERS_FILE $LOG_CLIENTS_FILE $COMMANDS_FILE
View
62 contrib/ec2-testing/examples/remotetest/remotetestparser.scala
@@ -0,0 +1,62 @@
+#!/bin/sh
+exec scala "$0" "$@"
+!#
+
+val ITERATOR_LINE_TAG = " iteration = "
+val ITERATIONS_TOTAL_LINE_TAG = "iterations : "
+val THROUGHPUT_LINE_TAG = "Throughput: "
+
+class Iteration {
+
+ var reads = 0.0
+
+ var writes = 0.0
+
+ var deletes = 0.0
+
+}
+
+var currentIterationMap = scala.collection.mutable.Map[String, Int]()
+val hostNamesMap = scala.collection.mutable.Map[String, scala.collection.mutable.Map[Int, Iteration]]()
+
+if (args.length > 0) {
+ for (line <- scala.io.Source.fromFile(args(0)).getLines) {
+ val hostName = line.split(":")(0)
+ var iterationMap = hostNamesMap.getOrElseUpdate(hostName, scala.collection.mutable.Map[Int, Iteration]())
+
+ var i = line.indexOf(ITERATOR_LINE_TAG)
+
+ if (i != -1) {
+ var iterationIndex = java.lang.Integer.parseInt(line.substring(i + ITERATOR_LINE_TAG.length()).replace("=", "").trim())
+ currentIterationMap.put(hostName, iterationIndex)
+ } else {
+ i = line.indexOf(THROUGHPUT_LINE_TAG)
+
+ if (i != -1) {
+ var iterationIndex = currentIterationMap.getOrElseUpdate(hostName, 0)
+ var iteration = iterationMap.getOrElseUpdate(iterationIndex, new Iteration)
+ val value = java.lang.Double.parseDouble(line.substring(i + THROUGHPUT_LINE_TAG.length(), line.indexOf(" ", i + THROUGHPUT_LINE_TAG.length())))
+
+ if (line.contains("reads"))
+ iteration.reads = value
+ else if (line.contains("writes"))
+ iteration.writes = value
+ else if (line.contains("deletes"))
+ iteration.deletes = value
+ }
+ }
+ }
+}
+
+hostNamesMap.foreach(arg => {
+ val hostName = arg._1
+ val iterationMap = arg._2
+ val iterationList:List[Int] = List.fromIterator(iterationMap.keys).sort(_ < _)
+
+ iterationList.foreach(iterationIndex => {
+ val iterationOption:Option[Iteration] = iterationMap.get(iterationIndex)
+ val iteration:Iteration = iterationOption.getOrElse(new Iteration)
+ println(hostName + " " + iterationIndex + " " + iteration.reads + " " + iteration.writes + " " + iteration.deletes)
+ })
+ }
+)
View
BIN contrib/ec2-testing/lib/typica.jar
Binary file not shown.
View
46 contrib/ec2-testing/resources/commands.properties
@@ -0,0 +1,46 @@
+# This file contains a list of named command-line commands that are executed
+# after substituting the appropriate dynamic values.
+#
+# Valid variables are:
+#
+# hostName External host name of remote host
+# hostUserId User ID for remote host
+# sshPrivateKey Local path to SSH private key to use to connect
+# destinationDirectory Relative directory on remote host under which
+# the files will be copied
+# voldemortRootDirectory Relative directory on remote host which points
+# to the Voldemort distribution; usually it's
+# ${destinationDirectory}/voldemort
+# voldemortHomeDirectory Relative directory on remote host which points
+# to the Voldemort configuration; usually it's
+# ${voldemortRootDirectory}/config/<some directory>
+# voldemortNodeId Node ID (as seen in cluster.xml) of the node
+# remoteTestArguments Arguments passed to the voldemort-remote-test.sh
+# command on the remote host
+# sourceDirectory Local path which points to the Voldemort
+# distribution; this is copied to
+# ${voldemortParentDirectory}
+#
+# Of course, feel free to update CommandLineClusterOperation.java and
+# CommandLineClusterConfig.java to add more variables, as needed.
+
+RsyncDeployer.rsync=rsync -vazc --delete --progress --exclude=.git \
+ -e "ssh -o StrictHostKeyChecking=no -i ${sshPrivateKey}" \
+ ${sourceDirectory} ${hostUserId}@${hostName}:${destinationDirectory}
+
+SshClusterCleaner.ssh=ssh -o StrictHostKeyChecking=no -i ${sshPrivateKey} ${hostUserId}@${hostName} \
+ "rm -rf ${voldemortHomeDirectory}/data "
+
+SshClusterStarter.ssh=ssh -o StrictHostKeyChecking=no -i ${sshPrivateKey} ${hostUserId}@${hostName} \
+ "export VOLDEMORT_HOME=$(pwd)/${voldemortHomeDirectory} ; \
+ export VOLDEMORT_NODE_ID=${voldemortNodeId} ; \
+ mv $VOLDEMORT_HOME/config/server.properties $VOLDEMORT_HOME/config/server.properties.bak ; \
+ grep -v "^node.id=" $VOLDEMORT_HOME/config/server.properties.bak > $VOLDEMORT_HOME/config/server.properties ; \
+ rm -rf ${voldemortHomeDirectory}/data ; \
+ cd ${voldemortRootDirectory} ; \
+ ./bin/voldemort-server.sh"
+
+SshClusterStopper.ssh=ssh -o StrictHostKeyChecking=no -i ${sshPrivateKey} ${hostUserId}@${hostName} \
+ "cd ${voldemortRootDirectory} ; ./bin/voldemort-stop.sh"
+
+SshRemoteTest.ssh=ssh -o StrictHostKeyChecking=no -i ${sshPrivateKey} ${hostUserId}@${hostName} "${testCommand}"
View
9 contrib/ec2-testing/resources/log4j.properties
@@ -0,0 +1,9 @@
+## This file controls logging for the voldemort server and voldemort client
+## For details on configuration see http://logging.apache.org/log4j
+
+log4j.rootLogger=INFO, stderr
+
+log4j.appender.stderr=org.apache.log4j.ConsoleAppender
+log4j.appender.stderr.target=System.err
+log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
+log4j.appender.stderr.layout.ConversionPattern=%m%n
View
54 contrib/ec2-testing/src/java/voldemort/utils/ClusterCleaner.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2009 LinkedIn, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package voldemort.utils;
+
+/**
+ * The ClusterCleaner class represents the task of remotely cleaning the data
+ * directory for a given Voldemort configuration directory. That is, given a
+ * value <i>myconfig</i> for the <i>Voldemort home</i> configuration attribute,
+ * the remote directory <i>myconfig/data</i> will be deleted. For example, if
+ * the remote Voldemort home directory is
+ * <i>voldemort/config/single_node_cluster</i>, the directory
+ * <i>voldemort/config/single_node_cluster/data</i> will be deleted by this
+ * operation.
+ *
+ * <p/>
+ *
+ * Implementation notes:
+ *
+ * <ol>
+ * <li>Implementations must provide a reasonable guarantee that the data was
+ * actually deleted. An error should be raised, therefore, if the directory was
+ * unable to be deleted, independent of cause (security permissions, for
+ * example).
+ * <li>In the event that the remote data directory was not already present, no
+ * error should be thrown. The operation should simply return. A warning message
+ * may be output but the application logic should continue.
+ * <li>The deletion should happen recursively and thus include all child files
+ * and sub-directories.
+ * <li>No precaution or assumption need be made as to the state of the data
+ * being deleted. For example, no checks are made to ensure that a Voldemort
+ * server is not currently running on the given remote host, using this data
+ * directory upon which this operation is being performed.
+ * </ol>
+ *
+ * @author Kirk True
+ */
+
+public interface ClusterCleaner extends RemoteOperation {
+
+}
View
178 contrib/ec2-testing/src/java/voldemort/utils/ClusterGenerator.java
@@ -0,0 +1,178 @@
+/*
+ * Copyright 2009 LinkedIn, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package voldemort.utils;
+
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.commons.lang.StringUtils;
+
+/**
+ * ClusterGenerator generates a cluster.xml file given either a list of hosts or
+ * a list of ClusterNodeDescriptor instances.
+ *
+ * <p/>
+ *
+ * This is largely the same as the generate_cluster_xml.py script, but was
+ * created because we need to be able to create new cluster.xml files
+ * dynamically from JUnit. It seemed overly kludgey to have a script that calls
+ * Java that then calls Python.
+ *
+ * <p/>
+ *
+ * <b>A note about host names</b>: the host name that is referred to in this
+ * class is a system's <i>internal</i> host name, rather than its external name.
+ * That is, depending on the network topology, a system may have an internal
+ * host name by which it is known by in its local network and an external host
+ * name by which it's known by systems external to that network.
+ *
+ * <p/>
+ *
+ * For example, EC2 systems have both internal host names and external host
+ * names. When a system external to EC2 (e.g. a developer's machine or a machine
+ * running a testing framework) wants to communicate with EC2 (via SSH, et al.),
+ * he must use the EC2 instance's external host name. However, for the EC2
+ * instances to communicate amongst themselves (e.g. when running the Voldemort
+ * tests), the Voldemort cluster nodes and Voldemort test nodes must use the
+ * internal host name. The external host name is used by the development/testing
+ * system to reach EC2 for orchestrating the testing. But the communication of
+ * the test and server nodes in the test are all on the same network, using the
+ * internal host name.
+ *
+ * @author Kirk True
+ *
+ * @see ClusterNodeDescriptor
+ */
+
+public class ClusterGenerator {
+
+ private final static long SEED = 5276239082346L;
+
+ /**
+ * Creates a list of ClusterNodeDescriptor instances given a list of host
+ * names. The returned list of ClusterNodeDescriptor have only the host
+ * name, ID, and partition list attributes set.
+ *
+ * <p/>
+ *
+ * The list of partitions is randomly distributed over the list of hosts
+ * using a hard-coded random seed. This mimics the behavior of
+ * generate_cluster_xml.py.
+ *
+ * <p/>
+ *
+ * <b>Please see "A note about host names"</b> in the class' JavaDoc for
+ * important clarification as to the type of host names used.
+ *
+ * @param hostNames <i>Internal</i> host name
+ * @param numPartitions Number of partitions <b>per host</b>
+ *
+ * @return List of ClusterNodeDescriptor
+ */
+
+ public List<ClusterNodeDescriptor> createClusterNodeDescriptors(List<String> hostNames,
+ int numPartitions) {
+ // Create a list of integers [0..totalPartitions).
+ int totalPartitions = hostNames.size() * numPartitions;
+ List<Integer> allPartitionIds = new ArrayList<Integer>();
+
+ for(int i = 0; i < totalPartitions; i++)
+ allPartitionIds.add(i);
+
+ Random random = new Random(SEED);
+ Collections.shuffle(allPartitionIds, random);
+
+ List<ClusterNodeDescriptor> list = new ArrayList<ClusterNodeDescriptor>();
+
+ for(int i = 0; i < hostNames.size(); i++) {
+ String hostName = hostNames.get(i);
+ List<Integer> partitions = allPartitionIds.subList(i * numPartitions, (i + 1)
+ * numPartitions);
+ Collections.sort(partitions);
+
+ ClusterNodeDescriptor cnd = new ClusterNodeDescriptor();
+ cnd.setHostName(hostName);
+ cnd.setId(i);
+ cnd.setPartitions(partitions);
+
+ list.add(cnd);
+ }
+
+ return list;
+ }
+
+ /**
+ * Creates a String representing the format used by cluster.xml given the
+ * cluster name, host names, and number of partitions for each host.
+ *
+ * @param clusterName Name of cluster
+ * @param hostNames <i>Internal</i> host name
+ * @param numPartitions Number of partitions <b>per host</b>
+ *
+ * @return String of formatted XML as used by cluster.xml
+ */
+
+ public String createClusterDescriptor(String clusterName,
+ List<String> hostNames,
+ int numPartitions) {
+ List<ClusterNodeDescriptor> clusterNodeDescriptors = createClusterNodeDescriptors(hostNames,
+ numPartitions);
+
+ return createClusterDescriptor(clusterName, clusterNodeDescriptors);
+ }
+
+ /**
+ * Creates a String representing the format used by cluster.xml given the
+ * cluster name and a list of ClusterNodeDescriptor instances.
+ *
+ * @param clusterName Name of cluster
+ * @param clusterNodeDescriptors List of ClusterNodeDescriptor instances
+ * with which the string is generated
+ *
+ * @return String of formatted XML as used by cluster.xml
+ */
+
+ public String createClusterDescriptor(String clusterName,
+ List<ClusterNodeDescriptor> clusterNodeDescriptors) {
+ StringWriter sw = new StringWriter();
+ PrintWriter pw = new PrintWriter(sw);
+
+ pw.println("<cluster>");
+ pw.println("\t<name>" + clusterName + "</name>");
+
+ for(ClusterNodeDescriptor cnd: clusterNodeDescriptors) {
+ String partitions = StringUtils.join(cnd.getPartitions(), ", ");
+
+ pw.println("\t<server>");
+ pw.println("\t\t<id>" + cnd.getId() + "</id>");
+ pw.println("\t\t<host>" + cnd.getHostName() + "</host>");
+ pw.println("\t\t<http-port>" + cnd.getHttpPort() + "</http-port>");
+ pw.println("\t\t<socket-port>" + cnd.getSocketPort() + "</socket-port>");
+ pw.println("\t\t<partitions>" + partitions + "</partitions>");
+ pw.println("\t</server>");
+ }
+
+ pw.println("</cluster>");
+
+ return sw.toString();
+ }
+
+}
View
171 contrib/ec2-testing/src/java/voldemort/utils/ClusterNodeDescriptor.java
@@ -0,0 +1,171 @@
+/*
+ * Copyright 2009 LinkedIn, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package voldemort.utils;
+
+import java.util.List;
+
+/**
+ * ClusterNodeDescriptor is a simple POJO for storing the attributes of a node
+ * as needed by the cluster.xml cluster descriptor file.
+ *
+ * @author Kirk True
+ *
+ * @see ClusterGenerator
+ */
+
+public class ClusterNodeDescriptor {
+
+ /**
+ * DEFAULT_HTTP_PORT is 8081 as seen in the examples.
+ */
+
+ public static final int DEFAULT_HTTP_PORT = 8081;
+
+ /**
+ * DEFAULT_SOCKET_PORT is 6666 as seen in the examples.
+ */
+
+ public static final int DEFAULT_SOCKET_PORT = 6666;
+
+ private String hostName;
+
+ private int id;
+
+ private int httpPort = DEFAULT_HTTP_PORT;
+
+ private int socketPort = DEFAULT_SOCKET_PORT;
+
+ private List<Integer> partitions;
+
+ /**
+ * Returns the host name (or IP address) of the node. This is the internal
+ * host name as seen by the other server nodes and clients on the same
+ * network.
+ *
+ * @return Host name (or IP address) or null if unset
+ */
+
+ public String getHostName() {
+ return hostName;
+ }
+
+ /**
+ * Assigns the host name (or IP address) of the node. This is the internal
+ * host name as seen by the other server nodes and clients on the same
+ * network.
+ *
+ * @param hostName Host name (or IP address)
+ */
+
+ public void setHostName(String hostName) {
+ this.hostName = hostName;
+ }
+
+ /**
+ * Returns the ID of the node. This is the node ID that is used in both the
+ * cluster.xml and server.properties configuration files.
+ *
+ * @return Node ID
+ */
+
+ public int getId() {
+ return id;
+ }
+
+ /**
+ * Assigns the ID of the node. This is the node ID that is used in both the
+ * cluster.xml and server.properties configuration files.
+ *
+ * @param id Node ID
+ */
+
+ public void setId(int id) {
+ this.id = id;
+ }
+
+ /**
+ * Retrieves the port used by the HTTP server. It defaults to
+ * DEFAULT_HTTP_PORT if left unset.
+ *
+ * @return Port used by the HTTP server
+ */
+
+ public int getHttpPort() {
+ return httpPort;
+ }
+
+ /**
+ * Assigns the port used by the HTTP server.
+ *
+ * @param httpPort Port used by the HTTP server
+ */
+
+ public void setHttpPort(int httpPort) {
+ this.httpPort = httpPort;
+ }
+
+ /**
+ * Retrieves the port used by the socket server. It defaults to
+ * DEFAULT_SOCKET_PORT if left unset.
+ *
+ * @return Port used by the socket server
+ */
+
+ public int getSocketPort() {
+ return socketPort;
+ }
+
+ /**
+ * Assigns the port used by the socket server. It defaults to
+ * DEFAULT_SOCKET_PORT if left unset.
+ *
+ * @param socketPort Port used by the socket server
+ */
+
+ public void setSocketPort(int socketPort) {
+ this.socketPort = socketPort;
+ }
+
+ /**
+ * Returns the list of partition IDs used by this particular node.
+ *
+ * @return List of partitions, or null if unset
+ */
+
+ public List<Integer> getPartitions() {
+ return partitions;
+ }
+
+ /**
+ * Assigns the list of partition IDs used by this particular node. Please
+ * ensure that the partitions provided to <i>this</i> node have no overlap
+ * with any partitions for any <i>other</i> nodes in the cluster. This class
+ * doesn't enforce/check overlaps; you'll find them at runtime ;)
+ *
+ * <p/>
+ *
+ * Also, if possible, provide the list in ascending sorted order. (This is
+ * an optimization for humans.)
+ *
+ * @param partitions List of partitions
+ */
+
+ public void setPartitions(List<Integer> partitions) {
+ this.partitions = partitions;
+ }
+
+}
View
49 contrib/ec2-testing/src/java/voldemort/utils/ClusterStarter.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2009 LinkedIn, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package voldemort.utils;
+
+/**
+ * ClusterStarter represents the operation of remotely starting up a set of
+ * Voldemort servers. The operation is <b>not</b> to start the remote hosts
+ * themselves.
+ *
+ * <p/>
+ *
+ * Implementation notes:
+ *
+ * <ol>
+ * <li>Implementations must provide a reasonable guarantee that the servers were
+ * actually started. An error should be raised, therefore, if the server could
+ * not be started, regardless of cause.
+ * <li>It is assumed that the remote host is properly set up to start the
+ * Voldemort server. This means that:
+ * <ul>
+ * <li>The Java environment is properly configured with $JAVA_HOME pointing at a
+ * valid JDK.
+ * <li>The Voldemort distribution is already present at a known location on the
+ * remote host.
+ * </ul>
+ * <li>The startup procedure should occur in parallel against the remote hosts,
+ * if possible.
+ * </ol>
+ *
+ * @author Kirk True
+ */
+
+public interface ClusterStarter extends RemoteOperation {
+
+}
View
52 contrib/ec2-testing/src/java/voldemort/utils/ClusterStopper.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright 2009 LinkedIn, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package voldemort.utils;
+
+/**
+ * ClusterStopper represents the operation of remotely stopping a set of
+ * Voldemort servers. The operation is <b>not</b> to stop/halt/shutdown the
+ * remote hosts themselves.
+ *
+ * <p/>
+ *
+ * Implementation notes:
+ *
+ * <ol>
+ * <li>Implementations must provide a reasonable guarantee that the servers were
+ * actually stopped. An error should be raised, therefore, if the server could
+ * not be stopped, regardless of cause.
+ * <li>However, if the server was not already running, no error should be
+ * raised. The application logic should simply continue unaffected. It is up to
+ * the implementation to decide the means in which this may be reported, if any.
+ * <li>It is assumed that the remote host is properly set up to stop the
+ * Voldemort server. This means that:
+ * <ul>
+ * <li>The Java environment is properly configured with $JAVA_HOME pointing at a
+ * valid JDK.
+ * <li>The Voldemort distribution is already present at a known location on the
+ * remote host.
+ * </ul>
+ * <li>The shutdown procedure should occur in parallel against the remote hosts,
+ * if possible.
+ * </ol>
+ *
+ * @author Kirk True
+ */
+
+public interface ClusterStopper extends RemoteOperation {
+
+}
View
48 contrib/ec2-testing/src/java/voldemort/utils/Deployer.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2009 LinkedIn, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package voldemort.utils;
+
+/**
+ * Deployer represents the operation of remotely deploying the Voldemort
+ * distribution to a set of Voldemort clients and/or servers. A <i>Voldemort
+ * distribution</i> is the set of binaries, configuration files, etc. that
+ * comprise the Voldemort system. For example, the binary distribution (.zip,
+ * .tar.gz, etc.) of Voldemort includes a top-level directory of the name
+ * <i>voldemort-&lt;version&gt;</i>, under which are directories such as
+ * <i>bin</i>, <i>config</i>, <i>dist</i>, etc. It is this top-level directory
+ * (<i>voldemort-&lt;version&gt;</i>) that is deployed.
+ *
+ * <p/>
+ *
+ * Implementation notes:
+ *
+ * <ol>
+ * <li>Implementations must provide a reasonable guarantee that the deployment
+ * actually occurred. An error should be raised, therefore, if the data could
+ * not be copied, regardless of cause.
+ * <li>No precaution or assumption need be made as to the state of the data
+ * being copied. For example, no checks are made to ensure that a Voldemort
+ * server is not currently running on the given remote host, using this data
+ * directory upon which this operation is being performed.
+ * </ol>
+ *
+ * @author Kirk True
+ */
+
+public interface Deployer extends RemoteOperation {
+
+}
View
98 contrib/ec2-testing/src/java/voldemort/utils/Ec2Connection.java
@@ -0,0 +1,98 @@
+/*
+ * Copyright 2009 LinkedIn, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package voldemort.utils;
+
+import java.util.List;
+
+/**
+ * Ec2Connection represents a connection to Amazon's EC2 API. The name of the
+ * API doesn't denote some sort of persistent, long-lived connection. This API
+ * simply serves as an abstraction of the actual underlying library/mechanism we
+ * use to talk to EC2.
+ *
+ * This API defines methods to create, delete, and retrieve a list of the remote
+ * hosts from EC2.
+ *
+ * @author Kirk True
+ */
+
+public interface Ec2Connection {
+
+ public enum Ec2InstanceType {
+
+ DEFAULT,
+ LARGE,
+ XLARGE,
+ MEDIUM_HCPU,
+ XLARGE_HCPU;
+
+ }
+
+ /**
+ * Retrieve the host names (external/internal) of the instances launched by
+ * the EC2 account. These are not filtered in any form but represent
+ * <b>all</b> instances owned by this account, regardless of launch time,
+ * instance size, etc.
+ *
+ * @return List of HostNamePair instances
+ *
+ * @throws Exception Thrown on errors
+ */
+
+ public List<HostNamePair> list() throws Exception;
+
+ /**
+ * Creates <code>instanceCount</code> number of instances of type
+ * <code>instanceType</code> using the given AMI and keypair ID. The
+ * implementation of this method must block until the instances are all
+ * successfully started and in the <i>running</i> state (as reported by
+ * EC2).
+ *
+ * <p/>
+ *
+ * Please note: the AMI and instance type must be compatible. For example,
+ * attempting to launch a XLARGE_HCPU instance using a 32-bit AMI will
+ * result in an error.
+ *
+ * @param ami Amazon Machine Image; can be public, private, etc.
+ * @param keypairId String representing keypair ID
+ * @param instanceType Instance type to launch
+ * @param instanceCount Number of instances to launch
+ *
+ * @return List of HostNamePair instances
+ *
+ * @throws Exception Thrown on errors
+ */
+
+ public List<HostNamePair> create(String ami,
+ String keypairId,
+ Ec2InstanceType instanceType,
+ int instanceCount) throws Exception;
+
+ /**
+ * Deletes the EC2 instances represented by the given external host names.
+ * Like the create method, this method blocks until the instances
+ * represented by the given host names have terminated.
+ *
+ * @param hostNames External host names of EC2 instances to terminate
+ *
+ * @throws Exception Thrown on errors
+ */
+
+ public void delete(List<String> hostNames) throws Exception;
+
+}
View
88 contrib/ec2-testing/src/java/voldemort/utils/HostNamePair.java
@@ -0,0 +1,88 @@
+/*
+ * Copyright 2009 LinkedIn, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package voldemort.utils;
+
+/**
+ * HostNamePair represents a pairing of a host's external and internal names.
+ * Depending on the network topology, a given server may be referenced by a
+ * different name to systems outside its local network than the name used
+ * internal to the local network.
+ *
+ * <p/>
+ *
+ * An EC2 instance, as an example, has two different host names. The external
+ * name (e.g. ec2-72-44-40-78.compute-1.amazonaws.com) and an internal one
+ * (domU-12-31-39-06-BE-25.compute-1.internal).
+ *
+ * <p/>
+ *
+ * For systems which have only one name, both the external and internal host
+ * names should be the same. That is, they should be identical and neither
+ * should be set to null.
+ *
+ * @author Kirk True
+ */
+
+public class HostNamePair {
+
+ private final String externalHostName;
+
+ private final String internalHostName;
+
+ public HostNamePair(String externalHostName, String internalHostName) {
+ if(externalHostName == null)
+ throw new IllegalArgumentException("externalHostName must be non-null");
+
+ if(internalHostName == null)
+ throw new IllegalArgumentException("externalHostName must be non-null");
+
+ this.externalHostName = externalHostName;
+ this.internalHostName = internalHostName;
+ }
+
+ public String getExternalHostName() {
+ return externalHostName;
+ }
+
+ public String getInternalHostName() {
+ return internalHostName;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + externalHostName.hashCode();
+ result = prime * result + internalHostName.hashCode();
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if(this == other)
+ return true;
+
+ if(!(other instanceof HostNamePair))
+ return false;
+
+ HostNamePair hnp = (HostNamePair) other;
+
+ return hnp.externalHostName.equals(externalHostName)
+ && hnp.internalHostName.equals(internalHostName);
+ }
+
+}
View
44 contrib/ec2-testing/src/java/voldemort/utils/RemoteOperation.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2009 LinkedIn, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package voldemort.utils;
+
+/**
+ * A RemoteOperation represents an operation that can be performed on a remote
+ * system. For example, it may represent the ability to remotely start/stop
+ * server and test nodes, deploy files, and so forth.
+ *
+ * <p/>
+ *
+ * Should the operation span multiple remote hosts, implementations should
+ * perform the operation in parallel with respect to the remote hosts.
+ *
+ * @author Kirk True
+ */
+
+public interface RemoteOperation {
+
+ /**
+ * Executes the specific remote operation which can span multiple remote
+ * hosts.
+ *
+ * @throws RemoteOperationException Thrown if an error occurred performing
+ * the remote operation
+ */
+
+ public void execute() throws RemoteOperationException;
+
+}
View
37 contrib/ec2-testing/src/java/voldemort/utils/RemoteOperationException.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2009 LinkedIn, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package voldemort.utils;
+
+public class RemoteOperationException extends Exception {
+
+ private static final long serialVersionUID = 1L;
+
+ public RemoteOperationException() {}
+
+ public RemoteOperationException(String message) {
+ super(message);
+ }
+
+ public RemoteOperationException(Throwable cause) {
+ super(cause);
+ }
+
+ public RemoteOperationException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+}
View
62 contrib/ec2-testing/src/java/voldemort/utils/RemoteTest.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright 2009 LinkedIn, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package voldemort.utils;
+
+/**
+ * RemoteTest represents the operation of invoking the voldemort-remote-test.sh
+ * script on remote machines in parallel.
+ *
+ * <p/>
+ *
+ * The remote tests are executed in parallel against the set of server nodes
+ * returned by the bootstrap Voldemort server's meta data.
+ *
+ * <p/>
+ *
+ * A list of RemoteTestResult instances is returned, one for each client. The
+ * result includes the values for the reads/second, writes/second, and
+ * deletes/second for each iteration. These raw numbers can then be manipulated
+ * as seen fit for the desired measurement.
+ *
+ * <p/>
+ *
+ * Implementation notes:
+ *
+ * <ol>
+ * <li>Implementations must provide a reasonable guarantee that the clients were
+ * actually started. An error should be raised, therefore, if the server could
+ * not be started, regardless of cause.
+ * <li>It is assumed that the remote host is properly set up to start the
+ * Voldemort test. This means that:
+ * <ul>
+ * <li>The Java environment is properly configured with $JAVA_HOME pointing at a
+ * valid JDK.
+ * <li>The Voldemort distribution is already present at a known location on the
+ * remote host.
+ * </ul>
+ * <li>The test run should occur in parallel against the remote hosts, if
+ * possible.
+ * </ol>
+ *
+ * @author Kirk True
+ *
+ * @see RemoteTestResult
+ */
+
+public interface RemoteTest extends RemoteOperation {
+
+}
View
206 contrib/ec2-testing/src/java/voldemort/utils/app/VoldemortApp.java
@@ -0,0 +1,206 @@
+/*
+ * Copyright 2009 LinkedIn, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package voldemort.utils.app;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import joptsimple.OptionException;
+import joptsimple.OptionParser;
+import joptsimple.OptionSet;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+import voldemort.utils.CmdUtils;
+import voldemort.utils.HostNamePair;
+
+public abstract class VoldemortApp {
+
+ protected final OptionParser parser = new OptionParser();
+
+ protected abstract String getScriptName();
+
+ protected abstract void run(String[] args) throws Exception;
+
+ protected void printUsage() {
+ System.err.println("Usage: $VOLDEMORT_HOME/contrib/ec2-testing/bin/" + getScriptName());
+
+ try {
+ parser.printHelpOn(System.err);
+ } catch(IOException e) {
+ e.printStackTrace();
+ }
+
+ System.exit(1);
+ }
+
+ protected OptionSet parse(String[] args) {
+ try {
+ OptionSet options = parser.parse(args);
+
+ if(options.has("help"))
+ printUsage();
+
+ setLogging(options);
+ return options;
+ } catch(OptionException e) {
+ System.err.println(e.getMessage());
+ printUsage();
+ return null;
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ protected void setLogging(OptionSet options) {
+ // "Options are \"debug\", \"info\" (default), \"warn\", \"error\", or \"off\"")
+ String levelString = CmdUtils.valueOf(options, "logging", "info");
+
+ Level level = null;
+
+ if(levelString.equals("debug"))
+ level = Level.DEBUG;
+ else if(levelString.equals("info"))
+ level = Level.INFO;
+ else if(levelString.equals("warn"))
+ level = Level.WARN;
+ else if(levelString.equals("error"))
+ level = Level.ERROR;
+ else if(levelString.equals("off"))
+ level = Level.OFF;
+ else
+ printUsage();
+
+ Logger rootLogger = Logger.getRootLogger();
+ rootLogger.setLevel(level);
+
+ Enumeration<Logger> e = rootLogger.getLoggerRepository().getCurrentLoggers();
+
+ while(e.hasMoreElements()) {
+ Logger logger = e.nextElement();
+ logger.setLevel(level);
+ }
+ }
+
+ protected String getRequiredString(OptionSet options, String argumentName) {
+ if(!options.has(argumentName)) {
+ System.err.println("Missing required argument " + argumentName);
+ printUsage();
+ }
+
+ return CmdUtils.valueOf(options, argumentName, "");
+ }
+
+ protected int getRequiredInt(OptionSet options, String argumentName) {
+ if(!options.has(argumentName)) {
+ System.err.println("Missing required argument " + argumentName);
+ printUsage();
+ }
+
+ return CmdUtils.valueOf(options, argumentName, 0);
+ }
+
+ protected long getRequiredLong(OptionSet options, String argumentName) {
+ if(!options.has(argumentName)) {
+ System.err.println("Missing required argument " + argumentName);
+ printUsage();
+ }
+
+ return Long.parseLong(CmdUtils.valueOf(options, argumentName, "0"));
+ }
+
+ protected File getRequiredInputFile(OptionSet options, String argumentName) {
+ String fileName = getRequiredString(options, argumentName);
+ File file = new File(fileName);
+
+ if(!file.canRead()) {
+ System.err.println("File " + fileName + " cannot be read");
+ System.exit(2);
+ }
+
+ return file;
+ }
+
+ protected File getInputFile(OptionSet options, String argumentName) {
+ if(!options.has(argumentName))
+ return null;
+
+ String fileName = CmdUtils.valueOf(options, argumentName, "");
+ File file = new File(fileName);
+
+ if(!file.canRead()) {
+ System.err.println("File " + fileName + " cannot be read");
+ System.exit(2);
+ }
+
+ return file;
+ }
+
+ protected List<HostNamePair> getHostNamesPairsFromFile(File file) {
+ Map<String, String> properties = getRequiredPropertiesFile(file);
+ List<HostNamePair> hostNamePairs = new ArrayList<HostNamePair>();
+
+ for(Map.Entry<String, String> entry: properties.entrySet()) {
+ String externalHostName = entry.getKey();
+ String internalHostName = entry.getValue() != null ? entry.getValue()
+ : externalHostName;
+ hostNamePairs.add(new HostNamePair(externalHostName, internalHostName));
+ }
+
+ return hostNamePairs;
+ }
+
+ protected Map<String, String> getRequiredPropertiesFile(File file) {
+ if(!file.canRead()) {
+ System.err.println("File " + file.getAbsolutePath() + " cannot be read");
+ System.exit(2);
+ }
+
+ Properties properties = new Properties();
+ InputStream is = null;
+
+ try {
+ is = new FileInputStream(file);
+ properties.load(is);
+ } catch(IOException e) {
+ throw new RuntimeException(e);
+ } finally {
+ IOUtils.closeQuietly(is);
+ }
+
+ Map<String, String> map = new HashMap<String, String>();
+
+ for(Map.Entry<Object, Object> entry: properties.entrySet()) {
+ String key = entry.getKey() != null ? entry.getKey().toString() : null;
+ String value = entry.getValue() != null ? entry.getValue().toString() : null;
+
+ map.put(key, value);
+ }
+
+ return map;
+ }
+
+}
View
71 contrib/ec2-testing/src/java/voldemort/utils/app/VoldemortClusterCleanerApp.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright 2009 LinkedIn, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package voldemort.utils.app;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+
+import joptsimple.OptionSet;
+import voldemort.utils.CmdUtils;
+import voldemort.utils.HostNamePair;
+import voldemort.utils.RemoteOperation;
+import voldemort.utils.impl.SshClusterCleaner;
+
+public class VoldemortClusterCleanerApp extends VoldemortApp {
+
+ public static void main(String[] args) throws Exception {
+ new VoldemortClusterCleanerApp().run(args);
+ }
+
+ @Override
+ protected String getScriptName() {
+ return "voldemort-clustercleaner.sh";
+ }
+
+ @Override
+ public void run(String[] args) throws Exception {
+ parser.accepts("help", "Prints this help");
+ parser.accepts("logging",
+ "Options are \"debug\", \"info\" (default), \"warn\", \"error\", or \"off\"")
+ .withRequiredArg();
+ parser.accepts("hostnames", "File containing host names").withRequiredArg();
+ parser.accepts("sshprivatekey", "File containing SSH private key").withRequiredArg();
+ parser.accepts("hostuserid", "User ID on remote host").withRequiredArg();
+ parser.accepts("voldemorthome", "Voldemort's home directory on remote host")
+ .withRequiredArg();
+
+ OptionSet options = parse(args);
+ File hostNamesFile = getRequiredInputFile(options, "hostnames");
+ File sshPrivateKey = getRequiredInputFile(options, "sshprivatekey");
+ String hostUserId = CmdUtils.valueOf(options, "hostuserid", "root");
+ String voldemortHomeDirectory = getRequiredString(options, "voldemorthome");
+
+ List<HostNamePair> hostNamePairs = getHostNamesPairsFromFile(hostNamesFile);
+ List<String> hostNames = new ArrayList<String>();
+
+ for(HostNamePair hostNamePair: hostNamePairs)
+ hostNames.add(hostNamePair.getExternalHostName());
+
+ RemoteOperation operation = new SshClusterCleaner(hostNames,
+ sshPrivateKey,
+ hostUserId,
+ voldemortHomeDirectory);
+ operation.execute();
+ }
+
+}
View
69 contrib/ec2-testing/src/java/voldemort/utils/app/VoldemortClusterGeneratorApp.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2009 LinkedIn, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package voldemort.utils.app;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+
+import joptsimple.OptionSet;
+import voldemort.utils.ClusterGenerator;
+import voldemort.utils.CmdUtils;
+import voldemort.utils.HostNamePair;
+
+public class VoldemortClusterGeneratorApp extends VoldemortApp {
+
+ public static void main(String[] args) throws Exception {
+ new VoldemortClusterGeneratorApp().run(args);
+ }
+
+ @Override
+ protected String getScriptName() {
+ return "voldemort-clustergenerator.sh";
+ }
+
+ @Override
+ public void run(String[] args) throws Exception {
+ parser.accepts("help", "Prints this help");
+ parser.accepts("logging",
+ "Options are \"debug\", \"info\" (default), \"warn\", \"error\", or \"off\"")
+ .withRequiredArg();
+ parser.accepts("hostnames", "File containing host names").withRequiredArg();
+ parser.accepts("partitions", "Number of partitions per cluster node")
+ .withRequiredArg()
+ .ofType(Integer.class);
+ parser.accepts("clustername", "Cluster name; defaults to mycluster").withRequiredArg();
+
+ OptionSet options = parse(args);
+ File hostNamesFile = getRequiredInputFile(options, "hostnames");
+ int partitions = getRequiredInt(options, "partitions");
+ String clusterName = CmdUtils.valueOf(options, "clustername", "mycluster");
+
+ List<HostNamePair> hostNamePairs = getHostNamesPairsFromFile(hostNamesFile);
+ List<String> hostNames = new ArrayList<String>();
+
+ for(HostNamePair hostNamePair: hostNamePairs)
+ hostNames.add(hostNamePair.getInternalHostName());
+
+ String clusterXml = new ClusterGenerator().createClusterDescriptor(clusterName,
+ hostNames,
+ partitions);
+
+ System.out.print(clusterXml);
+ }
+
+}
View
72 contrib/ec2-testing/src/java/voldemort/utils/app/VoldemortClusterRemoteTestApp.java
@@ -0,0 +1,72 @@
+/*
+ * Copyright 2009 LinkedIn, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package voldemort.utils.app;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import joptsimple.OptionSet;
+import voldemort.utils.CmdUtils;
+import voldemort.utils.HostNamePair;
+import voldemort.utils.RemoteOperation;
+import voldemort.utils.impl.SshRemoteTest;
+
+public class VoldemortClusterRemoteTestApp extends VoldemortApp {
+
+ public static void main(String[] args) throws Exception {
+ new VoldemortClusterRemoteTestApp().run(args);
+ }
+
+ @Override
+ protected String getScriptName() {
+ return "voldemort-clusterremotetest.sh";
+ }
+
+ @Override
+ public void run(String[] args) throws Exception {
+ parser.accepts("help", "Prints this help");
+ parser.accepts("logging",
+ "Options are \"debug\", \"info\" (default), \"warn\", \"error\", or \"off\"")
+ .withRequiredArg();
+ parser.accepts("hostnames", "File containing host names").withRequiredArg();
+ parser.accepts("sshprivatekey", "File containing SSH private key").withRequiredArg();
+ parser.accepts("hostuserid", "User ID on remote host").withRequiredArg();
+ parser.accepts("commands", "File containing per-host commands").withRequiredArg();
+
+ OptionSet options = parse(args);
+ List<HostNamePair> hostNamePairs = getHostNamesPairsFromFile(getRequiredInputFile(options,
+ "hostnames"));
+ List<String> hostNames = new ArrayList<String>();
+
+ for(HostNamePair hostNamePair: hostNamePairs)
+ hostNames.add(hostNamePair.getExternalHostName());
+
+ File sshPrivateKey = getRequiredInputFile(options, "sshprivatekey");
+ String hostUserId = CmdUtils.valueOf(options, "hostuserid", "root");
+ Map<String, String> commands = getRequiredPropertiesFile(getRequiredInputFile(options,
+ "commands"));
+
+ RemoteOperation operation = new SshRemoteTest(hostNames,
+ sshPrivateKey,
+ hostUserId,
+ commands);
+ operation.execute();
+ }
+
+}
View
169 contrib/ec2-testing/src/java/voldemort/utils/app/VoldemortClusterStarterApp.java
@@ -0,0 +1,169 @@
+/*
+ * Copyright 2009 LinkedIn, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package voldemort.utils.app;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+
+import joptsimple.OptionSet;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+import voldemort.utils.CmdUtils;
+import voldemort.utils.HostNamePair;
+import voldemort.utils.RemoteOperation;
+import voldemort.utils.RemoteOperationException;
+import voldemort.utils.impl.SshClusterStarter;
+import voldemort.utils.impl.SshClusterStopper;
+
+public class VoldemortClusterStarterApp extends VoldemortApp {
+
+ public static void main(String[] args) throws Exception {
+ new VoldemortClusterStarterApp().run(args);
+ }
+
+ @Override
+ protected String getScriptName() {
+ return "voldemort-clusterstarter.sh";
+ }
+
+ @Override
+ public void run(String[] args) throws Exception {
+ parser.accepts("help", "Prints this help");
+ parser.accepts("logging",
+ "Options are \"debug\", \"info\" (default), \"warn\", \"error\", or \"off\"")
+ .withRequiredArg();
+ parser.accepts("hostnames", "File containing host names").withRequiredArg();
+ parser.accepts("sshprivatekey", "File containing SSH private key").withRequiredArg();
+ parser.accepts("hostuserid", "User ID on remote host").withRequiredArg();
+ parser.accepts("voldemortroot", "Voldemort's root directory on remote host")
+ .withRequiredArg();
+ parser.accepts("voldemorthome", "Voldemort's home directory on remote host")
+ .withRequiredArg();
+ parser.accepts("clusterxml",
+ "Voldemort's cluster.xml file on the local file system; used to determine host names")
+ .withRequiredArg();
+
+ OptionSet options = parse(args);
+ File hostNamesFile = getRequiredInputFile(options, "hostnames");
+
+ List<HostNamePair> hostNamePairs = getHostNamesPairsFromFile(hostNamesFile);
+ final List<String> externalHostNames = new ArrayList<String>();
+ final List<String> internalHostNames = new ArrayList<String>();
+
+ for(HostNamePair hostNamePair: hostNamePairs) {
+ externalHostNames.add(hostNamePair.getExternalHostName());
+ internalHostNames.add(hostNamePair.getInternalHostName());
+ }
+
+ final File sshPrivateKey = getRequiredInputFile(options, "sshprivatekey");
+ final String hostUserId = CmdUtils.valueOf(options, "hostuserid", "root");
+ String voldemortHomeDirectory = getRequiredString(options, "voldemorthome");
+ final String voldemortRootDirectory = getRequiredString(options, "voldemortroot");
+
+