Permalink
Browse files

Clean up:

1. Removed log4j.properties that was conflicting with the "main" configuration.
2. Introduced Config pattern so that actual unit tests can be much slimmer.
3. Allowing for source directory to be different than "voldemort"
  • Loading branch information...
1 parent 233e2d0 commit 40c36088f67dbe7699d2ff3700d433aa46d9441e @kirktrue kirktrue committed Dec 13, 2009
@@ -44,4 +44,4 @@ if [ -z $VOLD_OPTS ]; then
fi
export CLASSPATH
-java $VOLD_OPTS -cp $CLASSPATH $@
+java $VOLD_OPTS -Dlog4j.configuration=log4j-stderr.properties -cp $CLASSPATH $@
@@ -63,7 +63,7 @@ if [ $exitCode -ne 0 ]
fi
# We can't easily check the exit code as we run this in the background...
-./contrib/ec2-testing/bin/voldemort-clusterstarter.sh --hostnames $hostsServers $sshPrivateKey --voldemortroot "server/voldemort" --voldemorthome "testconfig" --clusterxml $configDir/config/cluster.xml --logging debug 2>> $LOG_SERVERS_FILE &
+./contrib/ec2-testing/bin/voldemort-clusterstarter.sh --hostnames $hostsServers $sshPrivateKey --voldemortroot "server/$(basename `pwd`)" --voldemorthome "testconfig" --clusterxml $configDir/config/cluster.xml --logging debug 2>> $LOG_SERVERS_FILE &
sleep 10
@@ -85,7 +85,7 @@ do
startKeyIndex=$(($numRequests * $counter))
counter=$(($counter + 1))
externalHost=`echo $line | cut -d'=' -f1`
- echo "$externalHost=cd client/voldemort ; sleep $rampSeconds ; ./bin/voldemort-remote-test.sh -r -w -d --iterations $iterations --start-key-index $startKeyIndex tcp://${bootstrapHost}:6666 test $numRequests" >> $COMMANDS_FILE
+ echo "$externalHost=cd "client/$(basename `pwd`)" ; sleep $rampSeconds ; ./bin/voldemort-remote-test.sh -r -w -d --iterations $iterations --start-key-index $startKeyIndex tcp://${bootstrapHost}:6666 test $numRequests" >> $COMMANDS_FILE
done
./contrib/ec2-testing/bin/voldemort-clusterremotetest.sh --hostnames $hostsClients $sshPrivateKey --commands $COMMANDS_FILE --logging debug 2>> $LOG_SERVERS_FILE > $LOG_CLIENTS_FILE
@@ -6,4 +6,12 @@ log4j.rootLogger=INFO, stderr
log4j.appender.stderr=org.apache.log4j.ConsoleAppender
log4j.appender.stderr.target=System.err
log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
-log4j.appender.stderr.layout.ConversionPattern=%m%n
+log4j.appender.stderr.layout.ConversionPattern=[%d] %p %m (%c)%n
+
+# Turn on all our debugging info
+#log4j.logger=DEBUG
+#log4j.logger.httpclient.wire=DEBUG
+log4j.logger.org.mortbay.log=WARN
+log4j.logger.voldemort.store.routed=INFO
+log4j.logger.voldemort.server.niosocket=INFO
+log4j.logger.voldemort.utils=DEBUG
@@ -0,0 +1,240 @@
+/*
+ * Copyright 2009 LinkedIn, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package voldemort.utils;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
+
+import voldemort.VoldemortException;
+
+/**
+ * Testing on EC2 requires quite a few configuration properties; these are
+ * provided in a *.properties file, the path of which is provided in the
+ * "ec2PropertiesFile" System property. Below is a table of the properties:
+ *
+ * <table>
+ * <th>Name</th>
+ * <th>Description</th>
+ * <tr>
+ * <td>ec2AccessId</td>
+ * <td>EC2 access ID, provided by Amazon</td>
+ * </tr>
+ * <tr>
+ * <td>ec2SecretKey</td>
+ * <td>EC2 secret key, provided by Amazon</td>
+ * </tr>
+ * <tr>
+ * <td>ec2Ami</td>
+ * <td>ID of the EC2 AMI used for the instances that are started</td>
+ * </tr>
+ * <tr>
+ * <td>ec2KeyPairId</td>
+ * <td>Key pair ID</td>
+ * </tr>
+ * <tr>
+ * <td>ec2SshPrivateKeyPath</td>
+ * <td>SSH private key path to key used to connect to instances (optional)</td>
+ * </tr>
+ * <tr>
+ * <td>ec2HostUserId</td>
+ * <td>User ID on the hosts; for EC2 this is usually "root"</td>
+ * </tr>
+ * <tr>
+ * <td>ec2VoldemortRootDirectory</td>
+ * <td>Root directory on remote instances that points to the the Voldemort
+ * "distribution" directory; relative to the host user ID's home directory. For
+ * example, if the remote user's home directory is /root and the Voldemort
+ * distribution directory is /root/voldemort, ec2VoldemortRootDirectory would be
+ * "voldemort"</td>
+ * </tr>
+ * <tr>
+ * <td>ec2VoldemortHomeDirectory</td>
+ * <td>Home directory on remote instances that points to the configuration
+ * directory, relative to the host user ID's home directory. For example, if the
+ * remote user's home directory is /root and the Voldemort configuration
+ * directory is /root/voldemort/config/single_node_cluster,
+ * ec2VoldemortHomeDirectory would be "voldemort/config/single_node_cluster"</td>
+ * </tr>
+ * <tr>
+ * <td>ec2SourceDirectory</td>
+ * <td>Source directory on <b>local</b> machine from which to copy the Voldemort
+ * "distribution" to the remote hosts; e.g. "/home/kirk/voldemortdev/voldemort"</td>
+ * </tr>
+ * <tr>
+ * <td>ec2ParentDirectory</td>
+ * <td>Parent directory on the <b>remote</b> machine into which to copy the
+ * Voldemort "distribution". For example, if the remote user's home directory is
+ * /root and the Voldemort distribution directory is /root/voldemort,
+ * ec2ParentDirectory would be "." or "/root".</td>
+ * </tr>
+ * <tr>
+ * <td>ec2ClusterXmlFile</td>
+ * <td><b>Local</b> path to which cluster.xml will be written with EC2 hosts;
+ * this needs to live under the ec2SourceDirectory's configuration directory
+ * that is copied to the remote host.</td>
+ * </tr>
+ * <tr>
+ * <td>ec2InstanceCount</td>
+ * <td>The number of instances to create.</td>
+ * </tr>
+ * </table>
+ *
+ * @author Kirk True
+ */
+
+public class Ec2Config extends RemoteTestConfig {
+
+ protected String accessId;
+
+ protected String secretKey;
+
+ protected String ami;
+
+ protected String keyPairId;
+
+ protected int instanceCount;
+
+ public Ec2Config() {
+ Properties properties = getEc2Properties();
+ init(properties);
+ }
+
+ public String getAccessId() {
+ return accessId;
+ }
+
+ public void setAccessId(String accessId) {
+ this.accessId = accessId;
+ }
+
+ public String getSecretKey() {
+ return secretKey;
+ }
+
+ public void setSecretKey(String secretKey) {
+ this.secretKey = secretKey;
+ }
+
+ public String getAmi() {
+ return ami;
+ }
+
+ public void setAmi(String ami) {
+ this.ami = ami;
+ }
+
+ public String getKeyPairId() {
+ return keyPairId;
+ }
+
+ public void setKeyPairId(String keyPairId) {
+ this.keyPairId = keyPairId;
+ }
+
+ public int getInstanceCount() {
+ return instanceCount;
+ }
+
+ public void setInstanceCount(int instanceCount) {
+ this.instanceCount = instanceCount;
+ }
+
+ protected void init(Properties properties) {
+ setAccessId(properties.getProperty("ec2AccessId"));
+ setSecretKey(properties.getProperty("ec2SecretKey"));
+ setAmi(properties.getProperty("ec2Ami"));
+ setKeyPairId(properties.getProperty("ec2KeyPairId"));
+ setHostUserId(properties.getProperty("ec2HostUserId"));
+ setSshPrivateKey(getFileProperty(properties, "ec2SshPrivateKeyPath"));
+ setVoldemortRootDirectory(properties.getProperty("ec2VoldemortRootDirectory"));
+ setVoldemortHomeDirectory(properties.getProperty("ec2VoldemortHomeDirectory"));
+ setSourceDirectory(getFileProperty(properties, "ec2SourceDirectory"));
+ setParentDirectory(properties.getProperty("ec2ParentDirectory"));
+ setClusterXmlFile(getFileProperty(properties, "ec2ClusterXmlFile"));
+ setInstanceCount(getIntProperty(properties, "ec2InstanceCount"));
+ }
+
+ protected File getFileProperty(Properties properties, String propertyName) {
+ String value = properties.getProperty(propertyName);
+ return value != null ? new File(value) : null;
+ }
+
+ protected int getIntProperty(Properties properties, String propertyName) {
+ String value = properties.getProperty(propertyName);
+ return value != null ? Integer.parseInt(value) : 0;
+ }
+
+ protected Properties getEc2Properties() {
+ String propertiesFileName = System.getProperty("ec2PropertiesFile");
+
+ List<String> requireds = getRequiredPropertyNames();
+
+ if(propertiesFileName == null)
+ throw new VoldemortException("ec2PropertiesFile system property must be defined that "
+ + "provides the path to file containing the following "
+ + "required properties: "
+ + StringUtils.join(requireds, ", "));
+
+ Properties properties = new Properties();
+ InputStream in = null;
+
+ try {
+ in = new FileInputStream(propertiesFileName);
+ properties.load(in);
+ } catch(IOException e) {
+ throw new VoldemortException(e);
+ } finally {
+ IOUtils.closeQuietly(in);
+ }
+
+ for(String required: requireds) {
+ // Allow system properties to override
+ if(System.getProperties().containsKey(required))
+ properties.put(required, System.getProperty(required));
+
+ if(!properties.containsKey(required))
+ throw new VoldemortException("Required properties: "
+ + StringUtils.join(requireds, ", ") + "; missing "
+ + required);
+ }
+
+ return properties;
+ }
+
+ protected List<String> getRequiredPropertyNames() {
+ return Arrays.asList("ec2AccessId",
+ "ec2SecretKey",
+ "ec2Ami",
+ "ec2KeyPairId",
+ "ec2HostUserId",
+ "ec2VoldemortRootDirectory",
+ "ec2VoldemortHomeDirectory",
+ "ec2SourceDirectory",
+ "ec2ParentDirectory",
+ "ec2ClusterXmlFile",
+ "ec2InstanceCount");
+ }
+
+}
@@ -28,25 +28,25 @@
public class Ec2InstanceRemoteTestUtils {
- public static List<HostNamePair> createInstances(String ec2AccessId,
- String ec2SecretKey,
- String ec2Ami,
- String ec2KeyPairId,
- int count) throws Exception {
- Ec2Connection ec2 = new TypicaEc2Connection(ec2AccessId, ec2SecretKey);
- return ec2.create(ec2Ami, ec2KeyPairId, Ec2Connection.Ec2InstanceType.DEFAULT, count);
+ public static List<HostNamePair> createInstances(Ec2Config ec2Config) throws Exception {
+ Ec2Connection ec2 = new TypicaEc2Connection(ec2Config.getAccessId(),
+ ec2Config.getSecretKey());
+ return ec2.create(ec2Config.getAmi(),
+ ec2Config.getKeyPairId(),
+ Ec2Connection.Ec2InstanceType.DEFAULT,
+ ec2Config.getInstanceCount());
}
- public static void destroyInstances(String ec2AccessId,
- String ec2SecretKey,
- List<String> hostNames) throws Exception {
- Ec2Connection ec2 = new TypicaEc2Connection(ec2AccessId, ec2SecretKey);
+ public static void destroyInstances(List<String> hostNames, Ec2Config ec2Config)
+ throws Exception {
+ Ec2Connection ec2 = new TypicaEc2Connection(ec2Config.getAccessId(),
+ ec2Config.getSecretKey());
ec2.delete(hostNames);
}
- public static List<HostNamePair> listInstances(String ec2AccessId, String ec2SecretKey)
- throws Exception {
- Ec2Connection ec2 = new TypicaEc2Connection(ec2AccessId, ec2SecretKey);
+ public static List<HostNamePair> listInstances(Ec2Config ec2Config) throws Exception {
+ Ec2Connection ec2 = new TypicaEc2Connection(ec2Config.getAccessId(),
+ ec2Config.getSecretKey());
return ec2.list();
}
Oops, something went wrong.

0 comments on commit 40c3608

Please sign in to comment.