Skip to content
This repository
Browse code

Clean up:

1. Removed log4j.properties that was conflicting with the "main" configuration.
2. Introduced Config pattern so that actual unit tests can be much slimmer.
3. Allowing for source directory to be different than "voldemort"
  • Loading branch information...
commit 40c36088f67dbe7699d2ff3700d433aa46d9441e 1 parent 233e2d0
Kirk True authored December 12, 2009
2  contrib/ec2-testing/bin/run-class.sh
@@ -44,4 +44,4 @@ if [ -z $VOLD_OPTS ]; then
44 44
 fi
45 45
 
46 46
 export CLASSPATH
47  
-java $VOLD_OPTS -cp $CLASSPATH $@
  47
+java $VOLD_OPTS -Dlog4j.configuration=log4j-stderr.properties -cp $CLASSPATH $@
4  contrib/ec2-testing/examples/remotetest/remotetest.sh
@@ -63,7 +63,7 @@ if [ $exitCode -ne 0 ]
63 63
 fi
64 64
 
65 65
 # We can't easily check the exit code as we run this in the background...
66  
-./contrib/ec2-testing/bin/voldemort-clusterstarter.sh --hostnames $hostsServers $sshPrivateKey --voldemortroot "server/voldemort" --voldemorthome "testconfig" --clusterxml $configDir/config/cluster.xml --logging debug 2>> $LOG_SERVERS_FILE &
  66
+./contrib/ec2-testing/bin/voldemort-clusterstarter.sh --hostnames $hostsServers $sshPrivateKey --voldemortroot "server/$(basename `pwd`)" --voldemorthome "testconfig" --clusterxml $configDir/config/cluster.xml --logging debug 2>> $LOG_SERVERS_FILE &
67 67
 
68 68
 sleep 10
69 69
 
@@ -85,7 +85,7 @@ do
85 85
 	startKeyIndex=$(($numRequests * $counter))
86 86
 	counter=$(($counter + 1))
87 87
 	externalHost=`echo $line | cut -d'=' -f1`
88  
-	echo "$externalHost=cd client/voldemort ; sleep $rampSeconds ; ./bin/voldemort-remote-test.sh -r -w -d --iterations $iterations --start-key-index $startKeyIndex tcp://${bootstrapHost}:6666 test $numRequests" >> $COMMANDS_FILE
  88
+	echo "$externalHost=cd "client/$(basename `pwd`)" ; sleep $rampSeconds ; ./bin/voldemort-remote-test.sh -r -w -d --iterations $iterations --start-key-index $startKeyIndex tcp://${bootstrapHost}:6666 test $numRequests" >> $COMMANDS_FILE
89 89
 done
90 90
 
91 91
 ./contrib/ec2-testing/bin/voldemort-clusterremotetest.sh --hostnames $hostsClients $sshPrivateKey --commands $COMMANDS_FILE --logging debug 2>> $LOG_SERVERS_FILE > $LOG_CLIENTS_FILE
10  contrib/ec2-testing/resources/log4j.properties → ...rib/ec2-testing/resources/log4j-stderr.properties
@@ -6,4 +6,12 @@ log4j.rootLogger=INFO, stderr
6 6
 log4j.appender.stderr=org.apache.log4j.ConsoleAppender
7 7
 log4j.appender.stderr.target=System.err
8 8
 log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
9  
-log4j.appender.stderr.layout.ConversionPattern=%m%n
  9
+log4j.appender.stderr.layout.ConversionPattern=[%d] %p %m (%c)%n
  10
+
  11
+# Turn on all our debugging info
  12
+#log4j.logger=DEBUG
  13
+#log4j.logger.httpclient.wire=DEBUG
  14
+log4j.logger.org.mortbay.log=WARN
  15
+log4j.logger.voldemort.store.routed=INFO
  16
+log4j.logger.voldemort.server.niosocket=INFO
  17
+log4j.logger.voldemort.utils=DEBUG
240  contrib/ec2-testing/test/voldemort/utils/Ec2Config.java
... ...
@@ -0,0 +1,240 @@
  1
+/*
  2
+ * Copyright 2009 LinkedIn, Inc.
  3
+ * 
  4
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
  5
+ * use this file except in compliance with the License. You may obtain a copy of
  6
+ * the License at
  7
+ * 
  8
+ * http://www.apache.org/licenses/LICENSE-2.0
  9
+ * 
  10
+ * Unless required by applicable law or agreed to in writing, software
  11
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  12
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  13
+ * License for the specific language governing permissions and limitations under
  14
+ * the License.
  15
+ */
  16
+
  17
+package voldemort.utils;
  18
+
  19
+import java.io.File;
  20
+import java.io.FileInputStream;
  21
+import java.io.IOException;
  22
+import java.io.InputStream;
  23
+import java.util.Arrays;
  24
+import java.util.List;
  25
+import java.util.Properties;
  26
+
  27
+import org.apache.commons.io.IOUtils;
  28
+import org.apache.commons.lang.StringUtils;
  29
+
  30
+import voldemort.VoldemortException;
  31
+
  32
+/**
  33
+ * Testing on EC2 requires quite a few configuration properties; these are
  34
+ * provided in a *.properties file, the path of which is provided in the
  35
+ * "ec2PropertiesFile" System property. Below is a table of the properties:
  36
+ * 
  37
+ * <table>
  38
+ * <th>Name</th>
  39
+ * <th>Description</th>
  40
+ * <tr>
  41
+ * <td>ec2AccessId</td>
  42
+ * <td>EC2 access ID, provided by Amazon</td>
  43
+ * </tr>
  44
+ * <tr>
  45
+ * <td>ec2SecretKey</td>
  46
+ * <td>EC2 secret key, provided by Amazon</td>
  47
+ * </tr>
  48
+ * <tr>
  49
+ * <td>ec2Ami</td>
  50
+ * <td>ID of the EC2 AMI used for the instances that are started</td>
  51
+ * </tr>
  52
+ * <tr>
  53
+ * <td>ec2KeyPairId</td>
  54
+ * <td>Key pair ID</td>
  55
+ * </tr>
  56
+ * <tr>
  57
+ * <td>ec2SshPrivateKeyPath</td>
  58
+ * <td>SSH private key path to key used to connect to instances (optional)</td>
  59
+ * </tr>
  60
+ * <tr>
  61
+ * <td>ec2HostUserId</td>
  62
+ * <td>User ID on the hosts; for EC2 this is usually "root"</td>
  63
+ * </tr>
  64
+ * <tr>
  65
+ * <td>ec2VoldemortRootDirectory</td>
  66
+ * <td>Root directory on remote instances that points to the the Voldemort
  67
+ * "distribution" directory; relative to the host user ID's home directory. For
  68
+ * example, if the remote user's home directory is /root and the Voldemort
  69
+ * distribution directory is /root/voldemort, ec2VoldemortRootDirectory would be
  70
+ * "voldemort"</td>
  71
+ * </tr>
  72
+ * <tr>
  73
+ * <td>ec2VoldemortHomeDirectory</td>
  74
+ * <td>Home directory on remote instances that points to the configuration
  75
+ * directory, relative to the host user ID's home directory. For example, if the
  76
+ * remote user's home directory is /root and the Voldemort configuration
  77
+ * directory is /root/voldemort/config/single_node_cluster,
  78
+ * ec2VoldemortHomeDirectory would be "voldemort/config/single_node_cluster"</td>
  79
+ * </tr>
  80
+ * <tr>
  81
+ * <td>ec2SourceDirectory</td>
  82
+ * <td>Source directory on <b>local</b> machine from which to copy the Voldemort
  83
+ * "distribution" to the remote hosts; e.g. "/home/kirk/voldemortdev/voldemort"</td>
  84
+ * </tr>
  85
+ * <tr>
  86
+ * <td>ec2ParentDirectory</td>
  87
+ * <td>Parent directory on the <b>remote</b> machine into which to copy the
  88
+ * Voldemort "distribution". For example, if the remote user's home directory is
  89
+ * /root and the Voldemort distribution directory is /root/voldemort,
  90
+ * ec2ParentDirectory would be "." or "/root".</td>
  91
+ * </tr>
  92
+ * <tr>
  93
+ * <td>ec2ClusterXmlFile</td>
  94
+ * <td><b>Local</b> path to which cluster.xml will be written with EC2 hosts;
  95
+ * this needs to live under the ec2SourceDirectory's configuration directory
  96
+ * that is copied to the remote host.</td>
  97
+ * </tr>
  98
+ * <tr>
  99
+ * <td>ec2InstanceCount</td>
  100
+ * <td>The number of instances to create.</td>
  101
+ * </tr>
  102
+ * </table>
  103
+ * 
  104
+ * @author Kirk True
  105
+ */
  106
+
  107
+public class Ec2Config extends RemoteTestConfig {
  108
+
  109
+    protected String accessId;
  110
+
  111
+    protected String secretKey;
  112
+
  113
+    protected String ami;
  114
+
  115
+    protected String keyPairId;
  116
+
  117
+    protected int instanceCount;
  118
+
  119
+    public Ec2Config() {
  120
+        Properties properties = getEc2Properties();
  121
+        init(properties);
  122
+    }
  123
+
  124
+    public String getAccessId() {
  125
+        return accessId;
  126
+    }
  127
+
  128
+    public void setAccessId(String accessId) {
  129
+        this.accessId = accessId;
  130
+    }
  131
+
  132
+    public String getSecretKey() {
  133
+        return secretKey;
  134
+    }
  135
+
  136
+    public void setSecretKey(String secretKey) {
  137
+        this.secretKey = secretKey;
  138
+    }
  139
+
  140
+    public String getAmi() {
  141
+        return ami;
  142
+    }
  143
+
  144
+    public void setAmi(String ami) {
  145
+        this.ami = ami;
  146
+    }
  147
+
  148
+    public String getKeyPairId() {
  149
+        return keyPairId;
  150
+    }
  151
+
  152
+    public void setKeyPairId(String keyPairId) {
  153
+        this.keyPairId = keyPairId;
  154
+    }
  155
+
  156
+    public int getInstanceCount() {
  157
+        return instanceCount;
  158
+    }
  159
+
  160
+    public void setInstanceCount(int instanceCount) {
  161
+        this.instanceCount = instanceCount;
  162
+    }
  163
+
  164
+    protected void init(Properties properties) {
  165
+        setAccessId(properties.getProperty("ec2AccessId"));
  166
+        setSecretKey(properties.getProperty("ec2SecretKey"));
  167
+        setAmi(properties.getProperty("ec2Ami"));
  168
+        setKeyPairId(properties.getProperty("ec2KeyPairId"));
  169
+        setHostUserId(properties.getProperty("ec2HostUserId"));
  170
+        setSshPrivateKey(getFileProperty(properties, "ec2SshPrivateKeyPath"));
  171
+        setVoldemortRootDirectory(properties.getProperty("ec2VoldemortRootDirectory"));
  172
+        setVoldemortHomeDirectory(properties.getProperty("ec2VoldemortHomeDirectory"));
  173
+        setSourceDirectory(getFileProperty(properties, "ec2SourceDirectory"));
  174
+        setParentDirectory(properties.getProperty("ec2ParentDirectory"));
  175
+        setClusterXmlFile(getFileProperty(properties, "ec2ClusterXmlFile"));
  176
+        setInstanceCount(getIntProperty(properties, "ec2InstanceCount"));
  177
+    }
  178
+
  179
+    protected File getFileProperty(Properties properties, String propertyName) {
  180
+        String value = properties.getProperty(propertyName);
  181
+        return value != null ? new File(value) : null;
  182
+    }
  183
+
  184
+    protected int getIntProperty(Properties properties, String propertyName) {
  185
+        String value = properties.getProperty(propertyName);
  186
+        return value != null ? Integer.parseInt(value) : 0;
  187
+    }
  188
+
  189
+    protected Properties getEc2Properties() {
  190
+        String propertiesFileName = System.getProperty("ec2PropertiesFile");
  191
+
  192
+        List<String> requireds = getRequiredPropertyNames();
  193
+
  194
+        if(propertiesFileName == null)
  195
+            throw new VoldemortException("ec2PropertiesFile system property must be defined that "
  196
+                                         + "provides the path to file containing the following "
  197
+                                         + "required properties: "
  198
+                                         + StringUtils.join(requireds, ", "));
  199
+
  200
+        Properties properties = new Properties();
  201
+        InputStream in = null;
  202
+
  203
+        try {
  204
+            in = new FileInputStream(propertiesFileName);
  205
+            properties.load(in);
  206
+        } catch(IOException e) {
  207
+            throw new VoldemortException(e);
  208
+        } finally {
  209
+            IOUtils.closeQuietly(in);
  210
+        }
  211
+
  212
+        for(String required: requireds) {
  213
+            // Allow system properties to override
  214
+            if(System.getProperties().containsKey(required))
  215
+                properties.put(required, System.getProperty(required));
  216
+
  217
+            if(!properties.containsKey(required))
  218
+                throw new VoldemortException("Required properties: "
  219
+                                             + StringUtils.join(requireds, ", ") + "; missing "
  220
+                                             + required);
  221
+        }
  222
+
  223
+        return properties;
  224
+    }
  225
+
  226
+    protected List<String> getRequiredPropertyNames() {
  227
+        return Arrays.asList("ec2AccessId",
  228
+                             "ec2SecretKey",
  229
+                             "ec2Ami",
  230
+                             "ec2KeyPairId",
  231
+                             "ec2HostUserId",
  232
+                             "ec2VoldemortRootDirectory",
  233
+                             "ec2VoldemortHomeDirectory",
  234
+                             "ec2SourceDirectory",
  235
+                             "ec2ParentDirectory",
  236
+                             "ec2ClusterXmlFile",
  237
+                             "ec2InstanceCount");
  238
+    }
  239
+
  240
+}
28  contrib/ec2-testing/test/voldemort/utils/Ec2InstanceRemoteTestUtils.java
@@ -28,25 +28,25 @@
28 28
 
29 29
 public class Ec2InstanceRemoteTestUtils {
30 30
 
31  
-    public static List<HostNamePair> createInstances(String ec2AccessId,
32  
-                                                     String ec2SecretKey,
33  
-                                                     String ec2Ami,
34  
-                                                     String ec2KeyPairId,
35  
-                                                     int count) throws Exception {
36  
-        Ec2Connection ec2 = new TypicaEc2Connection(ec2AccessId, ec2SecretKey);
37  
-        return ec2.create(ec2Ami, ec2KeyPairId, Ec2Connection.Ec2InstanceType.DEFAULT, count);
  31
+    public static List<HostNamePair> createInstances(Ec2Config ec2Config) throws Exception {
  32
+        Ec2Connection ec2 = new TypicaEc2Connection(ec2Config.getAccessId(),
  33
+                                                    ec2Config.getSecretKey());
  34
+        return ec2.create(ec2Config.getAmi(),
  35
+                          ec2Config.getKeyPairId(),
  36
+                          Ec2Connection.Ec2InstanceType.DEFAULT,
  37
+                          ec2Config.getInstanceCount());
38 38
     }
39 39
 
40  
-    public static void destroyInstances(String ec2AccessId,
41  
-                                        String ec2SecretKey,
42  
-                                        List<String> hostNames) throws Exception {
43  
-        Ec2Connection ec2 = new TypicaEc2Connection(ec2AccessId, ec2SecretKey);
  40
+    public static void destroyInstances(List<String> hostNames, Ec2Config ec2Config)
  41
+            throws Exception {
  42
+        Ec2Connection ec2 = new TypicaEc2Connection(ec2Config.getAccessId(),
  43
+                                                    ec2Config.getSecretKey());
44 44
         ec2.delete(hostNames);
45 45
     }
46 46
 
47  
-    public static List<HostNamePair> listInstances(String ec2AccessId, String ec2SecretKey)
48  
-            throws Exception {
49  
-        Ec2Connection ec2 = new TypicaEc2Connection(ec2AccessId, ec2SecretKey);
  47
+    public static List<HostNamePair> listInstances(Ec2Config ec2Config) throws Exception {
  48
+        Ec2Connection ec2 = new TypicaEc2Connection(ec2Config.getAccessId(),
  49
+                                                    ec2Config.getSecretKey());
50 50
         return ec2.list();
51 51
     }
52 52
 
295  contrib/ec2-testing/test/voldemort/utils/Ec2SmokeTest.java
@@ -23,135 +23,36 @@
23 23
 import static voldemort.utils.RemoteTestUtils.generateClusterDescriptor;
24 24
 import static voldemort.utils.RemoteTestUtils.startClusterAsync;
25 25
 import static voldemort.utils.RemoteTestUtils.startClusterNode;
26  
-import static voldemort.utils.RemoteTestUtils.stopCluster;
27 26
 import static voldemort.utils.RemoteTestUtils.stopClusterNode;
28 27
 import static voldemort.utils.RemoteTestUtils.stopClusterQuiet;
29 28
 import static voldemort.utils.RemoteTestUtils.toHostNames;
30 29
 
31  
-import java.io.File;
32  
-import java.io.FileInputStream;
33  
-import java.io.InputStream;
  30
+import java.util.ArrayList;
  31
+import java.util.Arrays;
  32
+import java.util.HashMap;
34 33
 import java.util.List;
35 34
 import java.util.Map;
36 35
 import java.util.Properties;
37 36
 
38  
-import org.apache.commons.io.IOUtils;
39  
-import org.apache.commons.lang.StringUtils;
40 37
 import org.apache.log4j.Logger;
41 38
 import org.junit.After;
42 39
 import org.junit.AfterClass;
  40
+import org.junit.Before;
43 41
 import org.junit.BeforeClass;
44 42
 import org.junit.Test;
45 43
 
46 44
 /**
47 45
  * Ec2SmokeTest contains two examples that interact with EC2.
48 46
  * 
49  
- * There are quite a few properties that are needed which are provided in a
50  
- * *.properties file, the path of which is provided in the "ec2PropertiesFile"
51  
- * System property. Below is a table of the properties:
52  
- * 
53  
- * <table>
54  
- * <th>Name</th>
55  
- * <th>Description</th>
56  
- * <tr>
57  
- * <td>ec2AccessId</td>
58  
- * <td>EC2 access ID, provided by Amazon</td>
59  
- * </tr>
60  
- * <tr>
61  
- * <td>ec2SecretKey</td>
62  
- * <td>EC2 secret key, provided by Amazon</td>
63  
- * </tr>
64  
- * <tr>
65  
- * <td>ec2Ami</td>
66  
- * <td>ID of the EC2 AMI used for the instances that are started</td>
67  
- * </tr>
68  
- * <tr>
69  
- * <td>ec2KeyPairId</td>
70  
- * <td>Key pair ID</td>
71  
- * </tr>
72  
- * <tr>
73  
- * <td>ec2SshPrivateKeyPath</td>
74  
- * <td>SSH private key path to key used to connect to instances (optional)</td>
75  
- * </tr>
76  
- * <tr>
77  
- * <td>ec2HostUserId</td>
78  
- * <td>User ID on the hosts; for EC2 this is usually "root"</td>
79  
- * </tr>
80  
- * <tr>
81  
- * <td>ec2VoldemortRootDirectory</td>
82  
- * <td>Root directory on remote instances that points to the the Voldemort
83  
- * "distribution" directory; relative to the host user ID's home directory. For
84  
- * example, if the remote user's home directory is /root and the Voldemort
85  
- * distribution directory is /root/voldemort, ec2VoldemortRootDirectory would be
86  
- * "voldemort"</td>
87  
- * </tr>
88  
- * <tr>
89  
- * <td>ec2VoldemortHomeDirectory</td>
90  
- * <td>Home directory on remote instances that points to the configuration
91  
- * directory, relative to the host user ID's home directory. For example, if the
92  
- * remote user's home directory is /root and the Voldemort configuration
93  
- * directory is /root/voldemort/config/single_node_cluster,
94  
- * ec2VoldemortHomeDirectory would be "voldemort/config/single_node_cluster"</td>
95  
- * </tr>
96  
- * <tr>
97  
- * <td>ec2SourceDirectory</td>
98  
- * <td>Source directory on <b>local</b> machine from which to copy the Voldemort
99  
- * "distribution" to the remote hosts; e.g. "/home/kirk/voldemortdev/voldemort"</td>
100  
- * </tr>
101  
- * <tr>
102  
- * <td>ec2ParentDirectory</td>
103  
- * <td>Parent directory on the <b>remote</b> machine into which to copy the
104  
- * Voldemort "distribution". For example, if the remote user's home directory is
105  
- * /root and the Voldemort distribution directory is /root/voldemort,
106  
- * ec2ParentDirectory would be "." or "/root".</td>
107  
- * </tr>
108  
- * <tr>
109  
- * <td>ec2ClusterXmlFile</td>
110  
- * <td><b>Local</b> path to which cluster.xml will be written with EC2 hosts;
111  
- * this needs to live under the ec2SourceDirectory's configuration directory
112  
- * that is copied to the remote host.</td>
113  
- * </tr>
114  
- * <tr>
115  
- * <td>ec2InstanceCount</td>
116  
- * <td>The number of instances to create.</td>
117  
- * </tr>
118  
- * <tr>
119  
- * <td>ec2RampTime</td>
120  
- * <td>For the remote test, the number of seconds to wait for each instance
121  
- * before connecting to the server. Prevents the server nodes from being flooded
122  
- * all at once.</td>
123  
- * </tr>
124  
- * <tr>
125  
- * <td>ec2Iterations</td>
126  
- * <td>For the remote test, the number of remote test iterations.</td>
127  
- * </tr>
128  
- * <tr>
129  
- * <td>ec2NumRequests</td>
130  
- * <td>For the remote test, the number of remote test requests per each
131  
- * iteration.</td>
132  
- * </tr>
133  
- * </table>
  47
+ * There are quite a few properties that are needed which are provided for these
  48
+ * tests to run. Please see {@link Ec2SmokeTestConfig} for details.
134 49
  * 
135 50
  * @author Kirk True
136 51
  */
137 52
 
138 53
 public class Ec2SmokeTest {
139 54
 
140  
-    private static String accessId;
141  
-    private static String secretKey;
142  
-    private static String ami;
143  
-    private static String keyPairId;
144  
-    private static String sshPrivateKeyPath;
145  
-    private static String hostUserId;
146  
-    private static File sshPrivateKey;
147  
-    private static String voldemortRootDirectory;
148  
-    private static String voldemortHomeDirectory;
149  
-    private static File sourceDirectory;
150  
-    private static String parentDirectory;
151  
-    private static File clusterXmlFile;
152  
-    private static int rampTime;
153  
-    private static int iterations;
154  
-    private static int numRequests;
  55
+    private static Ec2SmokeTestConfig ec2SmokeTestConfig;
155 56
     private static List<HostNamePair> hostNamePairs;
156 57
     private static List<String> hostNames;
157 58
     private static Map<String, Integer> nodeIds;
@@ -160,29 +61,10 @@
160 61
 
161 62
     @BeforeClass
162 63
     public static void setUpClass() throws Exception {
163  
-        Properties properties = getEc2Properties();
164  
-        accessId = properties.getProperty("ec2AccessId");
165  
-        secretKey = properties.getProperty("ec2SecretKey");
166  
-        ami = properties.getProperty("ec2Ami");
167  
-        keyPairId = properties.getProperty("ec2KeyPairId");
168  
-        sshPrivateKeyPath = properties.getProperty("ec2SshPrivateKeyPath");
169  
-        hostUserId = properties.getProperty("ec2HostUserId");
170  
-        sshPrivateKey = sshPrivateKeyPath != null ? new File(sshPrivateKeyPath) : null;
171  
-        voldemortRootDirectory = properties.getProperty("ec2VoldemortRootDirectory");
172  
-        voldemortHomeDirectory = properties.getProperty("ec2VoldemortHomeDirectory");
173  
-        sourceDirectory = new File(properties.getProperty("ec2SourceDirectory"));
174  
-        parentDirectory = properties.getProperty("ec2ParentDirectory");
175  
-        clusterXmlFile = new File(properties.getProperty("ec2ClusterXmlFile"));
176  
-        int ec2InstanceCount = Integer.parseInt(properties.getProperty("ec2InstanceCount"));
177  
-        rampTime = Integer.parseInt(properties.getProperty("ec2RampTime"));
178  
-        iterations = Integer.parseInt(properties.getProperty("ec2Iterations"));
179  
-        numRequests = Integer.parseInt(properties.getProperty("ec2NumRequests"));
180  
-
181  
-        hostNamePairs = createInstances(accessId, secretKey, ami, keyPairId, ec2InstanceCount);
182  
-
  64
+        ec2SmokeTestConfig = new Ec2SmokeTestConfig();
  65
+        hostNamePairs = createInstances(ec2SmokeTestConfig);
183 66
         hostNames = toHostNames(hostNamePairs);
184  
-
185  
-        nodeIds = generateClusterDescriptor(hostNamePairs, "test", clusterXmlFile);
  67
+        nodeIds = generateClusterDescriptor(hostNamePairs, "test", ec2SmokeTestConfig);
186 68
 
187 69
         if(logger.isInfoEnabled())
188 70
             logger.info("Sleeping for 30 seconds to give EC2 instances some time to complete startup");
@@ -193,100 +75,109 @@ public static void setUpClass() throws Exception {
193 75
     @AfterClass
194 76
     public static void tearDownClass() throws Exception {
195 77
         if(hostNames != null)
196  
-            destroyInstances(accessId, secretKey, hostNames);
  78
+            destroyInstances(hostNames, ec2SmokeTestConfig);
  79
+    }
  80
+
  81
+    @Before
  82
+    public void setUp() throws Exception {
  83
+        deploy(hostNames, ec2SmokeTestConfig);
  84
+        startClusterAsync(hostNames, ec2SmokeTestConfig, nodeIds);
197 85
     }
198 86
 
199 87
     @After
200 88
     public void tearDown() throws Exception {
201  
-        stopClusterQuiet(hostNames, sshPrivateKey, hostUserId, voldemortRootDirectory);
  89
+        stopClusterQuiet(hostNames, ec2SmokeTestConfig);
202 90
     }
203 91
 
204 92
     @Test
205 93
     public void testRemoteTest() throws Exception {
206  
-        deploy(hostNames, sshPrivateKey, hostUserId, sourceDirectory, parentDirectory);
207  
-
208  
-        try {
209  
-            startClusterAsync(hostNames,
210  
-                              sshPrivateKey,
211  
-                              hostUserId,
212  
-                              voldemortRootDirectory,
213  
-                              voldemortHomeDirectory,
214  
-                              nodeIds);
215  
-
216  
-            executeRemoteTest(hostNamePairs,
217  
-                              voldemortRootDirectory,
218  
-                              sshPrivateKey,
219  
-                              hostUserId,
220  
-                              rampTime,
221  
-                              iterations,
222  
-                              numRequests);
223  
-        } finally {
224  
-            stopCluster(hostNames, sshPrivateKey, hostUserId, voldemortRootDirectory);
  94
+        Map<String, String> commands = new HashMap<String, String>();
  95
+        List<String> hostNames = new ArrayList<String>();
  96
+        String bootstrapHostName = hostNamePairs.get(0).getInternalHostName();
  97
+
  98
+        int i = 0;
  99
+
  100
+        for(HostNamePair hostNamePair: hostNamePairs)
  101
+            hostNames.add(hostNamePair.getExternalHostName());
  102
+
  103
+        for(HostNamePair hostNamePair: hostNamePairs) {
  104
+            String command = "cd " + ec2SmokeTestConfig.getVoldemortRootDirectory() + " ; sleep "
  105
+                             + (i * ec2SmokeTestConfig.rampTime)
  106
+                             + "; ./bin/voldemort-remote-test.sh -w -d --iterations "
  107
+                             + ec2SmokeTestConfig.iterations + " --start-key-index "
  108
+                             + (i * ec2SmokeTestConfig.numRequests) + " tcp://" + bootstrapHostName
  109
+                             + ":6666 test " + ec2SmokeTestConfig.numRequests;
  110
+            commands.put(hostNamePair.getExternalHostName(), command);
  111
+            i++;
225 112
         }
  113
+
  114
+        executeRemoteTest(hostNames, ec2SmokeTestConfig, commands);
226 115
     }
227 116
 
228 117
     @Test
229 118
     public void testTemporaryNodeOffline() throws Exception {
230  
-        deploy(hostNames, sshPrivateKey, hostUserId, sourceDirectory, parentDirectory);
  119
+        String offlineHostName = hostNames.get(0);
  120
+        Integer nodeId = nodeIds.get(offlineHostName);
231 121
 
232  
-        try {
233  
-            startClusterAsync(hostNames,
234  
-                              sshPrivateKey,
235  
-                              hostUserId,
236  
-                              voldemortRootDirectory,
237  
-                              voldemortHomeDirectory,
238  
-                              nodeIds);
239  
-
240  
-            String offlineHostName = hostNames.get(0);
241  
-
242  
-            stopClusterNode(offlineHostName, sshPrivateKey, hostUserId, voldemortRootDirectory);
243  
-
244  
-            startClusterNode(offlineHostName,
245  
-                             sshPrivateKey,
246  
-                             hostUserId,
247  
-                             voldemortRootDirectory,
248  
-                             voldemortHomeDirectory,
249  
-                             nodeIds.get(hostNames.get(0)));
250  
-        } finally {
251  
-            stopCluster(hostNames, sshPrivateKey, hostUserId, voldemortRootDirectory);
252  
-        }
  122
+        stopClusterNode(offlineHostName, ec2SmokeTestConfig);
  123
+        startClusterNode(offlineHostName, ec2SmokeTestConfig, nodeId);
253 124
     }
254 125
 
255  
-    private static Properties getEc2Properties() throws Exception {
256  
-        String propertiesFileName = System.getProperty("ec2PropertiesFile");
257  
-
258  
-        String[] requireds = { "ec2AccessId", "ec2SecretKey", "ec2Ami", "ec2KeyPairId",
259  
-                "ec2HostUserId", "ec2VoldemortRootDirectory", "ec2VoldemortHomeDirectory",
260  
-                "ec2SourceDirectory", "ec2ParentDirectory", "ec2ClusterXmlFile",
261  
-                "ec2InstanceCount", "ec2RampTime", "ec2Iterations", "ec2NumRequests" };
262  
-
263  
-        if(propertiesFileName == null)
264  
-            throw new Exception("ec2PropertiesFile system property must be defined that "
265  
-                                + "provides the path to file containing the following "
266  
-                                + "required Ec2SmokeTest properties: "
267  
-                                + StringUtils.join(requireds, ", "));
268  
-
269  
-        Properties properties = new Properties();
270  
-        InputStream in = null;
271  
-
272  
-        try {
273  
-            in = new FileInputStream(propertiesFileName);
274  
-            properties.load(in);
275  
-        } finally {
276  
-            IOUtils.closeQuietly(in);
  126
+    /**
  127
+     * Ec2SmokeTestConfig contains configuration for {@link Ec2SmokeTest}.
  128
+     * 
  129
+     * There are quite a few properties that are needed which are provided in a
  130
+     * *.properties file, the path of which is provided in the
  131
+     * "ec2PropertiesFile" System property. Below is a table of the properties
  132
+     * <i>in addition to those from {@link Ec2Config}</i>:
  133
+     * 
  134
+     * <table>
  135
+     * <th>Name</th>
  136
+     * <th>Description</th>
  137
+     * <tr>
  138
+     * <td>ec2RampTime</td>
  139
+     * <td>For the remote test, the number of seconds to wait for each instance
  140
+     * before connecting to the server. Prevents the server nodes from being
  141
+     * flooded all at once.</td>
  142
+     * </tr>
  143
+     * <tr>
  144
+     * <td>ec2Iterations</td>
  145
+     * <td>For the remote test, the number of remote test iterations.</td>
  146
+     * </tr>
  147
+     * <tr>
  148
+     * <td>ec2NumRequests</td>
  149
+     * <td>For the remote test, the number of remote test requests per each
  150
+     * iteration.</td>
  151
+     * </tr>
  152
+     * </table>
  153
+     * 
  154
+     * @author Kirk True
  155
+     */
  156
+
  157
+    private static class Ec2SmokeTestConfig extends Ec2Config {
  158
+
  159
+        private int rampTime;
  160
+
  161
+        private int iterations;
  162
+
  163
+        private int numRequests;
  164
+
  165
+        @Override
  166
+        protected void init(Properties properties) {
  167
+            super.init(properties);
  168
+
  169
+            rampTime = getIntProperty(properties, "ec2RampTime");
  170
+            iterations = getIntProperty(properties, "ec2Iterations");
  171
+            numRequests = getIntProperty(properties, "ec2NumRequests");
277 172
         }
278 173
 
279  
-        for(String required: requireds) {
280  
-            // Allow system properties to override
281  
-            if(System.getProperties().containsKey(required))
282  
-                properties.put(required, System.getProperty(required));
283  
-
284  
-            if(!properties.containsKey(required))
285  
-                throw new Exception("Required properties for Ec2SmokeTest: "
286  
-                                    + StringUtils.join(requireds, ", ") + "; missing " + required);
  174
+        @Override
  175
+        protected List<String> getRequiredPropertyNames() {
  176
+            List<String> requireds = super.getRequiredPropertyNames();
  177
+            requireds.addAll(Arrays.asList("ec2RampTime", "ec2Iterations", "ec2NumRequests"));
  178
+            return requireds;
287 179
         }
288 180
 
289  
-        return properties;
290 181
     }
291 182
 
292 183
 }
93  contrib/ec2-testing/test/voldemort/utils/RemoteTestConfig.java
... ...
@@ -0,0 +1,93 @@
  1
+/*
  2
+ * Copyright 2009 LinkedIn, Inc.
  3
+ * 
  4
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
  5
+ * use this file except in compliance with the License. You may obtain a copy of
  6
+ * the License at
  7
+ * 
  8
+ * http://www.apache.org/licenses/LICENSE-2.0
  9
+ * 
  10
+ * Unless required by applicable law or agreed to in writing, software
  11
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  12
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  13
+ * License for the specific language governing permissions and limitations under
  14
+ * the License.
  15
+ */
  16
+
  17
+package voldemort.utils;
  18
+
  19
+import java.io.File;
  20
+
  21
+public class RemoteTestConfig {
  22
+
  23
+    protected String hostUserId;
  24
+
  25
+    protected File sshPrivateKey;
  26
+
  27
+    protected String voldemortRootDirectory;
  28
+
  29
+    protected String voldemortHomeDirectory;
  30
+
  31
+    protected File sourceDirectory;
  32
+
  33
+    protected String parentDirectory;
  34
+
  35
+    protected File clusterXmlFile;
  36
+
  37
+    public String getHostUserId() {
  38
+        return hostUserId;
  39
+    }
  40
+
  41
+    public void setHostUserId(String hostUserId) {
  42
+        this.hostUserId = hostUserId;
  43
+    }
  44
+
  45
+    public File getSshPrivateKey() {
  46
+        return sshPrivateKey;
  47
+    }
  48
+
  49
+    public void setSshPrivateKey(File sshPrivateKey) {
  50
+        this.sshPrivateKey = sshPrivateKey;
  51
+    }
  52
+
  53
+    public String getVoldemortRootDirectory() {
  54
+        return voldemortRootDirectory;
  55
+    }
  56
+
  57
+    public void setVoldemortRootDirectory(String voldemortRootDirectory) {
  58
+        this.voldemortRootDirectory = voldemortRootDirectory;
  59
+    }
  60
+
  61
+    public String getVoldemortHomeDirectory() {
  62
+        return voldemortHomeDirectory;
  63
+    }
  64
+
  65
+    public void setVoldemortHomeDirectory(String voldemortHomeDirectory) {
  66
+        this.voldemortHomeDirectory = voldemortHomeDirectory;
  67
+    }
  68
+
  69
+    public File getSourceDirectory() {
  70
+        return sourceDirectory;
  71
+    }
  72
+
  73
+    public void setSourceDirectory(File sourceDirectory) {
  74
+        this.sourceDirectory = sourceDirectory;
  75
+    }
  76
+
  77
+    public String getParentDirectory() {
  78
+        return parentDirectory;
  79
+    }
  80
+
  81
+    public void setParentDirectory(String parentDirectory) {
  82
+        this.parentDirectory = parentDirectory;
  83
+    }
  84
+
  85
+    public File getClusterXmlFile() {
  86
+        return clusterXmlFile;
  87
+    }
  88
+
  89
+    public void setClusterXmlFile(File clusterXmlFile) {
  90
+        this.clusterXmlFile = clusterXmlFile;
  91
+    }
  92
+
  93
+}
140  contrib/ec2-testing/test/voldemort/utils/RemoteTestUtils.java
@@ -16,7 +16,6 @@
16 16
 
17 17
 package voldemort.utils;
18 18
 
19  
-import java.io.File;
20 19
 import java.util.ArrayList;
21 20
 import java.util.Arrays;
22 21
 import java.util.HashMap;
@@ -36,70 +35,32 @@
36 35
 
37 36
     private static final Logger logger = Logger.getLogger(RemoteTestUtils.class);
38 37
 
39  
-    public static void deploy(List<String> hostNames,
40  
-                              File sshPrivateKey,
41  
-                              String hostUserId,
42  
-                              File sourceDirectory,
43  
-                              String parentDirectory) throws Exception {
44  
-        new RsyncDeployer(hostNames, sshPrivateKey, hostUserId, sourceDirectory, parentDirectory).execute();
  38
+    public static void deploy(List<String> hostNames, RemoteTestConfig remoteTestConfig)
  39
+            throws Exception {
  40
+        new RsyncDeployer(hostNames,
  41
+                          remoteTestConfig.getSshPrivateKey(),
  42
+                          remoteTestConfig.getHostUserId(),
  43
+                          remoteTestConfig.getSourceDirectory(),
  44
+                          remoteTestConfig.getParentDirectory()).execute();
45 45
     }
46 46
 
47 47
     public static void executeRemoteTest(List<String> hostNames,
48  
-                                         File sshPrivateKey,
49  
-                                         String hostUserId,
  48
+                                         RemoteTestConfig remoteTestConfig,
50 49
                                          Map<String, String> commands) throws Exception {
51  
-        new SshRemoteTest(hostNames, sshPrivateKey, hostUserId, commands).execute();
52  
-    }
53  
-
54  
-    public static void executeRemoteTest(List<HostNamePair> hostNamePairs,
55  
-                                         String voldemortRootDirectory,
56  
-                                         File sshPrivateKey,
57  
-                                         String hostUserId,
58  
-                                         int rampTime,
59  
-                                         int iterations,
60  
-                                         int numRequests) throws Exception {
61  
-        Map<String, String> commands = new HashMap<String, String>();
62  
-        List<String> hostNames = new ArrayList<String>();
63  
-        String bootstrapHostName = hostNamePairs.get(0).getInternalHostName();
64  
-
65  
-        int i = 0;
66  
-
67  
-        for(HostNamePair hostNamePair: hostNamePairs)
68  
-            hostNames.add(hostNamePair.getExternalHostName());
69  
-
70  
-        for(HostNamePair hostNamePair: hostNamePairs) {
71  
-            String command = "cd " + voldemortRootDirectory + " ; sleep " + (i * rampTime)
72  
-                             + "; ./bin/voldemort-remote-test.sh -w -d --iterations " + iterations
73  
-                             + " --start-key-index " + (i * numRequests) + " tcp://"
74  
-                             + bootstrapHostName + ":6666 test " + numRequests;
75  
-            commands.put(hostNamePair.getExternalHostName(), command);
76  
-            i++;
77  
-        }
78  
-
79  
-        executeRemoteTest(hostNames, sshPrivateKey, hostUserId, commands);
  50
+        new SshRemoteTest(hostNames,
  51
+                          remoteTestConfig.getSshPrivateKey(),
  52
+                          remoteTestConfig.getHostUserId(),
  53
+                          commands).execute();
80 54
     }
81 55
 
82  
-    public static void startClusterAsync(final List<String> hostNames,
83  
-                                         final File sshPrivateKey,
84  
-                                         final String hostUserId,
85  
-                                         final String voldemortRootDirectory,
86  
-                                         final String voldemortHomeDirectory,
87  
-                                         final Map<String, Integer> nodeIds) throws Exception {
88  
-        startCluster(hostNames,
89  
-                     sshPrivateKey,
90  
-                     hostUserId,
91  
-                     voldemortRootDirectory,
92  
-                     voldemortHomeDirectory,
93  
-                     nodeIds,
94  
-                     true,
95  
-                     10);
  56
+    public static void startClusterAsync(List<String> hostNames,
  57
+                                         RemoteTestConfig remoteTestConfig,
  58
+                                         Map<String, Integer> nodeIds) throws Exception {
  59
+        startCluster(hostNames, remoteTestConfig, nodeIds, true, 10);
96 60
     }
97 61
 
98 62
     public static void startCluster(final List<String> hostNames,
99  
-                                    final File sshPrivateKey,
100  
-                                    final String hostUserId,
101  
-                                    final String voldemortRootDirectory,
102  
-                                    final String voldemortHomeDirectory,
  63
+                                    final RemoteTestConfig remoteTestConfig,
103 64
                                     final Map<String, Integer> nodeIds,
104 65
                                     boolean startAsynchronously,
105 66
                                     int waitSeconds) throws Exception {
@@ -109,10 +70,10 @@ public static void startCluster(final List<String> hostNames,
109 70
                 public void run() {
110 71
                     try {
111 72
                         new SshClusterStarter(hostNames,
112  
-                                              sshPrivateKey,
113  
-                                              hostUserId,
114  
-                                              voldemortRootDirectory,
115  
-                                              voldemortHomeDirectory,
  73
+                                              remoteTestConfig.getSshPrivateKey(),
  74
+                                              remoteTestConfig.getHostUserId(),
  75
+                                              remoteTestConfig.getVoldemortRootDirectory(),
  76
+                                              remoteTestConfig.getVoldemortHomeDirectory(),
116 77
                                               nodeIds).execute();
117 78
                     } catch(RemoteOperationException e) {
118 79
                         e.printStackTrace();
@@ -124,47 +85,39 @@ public void run() {
124 85
             Thread.sleep(waitSeconds * 1000);
125 86
         } else {
126 87
             new SshClusterStarter(hostNames,
127  
-                                  sshPrivateKey,
128  
-                                  hostUserId,
129  
-                                  voldemortRootDirectory,
130  
-                                  voldemortHomeDirectory,
  88
+                                  remoteTestConfig.getSshPrivateKey(),
  89
+                                  remoteTestConfig.getHostUserId(),
  90
+                                  remoteTestConfig.getVoldemortRootDirectory(),
  91
+                                  remoteTestConfig.getVoldemortHomeDirectory(),
131 92
                                   nodeIds).execute();
132 93
         }
133 94
     }
134 95
 
135 96
     public static void startClusterNode(String hostName,
136  
-                                        File sshPrivateKey,
137  
-                                        String hostUserId,
138  
-                                        String voldemortRootDirectory,
139  
-                                        String voldemortHomeDirectory,
  97
+                                        RemoteTestConfig remoteTestConfig,
140 98
                                         int nodeId) throws Exception {
141 99
         Map<String, Integer> nodeIds = new HashMap<String, Integer>();
142 100
         nodeIds.put(hostName, nodeId);
143 101
 
144  
-        startClusterAsync(new ArrayList<String>(Arrays.asList(hostName)),
145  
-                          sshPrivateKey,
146  
-                          hostUserId,
147  
-                          voldemortRootDirectory,
148  
-                          voldemortHomeDirectory,
149  
-                          nodeIds);
  102
+        startClusterAsync(new ArrayList<String>(Arrays.asList(hostName)), remoteTestConfig, nodeIds);
150 103
     }
151 104
 
152  
-    public static void stopCluster(List<String> hostNames,
153  
-                                   File sshPrivateKey,
154  
-                                   String hostUserId,
155  
-                                   String voldemortRootDirectory) throws Exception {
156  
-        new SshClusterStopper(hostNames, sshPrivateKey, hostUserId, voldemortRootDirectory, false).execute();
  105
+    public static void stopCluster(List<String> hostNames, RemoteTestConfig remoteTestConfig)
  106
+            throws Exception {
  107
+        new SshClusterStopper(hostNames,
  108
+                              remoteTestConfig.getSshPrivateKey(),
  109
+                              remoteTestConfig.getHostUserId(),
  110
+                              remoteTestConfig.getVoldemortRootDirectory(),
  111
+                              false).execute();
157 112
     }
158 113
 
159  
-    public static void stopClusterQuiet(List<String> hostNames,
160  
-                                        File sshPrivateKey,
161  
-                                        String hostUserId,
162  
-                                        String voldemortRootDirectory) throws Exception {
  114
+    public static void stopClusterQuiet(List<String> hostNames, RemoteTestConfig remoteTestConfig)
  115
+            throws Exception {
163 116
         try {
164 117
             new SshClusterStopper(hostNames,
165  
-                                  sshPrivateKey,
166  
-                                  hostUserId,
167  
-                                  voldemortRootDirectory,
  118
+                                  remoteTestConfig.getSshPrivateKey(),
  119
+                                  remoteTestConfig.getHostUserId(),
  120
+                                  remoteTestConfig.getVoldemortRootDirectory(),
168 121
                                   true).execute();
169 122
         } catch(Exception e) {
170 123
             if(logger.isEnabledFor(Level.WARN))
@@ -172,14 +125,9 @@ public static void stopClusterQuiet(List<String> hostNames,
172 125
         }
173 126
     }
174 127
 
175  
-    public static void stopClusterNode(String hostName,
176  
-                                       File sshPrivateKey,
177  
-                                       String hostUserId,
178  
-                                       String voldemortRootDirectory) throws Exception {
179  
-        stopCluster(new ArrayList<String>(Arrays.asList(hostName)),
180  
-                    sshPrivateKey,
181  
-                    hostUserId,
182  
-                    voldemortRootDirectory);
  128
+    public static void stopClusterNode(String hostName, RemoteTestConfig remoteTestConfig)
  129
+            throws Exception {
  130
+        stopCluster(Arrays.asList(hostName), remoteTestConfig);
183 131
     }
184 132
 
185 133
     public static List<String> toHostNames(List<HostNamePair> hostNamePairs) {
@@ -193,7 +141,7 @@ public static void stopClusterNode(String hostName,
193 141
 
194 142
     public static Map<String, Integer> generateClusterDescriptor(List<HostNamePair> hostNamePairs,
195 143
                                                                  String clusterName,
196  
-                                                                 File clusterXmlFile)
  144
+                                                                 RemoteTestConfig remoteTestConfig)
197 145
             throws Exception {
198 146
         List<String> hostNames = new ArrayList<String>();
199 147
 
@@ -204,7 +152,7 @@ public static void stopClusterNode(String hostName,
204 152
         List<ClusterNodeDescriptor> nodes = clusterGenerator.createClusterNodeDescriptors(hostNames,
205 153
                                                                                           3);
206 154
         String clusterXml = clusterGenerator.createClusterDescriptor(clusterName, nodes);
207  
-        FileUtils.writeStringToFile(clusterXmlFile, clusterXml);
  155
+        FileUtils.writeStringToFile(remoteTestConfig.getClusterXmlFile(), clusterXml);
208 156
         Map<String, Integer> nodeIds = new HashMap<String, Integer>();
209 157
 
210 158
         for(ClusterNodeDescriptor node: nodes) {

0 notes on commit 40c3608

Please sign in to comment.
Something went wrong with that request. Please try again.