Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Merging updated EC2 tests to support multiple EC2 regions

  • Loading branch information...
commit e88d32d42b889a4f796e2bda924e103a3fb231ea 1 parent 1065da5
@rsumbaly rsumbaly authored
Showing with 426 additions and 68 deletions.
  1. BIN  contrib/ec2-testing/lib/activation.jar
  2. BIN  contrib/ec2-testing/lib/httpclient-4.0.1.jar
  3. BIN  contrib/ec2-testing/lib/httpcore-4.0.1.jar
  4. BIN  contrib/ec2-testing/lib/jaxb-api.jar
  5. BIN  contrib/ec2-testing/lib/jaxb-impl.jar
  6. BIN  contrib/ec2-testing/lib/jaxb-xjc.jar
  7. BIN  contrib/ec2-testing/lib/jaxb1-impl.jar
  8. BIN  contrib/ec2-testing/lib/jsr173_1.0_api.jar
  9. BIN  contrib/ec2-testing/lib/typica.jar
  10. +4 −0 contrib/ec2-testing/resources/commands.properties
  11. +142 −42 contrib/ec2-testing/src/java/voldemort/utils/ClusterGenerator.java
  12. +28 −0 contrib/ec2-testing/src/java/voldemort/utils/ClusterNodeDescriptor.java
  13. +68 −0 contrib/ec2-testing/src/java/voldemort/utils/StoresGenerator.java
  14. +32 −0 contrib/ec2-testing/src/java/voldemort/utils/app/VoldemortApp.java
  15. +22 −7 contrib/ec2-testing/src/java/voldemort/utils/app/VoldemortClusterGeneratorApp.java
  16. +24 −11 contrib/ec2-testing/src/java/voldemort/utils/app/VoldemortClusterStarterApp.java
  17. +9 −1 contrib/ec2-testing/src/java/voldemort/utils/app/VoldemortEc2InstanceCreatorApp.java
  18. +9 −1 contrib/ec2-testing/src/java/voldemort/utils/app/VoldemortEc2InstanceTerminatorApp.java
  19. +71 −0 contrib/ec2-testing/src/java/voldemort/utils/app/VoldemortStoresGeneratorApp.java
  20. +8 −2 contrib/ec2-testing/src/java/voldemort/utils/impl/TypicaEc2Connection.java
  21. +9 −4 contrib/ec2-testing/test/voldemort/utils/Ec2RemoteTestUtils.java
View
BIN  contrib/ec2-testing/lib/activation.jar
Binary file not shown
View
BIN  contrib/ec2-testing/lib/httpclient-4.0.1.jar
Binary file not shown
View
BIN  contrib/ec2-testing/lib/httpcore-4.0.1.jar
Binary file not shown
View
BIN  contrib/ec2-testing/lib/jaxb-api.jar
Binary file not shown
View
BIN  contrib/ec2-testing/lib/jaxb-impl.jar
Binary file not shown
View
BIN  contrib/ec2-testing/lib/jaxb-xjc.jar
Binary file not shown
View
BIN  contrib/ec2-testing/lib/jaxb1-impl.jar
Binary file not shown
View
BIN  contrib/ec2-testing/lib/jsr173_1.0_api.jar
Binary file not shown
View
BIN  contrib/ec2-testing/lib/typica.jar
Binary file not shown
View
4 contrib/ec2-testing/resources/commands.properties
@@ -44,6 +44,8 @@ SshClusterStarter.ssh=ssh -o StrictHostKeyChecking=no -i ${sshPrivateKey} ${host
mv $VOLDEMORT_HOME/config/server.properties $VOLDEMORT_HOME/config/server.properties.bak ; \
grep -v "^node.id=" $VOLDEMORT_HOME/config/server.properties.bak > $VOLDEMORT_HOME/config/server.properties ; \
rm -rf ${voldemortHomeDirectory}/data ; \
+ rm -rf ${voldemortHomeDirectory}/config/.temp ; \
+ rm -rf ${voldemortHomeDirectory}/config/.version ; \
cd ${voldemortRootDirectory} ; \
./bin/voldemort-server.sh"
@@ -53,6 +55,8 @@ SshClusterStarter.ssh.nokey=ssh -o StrictHostKeyChecking=no ${hostUserId}@${host
mv $VOLDEMORT_HOME/config/server.properties $VOLDEMORT_HOME/config/server.properties.bak ; \
grep -v "^node.id=" $VOLDEMORT_HOME/config/server.properties.bak > $VOLDEMORT_HOME/config/server.properties ; \
rm -rf ${voldemortHomeDirectory}/data ; \
+ rm -rf ${voldemortHomeDirectory}/config/.temp ; \
+ rm -rf ${voldemortHomeDirectory}/config/.version ; \
cd ${voldemortRootDirectory} ; \
./bin/voldemort-server.sh"
View
184 contrib/ec2-testing/src/java/voldemort/utils/ClusterGenerator.java
@@ -20,12 +20,17 @@
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Random;
+import java.util.Set;
import org.apache.commons.lang.StringUtils;
+
import voldemort.cluster.Cluster;
import voldemort.cluster.Node;
+import voldemort.cluster.Zone;
/**
* ClusterGenerator generates a cluster.xml file given either a list of hosts or
@@ -88,11 +93,30 @@
*
* @return List of ClusterNodeDescriptor
*/
-
public List<ClusterNodeDescriptor> createClusterNodeDescriptors(List<String> hostNames,
int numPartitions) {
+ HashMap<Integer, List<String>> zoneToHostName = new HashMap<Integer, List<String>>();
+ zoneToHostName.put(Zone.DEFAULT_ZONE_ID, hostNames);
+ return createClusterNodeDescriptors(zoneToHostName, numPartitions);
+ }
+
+ /**
+ * Creates a list of ClusterNodeDescriptor instances
+ *
+ * @param zoneToHostNames A mapping from zone to list of host names
+ * @param numPartitions Number of partitions per host
+ *
+ * @return List of ClusterNodeDescriptor
+ */
+ private List<ClusterNodeDescriptor> createClusterNodeDescriptors(HashMap<Integer, List<String>> zoneToHostNames,
+ int numPartitions) {
+ int numHosts = 0;
+ for(List<String> hostNames: zoneToHostNames.values()) {
+ numHosts += hostNames.size();
+ }
+
// Create a list of integers [0..totalPartitions).
- int totalPartitions = hostNames.size() * numPartitions;
+ int totalPartitions = numHosts * numPartitions;
List<Integer> allPartitionIds = new ArrayList<Integer>();
for(int i = 0; i < totalPartitions; i++)
@@ -103,64 +127,107 @@
List<ClusterNodeDescriptor> list = new ArrayList<ClusterNodeDescriptor>();
- for(int i = 0; i < hostNames.size(); i++) {
- String hostName = hostNames.get(i);
- List<Integer> partitions = allPartitionIds.subList(i * numPartitions, (i + 1)
- * numPartitions);
- Collections.sort(partitions);
-
- ClusterNodeDescriptor cnd = new ClusterNodeDescriptor();
- cnd.setHostName(hostName);
- cnd.setId(i);
- cnd.setPartitions(partitions);
-
- list.add(cnd);
+ int nodeId = 0;
+ for(int zoneId = 0; zoneId < zoneToHostNames.size(); zoneId++) {
+ List<String> hostNames = zoneToHostNames.get(zoneId);
+
+ for(int i = 0; i < hostNames.size(); i++) {
+ String hostName = hostNames.get(i);
+ List<Integer> partitions = allPartitionIds.subList(nodeId * numPartitions,
+ (nodeId + 1) * numPartitions);
+ Collections.sort(partitions);
+
+ ClusterNodeDescriptor cnd = new ClusterNodeDescriptor();
+ cnd.setHostName(hostName);
+ cnd.setId(nodeId);
+ cnd.setPartitions(partitions);
+ cnd.setZoneId(zoneId);
+
+ nodeId++;
+ list.add(cnd);
+ }
}
return list;
}
- public List<ClusterNodeDescriptor> createClusterNodeDescriptors(List<String> hostNames,
+ /**
+ * Creates a list of ClusterNodeDescriptor instances
+ *
+ * @param zoneToHostNames A mapping from zone to list of host names
+ * @param cluster The cluster describing the various hosts
+ *
+ * @return List of ClusterNodeDescriptor
+ */
+ public List<ClusterNodeDescriptor> createClusterNodeDescriptors(HashMap<Integer, List<String>> zoneToHostNames,
Cluster cluster) {
- if (cluster.getNumberOfNodes() > hostNames.size())
+ if(cluster.getNumberOfZones() != zoneToHostNames.size())
+ throw new IllegalStateException("zone size does not match");
+
+ int numHosts = 0;
+ for(List<String> hostNames: zoneToHostNames.values()) {
+ numHosts += hostNames.size();
+ }
+
+ if(cluster.getNumberOfNodes() != numHosts)
throw new IllegalStateException("cluster size exceeds the number of available instances");
List<ClusterNodeDescriptor> list = new ArrayList<ClusterNodeDescriptor>();
- for(int i = 0; i < cluster.getNumberOfNodes(); i++) {
- Node node = cluster.getNodeById(i);
- String hostName = hostNames.get(i);
- List<Integer> partitions = node.getPartitionIds();
-
- ClusterNodeDescriptor cnd = new ClusterNodeDescriptor();
- cnd.setHostName(hostName);
- cnd.setId(i);
- cnd.setSocketPort(node.getSocketPort());
- cnd.setHttpPort(node.getHttpPort());
- cnd.setAdminPort(node.getAdminPort());
- cnd.setPartitions(partitions);
-
- list.add(cnd);
+
+ int nodeId = 0;
+ for(int zoneId = 0; zoneId < zoneToHostNames.size(); zoneId++) {
+ List<String> hostNames = zoneToHostNames.get(zoneId);
+
+ for(int i = 0; i < hostNames.size(); i++) {
+ Node node = cluster.getNodeById(nodeId);
+ String hostName = hostNames.get(i);
+ List<Integer> partitions = node.getPartitionIds();
+
+ ClusterNodeDescriptor cnd = new ClusterNodeDescriptor();
+ cnd.setHostName(hostName);
+ cnd.setId(nodeId);
+ cnd.setSocketPort(node.getSocketPort());
+ cnd.setHttpPort(node.getHttpPort());
+ cnd.setAdminPort(node.getAdminPort());
+ cnd.setPartitions(partitions);
+ cnd.setZoneId(zoneId);
+
+ nodeId++;
+ list.add(cnd);
+ }
}
return list;
}
+ /**
+ * @param hostNames <i>Internal</i> host name
+ * @param cluster Number of partitions <b>per host</b>
+ *
+ * @return List of ClusterNodeDescriptor
+ */
+ public List<ClusterNodeDescriptor> createClusterNodeDescriptors(List<String> hostNames,
+ Cluster cluster) {
+ HashMap<Integer, List<String>> zoneToHostName = new HashMap<Integer, List<String>>();
+ zoneToHostName.put(Zone.DEFAULT_ZONE_ID, hostNames);
+ return createClusterNodeDescriptors(zoneToHostName, cluster);
+ }
/**
* Creates a String representing the format used by cluster.xml given the
* cluster name, host names, and number of partitions for each host.
*
* @param clusterName Name of cluster
- * @param hostNames <i>Internal</i> host name
+ * @param zoneToHostNames Zone to list of HostNames map
* @param numPartitions Number of partitions <b>per host</b>
*
* @return String of formatted XML as used by cluster.xml
*/
public String createClusterDescriptor(String clusterName,
- List<String> hostNames,
+ HashMap<Integer, List<String>> zoneToHostNames,
int numPartitions) {
- List<ClusterNodeDescriptor> clusterNodeDescriptors = createClusterNodeDescriptors(hostNames,
+ List<ClusterNodeDescriptor> clusterNodeDescriptors = createClusterNodeDescriptors(zoneToHostNames,
numPartitions);
return createClusterDescriptor(clusterName, clusterNodeDescriptors);
@@ -185,22 +252,55 @@ public String createClusterDescriptor(String clusterName,
pw.println("<cluster>");
pw.println("\t<name>" + clusterName + "</name>");
+ StringBuffer nodesBuffer = new StringBuffer();
+ Set<Integer> zoneIds = new HashSet<Integer>();
for(ClusterNodeDescriptor cnd: clusterNodeDescriptors) {
String partitions = StringUtils.join(cnd.getPartitions(), ", ");
- pw.println("\t<server>");
- pw.println("\t\t<id>" + cnd.getId() + "</id>");
- pw.println("\t\t<host>" + cnd.getHostName() + "</host>");
- pw.println("\t\t<http-port>" + cnd.getHttpPort() + "</http-port>");
- pw.println("\t\t<socket-port>" + cnd.getSocketPort() + "</socket-port>");
- pw.println("\t\t<admin-port>" + cnd.getAdminPort() + "</admin-port>");
- pw.println("\t\t<partitions>" + partitions + "</partitions>");
- pw.println("\t</server>");
+ nodesBuffer.append("\t<server>\n");
+ nodesBuffer.append("\t\t<id>" + cnd.getId() + "</id>\n");
+ nodesBuffer.append("\t\t<host>" + cnd.getHostName() + "</host>\n");
+ nodesBuffer.append("\t\t<http-port>" + cnd.getHttpPort() + "</http-port>\n");
+ nodesBuffer.append("\t\t<socket-port>" + cnd.getSocketPort() + "</socket-port>\n");
+ nodesBuffer.append("\t\t<admin-port>" + cnd.getAdminPort() + "</admin-port>\n");
+ nodesBuffer.append("\t\t<partitions>" + partitions + "</partitions>\n");
+ nodesBuffer.append("\t\t<zone-id>" + cnd.getZoneId() + "</zone-id>\n");
+ nodesBuffer.append("\t</server>");
+
+ zoneIds.add(cnd.getZoneId());
}
+ // Insert zones
+ for(Integer zoneId: zoneIds) {
+ pw.println("\t<zone>");
+ pw.println("\t\t<zone-id>" + zoneId + "</zone-id>");
+ pw.println("\t\t<proximity-list>" + generateProximityList(zoneId, zoneIds.size())
+ + "</proximity-list>");
+ pw.println("\t</zone>");
+ }
+
+ // Insert servers
+ pw.println(nodesBuffer.toString());
pw.println("</cluster>");
return sw.toString();
}
-}
+ /**
+ * Generate sequential proximity list
+ *
+ * @param zoneId Id of the Zone for which the proximity List is being
+ * generated
+ * @param totalZones Total number of zones
+ * @return String of list of zones
+ */
+ private String generateProximityList(int zoneId, int totalZones) {
+ List<Integer> proximityList = new ArrayList<Integer>();
+ int currentZoneId = (zoneId + 1) % totalZones;
+ for(int i = 0; i < (totalZones - 1); i++) {
+ proximityList.add(currentZoneId);
+ currentZoneId = (currentZoneId + 1) % totalZones;
+ }
+ return StringUtils.join(proximityList, ", ");
+ }
+}
View
28 contrib/ec2-testing/src/java/voldemort/utils/ClusterNodeDescriptor.java
@@ -18,6 +18,8 @@
import java.util.List;
+import voldemort.cluster.Zone;
+
/**
* ClusterNodeDescriptor is a simple POJO for storing the attributes of a node
* as needed by the cluster.xml cluster descriptor file.
@@ -46,6 +48,12 @@
private static final int DEFAULT_ADMIN_PORT = DEFAULT_SOCKET_PORT + 1;
+ /**
+ * DEFAULT_ZONE_ID is Zone.DEFAULT_ZONE_ID
+ */
+
+ private static final int DEFAULT_ZONE_ID = Zone.DEFAULT_ZONE_ID;
+
private String hostName;
private int id;
@@ -56,9 +64,29 @@
private int adminPort = DEFAULT_ADMIN_PORT;
+ private int zoneId = DEFAULT_ZONE_ID;
+
private List<Integer> partitions;
/**
+ * Returns the Zone ID of this node
+ *
+ * @return Zone id of this node
+ */
+ public int getZoneId() {
+ return this.zoneId;
+ }
+
+ /**
+ * Assign the zone Id to this node.
+ *
+ * @param zoneId Zone Id
+ */
+ public void setZoneId(int zoneId) {
+ this.zoneId = zoneId;
+ }
+
+ /**
* Returns the host name (or IP address) of the node. This is the internal
* host name as seen by the other server nodes and clients on the same
* network.
View
68 contrib/ec2-testing/src/java/voldemort/utils/StoresGenerator.java
@@ -0,0 +1,68 @@
+package voldemort.utils;
+
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.List;
+
+import voldemort.routing.RoutingStrategyType;
+import voldemort.store.bdb.BdbStorageConfiguration;
+
+/**
+ * StoresGenerator generates the stores.xml file given either a store name along
+ * with zone specific replication factors. Currently hard-coding the serializers
+ * to string format
+ *
+ */
+public class StoresGenerator {
+
+ public String createStoreDescriptor(String storeName,
+ List<Integer> zoneRepFactor,
+ int requiredReads,
+ int requiredWrites,
+ int zoneCountReads,
+ int zoneCountWrites,
+ String routingStrategy) {
+ StringWriter sw = new StringWriter();
+ PrintWriter pw = new PrintWriter(sw);
+
+ pw.println("<stores>");
+ pw.println("\t<store>");
+ pw.println("\t\t<name>" + storeName + "</name>");
+ pw.println("\t\t<persistence>" + BdbStorageConfiguration.TYPE_NAME + "</persistence>");
+ pw.println("\t\t<routing>client</routing>");
+ pw.println("\t\t<routing-strategy>" + routingStrategy + "</routing-strategy>");
+
+ pw.println("\t\t<key-serializer>");
+ pw.println("\t\t\t<type>string</type>");
+ pw.println("\t\t\t<schema-info>UTF-8</schema-info>");
+ pw.println("\t\t</key-serializer>");
+ pw.println("\t\t<value-serializer>");
+ pw.println("\t\t\t<type>string</type>");
+ pw.println("\t\t\t<schema-info>UTF-8</schema-info>");
+ pw.println("\t\t</value-serializer>");
+
+ pw.println("\t\t<required-reads>" + requiredReads + "</required-reads>");
+ pw.println("\t\t<required-writes>" + requiredWrites + "</required-writes>");
+
+ pw.println("\t\t<zone-replication-factor>");
+ int zoneId = 0, totalReplicationFactor = 0;
+ for(int repFactor: zoneRepFactor) {
+ pw.println("\t\t\t<replication-factor zone-id=\"" + zoneId + "\">" + repFactor
+ + "</replication-factor>");
+ zoneId++;
+ totalReplicationFactor += repFactor;
+ }
+ pw.println("\t\t</zone-replication-factor>");
+ pw.println("\t\t<replication-factor>" + totalReplicationFactor + "</replication-factor>");
+
+ if(routingStrategy.compareTo(RoutingStrategyType.ZONE_STRATEGY) == 0) {
+ pw.println("\t\t<zone-count-reads>" + zoneCountReads + "</zone-count-reads>");
+ pw.println("\t\t<zone-count-writes>" + zoneCountWrites + "</zone-count-writes>");
+ }
+ pw.println("\t</store>");
+ pw.println("</stores>");
+
+ return sw.toString();
+ }
+
+}
View
32 contrib/ec2-testing/src/java/voldemort/utils/app/VoldemortApp.java
@@ -39,6 +39,8 @@
import voldemort.utils.CmdUtils;
import voldemort.utils.HostNamePair;
+import com.xerox.amazonws.ec2.RegionInfo;
+
public abstract class VoldemortApp {
protected final OptionParser parser = new OptionParser();
@@ -145,6 +147,19 @@ protected File getRequiredInputFile(OptionSet options, String argumentName) {
return file;
}
+ protected List<File> getRequiredInputFiles(OptionSet options, String argumentName) {
+ List<String> fileNames = (List<String>) options.valuesOf(argumentName);
+ List<File> returnFileList = new ArrayList<File>();
+ for(String name: fileNames) {
+ File file = new File(name);
+ if(file.canRead()) {
+ returnFileList.add(file);
+ }
+ }
+
+ return returnFileList;
+ }
+
protected File getInputFile(OptionSet options, String argumentName) {
if(!options.has(argumentName))
return null;
@@ -204,6 +219,14 @@ protected File getInputFile(OptionSet options, String argumentName) {
return map;
}
+ protected String getRegionUrl(OptionSet options) throws Exception {
+ if(options.has("region")) {
+ return CmdUtils.valueOf(options, "region", RegionInfo.REGIONURL_US_EAST);
+ } else {
+ return RegionInfo.REGIONURL_US_EAST;
+ }
+ }
+
protected String getAccessId(OptionSet options) throws Exception {
if(!options.has("accessid") && !options.has("accessidfile")) {
System.err.println("Missing required argument accessid or accessidfile");
@@ -238,4 +261,13 @@ protected String getSecretKey(OptionSet options) throws Exception {
return null;
}
+ protected List<Integer> getRequiredListIntegers(OptionSet options, String argumentName)
+ throws Exception {
+ if(!options.has(argumentName)) {
+ System.err.println("Missing required argument " + argumentName);
+ printUsage();
+ }
+ return (List<Integer>) options.valuesOf(argumentName);
+ }
+
}
View
29 contrib/ec2-testing/src/java/voldemort/utils/app/VoldemortClusterGeneratorApp.java
@@ -18,6 +18,7 @@
import java.io.File;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import joptsimple.OptionSet;
@@ -47,23 +48,37 @@ public void run(String[] args) throws Exception {
.withRequiredArg()
.ofType(Integer.class);
parser.accepts("clustername", "Cluster name; defaults to mycluster").withRequiredArg();
+ parser.accepts("useinternal", "Use internal host name; defaults to true")
+ .withRequiredArg()
+ .ofType(Boolean.class);
OptionSet options = parse(args);
- File hostNamesFile = getRequiredInputFile(options, "hostnames");
+ List<File> hostNamesFiles = getRequiredInputFiles(options, "hostnames");
int partitions = getRequiredInt(options, "partitions");
String clusterName = CmdUtils.valueOf(options, "clustername", "mycluster");
+ boolean useInternal = CmdUtils.valueOf(options, "useinternal", true);
+
+ HashMap<Integer, List<String>> zoneToHostNames = new HashMap<Integer, List<String>>();
- List<HostNamePair> hostNamePairs = getHostNamesPairsFromFile(hostNamesFile);
- List<String> hostNames = new ArrayList<String>();
+ int zoneId = 0;
+ // Every file specified is considered as one zone
+ for(File hostNameFile: hostNamesFiles) {
+ List<HostNamePair> hostNamePairs = getHostNamesPairsFromFile(hostNameFile);
+ List<String> hostNames = new ArrayList<String>();
- for(HostNamePair hostNamePair: hostNamePairs)
- hostNames.add(hostNamePair.getInternalHostName());
+ for(HostNamePair hostNamePair: hostNamePairs) {
+ if(useInternal)
+ hostNames.add(hostNamePair.getInternalHostName());
+ else
+ hostNames.add(hostNamePair.getExternalHostName());
+ }
+ zoneToHostNames.put(zoneId++, hostNames);
+ }
String clusterXml = new ClusterGenerator().createClusterDescriptor(clusterName,
- hostNames,
+ zoneToHostNames,
partitions);
System.out.print(clusterXml);
}
-
}
View
35 contrib/ec2-testing/src/java/voldemort/utils/app/VoldemortClusterStarterApp.java
@@ -66,6 +66,9 @@ public void run(String[] args) throws Exception {
parser.accepts("clusterxml",
"Voldemort's cluster.xml file on the local file system; used to determine host names")
.withRequiredArg();
+ parser.accepts("useinternal", "Use internal host name; defaults to true")
+ .withRequiredArg()
+ .ofType(Boolean.class);
OptionSet options = parse(args);
File hostNamesFile = getRequiredInputFile(options, "hostnames");
@@ -83,10 +86,14 @@ public void run(String[] args) throws Exception {
final String hostUserId = CmdUtils.valueOf(options, "hostuserid", "root");
String voldemortHomeDirectory = getRequiredString(options, "voldemorthome");
final String voldemortRootDirectory = getRequiredString(options, "voldemortroot");
+ boolean useInternal = CmdUtils.valueOf(options, "useinternal", true);
File clusterXmlFile = getRequiredInputFile(options, "clusterxml");
- Map<String, Integer> nodeIds = getNodeIds(hostNamesFile, clusterXmlFile, hostNamePairs);
+ Map<String, Integer> nodeIds = getNodeIds(hostNamesFile,
+ clusterXmlFile,
+ hostNamePairs,
+ useInternal);
Runtime.getRuntime().addShutdownHook(new Thread() {
@@ -117,31 +124,30 @@ public void run() {
private Map<String, Integer> getNodeIds(File hostNamesFile,
File clusterXmlFile,
- List<HostNamePair> hostNamePairs) throws Exception {
+ List<HostNamePair> hostNamePairs,
+ boolean useInternal) throws Exception {
Map<String, Integer> nodeIds = new HashMap<String, Integer>();
DocumentBuilder documentBuilder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
Document document = documentBuilder.parse(clusterXmlFile);
NodeList documentChildren = document.getChildNodes().item(0).getChildNodes();
-
for(int i = 0; i < documentChildren.getLength(); i++) {
Node documentChild = documentChildren.item(i);
-
if(documentChild.getNodeName().equals("server")) {
NodeList serverChildren = documentChild.getChildNodes();
- String internalHostName = null;
+ String hostName = null;
String id = null;
for(int j = 0; j < serverChildren.getLength(); j++) {
Node serverChild = serverChildren.item(j);
if(serverChild.getNodeName().equals("host"))
- internalHostName = serverChild.getTextContent();
+ hostName = serverChild.getTextContent();
else if(serverChild.getNodeName().equals("id"))
id = serverChild.getTextContent();
- }
- if(internalHostName != null && id != null) {
+ }
+ if(hostName != null && id != null) {
// Yes, this is super inefficient, but we have to assign the
// node ID to the *external* host name but the cluster.xml
// file contains the *internal* host name. So loop over all
@@ -149,8 +155,16 @@ else if(serverChild.getNodeName().equals("id"))
// matches the internal host name and assign the node ID to
// its corresponding external host name.
for(HostNamePair hostNamePair: hostNamePairs) {
- if(hostNamePair.getInternalHostName().equals(internalHostName))
- nodeIds.put(hostNamePair.getExternalHostName(), Integer.parseInt(id));
+ if(useInternal) {
+ if(hostNamePair.getInternalHostName().equals(hostName))
+ nodeIds.put(hostNamePair.getExternalHostName(),
+ Integer.parseInt(id));
+ } else {
+ if(hostNamePair.getExternalHostName().equals(hostName))
+ nodeIds.put(hostNamePair.getExternalHostName(),
+ Integer.parseInt(id));
+ }
+
}
} else {
throw new Exception(clusterXmlFile.getAbsolutePath()
@@ -158,7 +172,6 @@ else if(serverChild.getNodeName().equals("id"))
}
}
}
-
if(nodeIds.size() != hostNamePairs.size()) {
throw new Exception(clusterXmlFile.getAbsolutePath()
+ " appears to be corrupt; not all of the hosts from "
View
10 contrib/ec2-testing/src/java/voldemort/utils/app/VoldemortEc2InstanceCreatorApp.java
@@ -25,6 +25,8 @@
import voldemort.utils.HostNamePair;
import voldemort.utils.impl.TypicaEc2Connection;
+import com.xerox.amazonws.ec2.RegionInfo;
+
public class VoldemortEc2InstanceCreatorApp extends VoldemortApp {
public static void main(String[] args) throws Exception {
@@ -61,12 +63,18 @@ public void run(String[] args) throws Exception {
+ Ec2Connection.Ec2InstanceType.XLARGE + ", "
+ Ec2Connection.Ec2InstanceType.MEDIUM_HCPU + ", and "
+ Ec2Connection.Ec2InstanceType.XLARGE_HCPU).withRequiredArg();
+ parser.accepts("region",
+ "Region type; options are " + RegionInfo.REGIONURL_AP_SOUTHEAST + ", "
+ + RegionInfo.REGIONURL_EU_WEST + ", " + RegionInfo.REGIONURL_US_WEST
+ + ", " + RegionInfo.REGIONURL_US_EAST + " (default) ")
+ .withRequiredArg();
OptionSet options = parse(args);
String accessId = getAccessId(options);
String secretKey = getSecretKey(options);
String ami = getRequiredString(options, "ami");
String keypairId = getRequiredString(options, "keypairid");
+ String regionUrl = getRegionUrl(options);
int instanceCount = CmdUtils.valueOf(options, "instances", 1);
String securityGroups = CmdUtils.valueOf(options, "securitygroups", null);
List<String> securityGroupsList = (securityGroups != null)
@@ -82,7 +90,7 @@ public void run(String[] args) throws Exception {
printUsage();
}
- Ec2Connection ec2Connection = new TypicaEc2Connection(accessId, secretKey);
+ Ec2Connection ec2Connection = new TypicaEc2Connection(accessId, secretKey, null, regionUrl);
List<HostNamePair> hostNamePairs = ec2Connection.createInstances(ami,
keypairId,
instanceType,
View
10 contrib/ec2-testing/src/java/voldemort/utils/app/VoldemortEc2InstanceTerminatorApp.java
@@ -28,6 +28,8 @@
import voldemort.utils.HostNamePair;
import voldemort.utils.impl.TypicaEc2Connection;
+import com.xerox.amazonws.ec2.RegionInfo;
+
public class VoldemortEc2InstanceTerminatorApp extends VoldemortApp {
public static void main(String[] args) throws Exception {
@@ -54,12 +56,18 @@ public void run(String[] args) throws Exception {
.withRequiredArg();
parser.accepts("hostnames", "File containing host names").withRequiredArg();
parser.accepts("instances", "File containing instance IDs").withRequiredArg();
+ parser.accepts("region",
+ "Region type; options are " + RegionInfo.REGIONURL_AP_SOUTHEAST + ", "
+ + RegionInfo.REGIONURL_EU_WEST + ", " + RegionInfo.REGIONURL_US_WEST
+ + ", " + RegionInfo.REGIONURL_US_EAST + " (default) ")
+ .withRequiredArg();
OptionSet options = parse(args);
String accessId = getAccessId(options);
String secretKey = getSecretKey(options);
+ String regionUrl = getRegionUrl(options);
- Ec2Connection ec2Connection = new TypicaEc2Connection(accessId, secretKey);
+ Ec2Connection ec2Connection = new TypicaEc2Connection(accessId, secretKey, null, regionUrl);
List<String> hostNames = new ArrayList<String>();
File hostNamesFile = getInputFile(options, "hostnames");
View
71 contrib/ec2-testing/src/java/voldemort/utils/app/VoldemortStoresGeneratorApp.java
@@ -0,0 +1,71 @@
+package voldemort.utils.app;
+
+import java.util.List;
+
+import joptsimple.OptionSet;
+import voldemort.routing.RoutingStrategyType;
+import voldemort.utils.CmdUtils;
+import voldemort.utils.StoresGenerator;
+
+public class VoldemortStoresGeneratorApp extends VoldemortApp {
+
+ public static void main(String[] args) throws Exception {
+ new VoldemortStoresGeneratorApp().run(args);
+ }
+
+ @Override
+ protected String getScriptName() {
+ return "voldemort-storesgenerator.sh";
+ }
+
+ @Override
+ public void run(String[] args) throws Exception {
+ parser.accepts("help", "Prints this help");
+ parser.accepts("logging",
+ "Options are \"debug\", \"info\" (default), \"warn\", \"error\", or \"off\"")
+ .withRequiredArg();
+ parser.accepts("storename", "Store name; defaults to test").withRequiredArg();
+ parser.accepts("zonerepfactor",
+ "Zone specific replication factor as a list of comma separated integers")
+ .withRequiredArg()
+ .ofType(Integer.class)
+ .withValuesSeparatedBy(',');
+ parser.accepts("routing-strategy",
+ "Routing Strategy (" + RoutingStrategyType.CONSISTENT_STRATEGY
+ + " (default), " + RoutingStrategyType.ZONE_STRATEGY + ", "
+ + RoutingStrategyType.CONSISTENT_STRATEGY + ")").withRequiredArg();
+ parser.accepts("required-reads", "Required number of reads")
+ .withRequiredArg()
+ .ofType(Integer.class);
+ parser.accepts("required-writes", "Required number of writes")
+ .withRequiredArg()
+ .ofType(Integer.class);
+ parser.accepts("zone-count-read", "Number of zones to read from; default 0 ")
+ .withRequiredArg()
+ .ofType(Integer.class);
+ parser.accepts("zone-count-write", "Number of zones to write from; default 0 ")
+ .withRequiredArg()
+ .ofType(Integer.class);
+
+ OptionSet options = parse(args);
+ String storeName = CmdUtils.valueOf(options, "storename", "test");
+
+ List<Integer> zoneRepFactor = getRequiredListIntegers(options, "zonerepfactor");
+ int requiredReads = getRequiredInt(options, "required-reads");
+ int requiredWrites = getRequiredInt(options, "required-writes");
+ int zoneCountRead = CmdUtils.valueOf(options, "zone-count-read", 0);
+ int zoneCountWrite = CmdUtils.valueOf(options, "zone-count-write", 0);
+ String routingStrategy = CmdUtils.valueOf(options,
+ "routing-strategy",
+ RoutingStrategyType.CONSISTENT_STRATEGY);
+ String storesXml = new StoresGenerator().createStoreDescriptor(storeName,
+ zoneRepFactor,
+ requiredReads,
+ requiredWrites,
+ zoneCountRead,
+ zoneCountWrite,
+ routingStrategy);
+
+ System.out.print(storesXml);
+ }
+}
View
10 contrib/ec2-testing/src/java/voldemort/utils/impl/TypicaEc2Connection.java
@@ -50,12 +50,18 @@
private final Log logger = LogFactory.getLog(getClass());
public TypicaEc2Connection(String accessId, String secretKey) {
- this(accessId, secretKey, null);
+ this(accessId, secretKey, null, null);
}
- public TypicaEc2Connection(String accessId, String secretKey, Ec2ConnectionListener listener) {
+ public TypicaEc2Connection(String accessId,
+ String secretKey,
+ Ec2ConnectionListener listener,
+ String regionUrl) {
ec2 = new Jec2(accessId, secretKey);
this.listener = listener;
+
+ if(ec2 != null)
+ ec2.setRegionUrl(regionUrl);
}
public List<HostNamePair> list() throws Exception {
View
13 contrib/ec2-testing/test/voldemort/utils/Ec2RemoteTestUtils.java
@@ -36,14 +36,18 @@
return createInstances(ec2Config.getInstanceCount(), ec2Config);
}
- public static List<HostNamePair> createInstances(int instanceCount, Ec2RemoteTestConfig ec2Config) throws Exception{
+ public static List<HostNamePair> createInstances(int instanceCount,
+ Ec2RemoteTestConfig ec2Config)
+ throws Exception {
Ec2Connection ec2 = new TypicaEc2Connection(ec2Config.getAccessId(),
ec2Config.getSecretKey(),
- new Ec2Listener(ec2Config.getInstanceIdFile()));
+ new Ec2Listener(ec2Config.getInstanceIdFile()),
+ null);
return ec2.createInstances(ec2Config.getAmi(),
ec2Config.getKeyPairId(),
Ec2Connection.Ec2InstanceType.DEFAULT,
- instanceCount, null);
+ instanceCount,
+ null);
}
@@ -51,7 +55,8 @@ public static void destroyInstances(List<String> hostNames, Ec2RemoteTestConfig
throws Exception {
Ec2Connection ec2 = new TypicaEc2Connection(ec2Config.getAccessId(),
ec2Config.getSecretKey(),
- new Ec2Listener(ec2Config.getInstanceIdFile()));
+ new Ec2Listener(ec2Config.getInstanceIdFile()),
+ null);
ec2.deleteInstancesByHostName(hostNames);
}
Please sign in to comment.
Something went wrong with that request. Please try again.