Permalink
Browse files

Improved AdminTool, RebalanceUtils.

	* Added a shell script to invoke AdminTool.
	* AdminTool: no longer require store name.
	* RebalanceUtils: change RebalanceUtils.propagateCluster
	  to allow propagation to *some* of nodes in a cluster, vs.
	  all nodes in a cluster.
  • Loading branch information...
1 parent d6d487d commit f49223d4a558db1dc81add8e7b3c08ce4a6e3be2 @afeinberg afeinberg committed Mar 15, 2010
Showing with 101 additions and 23 deletions.
  1. +21 −0 bin/voldemort-admin-tool.sh
  2. +43 −13 src/java/voldemort/VoldemortAdminTool.java
  3. +37 −10 src/java/voldemort/utils/RebalanceUtils.java
View
21 bin/voldemort-admin-tool.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+#
+# Copyright 2008-2010 LinkedIn, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+base_dir=$(dirname $0)/..
+
+$base_dir/bin/run-class.sh voldemort.VoldemortAdminTool $@
View
56 src/java/voldemort/VoldemortAdminTool.java
@@ -1,15 +1,42 @@
+/*
+ * Copyright 2008-2010 LinkedIn, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
package voldemort;
import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
import joptsimple.OptionParser;
import joptsimple.OptionSet;
+import voldemort.annotations.concurrency.Immutable;
import voldemort.client.protocol.admin.AdminClient;
import voldemort.client.protocol.admin.AdminClientConfig;
+import voldemort.cluster.Cluster;
+import voldemort.cluster.Node;
import voldemort.store.StoreDefinition;
import voldemort.utils.CmdUtils;
+import voldemort.utils.Pair;
+import voldemort.utils.RebalanceUtils;
import voldemort.utils.Utils;
+import voldemort.versioning.VectorClock;
+import voldemort.versioning.Versioned;
+import voldemort.xml.ClusterMapper;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import java.util.Set;
@@ -47,7 +74,6 @@ public static void main (String [] args) throws Exception {
Set<String> missing = CmdUtils.missing(options,
"url",
- "store",
"node");
if (missing.size() > 0) {
System.err.println("Missing required arguments: " + Joiner.on(", ").join(missing));
@@ -59,6 +85,8 @@ public static void main (String [] args) throws Exception {
Integer nodeId = (Integer) options.valueOf("node");
Integer parallelism = CmdUtils.valueOf(options, "parallelism", 5);
+ AdminClient adminClient = new AdminClient(url, new AdminClientConfig());
+
String ops = "";
if (options.has("delete-partitions")) {
ops += "d";
@@ -67,26 +95,16 @@ public static void main (String [] args) throws Exception {
ops += "r";
}
if (ops.length() < 1) {
- Utils.croak("At least one of (delete-partitions, restore) must be specified");
+ Utils.croak("At least one of (delete-partitions, restore, add-node) must be specified");
}
- AdminClient adminClient = new AdminClient(url, new AdminClientConfig());
-
try {
if (ops.contains("d")) {
if (options.has("partition-ids")) {
System.out.println("Starting delete-partitions");
@SuppressWarnings("unchecked")
List<Integer> partitionIdList = (List<Integer>) options.valuesOf("partition-ids");
- List<StoreDefinition> storeDefinitionList = adminClient.getRemoteStoreDefList(nodeId).getValue();
- List<String> storeNames = new ArrayList<String>();
- for (StoreDefinition storeDefinition: storeDefinitionList) {
- storeNames.add(storeDefinition.getName());
- }
- for (String storeName: storeNames) {
- System.out.println("Deleting partitions " + Joiner.on(", ").join(partitionIdList) + " of " + storeName);
- adminClient.deletePartitions(nodeId, storeName, partitionIdList, null);
- }
+ executeDeletePartitions(nodeId, adminClient, partitionIdList);
System.out.println("Finished delete-partitions");
} else {
System.err.println("Not running delete-partitions: partition-ids must be specified when delete-partitions is invoked");
@@ -102,4 +120,16 @@ public static void main (String [] args) throws Exception {
Utils.croak(e.getMessage());
}
}
+
+ public static void executeDeletePartitions(Integer nodeId, AdminClient adminClient, List<Integer> partitionIdList) {
+ List<StoreDefinition> storeDefinitionList = adminClient.getRemoteStoreDefList(nodeId).getValue();
+ List<String> storeNames = new ArrayList<String>();
+ for (StoreDefinition storeDefinition: storeDefinitionList) {
+ storeNames.add(storeDefinition.getName());
+ }
+ for (String storeName: storeNames) {
+ System.out.println("Deleting partitions " + Joiner.on(", ").join(partitionIdList) + " of " + storeName);
+ adminClient.deletePartitions(nodeId, storeName, partitionIdList, null);
+ }
+ }
}
View
47 src/java/voldemort/utils/RebalanceUtils.java
@@ -177,28 +177,55 @@ private static void checkNotConcurrent(ArrayList<Versioned<Cluster>> clockList,
}
/**
- * propagate the cluster configuration to all nodes.<br>
- * throws an exception if failed to propagate on any of the required nodes.
- *
- * @param adminClient
- * @param masterNodeId
- * @param cluster
+ * Attempt to propagate cluster definition to all nodes in the cluster.
+ *
+ * @throws VoldemortException If we can't propagate to a list of require nodes.
+ * @param adminClient {@link voldemort.client.protocol.admin.AdminClient} instance to use
+ * @param cluster Cluster definition we wish to propagate
+ * @param clock Vector clock to attach to the cluster definition
+ * @param requireNodeIds If we can't propagate to these node ids, roll back and throw an exception
*/
public static void propagateCluster(AdminClient adminClient,
Cluster cluster,
VectorClock clock,
+ List<Integer> requireNodeIds) {
+ List<Integer> allNodeIds = new ArrayList<Integer>();
+ for (Node node: cluster.getNodes()) {
+ allNodeIds.add(node.getId());
+ }
+ propagateCluster(adminClient,
+ cluster,
+ clock,
+ allNodeIds,
+ requireNodeIds);
+ }
+
+ /**
+ * Attempt to propagate a cluster definition to specified nodes.
+ *
+ * @throws VoldemortException If we can't propagate to a list of require nodes.
+ * @param adminClient {@link voldemort.client.protocol.admin.AdminClient} instance to use.
+ * @param cluster Cluster definition we wish to propagate
+ * @param clock Vector clock to attach to the cluster definition
+ * @param attemptNodeIds Attempt to propagate to these node ids
+ * @param requiredNodeIds If we can't propagate can't propagate to these node ids, roll back and throw an exception
+ */
+ public static void propagateCluster(AdminClient adminClient,
+ Cluster cluster,
+ VectorClock clock,
+ List<Integer> attemptNodeIds,
List<Integer> requiredNodeIds) {
List<Integer> failures = new ArrayList<Integer>();
// copy everywhere else first
- for(Node node: cluster.getNodes()) {
- if(!requiredNodeIds.contains(node.getId())) {
+ for(int nodeId: attemptNodeIds) {
+ if(!requiredNodeIds.contains(nodeId)) {
try {
- adminClient.updateRemoteCluster(node.getId(), cluster, clock);
+ adminClient.updateRemoteCluster(nodeId, cluster, clock);
} catch(VoldemortException e) {
// ignore these
logger.debug("Failed to copy new cluster.xml(" + cluster
- + ") on non-required node:" + node, e);
+ + ") on non-required node:" + nodeId, e);
}
}
}

0 comments on commit f49223d

Please sign in to comment.