Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Partition-rebalance subTask 2: checked in.

  • Loading branch information...
commit 6010d30e8560a6383592cf39a0e0177f89d80e25 1 parent e1decd9
@bbansal bbansal authored
View
44 src/java/voldemort/store/InvalidMetadataException.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2008-2009 LinkedIn, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package voldemort.store;
+
+import voldemort.VoldemortException;
+
+/**
+ * Store should throw this Exception to indicate that the metadata (Cluster configuration/ Stores Configuration) at client is not in sync with this store metadata.
+ *
+ * @author bbansal
+ *
+ */
+public class InvalidMetadataException extends VoldemortException {
+
+ private static final long serialVersionUID = 1L;
+
+ public InvalidMetadataException(String s) {
+ super(s);
+ }
+
+ public InvalidMetadataException(String s, Throwable t) {
+ super(s, t);
+ }
+
+ @Override
+ public short getId() {
+ return 9;
+ }
+
+}
View
16 src/java/voldemort/store/StoreUtils.java
@@ -26,9 +26,12 @@
import org.apache.log4j.Logger;
+import voldemort.cluster.Node;
+import voldemort.routing.RoutingStrategy;
import voldemort.serialization.Serializer;
import voldemort.serialization.SerializerDefinition;
import voldemort.serialization.SerializerFactory;
+import voldemort.utils.ByteArray;
import voldemort.versioning.Versioned;
import com.google.common.collect.Maps;
@@ -111,6 +114,19 @@ public static void close(InputStream stream) {
}
}
+ public static void assertValidMetadata(ByteArray key,
+ RoutingStrategy routingStrategy,
+ int currentNodeId) {
+ List<Node> nodes = routingStrategy.routeRequest(key.get());
+ for(Node node: nodes) {
+ if(node.getId() == currentNodeId) {
+ return;
+ }
+ }
+
+ throw new InvalidMetadataException("client routing strategy not in sync with store routing strategy!");
+ }
+
/**
* This is a temporary measure until we have a type-safe solution for
* retrieving serializers from a SerializerFactory. It avoids warnings all
View
89 src/java/voldemort/store/rebalancing/InvalidMetadataCheckingStore.java
@@ -0,0 +1,89 @@
+/*
+ * Copyright 2008-2009 LinkedIn, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package voldemort.store.rebalancing;
+
+import java.util.List;
+
+import voldemort.VoldemortException;
+import voldemort.routing.RoutingStrategy;
+import voldemort.store.DelegatingStore;
+import voldemort.store.InvalidMetadataException;
+import voldemort.store.Store;
+import voldemort.store.StoreUtils;
+import voldemort.utils.ByteArray;
+import voldemort.versioning.Version;
+import voldemort.versioning.Versioned;
+
+/**
+ * An InvalidMetadataCheckingStore store is a store wrapper that delegates to an
+ * inner store, and throws {@link InvalidMetadataException} if a client requests
+ * a partition which is not or should not be available at this node.
+ *
+ * @author bbansal
+ *
+ */
+public class InvalidMetadataCheckingStore extends DelegatingStore<ByteArray, byte[]> {
+
+ private final int node;
+ private final RoutingStrategy routingStratgey;
+
+ /**
+ * Create a store which delegates its operations to its inner store and
+ * throws {@link InvalidMetadataException} if the partition for key
+ * requested should not lie with this node.
+ *
+ * @param node The id of the destination node
+ * @param innerStore The store which we delegate write operations to
+ * @param routingStrategy: the routing stratgey for this cluster
+ * configuration.
+ */
+ public InvalidMetadataCheckingStore(int node,
+ Store<ByteArray, byte[]> innerStore,
+ RoutingStrategy routingStrategy) {
+ super(innerStore);
+ this.node = node;
+ routingStratgey = routingStrategy;
+ }
+
+ @Override
+ public boolean delete(ByteArray key, Version version) throws VoldemortException {
+ StoreUtils.assertValidKey(key);
+ StoreUtils.assertValidMetadata(key, routingStratgey, node);
+
+ return getInnerStore().delete(key, version);
+ }
+
+ @Override
+ public void put(ByteArray key, Versioned<byte[]> value) throws VoldemortException {
+ StoreUtils.assertValidKey(key);
+ StoreUtils.assertValidMetadata(key, routingStratgey, node);
+
+ getInnerStore().put(key, value);
+ }
+
+ @Override
+ public List<Versioned<byte[]>> get(ByteArray key) throws VoldemortException {
+ StoreUtils.assertValidKey(key);
+ StoreUtils.assertValidMetadata(key, routingStratgey, node);
+
+ return getInnerStore().get(key);
+ }
+
+ public RoutingStrategy getRoutingStrategy() {
+ return routingStratgey;
+ }
+}
View
71 test/common/voldemort/TestUtils.java
@@ -23,13 +23,18 @@
import java.io.FileWriter;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Random;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import junit.framework.Assert;
import voldemort.client.RoutingTier;
import voldemort.cluster.Cluster;
+import voldemort.cluster.Node;
import voldemort.routing.ConsistentRoutingStrategy;
import voldemort.routing.RoutingStrategy;
import voldemort.serialization.SerializerDefinition;
@@ -281,4 +286,70 @@ public static String createReadOnlyIndex(Cluster cluster,
return dataDir.getAbsolutePath();
}
+ public static RoutingStrategy getRoutingStrategy(int[][] partitionMap, int numReplication) {
+ return new ConsistentRoutingStrategy(createNodes(partitionMap), numReplication);
+ }
+
+ public static Collection<Node> createNodes(int[][] partitionMap) {
+ ArrayList<Node> nodes = new ArrayList<Node>(partitionMap.length);
+ ArrayList<Integer> partitionList = new ArrayList<Integer>();
+
+ for(int i = 0; i < partitionMap.length; i++) {
+ partitionList.clear();
+ for(int p = 0; p < partitionMap[i].length; p++) {
+ partitionList.add(partitionMap[i][p]);
+ }
+ nodes.add(new Node(i, "localhost", 8880 + i, 6666 + i, partitionList));
+ }
+
+ return nodes;
+ }
+
+ public static void checkClusterMatch(Cluster A, Cluster B) {
+ Assert.assertEquals("num nodes do not match.", A.getNodes().size(), B.getNodes().size());
+
+ ArrayList<Node> nodeAList = new ArrayList<Node>(A.getNodes());
+
+ for(int i = 0; i < A.getNodes().size(); i++) {
+ Node nodeA = nodeAList.get(i);
+ Node nodeB = B.getNodeById(nodeA.getId());
+ Assert.assertEquals("NodeId do not match", nodeA.getId(), nodeB.getId());
+ Assert.assertEquals("num partitions for Node:" + nodeA.getId() + " Do not match",
+ nodeA.getNumberOfPartitions(),
+ nodeB.getNumberOfPartitions());
+
+ for(int j = 0; j < nodeA.getNumberOfPartitions(); j++) {
+ Assert.assertEquals("partitionList do not match",
+ nodeA.getPartitionIds(),
+ nodeB.getPartitionIds());
+ }
+ }
+
+ }
+
+ public static int getPartitionsDiff(Cluster orig, Cluster updated) {
+ int diffPartition = 0;
+ ArrayList<Node> nodeAList = new ArrayList<Node>(orig.getNodes());
+
+ for(int i = 0; i < orig.getNodes().size(); i++) {
+ Node nodeA = nodeAList.get(i);
+ Node nodeB;
+ try {
+ nodeB = updated.getNodeById(nodeA.getId());
+ } catch(VoldemortException e) {
+ // add the partition in this node
+ diffPartition += nodeA.getNumberOfPartitions();
+ continue;
+ }
+
+ SortedSet<Integer> BpartitonSet = new TreeSet<Integer>(nodeB.getPartitionIds());
+ for(int p: nodeA.getPartitionIds()) {
+ if(!BpartitonSet.contains(new Integer(p))) {
+ diffPartition++;
+ }
+
+ }
+ }
+ return diffPartition;
+ }
}
View
67 test/common/voldemort/store/DoNothingStore.java
@@ -0,0 +1,67 @@
+/*
+ * Copyright 2008-2009 LinkedIn, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package voldemort.store;
+
+import java.util.List;
+import java.util.Map;
+
+import voldemort.VoldemortException;
+import voldemort.utils.Utils;
+import voldemort.versioning.Version;
+import voldemort.versioning.Versioned;
+
+/**
+ * A store that does no Harm :)
+ *
+ * @author bbansal
+ *
+ */
+public class DoNothingStore<K, V> implements Store<K, V> {
+
+ private final String name;
+
+ public DoNothingStore(String name) {
+ this.name = Utils.notNull(name);
+ }
+
+ public void close() throws VoldemortException {
+ // Do nothing;
+ }
+
+ public List<Versioned<V>> get(K key) throws VoldemortException {
+ // do nothing
+ return null;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public boolean delete(K key, Version value) throws VoldemortException {
+ // Do nothing
+ return true;
+ }
+
+ public void put(K key, Versioned<V> value) throws VoldemortException {
+ // Do nothing
+ }
+
+ public Map<K, List<Versioned<V>>> getAll(Iterable<K> keys) throws VoldemortException {
+ return null;
+ }
+
+}
View
130 test/unit/voldemort/store/rebalancing/InvalidMetadataCheckingStoreTest.java
@@ -0,0 +1,130 @@
+/*
+ * Copyright 2008-2009 LinkedIn, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package voldemort.store.rebalancing;
+
+import java.util.List;
+
+import junit.framework.TestCase;
+import voldemort.TestUtils;
+import voldemort.cluster.Node;
+import voldemort.routing.RoutingStrategy;
+import voldemort.store.DoNothingStore;
+import voldemort.store.InvalidMetadataException;
+import voldemort.store.Store;
+import voldemort.utils.ByteArray;
+import voldemort.utils.ByteUtils;
+import voldemort.versioning.Versioned;
+
+/**
+ * @author bbansal
+ *
+ */
+public class InvalidMetadataCheckingStoreTest extends TestCase {
+
+ private static int LOOP_COUNT = 1000;
+
+ public void testValidMetaData() {
+ RoutingStrategy routingStrategy = TestUtils.getRoutingStrategy(new int[][] {
+ { 0, 1, 2, 3 }, { 4, 5, 6, 7 }, { 8, 9, 10, 11 } }, 1);
+ InvalidMetadataCheckingStore store = new InvalidMetadataCheckingStore(0,
+ new DoNothingStore<ByteArray, byte[]>("test"),
+ routingStrategy);
+
+ try {
+ doOperations(0, store, store.getRoutingStrategy());
+ } catch(InvalidMetadataException e) {
+ throw new RuntimeException("Should not see any InvalidMetaDataException", e);
+ }
+ }
+
+ /**
+ * NOTE: the total number of partitions should remain same for hash
+ * consistency
+ */
+ public void testAddingPartition() {
+ RoutingStrategy routingStrategy = TestUtils.getRoutingStrategy(new int[][] {
+ { 0, 1, 2, 3 }, { 4, 5, 6, 7 }, { 8, 9, 10, 11 } }, 1);
+
+ RoutingStrategy updatedRoutingStrategy = TestUtils.getRoutingStrategy(new int[][] {
+ { 0, 1, 2, 3, 11 }, { 4, 5, 6, 7 }, { 8, 9, 10 } }, 1);
+
+ InvalidMetadataCheckingStore store = new InvalidMetadataCheckingStore(0,
+ new DoNothingStore<ByteArray, byte[]>("test"),
+ updatedRoutingStrategy);
+ try {
+ doOperations(0, store, routingStrategy);
+ } catch(InvalidMetadataException e) {
+ throw new RuntimeException("Should not see any InvalidMetaDataException", e);
+ }
+ }
+
+ public void testRemovingPartition() {
+ boolean sawException = false;
+ RoutingStrategy routingStrategy = TestUtils.getRoutingStrategy(new int[][] {
+ { 0, 1, 2, 3 }, { 4, 5, 6, 7 }, { 8, 9, 10, 11 } }, 1);
+
+ RoutingStrategy updatedRoutingStrategy = TestUtils.getRoutingStrategy(new int[][] {
+ { 0, 1, 2 }, { 4, 5, 6, 7, 3 }, { 8, 9, 10, 11 } }, 1);
+
+ InvalidMetadataCheckingStore store = new InvalidMetadataCheckingStore(0,
+ new DoNothingStore<ByteArray, byte[]>("test"),
+ updatedRoutingStrategy);
+ try {
+ doOperations(0, store, routingStrategy);
+ } catch(InvalidMetadataException e) {
+ sawException = true;
+ }
+
+ assertEquals("Should see InvalidMetaDataException", true, sawException);
+ }
+
+ private boolean containsNodeId(List<Node> nodes, int nodeId) {
+ for(Node node: nodes) {
+ if(nodeId == node.getId()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private void doOperations(int nodeId,
+ Store<ByteArray, byte[]> store,
+ RoutingStrategy routingStrategy) {
+ for(int i = 0; i < LOOP_COUNT;) {
+ ByteArray key = new ByteArray(ByteUtils.md5(Integer.toString((int) (Math.random() * Integer.MAX_VALUE))
+ .getBytes()));
+ byte[] value = "value".getBytes();
+
+ if(containsNodeId(routingStrategy.routeRequest(key.get()), nodeId)) {
+ i++; // increment count
+ switch(i % 3) {
+ case 0:
+ store.get(key);
+ break;
+ case 1:
+ store.delete(key, null);
+ break;
+ case 2:
+ store.put(key, new Versioned<byte[]>(value));
+ break;
+
+ }
+ }
+ }
+
+ }
+}
Please sign in to comment.
Something went wrong with that request. Please try again.