Permalink
Browse files

merging voldemort master branch into server-transforms

  • Loading branch information...
2 parents 2e4df16 + eb5dcb6 commit 01b82c43e095d97c63c709fa7daa6f6f9b58f571 @nehanarkhede nehanarkhede committed Jun 12, 2010
Showing with 1,578 additions and 314 deletions.
  1. +7 −1 .classpath
  2. +1 −1 bin/run-class.sh
  3. +7 −0 clients/python/voldemort_admin_pb2.py
  4. +1 −0 contrib/hadoop-store-builder/lib/README
  5. BIN contrib/hadoop-store-builder/lib/hadoop-0.18.1-core.jar
  6. BIN contrib/hadoop-store-builder/lib/hadoop-0.20.2-core.jar
  7. +3 −1 contrib/hadoop-store-builder/src/java/voldemort/store/readonly/checksum/CheckSum.java
  8. +8 −5 contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/HadoopStoreBuilderReducer.java
  9. +20 −6 contrib/hadoop-store-builder/test/voldemort/store/readonly/mr/HadoopStoreBuilderTest.java
  10. +48 −0 contrib/hadoop/bin/hadoop-word-count.sh
  11. +52 −0 contrib/hadoop/bin/voldemort-pig.sh
  12. BIN contrib/hadoop/lib/pig-0.7.1-dev-core.jar
  13. +46 −0 contrib/hadoop/src/java/voldemort/hadoop/VoldemortHadoopConfig.java
  14. +81 −0 contrib/hadoop/src/java/voldemort/hadoop/VoldemortInputFormat.java
  15. +97 −0 contrib/hadoop/src/java/voldemort/hadoop/VoldemortInputSplit.java
  16. +85 −0 contrib/hadoop/src/java/voldemort/hadoop/VoldemortRecordReader.java
  17. +90 −0 contrib/hadoop/src/java/voldemort/hadoop/pig/VoldemortStore.java
  18. +186 −0 contrib/hadoop/test/voldemort/hadoop/VoldemortWordCount.java
  19. +10 −0 contrib/krati/config/single_node_cluster/config/cluster.xml
  20. +29 −0 contrib/krati/config/single_node_cluster/config/server.properties
  21. +16 −0 contrib/krati/config/single_node_cluster/config/stores.xml
  22. BIN contrib/krati/lib/krati-0.3.3.jar
  23. +53 −0 contrib/krati/src/java/voldemort/store/krati/KratiStorageConfiguration.java
  24. +312 −0 contrib/krati/src/java/voldemort/store/krati/KratiStorageEngine.java
  25. +31 −0 contrib/krati/test/voldemort/store/krati/KratiStorageEngineTest.java
  26. +3 −2 src/java/voldemort/VoldemortAdminTool.java
  27. +20 −5 src/java/voldemort/client/protocol/admin/AdminClient.java
  28. +29 −0 src/java/voldemort/client/protocol/admin/filter/MasterOnlyVoldemortFilter.java
  29. +106 −66 src/java/voldemort/client/protocol/pb/VAdminProto.java
  30. +66 −63 src/java/voldemort/server/protocol/admin/AdminServiceRequestHandler.java
  31. +12 −4 src/java/voldemort/server/protocol/admin/FetchStreamRequestHandler.java
  32. +3 −3 src/java/voldemort/server/rebalance/RebalancerState.java
  33. +1 −1 src/java/voldemort/server/storage/StorageService.java
  34. +2 −0 src/java/voldemort/store/readonly/ChunkedFileSet.java
  35. +93 −128 src/java/voldemort/store/rebalancing/RedirectingStore.java
  36. +2 −2 src/java/voldemort/utils/VoldemortAdminClientShell.java
  37. +1 −0 src/proto/voldemort-admin.proto
  38. +2 −1 test/integration/voldemort/performance/AdminTest.java
  39. +40 −18 test/integration/voldemort/performance/ReadOnlyStorePerformanceTest.java
  40. +2 −1 test/unit/voldemort/client/AbstractAdminServiceFilterTest.java
  41. +4 −2 test/unit/voldemort/client/AdminServiceBasicTest.java
  42. +7 −2 test/unit/voldemort/client/AdminServiceFailureTest.java
  43. +1 −1 test/unit/voldemort/client/rebalance/RebalanceTest.java
  44. +1 −1 test/unit/voldemort/server/gossip/GossiperTest.java
View
@@ -13,6 +13,10 @@
<classpathentry kind="src" path="test/integration"/>
<classpathentry kind="src" path="test/common"/>
<classpathentry kind="src" path="example/java"/>
+ <classpathentry kind="src" path="contrib/krati/src/java"/>
+ <classpathentry kind="src" path="contrib/krati/test"/>
+ <classpathentry kind="src" path="contrib/hadoop/test"/>
+ <classpathentry kind="src" path="contrib/hadoop/src/java"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
<classpathentry kind="lib" path="lib/catalina-ant.jar"/>
<classpathentry kind="lib" path="lib/commons-codec-1.3.jar"/>
@@ -21,7 +25,7 @@
<classpathentry kind="lib" path="lib/colt-1.2.0.jar"/>
<classpathentry kind="lib" path="contrib/mongodb/lib/mongo-xjdm.jar"/>
<classpathentry kind="lib" path="contrib/hadoop-store-builder/lib/commons-cli-2.0-SNAPSHOT.jar"/>
- <classpathentry kind="lib" path="contrib/hadoop-store-builder/lib/hadoop-0.18.1-core.jar"/>
+ <classpathentry kind="lib" path="contrib/hadoop-store-builder/lib/hadoop-0.20.2-core.jar"/>
<classpathentry kind="lib" path="lib/junit-4.6.jar"/>
<classpathentry kind="lib" path="lib/log4j-1.2.15.jar"/>
<classpathentry kind="lib" path="lib/jetty-6.1.18.jar"/>
@@ -48,5 +52,7 @@
<classpathentry kind="lib" path="lib/jackson-mapper-asl-1.4.0.jar"/>
<classpathentry kind="lib" path="lib/jackson-core-asl-1.4.0.jar"/>
<classpathentry kind="lib" path="lib/avro-modified-jdk5-1.3.0.jar"/>
+ <classpathentry kind="lib" path="contrib/krati/lib/krati-0.3.3.jar"/>
+ <classpathentry kind="lib" path="contrib/hadoop/lib/pig-0.7.1-dev-core.jar"/>
<classpathentry kind="output" path="classes"/>
</classpath>
View
@@ -39,4 +39,4 @@ if [ -z "$VOLD_OPTS" ]; then
fi
export CLASSPATH
-java -Dlog4j.configuration=src/java/log4j.properties $VOLD_OPTS -cp $CLASSPATH $@
+java -Dlog4j.configuration=$base_dir/src/java/log4j.properties $VOLD_OPTS -cp $CLASSPATH $@
@@ -331,6 +331,13 @@
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
+ descriptor.FieldDescriptor(
+ name='fetch_master_entries', full_name='voldemort.FetchPartitionEntriesRequest.fetch_master_entries', index=4,
+ number=5, type=8, cpp_type=7, label=1,
+ default_value=False,
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
],
extensions=[
],
@@ -0,0 +1 @@
+Java 5 users would need to exclude Hadoop 0.20.2 manually while replacing it with Hadoop 0.18.*
Binary file not shown.
Binary file not shown.
@@ -100,8 +100,10 @@ public static String toString(CheckSumType type) {
return "crc32";
} else if(type == CheckSumType.MD5) {
return "md5";
+ } else if(type == CheckSumType.NONE) {
+ return "none";
}
- return null;
+ return "none";
}
public static CheckSumType fromString(String input) {
@@ -81,17 +81,20 @@ public void reduce(BytesWritable key,
// Write key and position
this.indexFileStream.write(key.get(), 0, key.getSize());
- this.checkSumDigestIndex.update(key.get(), 0, key.getSize());
this.indexFileStream.writeInt(this.position);
- this.checkSumDigestIndex.update(this.position);
+ if(this.checkSumDigestIndex != null) {
+ this.checkSumDigestIndex.update(key.get(), 0, key.getSize());
+ this.checkSumDigestIndex.update(this.position);
+ }
// Write length and value
int valueLength = writable.getSize() - 4;
this.valueFileStream.writeInt(valueLength);
- this.checkSumDigestValue.update(valueLength);
this.valueFileStream.write(valueBytes, 4, valueLength);
- this.checkSumDigestValue.update(valueBytes, 4, valueLength);
-
+ if(this.checkSumDigestValue != null) {
+ this.checkSumDigestValue.update(valueLength);
+ this.checkSumDigestValue.update(valueBytes, 4, valueLength);
+ }
this.position += 4 + valueLength;
if(this.position < 0)
throw new VoldemortException("Chunk overflow exception: chunk " + chunkId
@@ -82,8 +82,8 @@ public void testHadoopBuild() throws Exception {
// create test data
Map<String, String> values = new HashMap<String, String>();
File testDir = TestUtils.createTempDir();
- File tempDir = new File(testDir, "temp");
- File outputDir = new File(testDir, "output");
+ File tempDir = new File(testDir, "temp"), tempDir2 = new File(testDir, "temp2");
+ File outputDir = new File(testDir, "output"), outputDir2 = new File(testDir, "output2");
File storeDir = TestUtils.createTempDir(testDir);
for(int i = 0; i < 200; i++)
values.put(Integer.toString(i), Integer.toBinaryString(i));
@@ -99,6 +99,8 @@ public void testHadoopBuild() throws Exception {
String storeName = "test";
SerializerDefinition serDef = new SerializerDefinition("string");
Cluster cluster = ServerTestUtils.getLocalCluster(1);
+
+ // Test backwards compatibility
StoreDefinition def = new StoreDefinitionBuilder().setName(storeName)
.setType(ReadOnlyStorageConfiguration.TYPE_NAME)
.setKeySerializer(serDef)
@@ -118,10 +120,22 @@ public void testHadoopBuild() throws Exception {
def,
2,
64 * 1024,
- new Path(tempDir.getAbsolutePath()),
- new Path(outputDir.getAbsolutePath()),
- new Path(inputFile.getAbsolutePath()),
- CheckSumType.MD5);
+ new Path(tempDir2.getAbsolutePath()),
+ new Path(outputDir2.getAbsolutePath()),
+ new Path(inputFile.getAbsolutePath()));
+ builder.build();
+
+ builder = new HadoopStoreBuilder(new Configuration(),
+ TextStoreMapper.class,
+ TextInputFormat.class,
+ cluster,
+ def,
+ 2,
+ 64 * 1024,
+ new Path(tempDir.getAbsolutePath()),
+ new Path(outputDir.getAbsolutePath()),
+ new Path(inputFile.getAbsolutePath()),
+ CheckSumType.MD5);
builder.build();
// Check if checkSum is generated in outputDir
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+#
+# Copyright 2010 LinkedIn, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+if [ $# -ne 2 ];
+then
+ echo 'USAGE: bin/hadoop-word-count.sh store_name admin_bootstrap_url'
+ exit 1
+fi
+
+base_dir=$(dirname $0)/../../..
+
+for file in $base_dir/dist/*.jar;
+do
+ HADOOP_CLASSPATH=$HADOOP_CLASSPATH:$file
+done
+
+for file in $base_dir/lib/*.jar;
+do
+ HADOOP_CLASSPATH=$HADOOP_CLASSPATH:$file
+done
+
+if [ -n "${HADOOP_HOME}" ];
+then
+ HADOOP_CLASSPATH=$HADOOP_CLASSPATH:$HADOOP_HOME
+else
+ echo 'Missing HADOOP_HOME'
+ exit 1
+fi
+
+HADOOP_CLASSPATH=$HADOOP_CLASSPATH:$base_dir/dist/resources:$VOLDEMORT_HOME/lib/
+export HADOOP_CLASSPATH
+
+$HADOOP_HOME/bin/hadoop voldemort.hadoop.VoldemortWordCount $@
@@ -0,0 +1,52 @@
+#!/bin/bash
+
+#
+# Copyright 2010 LinkedIn, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+base_dir=$(dirname $0)/../../..
+
+CLASSPATH=$home_dir/common/hadoop-conf/dev/conf:$home_dir/dist/jobs/
+
+for file in $base_dir/dist/*.jar;
+do
+ CLASSPATH=$CLASSPATH:$file
+done
+
+for file in $base_dir/lib/*.jar;
+do
+ CLASSPATH=$CLASSPATH:$file
+done
+
+for file in $base_dir/contrib/hadoop/lib/*.jar;
+do
+ CLASSPATH=$CLASSPATH:$file
+done
+
+if [ "x$PIG_HOME" = "x" ]; then
+ echo 'Please set PIG_HOME'
+ exit 1
+fi
+
+JAR=$PIG_HOME/pig.jar
+if [ ! -e $JAR ]; then
+ echo 'Could not find Pig jar'
+ exit 1
+fi
+CLASSPATH=$CLASSPATH:$JAR
+
+export CLASSPATH
+
+java $PIG_JAVA_OPTS -Dudf.import.list=voldemort.hadoop.pig -cp $CLASSPATH org.apache.pig.Main $*
Binary file not shown.
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2010 LinkedIn, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package voldemort.hadoop;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Common class shared by InputFormat
+ *
+ */
+public class VoldemortHadoopConfig {
+
+ private static final String URL = "voldemort.url";
+ private static final String STORE_NAME = "voldemort.store.name";
+
+ public static void setVoldemortURL(Configuration conf, String url) {
+ conf.set(URL, url);
+ }
+
+ public static void setVoldemortStoreName(Configuration conf, String storeName) {
+ conf.set(STORE_NAME, storeName);
+ }
+
+ public static String getVoldemortURL(Configuration conf) {
+ return conf.get(URL);
+ }
+
+ public static String getVoldemortStoreName(Configuration conf) {
+ return conf.get(STORE_NAME);
+ }
+
+}
@@ -0,0 +1,81 @@
+/*
+ * Copyright 2010 LinkedIn, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package voldemort.hadoop;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+
+import voldemort.client.protocol.admin.AdminClient;
+import voldemort.client.protocol.admin.AdminClientConfig;
+import voldemort.cluster.Cluster;
+import voldemort.cluster.Node;
+import voldemort.utils.ByteArray;
+import voldemort.versioning.Versioned;
+
+public class VoldemortInputFormat extends InputFormat<ByteArray, Versioned<byte[]>> {
+
+ /**
+ * Create a new connection to admin client and give it to RecordReader.
+ * Called on the TaskTracker
+ */
+ @Override
+ public RecordReader<ByteArray, Versioned<byte[]>> createRecordReader(InputSplit currentSplit,
+ TaskAttemptContext taskContext)
+ throws IOException, InterruptedException {
+ return new VoldemortRecordReader();
+ }
+
+ /**
+ * One mapper for every node. Every InputSplit then connects to the
+ * particular node. Called on JobClient
+ */
+ @Override
+ public List<InputSplit> getSplits(JobContext context) throws IOException, InterruptedException {
+ Configuration conf = context.getConfiguration();
+ String bootstrapURL = VoldemortHadoopConfig.getVoldemortURL(conf);
+ String storeName = VoldemortHadoopConfig.getVoldemortStoreName(conf);
+
+ AdminClient adminClient = new AdminClient(bootstrapURL, new AdminClientConfig());
+
+ /**
+ * TODO: To put check to see if store exists
+ */
+ Cluster cluster = adminClient.getAdminClientCluster();
+ Collection<Node> nodes = cluster.getNodes();
+ Iterator<Node> nodeIter = nodes.iterator();
+
+ List<InputSplit> splits = new ArrayList<InputSplit>();
+ while(nodeIter.hasNext()) {
+ Node currentNode = nodeIter.next();
+ VoldemortInputSplit split = new VoldemortInputSplit(storeName, currentNode);
+ splits.add(split);
+ }
+
+ adminClient.stop();
+ return splits;
+ }
+}
Oops, something went wrong.

0 comments on commit 01b82c4

Please sign in to comment.