Permalink
Browse files

Add support for protocol buffers based network format. Add new client…

… config to avoid crazy constructors on the client. Add alpha support for server-side routing (still needs support on the client, though).
  • Loading branch information...
1 parent 5ba2b99 commit c49df1b09aa617de5f0180ae2da943313c4d7390 @jkreps jkreps committed Apr 27, 2009
Showing with 8,649 additions and 1,516 deletions.
  1. +1 −0 .classpath
  2. +15 −1 NOTES
  3. +1 −1 bin/run-class.sh
  4. +2 −1 build.properties
  5. +23 −10 build.xml
  6. +16 −16 contrib/batch-indexer/src/java/voldemort/contrib/batchindexer/ReadOnlyBatchIndexMapper.java
  7. +28 −28 contrib/batch-indexer/src/java/voldemort/contrib/batchindexer/ReadOnlyBatchIndexReducer.java
  8. +1 −1 contrib/batch-indexer/test/voldemort/contrib/batchindexer/performance/BdbBuildPerformanceTest.java
  9. +1 −1 contrib/batch-indexer/test/voldemort/contrib/batchindexer/performance/MysqlBuildPerformanceTest.java
  10. +2 −2 contrib/batch-indexer/test/voldemort/contrib/batchswapper/ReadOnlyHadoopSwapperTest.java
  11. +2 −2 contrib/batch-indexer/test/voldemort/contrib/batchswapper/ReadOnlySimpleSwapperTest.java
  12. +41 −63 contrib/mongodb/example/MongoDBClient.java
  13. +31 −31 contrib/mongodb/src/java/voldemort/store/mongodb/MongoDBStorageEngine.java
  14. +7 −7 contrib/mongodb/test/voldemort/store/mongodb/MongoDBStorageEngineTest.java
  15. +2 −10 example/java/voldemort/examples/ClientExample.java
  16. +21 −8 src/java/voldemort/VoldemortClientShell.java
  17. +57 −33 src/java/voldemort/client/AbstractStoreClientFactory.java
  18. +278 −0 src/java/voldemort/client/ClientConfig.java
  19. +44 −0 src/java/voldemort/client/ClientThreadPool.java
  20. +25 −70 src/java/voldemort/client/HttpStoreClientFactory.java
  21. +24 −136 src/java/voldemort/client/SocketStoreClientFactory.java
  22. +54 −0 src/java/voldemort/client/protocol/RequestFormat.java
  23. +27 −0 src/java/voldemort/client/protocol/RequestFormatFactory.java
  24. +30 −0 src/java/voldemort/client/protocol/RequestFormatType.java
  25. +5 −0 src/java/voldemort/client/protocol/package.html
  26. +141 −0 src/java/voldemort/client/protocol/pb/ProtoBuffClientRequestFormat.java
  27. +75 −0 src/java/voldemort/client/protocol/pb/ProtoUtils.java
  28. +5,141 −0 src/java/voldemort/client/protocol/pb/VProto.java
  29. +5 −0 src/java/voldemort/client/protocol/pb/package.html
  30. +153 −0 src/java/voldemort/client/protocol/vold/VoldemortNativeClientRequestFormat.java
  31. +13 −1 src/java/voldemort/cluster/Cluster.java
  32. +1 −0 src/java/voldemort/serialization/VoldemortOpCode.java
  33. +8 −7 src/java/voldemort/server/AbstractService.java
  34. +26 −0 src/java/voldemort/server/ServiceType.java
  35. +181 −0 src/java/voldemort/server/StoreRepository.java
  36. +127 −12 src/java/voldemort/server/VoldemortConfig.java
  37. +50 −54 src/java/voldemort/server/VoldemortServer.java
  38. +2 −2 src/java/voldemort/server/VoldemortService.java
  39. +26 −5 src/java/voldemort/server/http/HttpService.java
  40. +17 −116 src/java/voldemort/server/http/StoreServlet.java
  41. +1 −1 src/java/voldemort/server/http/gui/AdminServlet.java
  42. +3 −1 src/java/voldemort/server/http/gui/QueryServlet.java
  43. +23 −6 src/java/voldemort/server/http/gui/ReadOnlyStoreManagementServlet.java
  44. +1 −1 src/java/voldemort/server/http/gui/templates/admin.vm
  45. +14 −11 src/java/voldemort/server/jmx/JmxService.java
  46. +36 −0 src/java/voldemort/server/protocol/AbstractRequestHandler.java
  47. +18 −0 src/java/voldemort/server/protocol/RequestHandler.java
  48. +36 −0 src/java/voldemort/server/protocol/RequestHandlerFactory.java
  49. +5 −0 src/java/voldemort/server/protocol/package.html
  50. +121 −0 src/java/voldemort/server/protocol/pb/ProtoBuffRequestHandler.java
  51. +5 −0 src/java/voldemort/server/protocol/pb/package.html
  52. +163 −0 src/java/voldemort/server/protocol/vold/VoldemortNativeRequestHandler.java
  53. +9 −14 src/java/voldemort/server/scheduler/RebalancingJob.java
  54. +3 −2 src/java/voldemort/server/scheduler/SchedulerService.java
  55. +12 −18 src/java/voldemort/server/scheduler/SlopPusherJob.java
  56. +44 −68 src/java/voldemort/server/socket/SocketServer.java
  57. +54 −0 src/java/voldemort/server/socket/SocketServerSession.java
  58. +7 −10 src/java/voldemort/server/socket/SocketService.java
  59. +0 −141 src/java/voldemort/server/socket/StreamStoreRequestHandler.java
  60. +179 −148 src/java/voldemort/server/storage/StorageService.java
  61. +12 −4 src/java/voldemort/store/StoreDefinition.java
  62. +5 −1 src/java/voldemort/store/bdb/BdbStorageEngine.java
  63. +86 −92 src/java/voldemort/store/http/HttpStore.java
  64. +4 −0 src/java/voldemort/store/memory/InMemoryStorageEngine.java
  65. +35 −43 src/java/voldemort/store/metadata/MetadataStore.java
  66. +2 −2 src/java/voldemort/store/routed/RoutedStore.java
  67. +71 −6 src/java/voldemort/store/socket/SocketPool.java
  68. +30 −17 src/java/voldemort/store/socket/SocketPoolableObjectFactory.java
  69. +48 −67 src/java/voldemort/store/socket/SocketStore.java
  70. +1 −1 src/java/voldemort/store/stats/StatTrackingStore.java
  71. +4 −3 src/java/voldemort/{client → utils}/DaemonThreadFactory.java
  72. +20 −0 src/java/voldemort/utils/Props.java
  73. +41 −0 src/java/voldemort/utils/Utils.java
  74. +4 −0 src/java/voldemort/versioning/VectorClock.java
  75. +8 −4 src/java/voldemort/versioning/Versioned.java
  76. +13 −0 src/java/voldemort/xml/ClusterMapper.java
  77. +15 −2 src/java/voldemort/xml/StoreDefinitionsMapper.java
  78. +84 −0 src/proto/voldemort-client.proto
  79. +83 −0 src/proto/voldemort_protocol.proto
  80. +65 −31 test/common/voldemort/ServerTestUtils.java
  81. +20 −0 test/common/voldemort/TestUtils.java
  82. +2 −8 test/integration/voldemort/TestClientShutdown.java
  83. +4 −12 test/integration/voldemort/performance/RemoteHttpStoreLoadTest.java
  84. +31 −11 test/integration/voldemort/performance/RemoteStoreComparisonTest.java
  85. +3 −10 test/integration/voldemort/performance/RemoteTest.java
  86. +2 −2 test/unit/voldemort/client/DefaultStoreClientTest.java
  87. +8 −13 test/unit/voldemort/client/HttpStoreClientFactoryTest.java
  88. +6 −17 test/unit/voldemort/client/SocketStoreClientFactoryTest.java
  89. +249 −0 test/unit/voldemort/protocol/AbstractRequestFormatTest.java
  90. +12 −0 test/unit/voldemort/protocol/pb/ProtocolBuffersRequestFormatTest.java
  91. +12 −0 test/unit/voldemort/protocol/vold/VoldemortNativeRequestFormatTest.java
  92. +55 −30 test/unit/voldemort/scheduled/SlopPusherTest.java
  93. +2 −2 test/unit/voldemort/server/ServiceTest.java
  94. +24 −57 test/unit/voldemort/server/socket/SocketPoolTest.java
  95. +65 −0 test/unit/voldemort/server/storage/StorageServiceTest.java
  96. +7 −0 test/unit/voldemort/store/AbstractStoreTest.java
  97. +58 −12 test/unit/voldemort/store/bdb/BdbStorageEngineTest.java
  98. +16 −2 test/unit/voldemort/store/http/HttpStoreTest.java
  99. +0 −27 test/unit/voldemort/store/metadata/MetadataStoreTest.java
  100. +4 −1 test/unit/voldemort/store/socket/SocketStoreTest.java
View
@@ -1,6 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" path="src/java"/>
+ <classpathentry kind="src" path="contrib/mongodb/example"/>
<classpathentry kind="src" path="contrib/batch-indexer/test"/>
<classpathentry kind="src" path="contrib/mongodb/src/java"/>
<classpathentry kind="src" path="contrib/mongodb/test"/>
View
16 NOTES
@@ -57,4 +57,18 @@ Background Resources
- Origin of vector clocks: http://research.microsoft.com/users/lamport/pubs/time-clocks.pdf
- Brewer's conjecture: http://citeseer.ist.psu.edu/544596.html
-Current build is from r19
+Current build is from r19
+
+
+Supporting other clients
+- Each store is available via all protcols, they are seperated by port
+- Wire format vs. protocol (HTTP vs. Tcp/IP), protocol buffers
+- How to abstract wire format?
+- A given serialization type may or may not be supported by the client language
+
+Socket servers share threadpool?
+
+A WireProtocol takes bytes and creates a voldemort request, and takes objects to create a voldemort response.
+Likewise the client does the opposite
+
+Connector.handleRequest()
View
@@ -30,4 +30,4 @@ done
CLASSPATH=$CLASSPATH:$base_dir/dist/resources
export CLASSPATH
-java -Xmx2G -server -cp $CLASSPATH ${1} ${2} ${3} ${4} ${5} ${6} ${7}
+java -Xmx2G -server -Dcom.sun.management.jmxremote -cp $CLASSPATH ${1} ${2} ${3} ${4} ${5} ${6} ${7}
View
@@ -1,6 +1,7 @@
## Main source
src.dir=src
java.dir=src/java
+protobuff.dir=src/proto
lib.dir=lib
classes.dir=dist/classes
resources.dir=dist/resources
@@ -31,4 +32,4 @@ tomcat.manager.password=tomcat
tomcat.context=/voldemort
## Release
-curr.release=0.51
+curr.release=0.52
View
@@ -94,6 +94,19 @@
<fileset dir="${testclasses.dir}" />
</jar>
</target>
+
+ <target name="protobuff" description="Generate source files from .proto files">
+ <path id="proto-files">
+ <fileset dir="${protobuff.dir}" />
+ </path>
+ <property name="proto.path" location="${protobuff.dir}"/>
+ <property name="javaout.path" location="${java.dir}"/>
+ <exec executable="protoc" failonerror="true">
+ <arg value="--proto_path=${proto.path}"/>
+ <arg value="--java_out=${javaout.path}"/>
+ <arg pathref="proto-files"/>
+ </exec>
+ </target>
<target name="jar" depends="build" description="Build server jar file">
<jar destfile="${dist.dir}/${name}-${curr.release}.jar">
@@ -115,6 +128,16 @@
</fileset>
</jar>
</target>
+
+ <target name="war" depends="build" description="Build server war file">
+ <war destfile="${dist.dir}/${name}.war" webxml="web.xml" basedir="${classes.dir}">
+ <classes dir="${classes.dir}"/>
+ <lib dir="${lib.dir}">
+ <exclude name="${lib.dir}/servlet-api.jar"/>
+ <exclude name="${lib.dir}/src/**"/>
+ </lib>
+ </war>
+ </target>
<target name="contrib-build" depends="build, buildtest" description="Compile contrib packages (java and test) ">
<replace-dir dir="${contrib.classes.dir}" />
@@ -206,16 +229,6 @@
<target name="junit-all" depends="junit, contrib-junit" description="Run All junit tests including contrib.">
</target>
- <target name="war" depends="build" description="Build server war file">
- <war destfile="${dist.dir}/${name}.war" webxml="web.xml" basedir="${classes.dir}">
- <classes dir="${classes.dir}" />
- <lib dir="${lib.dir}">
- <exclude name="${lib.dir}/servlet-api.jar" />
- <exclude name="${lib.dir}/src/**" />
- </lib>
- </war>
- </target>
-
<macrodef name="make-javadocs">
<attribute name="dir" />
<attribute name="packagenames" />
@@ -37,11 +37,11 @@
public abstract class ReadOnlyBatchIndexMapper<K, V> implements
Mapper<K, V, BytesWritable, BytesWritable> {
- private Cluster _cluster = null;
- private StoreDefinition _storeDef = null;
- private ConsistentRoutingStrategy _routingStrategy = null;
- private Serializer<Object> _keySerializer;
- private Serializer<Object> _valueSerializer;
+ private Cluster cluster = null;
+ private StoreDefinition storeDef = null;
+ private ConsistentRoutingStrategy routingStrategy = null;
+ private Serializer<Object> keySerializer;
+ private Serializer<Object> valueSerializer;
public abstract Object getKeyBytes(K key, V value);
@@ -51,10 +51,10 @@ public void map(K key,
V value,
OutputCollector<BytesWritable, BytesWritable> output,
Reporter reporter) throws IOException {
- byte[] keyBytes = _keySerializer.toBytes(getKeyBytes(key, value));
- byte[] valBytes = _valueSerializer.toBytes(getValueBytes(key, value));
+ byte[] keyBytes = keySerializer.toBytes(getKeyBytes(key, value));
+ byte[] valBytes = valueSerializer.toBytes(getValueBytes(key, value));
- List<Node> nodes = _routingStrategy.routeRequest(keyBytes);
+ List<Node> nodes = routingStrategy.routeRequest(keyBytes);
for(Node node: nodes) {
ByteArrayOutputStream versionedValue = new ByteArrayOutputStream();
DataOutputStream valueDin = new DataOutputStream(versionedValue);
@@ -82,17 +82,17 @@ public void configure(JobConf conf) {
}
// get Cluster and Store details
- _cluster = ContribUtils.getVoldemortClusterDetails(clusterFilePath);
- _storeDef = ContribUtils.getVoldemortStoreDetails(storeFilePath,
- conf.get("voldemort.store.name"));
+ cluster = ContribUtils.getVoldemortClusterDetails(clusterFilePath);
+ storeDef = ContribUtils.getVoldemortStoreDetails(storeFilePath,
+ conf.get("voldemort.store.name"));
- _keySerializer = (Serializer<Object>) new DefaultSerializerFactory().getSerializer(_storeDef.getKeySerializer());
- _valueSerializer = (Serializer<Object>) new DefaultSerializerFactory().getSerializer(_storeDef.getValueSerializer());
+ keySerializer = (Serializer<Object>) new DefaultSerializerFactory().getSerializer(storeDef.getKeySerializer());
+ valueSerializer = (Serializer<Object>) new DefaultSerializerFactory().getSerializer(storeDef.getValueSerializer());
- _routingStrategy = new ConsistentRoutingStrategy(_cluster.getNodes(),
- _storeDef.getReplicationFactor());
+ routingStrategy = new ConsistentRoutingStrategy(cluster.getNodes(),
+ storeDef.getReplicationFactor());
- if(_routingStrategy == null) {
+ if(routingStrategy == null) {
throw new RuntimeException("Failed to create routing strategy");
}
} catch(Exception e) {
@@ -20,14 +20,14 @@
public class ReadOnlyBatchIndexReducer implements Reducer<BytesWritable, BytesWritable, Text, Text> {
- private DataOutputStream _indexFileStream = null;
- private DataOutputStream _valueFileStream = null;
+ private DataOutputStream indexFileStream = null;
+ private DataOutputStream valueFileStream = null;
- private long _position = 0;
+ private long position = 0;
- private JobConf _conf = null;
- private String _taskId = null;
- private int _nodeId = -1;
+ private JobConf conf = null;
+ private String taskId = null;
+ private int nodeId = -1;
Path taskIndexFileName;
Path taskValueFileName;
@@ -46,21 +46,21 @@ public void reduce(BytesWritable key,
BytesWritable value = values.next();
byte[] valBytes = ByteUtils.copy(value.get(), 0, value.getSize());
- if(_nodeId == -1) {
+ if(nodeId == -1) {
DataInputStream buffer = new DataInputStream(new ByteArrayInputStream(valBytes));
- _nodeId = buffer.readInt();
+ nodeId = buffer.readInt();
}
// strip first 4 bytes as node_id
byte[] value1 = ByteUtils.copy(valBytes, 4, valBytes.length);
// Write Index Key/ position
- _indexFileStream.write(keyBytes);
- _indexFileStream.writeLong(_position);
- _valueFileStream.writeInt(value1.length);
- _valueFileStream.write(value1);
- _position += value1.length + 4;
+ indexFileStream.write(keyBytes);
+ indexFileStream.writeLong(position);
+ valueFileStream.writeInt(value1.length);
+ valueFileStream.write(value1);
+ position += value1.length + 4;
- if(_position < 0) {
+ if(position < 0) {
throw new RuntimeException("Position bigger than Integer size, split input files.");
}
}
@@ -69,34 +69,34 @@ public void reduce(BytesWritable key,
public void configure(JobConf job) {
try {
- _position = 0;
- _conf = job;
+ position = 0;
+ conf = job;
- _taskId = job.get("mapred.task.id");
+ taskId = job.get("mapred.task.id");
- taskIndexFileName = new Path(FileOutputFormat.getOutputPath(_conf),
- _conf.get("voldemort.index.filename") + "_" + _taskId);
- taskValueFileName = new Path(FileOutputFormat.getOutputPath(_conf),
- _conf.get("voldemort.data.filename") + "_" + _taskId);
+ taskIndexFileName = new Path(FileOutputFormat.getOutputPath(conf),
+ conf.get("voldemort.index.filename") + "_" + taskId);
+ taskValueFileName = new Path(FileOutputFormat.getOutputPath(conf),
+ conf.get("voldemort.data.filename") + "_" + taskId);
FileSystem fs = taskIndexFileName.getFileSystem(job);
- _indexFileStream = fs.create(taskIndexFileName, (short) 1);
- _valueFileStream = fs.create(taskValueFileName, (short) 1);
+ indexFileStream = fs.create(taskIndexFileName, (short) 1);
+ valueFileStream = fs.create(taskValueFileName, (short) 1);
} catch(IOException e) {
throw new RuntimeException("Failed to open Input/OutputStream", e);
}
}
public void close() throws IOException {
- _indexFileStream.close();
- _valueFileStream.close();
+ indexFileStream.close();
+ valueFileStream.close();
- Path hdfsIndexFile = new Path(FileOutputFormat.getOutputPath(_conf), _nodeId + ".index");
- Path hdfsValueFile = new Path(FileOutputFormat.getOutputPath(_conf), _nodeId + ".data");
+ Path hdfsIndexFile = new Path(FileOutputFormat.getOutputPath(conf), nodeId + ".index");
+ Path hdfsValueFile = new Path(FileOutputFormat.getOutputPath(conf), nodeId + ".data");
- FileSystem fs = hdfsIndexFile.getFileSystem(_conf);
+ FileSystem fs = hdfsIndexFile.getFileSystem(conf);
fs.rename(taskIndexFileName, hdfsIndexFile);
fs.rename(taskValueFileName, hdfsValueFile);
}
@@ -74,7 +74,7 @@ public void doOperation(int index) throws Exception {
reader.next(key, value);
store.put(new ByteArray(ByteUtils.copy(key.get(), 0, key.getSize())),
- Versioned.of(ByteUtils.copy(value.get(), 0, value.getSize())));
+ Versioned.value(ByteUtils.copy(value.get(), 0, value.getSize())));
} catch(ObsoleteVersionException e) {
obsoletes.incrementAndGet();
}
@@ -74,7 +74,7 @@ public void doOperation(int index) throws Exception {
reader.next(key, value);
store.put(new ByteArray(ByteUtils.copy(key.get(), 0, key.getSize())),
- Versioned.of(ByteUtils.copy(value.get(), 0, value.getSize())));
+ Versioned.value(ByteUtils.copy(value.get(), 0, value.getSize())));
} catch(ObsoleteVersionException e) {
obsoletes.incrementAndGet();
}
@@ -103,8 +103,8 @@ private String makeReadOnlyIndex(int minKey, int maxKey) throws Exception {
public void testswap() throws Throwable {
// assert that read-only store is working
- Store<ByteArray, byte[]> store1 = server1.getStoreMap().get(storeName);
- Store<ByteArray, byte[]> store2 = server2.getStoreMap().get(storeName);
+ Store<ByteArray, byte[]> store1 = server1.getStoreRepository().getStorageEngine(storeName);
+ Store<ByteArray, byte[]> store2 = server2.getStoreRepository().getStorageEngine(storeName);
SerializerDefinition serDef = new SerializerDefinition("json", "'string'");
Serializer<Object> serializer = StoreUtils.unsafeGetSerializer(new DefaultSerializerFactory(),
@@ -97,8 +97,8 @@ private String makeReadOnlyIndex(int minKey, int maxKey) throws Exception {
public void testswap() throws Throwable {
// assert that read-only store is working
- Store<ByteArray, byte[]> store1 = server1.getStoreMap().get(storeName);
- Store<ByteArray, byte[]> store2 = server2.getStoreMap().get(storeName);
+ Store<ByteArray, byte[]> store1 = server1.getStoreRepository().getStorageEngine(storeName);
+ Store<ByteArray, byte[]> store2 = server2.getStoreRepository().getStorageEngine(storeName);
SerializerDefinition serDef = new SerializerDefinition("json", "'string'");
Serializer<Object> serializer = StoreUtils.unsafeGetSerializer(new DefaultSerializerFactory(),
Oops, something went wrong.

0 comments on commit c49df1b

Please sign in to comment.