Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Updating branch to bring it up to date with master

  • Loading branch information...
commit acc46cdcbd7aa110e4fcd07bb3f162a26018b633 2 parents ee9c7bc + 36bd7f1
@rsumbaly authored
Showing with 22,102 additions and 7,262 deletions.
  1. +5 −3 .classpath
  2. +5 −0 CONTRIBUTORS
  3. +66 −0 bin/grandfather-readonly.sh
  4. +0 −21 bin/voldemort-rebalance-configure.sh
  5. +0 −21 bin/voldemort-remote-test.sh
  6. +1 −0  build.properties
  7. +71 −71 build.xml
  8. +1 −0  clients/cpp/utils/stress.cpp
  9. +63 −0 clients/python/README
  10. +5 −0 clients/python/setup.cfg
  11. +15 −0 clients/python/setup.py
  12. +0 −59 clients/python/test.py
  13. +302 −0 clients/python/tests/test_client.py
  14. +11 −0 clients/python/tests/voldemort_config/config/cluster.xml
  15. +27 −0 clients/python/tests/voldemort_config/config/server.properties
  16. +32 −0 clients/python/tests/voldemort_config/config/stores.xml
  17. +0 −399 clients/python/voldemort.py
  18. +1 −0  clients/python/voldemort/__init__.py
  19. +479 −0 clients/python/voldemort/client.py
  20. 0  clients/python/voldemort/protocol/__init__.py
  21. +88 −0 clients/python/voldemort/protocol/slop_pb2.py
  22. +2,139 −0 clients/python/voldemort/protocol/voldemort_admin_pb2.py
  23. +275 −81 clients/python/{ → voldemort/protocol}/voldemort_client_pb2.py
  24. +9 −0 clients/python/voldemort/serialization/__init__.py
  25. +3 −0  clients/python/voldemort/serialization/common.py
  26. +1,252 −0 clients/python/voldemort/serialization/json_serializer.py
  27. +69 −0 clients/python/voldemort/serialization/ordered_dict.py
  28. +10 −0 clients/python/voldemort/serialization/string_serializer.py
  29. +18 −0 clients/python/voldemort/serialization/unimplemented_serializer.py
  30. +0 −1,303 clients/python/voldemort_admin_pb2.py
  31. +1 −0  config/single_node_cluster/config/server.properties
  32. +3 −0  config/single_node_cluster/config/stores.xml
  33. +1 −1  contrib/ec2-testing/bin/voldemort-deployer.sh
  34. +2 −2 contrib/ec2-testing/src/java/voldemort/utils/ClusterGenerator.java
  35. +0 −84 contrib/ec2-testing/test/voldemort/utils/Ec2InstanceRemoteTestUtils.java
  36. +15 −9 contrib/ec2-testing/test/voldemort/utils/Ec2RebalanceTest.java
  37. +2 −2 contrib/hadoop-store-builder/perf/voldemort/contrib/batchindexer/performance/BdbBuildPerformanceTest.java
  38. +2 −2 contrib/hadoop-store-builder/perf/voldemort/contrib/batchindexer/performance/MysqlBuildPerformanceTest.java
  39. +64 −37 contrib/hadoop-store-builder/src/java/voldemort/store/readonly/fetcher/HdfsFetcher.java
  40. +140 −0 contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mapreduce/AbstractHadoopStoreBuilderMapper.java
  41. +282 −0 contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mapreduce/HadoopStoreBuilder.java
  42. +58 −0 contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mapreduce/HadoopStoreBuilderPartitioner.java
  43. +191 −0 contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mapreduce/HadoopStoreBuilderReducer.java
  44. +262 −0 contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mapreduce/HadoopStoreJobRunner.java
  45. +13 −9 contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/AbstractHadoopStoreBuilderMapper.java
  46. +91 −14 contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/HadoopStoreBuilder.java
  47. +2 −2 contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/HadoopStoreBuilderPartitioner.java
  48. +13 −8 contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/HadoopStoreBuilderReducer.java
  49. +1 −1  contrib/hadoop-store-builder/src/java/voldemort/store/readwrite/benchmark/BuildTestRWStore.java
  50. +52 −7 contrib/hadoop-store-builder/src/java/voldemort/store/readwrite/mr/AbstractRWHadoopStoreBuilderMapper.java
  51. +25 −43 contrib/hadoop-store-builder/src/java/voldemort/store/readwrite/mr/HadoopRWStoreBuilder.java
  52. +40 −27 contrib/hadoop-store-builder/src/java/voldemort/store/readwrite/mr/HadoopRWStoreBuilderReducer.java
  53. +29 −19 contrib/hadoop-store-builder/src/java/voldemort/store/readwrite/mr/HadoopRWStoreJobRunner.java
  54. +20 −10 contrib/hadoop-store-builder/test/voldemort/store/readonly/checksum/CheckSumTests.java
  55. +26 −18 contrib/hadoop-store-builder/test/voldemort/store/readonly/fetcher/HdfsFetcherTest.java
  56. +195 −0 contrib/hadoop-store-builder/test/voldemort/store/readonly/mapreduce/HadoopStoreBuilderTest.java
  57. +30 −11 contrib/hadoop-store-builder/test/voldemort/store/readonly/mr/HadoopStoreBuilderTest.java
  58. +2 −1  contrib/hadoop-store-builder/test/voldemort/store/readwrite/mr/HadoopRWStoreBuilderTest.java
  59. +1 −3 contrib/hadoop/src/java/voldemort/hadoop/pig/VoldemortStore.java
  60. +1 −1  contrib/krati/src/java/voldemort/store/krati/KratiStorageConfiguration.java
  61. +11 −8 contrib/krati/src/java/voldemort/store/krati/KratiStorageEngine.java
  62. +2 −2 contrib/krati/test/voldemort/store/krati/KratiStorageEngineTest.java
  63. BIN  lib/je-4.0.103.jar
  64. BIN  lib/je-4.0.92.jar
  65. BIN  lib/jna.jar
  66. BIN  lib/mockito-all-1.8.5.jar
  67. BIN  lib/protobuf-java-2.2.0.jar
  68. BIN  lib/protobuf-java-2.3.0.jar
  69. +73 −6 src/java/voldemort/VoldemortAdminTool.java
  70. +14 −5 src/java/voldemort/VoldemortClientShell.java
  71. +81 −34 src/java/voldemort/client/AbstractStoreClientFactory.java
  72. +35 −3 src/java/voldemort/client/ClientConfig.java
  73. +76 −4 src/java/voldemort/client/DefaultStoreClient.java
  74. +6 −7 src/java/voldemort/client/HttpStoreClientFactory.java
  75. +112 −8 src/java/voldemort/client/MockStoreClientFactory.java
  76. +22 −5 src/java/voldemort/client/SocketStoreClientFactory.java
  77. +35 −0 src/java/voldemort/client/StoreClient.java
  78. +5 −3 src/java/voldemort/client/StoreClientFactory.java
  79. +3 −0  src/java/voldemort/client/protocol/RequestFormat.java
  80. +4 −0 src/java/voldemort/client/protocol/RequestFormatFactory.java
  81. +2 −0  src/java/voldemort/client/protocol/RequestFormatType.java
  82. +445 −46 src/java/voldemort/client/protocol/admin/AdminClient.java
  83. +15 −1 src/java/voldemort/client/protocol/admin/AdminClientConfig.java
  84. +1 −1  src/java/voldemort/client/protocol/admin/SocketPool.java
  85. +41 −2 src/java/voldemort/client/protocol/admin/SocketResourceFactory.java
  86. +24 −2 src/java/voldemort/client/protocol/pb/ProtoBuffClientRequestFormat.java
  87. +4 −0 src/java/voldemort/client/protocol/pb/ProtoUtils.java
  88. +8,774 −3,227 src/java/voldemort/client/protocol/pb/VAdminProto.java
  89. +974 −177 src/java/voldemort/client/protocol/pb/VProto.java
  90. +44 −9 src/java/voldemort/client/protocol/vold/VoldemortNativeClientRequestFormat.java
  91. +102 −0 src/java/voldemort/client/rebalance/MigratePartitions.java
  92. +20 −19 src/java/voldemort/client/rebalance/RebalanceCLI.java
  93. +0 −250 src/java/voldemort/client/rebalance/RebalanceClusterBuilder.java
  94. +131 −14 src/java/voldemort/client/rebalance/RebalanceClusterPlan.java
  95. +143 −108 src/java/voldemort/client/rebalance/RebalanceClusterTool.java
  96. +149 −62 src/java/voldemort/client/rebalance/RebalanceController.java
  97. +70 −60 src/java/voldemort/client/rebalance/RebalancePartitionsInfo.java
  98. +10 −0 src/java/voldemort/cluster/Cluster.java
  99. +6 −1 src/java/voldemort/cluster/Zone.java
  100. +1 −7 src/java/voldemort/cluster/failuredetector/AsyncRecoveryFailureDetector.java
  101. +5 −5 src/java/voldemort/cluster/failuredetector/BasicStoreVerifier.java
  102. +5 −5 src/java/voldemort/cluster/failuredetector/ClientStoreVerifier.java
  103. +4 −4 src/java/voldemort/cluster/failuredetector/ServerStoreVerifier.java
  104. +27 −25 src/java/voldemort/serialization/SlopSerializer.java
  105. +547 −0 src/java/voldemort/serialization/VSlopProto.java
  106. +4 −4 src/java/voldemort/serialization/json/JsonWriter.java
  107. +53 −47 src/java/voldemort/server/StoreRepository.java
  108. +128 −23 src/java/voldemort/server/VoldemortConfig.java
  109. +3 −1 src/java/voldemort/server/VoldemortServer.java
  110. +0 −6 src/java/voldemort/server/http/gui/QueryServlet.java
  111. +78 −24 src/java/voldemort/server/http/gui/ReadOnlyStoreManagementServlet.java
  112. +8 −8 src/java/voldemort/server/http/gui/StatusServlet.java
  113. +4 −4 src/java/voldemort/server/http/gui/templates/status.vm
  114. +1 −1  src/java/voldemort/server/jmx/JmxService.java
  115. +1 −1  src/java/voldemort/server/protocol/AbstractRequestHandler.java
  116. +9 −7 src/java/voldemort/server/protocol/SocketRequestHandlerFactory.java
  117. +305 −90 src/java/voldemort/server/protocol/admin/AdminServiceRequestHandler.java
  118. +4 −3 src/java/voldemort/server/protocol/admin/FetchEntriesStreamRequestHandler.java
  119. +3 −2 src/java/voldemort/server/protocol/admin/FetchKeysStreamRequestHandler.java
  120. +2 −2 src/java/voldemort/server/protocol/admin/FetchMasterEntriesStreamRequestHandler.java
  121. +140 −0 src/java/voldemort/server/protocol/admin/FetchPartitionFileStreamRequestHandler.java
  122. +10 −2 src/java/voldemort/server/protocol/admin/FetchStreamRequestHandler.java
  123. +2 −2 src/java/voldemort/server/protocol/admin/UpdatePartitionEntriesStreamRequestHandler.java
  124. +165 −0 src/java/voldemort/server/protocol/admin/UpdateSlopEntriesRequestHandler.java
  125. +31 −9 src/java/voldemort/server/protocol/pb/ProtoBuffRequestHandler.java
  126. +44 −10 src/java/voldemort/server/protocol/vold/VoldemortNativeRequestHandler.java
  127. +76 −55 src/java/voldemort/server/rebalance/RebalanceAsyncOperation.java
  128. +99 −28 src/java/voldemort/server/rebalance/Rebalancer.java
  129. +1 −1  src/java/voldemort/server/rebalance/RebalancerService.java
  130. +3 −3 src/java/voldemort/server/scheduler/DataCleanupJob.java
  131. +3 −3 src/java/voldemort/server/scheduler/RebalancingJob.java
  132. +88 −5 src/java/voldemort/server/scheduler/SchedulerService.java
  133. +0 −108 src/java/voldemort/server/scheduler/SlopPusherJob.java
  134. +232 −0 src/java/voldemort/server/scheduler/slop/BlockingSlopPusherJob.java
  135. +126 −0 src/java/voldemort/server/scheduler/slop/RepairJob.java
  136. +450 −0 src/java/voldemort/server/scheduler/slop/StreamingSlopPusherJob.java
  137. +157 −78 src/java/voldemort/server/storage/StorageService.java
  138. +11 −10 src/java/voldemort/store/DelegatingStore.java
  139. +1 −1  src/java/voldemort/store/StorageConfiguration.java
  140. +2 −1  src/java/voldemort/store/StorageEngine.java
  141. +5 −4 src/java/voldemort/store/Store.java
  142. +67 −10 src/java/voldemort/store/StoreDefinition.java
  143. +49 −6 src/java/voldemort/store/StoreDefinitionBuilder.java
  144. +1 −1  src/java/voldemort/store/StoreRequest.java
  145. +10 −5 src/java/voldemort/store/StoreUtils.java
  146. +19 −6 src/java/voldemort/store/bdb/BdbStorageConfiguration.java
  147. +14 −9 src/java/voldemort/store/bdb/BdbStorageEngine.java
  148. +24 −9 src/java/voldemort/store/compress/CompressingStore.java
  149. +8 −5 src/java/voldemort/store/configuration/ConfigurationStorageEngine.java
  150. +8 −6 src/java/voldemort/store/gzip/GzipStore.java
  151. +9 −4 src/java/voldemort/store/http/HttpStore.java
  152. +14 −13 src/java/voldemort/store/invalidmetadata/InvalidMetadataCheckingStore.java
  153. +11 −11 src/java/voldemort/store/logging/LoggingStore.java
  154. +2 −2 src/java/voldemort/store/memory/CacheStorageConfiguration.java
  155. +3 −3 src/java/voldemort/store/memory/InMemoryStorageConfiguration.java
  156. +7 −6 src/java/voldemort/store/memory/InMemoryStorageEngine.java
  157. +94 −46 src/java/voldemort/store/metadata/MetadataStore.java
  158. +8 −0 src/java/voldemort/store/metadata/MetadataStoreListener.java
  159. +1 −1  src/java/voldemort/store/mysql/MysqlStorageConfiguration.java
  160. +10 −8 src/java/voldemort/store/mysql/MysqlStorageEngine.java
  161. +18 −5 src/java/voldemort/store/nonblockingstore/NonblockingStore.java
  162. +76 −27 src/java/voldemort/store/nonblockingstore/ThreadPoolBasedNonblockingStoreImpl.java
  163. +136 −7 src/java/voldemort/store/readonly/ChunkedFileSet.java
  164. +2 −1  src/java/voldemort/store/readonly/FileFetcher.java
  165. +129 −5 src/java/voldemort/store/readonly/JsonStoreBuilder.java
  166. +9 −1 src/java/voldemort/store/readonly/ReadOnlyStorageConfiguration.java
  167. +211 −134 src/java/voldemort/store/readonly/ReadOnlyStorageEngine.java
  168. +53 −0 src/java/voldemort/store/readonly/ReadOnlyStorageFormat.java
  169. +129 −0 src/java/voldemort/store/readonly/ReadOnlyStorageMetadata.java
  170. +143 −0 src/java/voldemort/store/readonly/ReadOnlyUtils.java
  171. +17 −6 src/java/voldemort/store/readonly/swapper/AdminStoreSwapper.java
  172. +37 −3 src/java/voldemort/store/readonly/swapper/HttpStoreSwapper.java
Sorry, we could not display the entire diff because too many files (341) changed.
View
8 .classpath
@@ -39,16 +39,18 @@
<classpathentry kind="lib" path="lib/commons-logging-1.1.1.jar"/>
<classpathentry kind="lib" path="lib/jline-0.9.94.jar"/>
<classpathentry kind="lib" path="lib/commons-pool-1.5.2.jar"/>
- <classpathentry kind="lib" path="lib/protobuf-java-2.2.0.jar"/>
+ <classpathentry kind="lib" path="lib/protobuf-java-2.3.0.jar"/>
<classpathentry kind="lib" path="contrib/ec2-testing/lib/typica.jar"/>
<classpathentry kind="lib" path="lib/libthrift-0.2.0.jar"/>
<classpathentry kind="lib" path="lib/google-collect-1.0.jar"/>
- <classpathentry kind="lib" path="lib/je-4.0.92.jar"/>
+ <classpathentry kind="lib" path="lib/je-4.0.103.jar"/>
<classpathentry kind="lib" path="lib/paranamer-2.1.jar"/>
<classpathentry kind="lib" path="lib/jackson-mapper-asl-1.4.0.jar"/>
<classpathentry kind="lib" path="lib/jackson-core-asl-1.4.0.jar"/>
<classpathentry kind="lib" path="lib/avro-modified-jdk5-1.3.0.jar"/>
<classpathentry kind="lib" path="contrib/hadoop/lib/pig-0.7.1-dev-core.jar"/>
<classpathentry kind="lib" path="contrib/krati/lib/krati-0.3.4.jar"/>
- <classpathentry kind="output" path="classes"/>
+ <classpathentry kind="lib" path="lib/jna.jar"/>
+ <classpathentry kind="lib" path="lib/mockito-all-1.8.5.jar" />
+ <classpathentry kind="output" path="classes"/>
</classpath>
View
5 CONTRIBUTORS
@@ -2,19 +2,24 @@ Alex Feinberg
Antoine Toulme
Anthony Lauzon
Bhupesh Bansal
+Bruce Ritchie
Chris Riccomini
Claudio Cherubino
Dain Sundstrom
+Dave Brosius
Elias Torres
Eric Evans
Geir Magnusson Jr.
Ismael Juma
+Jakob Homan
Janne Hietamäki
Jay Kreps
+Jonathan Traupman
Joshua Tuberville
Kirk True
Michael R. Head
Mike Frost
+Neha Narkhede
Padraig O'Sullivan
Paul Lindner
Rob Adams
View
66 bin/grandfather-readonly.sh
@@ -0,0 +1,66 @@
+#!/bin/bash
+
+#
+# Copyright 2010 LinkedIn, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# One time grandfathering script to convert all read-only store dirs to
+# new format
+if [ $# -ne 1 ];
+then
+ echo 'USAGE: bin/grandfather-readonly.sh [readonly-folder]'
+ exit 1
+fi
+# Read args
+READ_ONLY_DIR=$1
+
+# Create temporary metadata file
+METADATA_FILE="$(basename $0).$$.tmp"
+echo "{\"format\":\"ro0\"}" > $METADATA_FILE
+
+for stores in $READ_ONLY_DIR/*
+do
+ if [ -d $stores ]; then
+
+ echo ---Working on store ${stores} ---
+ # Convert all to .temp
+ numVersions=`find $stores -name version-* | grep -v .bak | grep -v .temp | wc -l`
+ maxVersion=`find $stores -name version-* | grep -v .bak | grep -v .temp | sed 's/^\(.*\)\-\([0-9]*\)$/\2/' | sort -n | tail -1`
+ if [ $numVersions -eq 1 ]; then
+ cp $METADATA_FILE ${stores}/version-${maxVersion}/.metadata
+ echo Added metadata to ${stores}/version-${maxVersion}
+ fi
+ if [ $numVersions -gt 1 ]; then
+ for versionDirNo in `find $stores -name version-* | grep -v .bak | grep -v .temp | sed 's/^\(.*\)\-\([0-9]*\)$/\2/' | sort -n`
+ do
+ mv ${stores}/version-${versionDirNo} ${stores}/version-${maxVersion}.temp
+ echo Moved ${stores}/version-${versionDirNo} to ${stores}/version-${maxVersion}.temp
+ cp $METADATA_FILE ${stores}/version-${maxVersion}.temp/.metadata
+ echo Added metadata to ${stores}/version-${maxVersion}.temp
+ let maxVersion=maxVersion-1
+ done
+ fi
+
+ # Convert all .temp to normal
+ numVersionsTmp=`find $stores -name version-*.temp | grep -v .bak | wc -l`
+ if [ $numVersionsTmp -gt 1 ]; then
+ for versionDir in `find $stores -name version-*.temp | grep -v .bak | sed 's/^\(.*\-\)\([0-9]*\).temp$/\1\2/'`
+ do
+ mv ${versionDir}.temp ${versionDir}
+ echo Moved ${versionDir}.temp to ${versionDir}
+ done
+ fi
+ fi
+done
+
View
21 bin/voldemort-rebalance-configure.sh
@@ -1,21 +0,0 @@
-#!/bin/bash
-
-#
-# Copyright 2008-2010 LinkedIn, Inc
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-base_dir=$(dirname $0)/..
-
-$base_dir/bin/run-class.sh voldemort.client.rebalance.RebalanceClusterBuilder $@
View
21 bin/voldemort-remote-test.sh
@@ -1,21 +0,0 @@
-#!/bin/bash
-
-#
-# Copyright 2008-2009 LinkedIn, Inc
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-bin_dir=$(dirname $0)
-
-${bin_dir}/run-class.sh voldemort.performance.RemoteTest $@
View
1  build.properties
@@ -2,6 +2,7 @@
src.dir=src
java.dir=src/java
python.dir=clients/python
+python.proto.dir=clients/python/voldemort/protocol
protobuff.dir=src/proto
lib.dir=lib
classes.dir=dist/classes
View
142 build.xml
@@ -22,7 +22,7 @@
<isset property="env.BUILD_NUMBER" />
<not>
<equals arg1="" arg2="${env.BUILD_NUMBER}" trim="yes"/>
- </not>
+ </not>
</and>
</condition>
@@ -79,7 +79,7 @@
<!-- place to put log4j.properties -->
<replace-dir dir="${resources.dir}"/>
<copy file="${java.dir}/log4j.properties" todir="${resources.dir}"/>
- <javac destdir="${classes.dir}" target="1.5" debug="true" deprecation="false" failonerror="true">
+ <javac destdir="${classes.dir}" target="1.5" debug="true" deprecation="false" failonerror="true" includeantruntime="false">
<src path="${java.dir}" />
<classpath refid="main-classpath" />
</javac>
@@ -99,7 +99,7 @@
<exclude name="**/*.html" />
</fileset>
</copy>
- <javac destdir="${testclasses.dir}" target="1.5" debug="true" deprecation="false" failonerror="true">
+ <javac destdir="${testclasses.dir}" target="1.5" debug="true" deprecation="false" failonerror="true" includeantruntime="false">
<src path="${unittestsrc.dir}" />
<src path="${inttestsrc.dir}" />
<src path="${commontestsrc.dir}" />
@@ -112,7 +112,7 @@
<fileset dir="${testclasses.dir}" />
</jar>
</target>
-
+
<target name="protobuff" description="Generate source files from .proto files">
<pathconvert property="proto.sources" pathsep=" ">
<path id="proto-files">
@@ -120,9 +120,9 @@
</path>
</pathconvert>
- <property name="proto.path" location="${protobuff.dir}"/>
+ <property name="proto.path" location="${protobuff.dir}"/>
<property name="javaout.path" location="${java.dir}"/>
- <property name="pythonout.path" location="${python.dir}"/>
+ <property name="pythonout.path" location="${python.proto.dir}"/>
<exec executable="protoc" failonerror="true">
<arg value="--proto_path=${proto.path}"/>
<arg value="--java_out=${javaout.path}"/>
@@ -151,7 +151,7 @@
</fileset>
</jar>
</target>
-
+
<target name="alljar" depends="build, contrib-build" description="Build a jar file that includes all contrib code.">
<jar destfile="${dist.dir}/${name}-${curr.release}-all.jar">
<fileset dir="${classes.dir}">
@@ -166,7 +166,7 @@
</fileset>
</jar>
</target>
-
+
<target name="war" depends="build" description="Build server war file">
<war destfile="${dist.dir}/${name}.war" webxml="web.xml" basedir="${classes.dir}">
<classes dir="${classes.dir}"/>
@@ -179,7 +179,7 @@
<target name="contrib-build" depends="build, buildtest" description="Compile contrib packages (java and test) ">
<replace-dir dir="${contrib.classes.dir}" />
- <javac destdir="${contrib.classes.dir}" target="1.5" debug="true" deprecation="false" failonerror="true">
+ <javac destdir="${contrib.classes.dir}" target="1.5" debug="true" deprecation="false" failonerror="true" includeantruntime="false">
<src path="${contrib.root.dir}" />
<src path="${contrib.root.dir}" />
<classpath refid="main-classpath" />
@@ -231,7 +231,7 @@
</junitreport>
</target>
- <target name="findbugs" depends="jar">
+ <target name="findbugs" depends="jar">
<taskdef name="findbugs" classname="edu.umd.cs.findbugs.anttask.FindBugsTask"
classpath="${findbugs.lib}/findbugs-ant-1.3.9.jar" />
@@ -250,7 +250,7 @@
</findbugs>
</target>
- <target name="ec2testing-junit" depends="contrib-jar" description="Run EC2 testing contrib junit tests.">
+ <target name="ec2testing-junit" depends="all" description="Run EC2 testing contrib junit tests.">
<copy todir="${testclasses.dir}">
<fileset dir="${contrib.root.dir}/ec2-testing/resources" />
</copy>
@@ -264,7 +264,63 @@
<formatter type="xml" />
<batchtest fork="yes" todir="${contribtestreport.dir}">
<fileset dir="${contrib.classes.dir}">
- <include name="**/Ec2*Test.class" />
+ <include name="**/Ec2SmokeTest.class" />
+ </fileset>
+ </batchtest>
+ </junit>
+ <junitreport todir="${contribtesthtml.dir}">
+ <fileset dir="${contribtestreport.dir}">
+ <include name="TEST-*.xml" />
+ </fileset>
+ <report todir="${contribtesthtml.dir}" format="frames" />
+ </junitreport>
+ </target>
+
+ <target name="ec2testing-gossip" depends="contrib-jar" description="Run gossip tests on EC2.">
+ <copy todir="${testclasses.dir}">
+ <fileset dir="${contrib.root.dir}/ec2-testing/resources" />
+ </copy>
+ <replace-dir dir="${contribtestreport.dir}" />
+ <replace-dir dir="${contribtesthtml.dir}" />
+ <junit printsummary="yes" maxmemory="2048m" showoutput="true" failureProperty="test.failure">
+ <syspropertyset>
+ <propertyref prefix="ec2" />
+ <propertyref prefix="log4j" />
+ <propertyref prefix="gossip" />
+ </syspropertyset>
+ <classpath refid="contrib-test-classpath" />
+ <formatter type="xml" />
+ <batchtest fork="yes" todir="${contribtestreport.dir}">
+ <fileset dir="${contrib.classes.dir}">
+ <include name="**/Ec2GossipTest.class" />
+ </fileset>
+ </batchtest>
+ </junit>
+ <junitreport todir="${contribtesthtml.dir}">
+ <fileset dir="${contribtestreport.dir}">
+ <include name="TEST-*.xml" />
+ </fileset>
+ <report todir="${contribtesthtml.dir}" format="frames" />
+ </junitreport>
+ </target>
+
+ <target name="ec2testing-rebalancing" depends="contrib-jar" description="Run rebalancing tests on EC2.">
+ <copy todir="${testclasses.dir}">
+ <fileset dir="${contrib.root.dir}/ec2-testing/resources" />
+ </copy>
+ <replace-dir dir="${contribtestreport.dir}" />
+ <replace-dir dir="${contribtesthtml.dir}" />
+ <junit printsummary="yes" maxmemory="2048m" showoutput="true" failureProperty="test.failure">
+ <syspropertyset>
+ <propertyref prefix="ec2" />
+ <propertyref prefix="log4j" />
+ <propertyref prefix="rebalancing" />
+ </syspropertyset>
+ <classpath refid="contrib-test-classpath" />
+ <formatter type="xml" />
+ <batchtest fork="yes" todir="${contribtestreport.dir}">
+ <fileset dir="${contrib.classes.dir}">
+ <include name="**/Ec2RebalancingTest.class" />
</fileset>
</batchtest>
</junit>
@@ -276,62 +332,6 @@
</junitreport>
</target>
- <target name="gossip-ec2test" depends="contrib-jar" description="Run gossip tests on EC2.">
- <copy todir="${testclasses.dir}">
- <fileset dir="${contrib.root.dir}/ec2-testing/resources" />
- </copy>
- <replace-dir dir="${contribtestreport.dir}" />
- <replace-dir dir="${contribtesthtml.dir}" />
- <junit printsummary="yes" maxmemory="2048m" showoutput="true" failureProperty="test.failure">
- <syspropertyset>
- <propertyref prefix="ec2" />
- <propertyref prefix="log4j" />
- <propertyref prefix="gossip" />
- </syspropertyset>
- <classpath refid="contrib-test-classpath" />
- <formatter type="xml" />
- <batchtest fork="yes" todir="${contribtestreport.dir}">
- <fileset dir="${contrib.classes.dir}">
- <include name="**/Ec2GossipTest.class" />
- </fileset>
- </batchtest>
- </junit>
- <junitreport todir="${contribtesthtml.dir}">
- <fileset dir="${contribtestreport.dir}">
- <include name="TEST-*.xml" />
- </fileset>
- <report todir="${contribtesthtml.dir}" format="frames" />
- </junitreport>
- </target>
-
- <target name="rebalancing-ec2test" depends="contrib-jar" description="Run rebalancing tests on EC2.">
- <copy todir="${testclasses.dir}">
- <fileset dir="${contrib.root.dir}/ec2-testing/resources" />
- </copy>
- <replace-dir dir="${contribtestreport.dir}" />
- <replace-dir dir="${contribtesthtml.dir}" />
- <junit printsummary="yes" maxmemory="2048m" showoutput="true" failureProperty="test.failure">
- <syspropertyset>
- <propertyref prefix="ec2" />
- <propertyref prefix="log4j" />
- <propertyref prefix="rebalancing" />
- </syspropertyset>
- <classpath refid="contrib-test-classpath" />
- <formatter type="xml" />
- <batchtest fork="yes" todir="${contribtestreport.dir}">
- <fileset dir="${contrib.classes.dir}">
- <include name="**/Ec2RebalancingTest.class" />
- </fileset>
- </batchtest>
- </junit>
- <junitreport todir="${contribtesthtml.dir}">
- <fileset dir="${contribtestreport.dir}">
- <include name="TEST-*.xml" />
- </fileset>
- <report todir="${contribtesthtml.dir}" format="frames" />
- </junitreport>
- </target>
-
<macrodef name="create-release-artifacts">
<attribute name="version" />
<sequential>
@@ -359,12 +359,12 @@
<target name="snapshot" description="Create a release-snapshot zip file with everything pre-built.">
<create-release-artifacts version="${curr.release.snapshot}" />
</target>
-
+
<target name="release" description="Create a release zip file with everything pre-built.">
<create-release-artifacts version="${curr.release}" />
</target>
-
- <target name="hadoop-benchmark-jar" depends="build, contrib-build"
+
+ <target name="hadoop-benchmark-jar" depends="build, contrib-build"
description="Build a jar file that includes all contrib code plus the necessary jars for running the hadoop benchmark.">
<jar destfile="${dist.dir}/hadoop-benchmark.jar">
<fileset dir="${classes.dir}">
View
1  clients/cpp/utils/stress.cpp
@@ -24,6 +24,7 @@
#include <vector>
#include <sstream>
#include <iostream>
+#include <cstring>
#include <voldemort/voldemort.h>
#include <boost/thread/thread.hpp>
View
63 clients/python/README
@@ -0,0 +1,63 @@
+ABOUT
+
+This directory contains a pure Python implementation of a voldemort client.
+It supports both raw (string) and JSON serialized stores. Only the protocol
+buffer interface over TCP is supported for talking to the server. Only
+server-side routing is supported.
+
+INSTALLING
+
+To install the module, you will need the following dependencies:
+- nose >= 0.11
+- simplejson >= 2.1.1
+- Google protobuf > 2.3.0
+
+The setup process will automatically install nose and simplejson, since they
+are well behaved Python packages. The protobuf module will need to be downloaded
+from https://code.google.com/p/protobuf/downloads/list and installed manually.
+
+Once the dependencies, run the test suite to sanity check things. You need to
+first start up a Voldemort server locally, pointing to the config files in
+tests/voldemort_config. From the root voldemort of the voldemort source tree, run:
+
+> bin/voldemort-server clients/python/tests/voldemort_config
+
+In a separate shell, change into the clients/python directory and run:
+
+> python setup.py nosetests
+
+If all tests pass, you can install the package with the command:
+
+> python setup.py install
+
+This may need to be run as root if you don't have permissions to install to your
+local python library.
+
+USING THE MODULE
+
+To use the client, simple import it into your program with the statement:
+
+import voldemort
+
+To create a client connection, instantiate a StoreClient object:
+
+client = voldemort.StoreClient('store_name', [('node1', 6666), ('node2', 6666)])
+
+The values of the store name and cluster nodes/ports will depend on your particular
+Voldemort setup. The key and value serialization type will be determined
+automatically during client initialization using the values in your cluster's
+stores.xml file.
+
+The StoreClient object implements the get(), get_all(), put(), maybe_put(),
+and delete() methods. For example:
+
+> v1 = client.put("foo", "hello")
+> resp = client.get("foo")
+> resp[0][0]
+"hello"
+
+> client.delete("foo")
+> client.get("foo")
+[]
+
+The test suite contains many other usage examples.
View
5 clients/python/setup.cfg
@@ -0,0 +1,5 @@
+[nosetests]
+detailed-errors=1
+with-doctest=1
+
+
View
15 clients/python/setup.py
@@ -0,0 +1,15 @@
+from setuptools import setup
+
+setup(name='voldemort',
+ version='0.1.1',
+ description='Python Voldemort Client',
+ long_description=('Pure python client for accessing Voldemort key/value stores. ' +
+ 'Supports both raw and JSON stores. Only supports the tcp protocol ' +
+ 'buffer interface with server-side routing.'),
+ packages=['voldemort', 'voldemort.protocol', 'voldemort.serialization'],
+ author='LinkedIn Corporation',
+ license='Apache 2.0',
+ url='http://project-voldemort.com',
+ install_requires=['protobuf>=2.3.0', 'simplejson>=2.1.1'],
+ setup_requires=['nose>=0.11'],
+)
View
59 clients/python/test.py
@@ -1,59 +0,0 @@
-import logging
-import time
-from voldemort import StoreClient
-
-if __name__ == '__main__':
-
- logging.basicConfig(level=logging.INFO,)
-
- ## some random tests
- s = StoreClient('test', [('localhost', 6666)])
- version = s.put("hello", "1")
- assert s.get("hello")[0][0] == "1"
- s.put("hello", "2", version)
- assert s.get("hello")[0][0] == "2"
- s.put("hello", "3")
- assert s.get("hello")[0][0] == "3"
- s.delete("hello")
- assert len(s.get("hello")) == 0
-
- ## test get_all
- pairs = [("a1", "1"), ("a2", "2"), ("a3", "3"), ("a4", "4")]
- for k, v in pairs:
- s.put(k, v)
-
- vals = s.get_all([k for k, v in pairs])
- for k, v in pairs:
- assert vals[k][0][0] == v
-
- requests = 10000
-
- ## Time get requests
- s.put("hello", "world")
- start = time.time()
- for i in xrange(requests):
- s.get('hello')
- print requests/(time.time() - start), ' get requests per second'
-
- ## Time put requests
- version = s.put('abc', 'def')
- start = time.time()
- for i in xrange(requests):
- version = s.put('abc', 'def', version)
- print requests/(time.time() - start), ' put requests per second'
-
- ## Time get_all requests
- keys = [k for k,v in pairs]
- start = time.time()
- for i in xrange(requests):
- vals = s.get_all(keys)
- print requests/(time.time() - start), ' get_all requests per second'
-
- ## Time delete requests
- version = None
- for i in xrange(requests):
- version = s.put(str(i), str(i))
- start = time.time()
- for i in xrange(requests):
- vals = s.delete(str(i), version)
- print requests/(time.time() - start), ' delete requests per second'
View
302 clients/python/tests/test_client.py
@@ -0,0 +1,302 @@
+# Tests of the client code.
+#
+# To run these tests, you must have a local Voldemort server running using the configuration files
+# in tests/voldemort_config.
+
+import unittest
+import datetime
+
+from voldemort import StoreClient, VoldemortException
+
+def _vector_clock_equal(clock1, clock2):
+ """
+ Compares two vector clocks, ignoring the timestamp field, which may be skewed.
+ """
+
+ clock1_entries = dict((entry.node_id, entry.version) for entry in clock1.entries)
+ clock2_entries = dict((entry.node_id, entry.version) for entry in clock2.entries)
+ return clock1_entries == clock2_entries
+
+class VoldemortClientTest(unittest.TestCase):
+ def _reinit_raw_client(self):
+ s = StoreClient('test', [('localhost', 6666)])
+ for k in ['a', 'b', 'c']:
+ s.delete(k)
+ return s
+
+ def _reinit_json_client(self):
+ s = StoreClient('json_test', [('localhost', 6666)])
+ for k in [1, 2, 3]:
+ s.delete(k)
+ return s
+
+ def test_raw_get(self):
+ """
+ Tests basic puts/gets in raw (non-serialized) mode.
+ """
+
+ s = self._reinit_raw_client()
+
+ s.put('a', '1')
+ resp = s.get('a')
+ self.assertEquals(len(resp), 1)
+ self.assertEquals(len(resp[0]), 2)
+ self.assertEquals(resp[0][0], '1')
+
+ s.put('b', '2')
+ resp = s.get('b')
+ self.assertEquals(len(resp), 1)
+ self.assertEquals(len(resp[0]), 2)
+ self.assertEquals(resp[0][0], '2')
+
+
+ def test_raw_get_all(self):
+ """
+ Tests the get_all() method in raw mode.
+ """
+
+ s = self._reinit_raw_client()
+
+ pairs = [('a', '1'), ('b', '2'), ('c', '3')]
+ for k, v in pairs:
+ s.put(k, v)
+
+ resp = s.get_all([k for k, v in pairs])
+ self.assertEquals(len(resp), len(pairs))
+
+ for k, v in pairs:
+ self.assertTrue(k in resp)
+ self.assertEquals(len(resp[k]), 1)
+ self.assertEquals(len(resp[k][0]), 2)
+
+ self.assertEquals(resp[k][0][0], v)
+
+
+ def test_raw_versions(self):
+ """
+ Tests the put_maybe() method in raw mode.
+ """
+
+ s = self._reinit_raw_client()
+
+ v1 = s.put('a', '1')
+ self.assertTrue(v1 is not None)
+
+ v2 = s.put('a', '2')
+ self.assertTrue(v2 is not None)
+ self.assertFalse(_vector_clock_equal(v2, v1))
+
+ v3 = s.put('a', '3')
+ self.assertTrue(v3 is not None)
+ self.assertFalse(_vector_clock_equal(v3, v1))
+ self.assertFalse(_vector_clock_equal(v3, v2))
+
+ resp = s.get('a')
+ self.assertEquals(resp[0][0], '3')
+
+ # put() should fail because v2 is not the current version
+ self.assertRaises(VoldemortException, s.put, 'a', '4', version=v2)
+
+ # maybe_put() won't raise an exception, but will return None
+ v4 = s.maybe_put('a', '4', version=v2)
+ self.assertTrue(v4 is None)
+
+ # this put() should succeed
+ v4 = s.put('a', '4', version=v3)
+ self.assertTrue(v4 is not None)
+ self.assertFalse(_vector_clock_equal(v4, v1))
+ self.assertFalse(_vector_clock_equal(v4, v2))
+ self.assertFalse(_vector_clock_equal(v4, v2))
+
+ # and this maybe_put() should not return None
+ v5 = s.maybe_put('a', '5', version=v4)
+ self.assertTrue(v5 is not None)
+ self.assertFalse(_vector_clock_equal(v5, v1))
+ self.assertFalse(_vector_clock_equal(v5, v2))
+ self.assertFalse(_vector_clock_equal(v5, v3))
+ self.assertFalse(_vector_clock_equal(v5, v4))
+
+ # the value at the latest version should be "5"
+ resp = s.get('a')
+ self.assertEquals(resp[0][0], '5')
+ self.assertTrue(_vector_clock_equal(resp[0][1], v5))
+
+ # deleting old versions should have no effect
+ s.delete('a', version=v3)
+ resp = s.get('a')
+ self.assertEquals(len(resp), 1)
+ self.assertEquals(resp[0][0], '5')
+ self.assertTrue(_vector_clock_equal(resp[0][1], v5))
+
+ # deleting the current version should erase the entry
+ s.delete('a', version=v5)
+ resp = s.get('a')
+ self.assertEquals(resp, [])
+
+ val1 = {
+ 'a': 0.25,
+ 'b': [1,2,3],
+ 'c': u'foo',
+ 'd': { 'foo': True,
+ 'bar': datetime.datetime(2010, 11, 24, 20, 8, 7, 155000)
+ }
+ }
+
+ val2 = {
+ 'a': 4.0,
+ 'b': [5,6],
+ 'c': u'bar',
+ 'd': { 'foo': None,
+ 'bar': datetime.datetime(2003, 5, 5, 1, 23, 45, 678000)
+ }
+ }
+
+ val3 = {
+ 'a': 8.0,
+ 'b': [],
+ 'c': u'',
+ 'd': None
+ }
+
+ val4 = {
+ 'a': 4.0,
+ 'b': [5,6],
+ 'c': 'bar',
+ 'd': { 'foo': True,
+ 'bar': datetime.datetime(2003, 5, 5, 1, 23, 45, 678123)
+ }
+ }
+
+ def test_json_get(self):
+ """
+ Tests the JSON serialization with put()/get()
+ """
+
+ s = self._reinit_json_client()
+
+ s.put(1, self.val1)
+ resp = s.get(1)
+ self.assertEquals(len(resp), 1)
+ self.assertEquals(len(resp[0]), 2)
+ self.assertEquals(resp[0][0], self.val1)
+
+ s.put(2, self.val2)
+ resp = s.get(2)
+ self.assertEquals(len(resp), 1)
+ self.assertEquals(len(resp[0]), 2)
+ self.assertEquals(resp[0][0], self.val2)
+
+ def test_json_get_all(self):
+ """
+ Tests JSON serialized get_all()
+ """
+ s = self._reinit_json_client()
+
+ pairs = [(1, self.val1), (2, self.val2), (3, self.val3)]
+ for k, v in pairs:
+ s.put(k, v)
+
+ resp = s.get_all([k for k, v in pairs])
+ self.assertEquals(len(resp), len(pairs))
+
+ for k, v in pairs:
+ self.assertTrue(k in resp)
+ self.assertEquals(len(resp[k]), 1)
+ self.assertEquals(len(resp[k][0]), 2)
+
+ self.assertEquals(resp[k][0][0], v)
+
+ def test_json_mismatches(self):
+ """
+ Sometimes the result we get out of Voldemort is a little different than what
+ went in, but it's not always a problem.
+ """
+
+ s = self._reinit_json_client()
+
+ s.put(1, self.val4)
+ resp = s.get(1)
+ self.assertEquals(len(resp), 1)
+ self.assertEquals(len(resp[0]), 2)
+
+ output = resp[0][0]
+ # the input and output won't be the same
+ self.assertNotEquals(output, self.val4)
+
+ # the float should have survived the trip
+ self.assertEquals(output['a'], self.val4['a'])
+
+ # as should the list
+ self.assertEquals(output['b'], self.val4['b'])
+
+ # the string is now a unicode, but it should still compare equal to the original
+ self.assertTrue(isinstance(output['c'], unicode))
+ self.assertEquals(output['c'], self.val4['c'])
+
+ # the boolean should be the same
+ self.assertEquals(output['d']['foo'], self.val4['d']['foo'])
+
+ # but the date gets truncated:
+ self.assertNotEquals(output['d']['bar'], self.val4['d']['bar'])
+
+ # the difference should be small
+ td = self.val4['d']['bar'] - output['d']['bar']
+ self.assertEquals(td.days, 0)
+ self.assertEquals(td.seconds, 0)
+ self.assertEquals(td.microseconds, 123)
+
+ def test_raw_versions(self):
+ """
+ Tests versioning in JSON mode.
+ """
+
+ s = self._reinit_json_client()
+
+ v1 = s.put(1, self.val4)
+ self.assertTrue(v1 is not None)
+
+ v2 = s.put(1, self.val3)
+ self.assertTrue(v2 is not None)
+ self.assertFalse(_vector_clock_equal(v2, v1))
+
+ resp = s.get(1)
+ self.assertEquals(resp[0][0], self.val3)
+ self.assertTrue(_vector_clock_equal(resp[0][1], v2))
+
+ # put() should fail because v1 is not the current version
+ self.assertRaises(VoldemortException, s.put, 1, self.val2, version=v1)
+
+ # maybe_put() won't raise an exception, but will return None
+ v3 = s.maybe_put(1, self.val2, version=v1)
+ self.assertTrue(v3 is None)
+
+ # this put() should succeed
+ v3 = s.put(1, self.val2, version=v2)
+ self.assertTrue(v3 is not None)
+ self.assertFalse(_vector_clock_equal(v3, v1))
+ self.assertFalse(_vector_clock_equal(v3, v2))
+
+ # and this maybe_put() should not return None
+ v4 = s.maybe_put(1, self.val1, version=v3)
+ self.assertTrue(v4 is not None)
+ self.assertFalse(_vector_clock_equal(v4, v1))
+ self.assertFalse(_vector_clock_equal(v4, v2))
+ self.assertFalse(_vector_clock_equal(v4, v3))
+
+ # the value at the latest version should be val1 at version v4
+ resp = s.get(1)
+ self.assertEquals(resp[0][0], self.val1)
+ self.assertTrue(_vector_clock_equal(resp[0][1], v4))
+
+ # deleting old versions should have no effect
+ s.delete(1, version=v2)
+ resp = s.get(1)
+ self.assertEquals(len(resp), 1)
+ self.assertEquals(resp[0][0], self.val1)
+ self.assertTrue(_vector_clock_equal(resp[0][1], v4))
+
+ # deleting the current version should erase the entry
+ s.delete(1, version=v4)
+ resp = s.get(1)
+ self.assertEquals(resp, [])
+
View
11 clients/python/tests/voldemort_config/config/cluster.xml
@@ -0,0 +1,11 @@
+<cluster>
+ <name>mycluster</name>
+ <server>
+ <id>0</id>
+ <host>localhost</host>
+ <http-port>8081</http-port>
+ <socket-port>6666</socket-port>
+ <partitions>0, 1</partitions>
+ </server>
+</cluster>
+
View
27 clients/python/tests/voldemort_config/config/server.properties
@@ -0,0 +1,27 @@
+# The ID of *this* particular cluster node
+node.id=0
+
+max.threads=100
+
+############### DB options ######################
+
+http.enable=true
+socket.enable=true
+
+# BDB
+bdb.write.transactions=false
+bdb.flush.transactions=false
+bdb.cache.size=1G
+
+# Mysql
+mysql.host=localhost
+mysql.port=1521
+mysql.user=root
+mysql.password=3306
+mysql.database=test
+
+#NIO connector settings.
+enable.nio.connector=false
+
+request.format=vp3
+storage.configs=voldemort.store.bdb.BdbStorageConfiguration, voldemort.store.readonly.ReadOnlyStorageConfiguration
View
32 clients/python/tests/voldemort_config/config/stores.xml
@@ -0,0 +1,32 @@
+<stores>
+ <store>
+ <name>test</name>
+ <persistence>bdb</persistence>
+ <routing>client</routing>
+ <replication-factor>1</replication-factor>
+ <required-reads>1</required-reads>
+ <required-writes>1</required-writes>
+ <key-serializer>
+ <type>string</type>
+ </key-serializer>
+ <value-serializer>
+ <type>string</type>
+ </value-serializer>
+ </store>
+ <store>
+ <name>json_test</name>
+ <persistence>bdb</persistence>
+ <routing>client</routing>
+ <replication-factor>1</replication-factor>
+ <required-reads>1</required-reads>
+ <required-writes>1</required-writes>
+ <key-serializer>
+ <type>json</type>
+ <schema-info version="0">"int32"</schema-info>
+ </key-serializer>
+ <value-serializer>
+ <type>json</type>
+ <schema-info version="0">{ "a":"float32", "b":["int16"], "c":"string", "d":{ "foo":"boolean", "bar":"date" }}</schema-info>
+ </value-serializer>
+ </store>
+</stores>
View
399 clients/python/voldemort.py
@@ -1,399 +0,0 @@
-import socket
-import struct
-import time
-import sys
-import re
-import random
-import logging
-import sets
-
-import voldemort_client_pb2 as protocol
-from xml.dom import minidom
-from datetime import datetime
-
-##################################################################
-# A Voldemort client. Each client uses a single connection to one
-# Voldemort server. All routing is done server-side.
-##################################################################
-
-
-## Extract all the child text of the given element
-def _extract_text(elm):
- if elm.nodeType == minidom.Node.TEXT_NODE:
- return elm.data
- elif elm.nodeType == minidom.Node.ELEMENT_NODE:
- text = ""
- for child in elm.childNodes:
- text += _extract_text(child)
- return text
-
-
-## Get a single child from the element, if there are multiple children, explode.
-def _child(elmt, name):
- children = elmt.getElementsByTagName(name)
- if len(children) != 1:
- raise Exception, "No child '" + str(name) + "' for element " + str(elmt.nodeName)
- return children[0]
-
-
-## Get the child text of a single element
-def _child_text(elmt, name):
- return _extract_text(_child(elmt, name))
-
-
-
-##################################################################
-# A node class representing a single voldemort server in the
-# cluster. The cluster itself is just a list of nodes
-##################################################################
-class Node:
- """A Voldemort node with the appropriate host and port information for contacting that node"""
-
- def __init__(self, id, host, socket_port, http_port, partitions, is_available = True, last_contact = None):
- self.id = id
- self.host = host
- self.socket_port = socket_port
- self.http_port = http_port
- self.partitions = partitions
- self.is_available = True
- if not last_contact:
- self.last_contact = time.clock()
-
- def __repr__(self):
- return 'node(id = ' + str(self.id) + ', host = ' + self.host + ', socket_port = ' + str(self.socket_port) + \
- ', http_port = ' + str(self.http_port) + ', partitions = ' + ', '.join(map(str, self.partitions)) + ')'
-
- @staticmethod
- def parse_cluster(xml):
- """Parse the cluster.xml file and return a dictionary of the nodes in the cluster indexed by node id """
- doc = minidom.parseString(xml)
- nodes = {}
- for curr in doc.getElementsByTagName('server'):
- id = int(_child_text(curr, 'id'))
- host = _child_text(curr, 'host')
- http_port = int(_child_text(curr, 'http-port'))
- socket_port = int(_child_text(curr, 'socket-port'))
- partition_str = _child_text(curr, 'partitions')
- partitions = [int(p) for p in re.split('[\s,]+', partition_str)]
- nodes[id] = Node(id = id, host = host, socket_port = socket_port, http_port = http_port, partitions = partitions)
- return nodes
-
-
-
-class VoldemortException(Exception):
- def __init__(self, message, code = 1):
- self.code = code
- self.message = message
-
- def __str__(self):
- return repr(self.message)
-
-
-
-class StoreClient:
- """A simple Voldemort client. It is single-threaded and supports only server-side routing."""
-
- def __init__(self, store_name, bootstrap_urls, reconnect_interval = 500, conflict_resolver = None):
- self.store_name = store_name
- self.request_count = 0
- self.conflict_resolver = conflict_resolver
- self.nodes = self._bootstrap_metadata(bootstrap_urls)
- self.node_id = random.randint(0, len(self.nodes) - 1)
- self.node_id, self.connection = self._reconnect()
- self.reconnect_interval = reconnect_interval
- self.open = True
-
- def _make_connection(self, host, port):
- protocol = 'pb0'
- logging.debug('Attempting to connect to ' + host + ':' + str(port))
- connection = socket.socket()
- connection.connect((host, port))
- logging.debug('Connection succeeded, negotiating protocol')
- connection.send(protocol)
- resp = connection.recv(2)
- if resp != 'ok':
- raise VoldemortException('Server does not understand the protocol ' + protocol)
- logging.debug('Protocol negotiation suceeded')
- return connection
-
-
- ## Connect to a the next available node in the cluster
- ## returns a tuple of (node_id, connection)
- def _reconnect(self):
- num_nodes = len(self.nodes)
- attempts = 0
- new_node_id = self.node_id
- while attempts < num_nodes:
- new_node_id = (new_node_id + 1) % num_nodes
- new_node = self.nodes[new_node_id]
- connection = None
- try:
- connection = self._make_connection(new_node.host, new_node.socket_port)
- self.request_count = 0
- return new_node_id, connection
- except socket.error, (err_num, message):
- self._close_socket(connection)
- logging.warn('Error connecting to node ' + str(new_node_id) + ': ' + message)
- attempts += 1
-
- # If we get here all nodes have failed us, explode
- raise VoldemortException('Connections to all nodes failed.')
-
-
- ## Safely close the socket, catching and logging any exceptions
- def _close_socket(self, socket):
- try:
- if socket:
- socket.close()
- except socket.error, exp:
- logging.error('Error while closing socket: ' + str(exp))
-
-
- ## Check if the the number of requests made on this connection is greater than the reconnect interval.
- ## If so reconnect to a random node in the cluster. No attempt is made at preventing the reconnecting
- ## from going back to the same node
- def _maybe_reconnect(self):
- if self.request_count >= self.reconnect_interval:
- logging.debug('Completed ' + str(self.request_count) + ' requests using this connection, reconnecting...')
- self._close_socket(self.connection)
- self.node_id, self.connection = self._reconnect()
-
-
- ## send a request to the server using the given connection
- def _send_request(self, connection, req_bytes):
- connection.send(struct.pack('>i', len(req_bytes)) + req_bytes)
- self.request_count += 1
-
-
- ## read a response from the connection
- def _receive_response(self, connection):
- size_bytes = connection.recv(4)
- size = struct.unpack('>i', size_bytes)
-
- if not size[0]:
- return ''
-
- return connection.recv(size[0])
-
- ## Bootstrap cluster metadata from a list of urls of nodes in the cluster.
- ## The urls are tuples in the form (host, port).
- ## A dictionary of node_id => node is returned.
- def _bootstrap_metadata(self, bootstrap_urls):
- random.shuffle(bootstrap_urls)
- for host, port in bootstrap_urls:
- logging.debug('Attempting to bootstrap metadata from ' + host + ':' + str(port))
- connection = None
- try:
- connection = self._make_connection(host, port)
- cluster_xmls = self._get_with_connection(connection, 'metadata', 'cluster.xml', should_route = False)
- if len(cluster_xmls) != 1:
- raise VoldemortException('Expected exactly one version of the metadata but found ' + str(cluster_xmls))
- nodes = Node.parse_cluster(cluster_xmls[0][0])
- logging.debug('Bootstrap from ' + host + ':' + str(port) + ' succeeded, found ' + str(len(nodes)) + " nodes.")
- return nodes
- except socket.error, (err_num, message):
- logging.warn('Metadata bootstrap from ' + host + ':' + str(port) + " failed: " + message)
- finally:
- self._close_socket(connection)
- raise VoldemortException('All bootstrap attempts failed')
-
-
- ## check if the server response has an error, if so throw an exception
- def _check_error(self, resp):
- if resp._has_error:
- raise VoldemortException(resp.error.error_message, resp.error.error_code)
-
-
- ## Increment the version for a vector clock
- def _increment(self, clock):
- # See if we already have a version for this guy, if so increment it
- for entry in clock.entries:
- if entry.node_id == self.node_id:
- entry.version += 1
- return
- # Otherwise add a version
- entry = clock.entries.add()
- entry.node_id = self.node_id
- entry.version = 1
- clock.timestamp = int(time.time() * 1000)
-
-
- ## Take a list of versions, and, if a conflict resolver has been given, resolve any conflicts that can be resolved
- def _resolve_conflicts(self, versions):
- if self.conflict_resolver and versions:
- return self.conflict_resolver(versions)
- else:
- return versions
-
-
- ## Turn a protocol buffer list of versioned items into a python list of items
- def _extract_versions(self, pb_versioneds):
- versions = []
- for versioned in pb_versioneds:
- versions.append((versioned.value, versioned.version))
- return self._resolve_conflicts(versions)
-
- ## A basic request wrapper, that handles reconnection logic and failures
- def _execute_request(self, fun, args):
- assert self.open, 'Store has been closed.'
- self._maybe_reconnect()
-
- failures = 0
- num_nodes = len(self.nodes)
- while failures < num_nodes:
- try:
- return apply(fun, args)
- except socket.error, (err_num, message):
- logging.warn('Error while performing ' + fun.__name__ + ' on node ' + str(self.node_id) + ': ' + message)
- self._reconnect()
- failures += 1
- raise VoldemortException('All nodes are down, ' + fun.__name__ + ' failed.')
-
-
- ## An internal get function that take the connection and store name as parameters. This is
- ## used by both the public get() method and also the metadata bootstrap process
- def _get_with_connection(self, connection, store_name, key, should_route):
- """Execute get request to the given store. Returns a (value, version) pair."""
-
- req = protocol.VoldemortRequest()
- req.should_route = should_route
- req.store = store_name
- req.type = protocol.GET
- req.get.key = key
-
- # send request
- self._send_request(connection, req.SerializeToString())
-
- # read and parse response
- resp_str = self._receive_response(connection)
- resp = protocol.GetResponse()
- resp.ParseFromString(resp_str)
- self._check_error(resp)
-
- return self._extract_versions(resp.versioned)
-
-
- ## Inner helper function for get
- def _get(self, key):
- return self._get_with_connection(self.connection, self.store_name, key, True)
-
-
- def get(self, key):
- """Execute a get request. Returns a list of (value, version) pairs."""
- return self._execute_request(self._get, [key])
-
-
- ## Inner get_all method that takes the connection and store_name as parameters
- def _get_all(self, keys):
- req = protocol.VoldemortRequest()
- req.should_route = True
- req.store = self.store_name
- req.type = protocol.GET_ALL
- for key in keys:
- req.getAll.keys.append(key)
-
- # send request
- self._send_request(self.connection, req.SerializeToString())
-
- # read and parse response
- resp_str = self._receive_response(self.connection)
- resp = protocol.GetAllResponse()
- resp.ParseFromString(resp_str)
- self._check_error(resp)
- values = {}
- for key_val in resp.values:
- values[key_val.key] = self._extract_versions(key_val.versions)
- return values
-
-
- def get_all(self, keys):
- """Execute get request for multiple keys given as a list or tuple.
- Returns a dictionary of key => [(value, version), ...] pairs."""
- return self._execute_request(self._get_all, [keys])
-
-
- ## Get the current version of the given key by doing a get request to the store
- def _fetch_version(self, key):
- versioned = self.get(key)
- if versioned:
- version = versioned[0][1]
- else:
- version = protocol.VectorClock()
- version.timestamp = int(time.time() * 1000)
- return version
-
-
- def _put(self, key, value, version):
- req = protocol.VoldemortRequest()
- req.should_route = True
- req.store = self.store_name
- req.type = protocol.PUT
- req.put.key = key
- req.put.versioned.value = value
- req.put.versioned.version.MergeFrom(version)
-
- # send request
- self._send_request(self.connection, req.SerializeToString())
-
- # read and parse response
- resp_str = self._receive_response(self.connection)
- resp = protocol.PutResponse()
- resp.ParseFromString(resp_str)
- self._check_error(resp)
- self._increment(version)
- return version
-
- def put (self, key, value, version = None):
- """Execute a put request using the given key and value. If no version is specified a get(key) request
- will be done to get the current version. The updated version is returned."""
- # if we don't have a version, fetch one
- if not version:
- version = self._fetch_version(key)
- return self._execute_request(self._put, [key, value, version])
-
-
- def maybe_put(self, key, value, version = None):
- """Execute a put request using the given key and value. If the version being put is obsolete,
- no modification will be made and this function will return None. Otherwise it will return the new version."""
- try:
- return self.put(key, value, version)
- except:
- return None
-
- def _delete(self, key, version):
- req = protocol.VoldemortRequest()
- req.should_route = True
- req.store = self.store_name
- req.type = protocol.DELETE
- req.delete.key = key
- req.delete.version.MergeFrom(version)
-
- # send request
- self._send_request(self.connection, req.SerializeToString())
-
- # read and parse response
- resp_str = self._receive_response(self.connection)
- resp = protocol.DeleteResponse()
- resp.ParseFromString(resp_str)
- self._check_error(resp)
-
- return resp.success
-
-
- def delete(self, key, version = None):
- """Execute a delete request, deleting all keys up to and including the given version.
- If no version is given a get(key) request will be done to find the latest version."""
- # if we don't have a version, fetch one
- if version == None:
- version = self._fetch_version(key)
- return self._execute_request(self._delete, [key, version])
-
-
- def close(self):
- """Close the connection this store maintains."""
- self.open = False
- self.connection.close()
-
-
-
-
View
1  clients/python/voldemort/__init__.py
@@ -0,0 +1 @@
+from client import StoreClient, VoldemortException
View
479 clients/python/voldemort/client.py
@@ -0,0 +1,479 @@
+import socket
+import struct
+import time
+import sys
+import re
+import random
+import logging
+
+import protocol.voldemort_client_pb2 as protocol
+from xml.dom import minidom
+from datetime import datetime
+
+import serialization
+
+##################################################################
+# A Voldemort client. Each client uses a single connection to one
+# Voldemort server. All routing is done server-side.
+##################################################################
+
+
+## Extract all the child text of the given element
+def _extract_text(elm):
+ if elm.nodeType == minidom.Node.TEXT_NODE:
+ return elm.data
+ elif elm.nodeType == minidom.Node.ELEMENT_NODE:
+ text = ""
+ for child in elm.childNodes:
+ text += _extract_text(child)
+ return text
+
+
+## Get a single child from the element, if there are multiple children, explode.
+def _child(elmt, name, required=True):
+ children = [child for child in elmt.childNodes
+ if child.nodeType == minidom.Node.ELEMENT_NODE and child.tagName == name]
+ if not children:
+ if required:
+ raise VoldemortException("No child '%s' for element '%s'." % (name, elmt.nodeName))
+ else:
+ return None
+
+ if len(children) > 1:
+ raise VoldemortException("Multiple children '%s' for element '%s'." % (name, elmt.nodeName))
+ return children[0]
+
+
+## Get the child text of a single element
+def _child_text(elmt, name, required=True, default=None):
+ if default:
+ required = False
+
+ child = _child(elmt, name, required=required)
+ if not child:
+ return default
+
+ return _extract_text(child)
+
+
+def _int_or_none(s):
+ if s is None:
+ return s
+ return int(s)
+
+
+##################################################################
+# A node class representing a single voldemort server in the
+# cluster. The cluster itself is just a list of nodes
+##################################################################
+class Node:
+ """A Voldemort node with the appropriate host and port information for contacting that node"""
+
+ def __init__(self, id, host, socket_port, http_port, partitions, is_available = True, last_contact = None):
+ self.id = id
+ self.host = host
+ self.socket_port = socket_port
+ self.http_port = http_port
+ self.partitions = partitions
+ self.is_available = True
+ if not last_contact:
+ self.last_contact = time.clock()
+
+ def __repr__(self):
+ return 'node(id = ' + str(self.id) + ', host = ' + self.host + ', socket_port = ' + str(self.socket_port) + \
+ ', http_port = ' + str(self.http_port) + ', partitions = ' + ', '.join(map(str, self.partitions)) + ')'
+
+ @staticmethod
+ def parse_cluster(xml):
+ """Parse the cluster.xml file and return a dictionary of the nodes in the cluster indexed by node id """
+ doc = minidom.parseString(xml)
+ nodes = {}
+ for curr in doc.getElementsByTagName('server'):
+ id = int(_child_text(curr, 'id'))
+ host = _child_text(curr, 'host')
+ http_port = int(_child_text(curr, 'http-port'))
+ socket_port = int(_child_text(curr, 'socket-port'))
+ partition_str = _child_text(curr, 'partitions')
+ partitions = [int(p) for p in re.split('[\s,]+', partition_str)]
+ nodes[id] = Node(id = id, host = host, socket_port = socket_port, http_port = http_port, partitions = partitions)
+ return nodes
+
+
+class Store:
+ def __init__(self, store_node):
+ self.name = _child_text(store_node, "name")
+ self.persistence = _child_text(store_node, "persistence")
+ self.routing = _child_text(store_node, "routing")
+ self.routing_strategy = _child_text(store_node, "routing-strategy", default="consistent-routing")
+ self.replication_factor = int(_child_text(store_node, "replication-factor"))
+ self.required_reads = int(_child_text(store_node, "required-reads"))
+ self.preferred_reads = _int_or_none(_child_text(store_node, "preferred-reads", required=False))
+ self.required_writes = int(_child_text(store_node, "required-writes"))
+ self.preferred_writes = _int_or_none(_child_text(store_node, "preferred-writes", required=False))
+
+ key_serializer_node = _child(store_node, "key-serializer")
+ self.key_serializer_type = _child_text(key_serializer_node, "type")
+ self.key_serializer = self._create_serializer(self.key_serializer_type, key_serializer_node)
+
+ value_serializer_node = _child(store_node, "value-serializer")
+ self.value_serializer_type = _child_text(value_serializer_node, "type")
+ self.value_serializer = self._create_serializer(self.value_serializer_type, value_serializer_node)
+
+ def _create_serializer(self, serializer_type, serializer_node):
+ if serializer_type not in serialization.SERIALIZER_CLASSES:
+ return serialization.UnimplementedSerializer(serializer_type)
+
+ return serialization.SERIALIZER_CLASSES[serializer_type].create_from_xml(serializer_node)
+
+ @staticmethod
+ def parse_stores_xml(xml):
+ doc = minidom.parseString(xml)
+ stores = [Store(store_node) for store_node in doc.getElementsByTagName("store")]
+ return dict((store.name, store) for store in stores)
+
+class VoldemortException(Exception):
+ def __init__(self, msg, code = 1):
+ self.code = code
+ self.msg = msg
+
+ def __str__(self):
+ return repr(self.msg)
+
+
+class StoreClient:
+ """A simple Voldemort client. It is single-threaded and supports only server-side routing."""
+
+ def __init__(self, store_name, bootstrap_urls, reconnect_interval = 500, conflict_resolver = None):
+ self.store_name = store_name
+ self.request_count = 0
+ self.conflict_resolver = conflict_resolver
+ self.nodes, self.stores = self._bootstrap_metadata(bootstrap_urls)
+ self.node_id = random.randint(0, len(self.nodes) - 1)
+ self.node_id, self.connection = self._reconnect()
+ self.reconnect_interval = reconnect_interval
+ self.open = True
+ self.key_serializer = self.stores[store_name].key_serializer
+ self.value_serializer = self.stores[store_name].value_serializer
+
+ def _make_connection(self, host, port):
+ protocol = 'pb0'
+ logging.debug('Attempting to connect to ' + host + ':' + str(port))
+ connection = socket.socket()
+ connection.connect((host, port))
+ logging.debug('Connection succeeded, negotiating protocol')
+ connection.send(protocol)
+ resp = connection.recv(2)
+ if resp != 'ok':
+ raise VoldemortException('Server does not understand the protocol ' + protocol)
+ logging.debug('Protocol negotiation suceeded')
+ return connection
+
+
+ ## Connect to a the next available node in the cluster
+ ## returns a tuple of (node_id, connection)
+ def _reconnect(self):
+ num_nodes = len(self.nodes)
+ attempts = 0
+ new_node_id = self.node_id
+ while attempts < num_nodes:
+ new_node_id = (new_node_id + 1) % num_nodes
+ new_node = self.nodes[new_node_id]
+ connection = None
+ try:
+ connection = self._make_connection(new_node.host, new_node.socket_port)
+ self.request_count = 0
+ return new_node_id, connection
+ except socket.error, (err_num, message):
+ self._close_socket(connection)
+ logging.warn('Error connecting to node ' + str(new_node_id) + ': ' + message)
+ attempts += 1
+
+ # If we get here all nodes have failed us, explode
+ raise VoldemortException('Connections to all nodes failed.')
+
+
+ ## Safely close the socket, catching and logging any exceptions
+ def _close_socket(self, socket):
+ try:
+ if socket:
+ socket.close()
+ except socket.error, exp:
+ logging.error('Error while closing socket: ' + str(exp))
+
+
+ ## Check if the the number of requests made on this connection is greater than the reconnect interval.
+ ## If so reconnect to a random node in the cluster. No attempt is made at preventing the reconnecting
+ ## from going back to the same node
+ def _maybe_reconnect(self):
+ if self.request_count >= self.reconnect_interval:
+ logging.debug('Completed ' + str(self.request_count) + ' requests using this connection, reconnecting...')
+ self._close_socket(self.connection)
+ self.node_id, self.connection = self._reconnect()
+
+
+ ## send a request to the server using the given connection
+ def _send_request(self, connection, req_bytes):
+ connection.send(struct.pack('>i', len(req_bytes)) + req_bytes)
+ self.request_count += 1
+
+
+ ## read a response from the connection
+ def _receive_response(self, connection):
+ size_bytes = connection.recv(4)
+ size = struct.unpack('>i', size_bytes)[0]
+
+ bytes_read = 0
+ data = []
+
+ while size and bytes_read < size:
+ chunk = connection.recv(size - bytes_read)
+ bytes_read += len(chunk)
+ data.append(chunk)
+
+ return ''.join(data)
+
+
+ ## Bootstrap cluster metadata from a list of urls of nodes in the cluster.
+ ## The urls are tuples in the form (host, port).
+ ## A dictionary of node_id => node is returned.
+ def _bootstrap_metadata(self, bootstrap_urls):
+ random.shuffle(bootstrap_urls)
+ for host, port in bootstrap_urls:
+ logging.debug('Attempting to bootstrap metadata from ' + host + ':' + str(port))
+ connection = None
+ try:
+ connection = self._make_connection(host, port)
+ cluster_xmls = self._get_with_connection(connection, 'metadata', 'cluster.xml', should_route = False)
+ if len(cluster_xmls) != 1:
+ raise VoldemortException('Expected exactly one version of the metadata but found ' + str(cluster_xmls))
+ nodes = Node.parse_cluster(cluster_xmls[0][0])
+ logging.debug('Bootstrap from ' + host + ':' + str(port) + ' succeeded, found ' + str(len(nodes)) + " nodes.")
+ stores_xml = self._get_with_connection(connection, 'metadata', 'stores.xml', should_route=False)[0][0]
+ stores = Store.parse_stores_xml(stores_xml)
+
+ return nodes, stores
+ except socket.error, (err_num, message):
+ logging.warn('Metadata bootstrap from ' + host + ':' + str(port) + " failed: " + message)
+ finally:
+ self._close_socket(connection)
+ raise VoldemortException('All bootstrap attempts failed')
+
+
+ ## check if the server response has an error, if so throw an exception
+ def _check_error(self, resp):
+ if resp.error and resp.error.error_code != 0:
+ raise VoldemortException(resp.error.error_message, resp.error.error_code)
+
+
+ ## Increment the version for a vector clock
+ def _increment(self, clock):
+ new_clock = protocol.VectorClock()
+ new_clock.MergeFrom(clock)
+
+ # See if we already have a version for this guy, if so increment it
+ for entry in new_clock.entries:
+ if entry.node_id == self.node_id:
+ entry.version += 1
+ return new_clock
+
+ # Otherwise add a version
+ entry = new_clock.entries.add()
+ entry.node_id = self.node_id
+ entry.version = 1
+ new_clock.timestamp = int(time.time() * 1000)
+
+ return new_clock
+
+ ## Take a list of versions, and, if a conflict resolver has been given, resolve any conflicts that can be resolved
+ def _resolve_conflicts(self, versions):
+ if self.conflict_resolver and versions:
+ return self.conflict_resolver(versions)
+ else:
+ return versions
+
+
+ ## Turn a protocol buffer list of versioned items into a python list of items
+ def _extract_versions(self, pb_versioneds):
+ versions = []
+ for versioned in pb_versioneds:
+ versions.append((versioned.value, versioned.version))
+ return self._resolve_conflicts(versions)
+
+ ## A basic request wrapper, that handles reconnection logic and failures
+ def _execute_request(self, fun, args):
+ assert self.open, 'Store has been closed.'
+ self._maybe_reconnect()
+
+ failures = 0
+ num_nodes = len(self.nodes)
+ while failures < num_nodes:
+ try:
+ return apply(fun, args)
+ except socket.error, (err_num, message):
+ logging.warn('Error while performing ' + fun.__name__ + ' on node ' + str(self.node_id) + ': ' + message)
+ self._reconnect()
+ failures += 1
+ raise VoldemortException('All nodes are down, ' + fun.__name__ + ' failed.')
+
+
+ ## An internal get function that take the connection and store name as parameters. This is
+ ## used by both the public get() method and also the metadata bootstrap process
+ def _get_with_connection(self, connection, store_name, key, should_route):
+ """Execute get request to the given store. Returns a (value, version) pair."""
+
+ req = protocol.VoldemortRequest()
+ req.should_route = should_route
+ req.store = store_name
+ req.type = protocol.GET
+ req.get.key = key
+
+ # send request
+ self._send_request(connection, req.SerializeToString())
+
+ # read and parse response
+ resp_str = self._receive_response(connection)
+ resp = protocol.GetResponse()
+ resp.ParseFromString(resp_str)
+ self._check_error(resp)
+
+ return self._extract_versions(resp.versioned)
+
+
+ ## Inner helper function for get
+ def _get(self, key):
+ return self._get_with_connection(self.connection, self.store_name, key, True)
+
+
+ def get(self, key):
+ """Execute a get request. Returns a list of (value, version) pairs."""
+
+ raw_key = self.key_serializer.writes(key)
+ return [(self.value_serializer.reads(value), version)
+ for value, version in self._execute_request(self._get, [raw_key])]
+
+
+ ## Inner get_all method that takes the connection and store_name as parameters
+ def _get_all(self, keys):
+ req = protocol.VoldemortRequest()
+ req.should_route = True
+ req.store = self.store_name
+ req.type = protocol.GET_ALL
+ for key in keys:
+ req.getAll.keys.append(key)
+
+ # send request
+ self._send_request(self.connection, req.SerializeToString())
+
+ # read and parse response
+ resp_str = self._receive_response(self.connection)
+ resp = protocol.GetAllResponse()
+ resp.ParseFromString(resp_str)
+ self._check_error(resp)
+ values = {}
+ for key_val in resp.values:
+ values[key_val.key] = self._extract_versions(key_val.versions)
+ return values
+
+
+ def get_all(self, keys):
+ """Execute get request for multiple keys given as a list or tuple.
+ Returns a dictionary of key => [(value, version), ...] pairs."""
+
+ raw_keys = [self.key_serializer.writes(key) for key in keys]
+ return dict((self.key_serializer.reads(key), [(self.value_serializer.reads(value), version)
+ for value, version in versioned_values])
+ for key, versioned_values in self._execute_request(self._get_all, [raw_keys]).iteritems())
+
+
+ ## Get the current version of the given key by doing a get request to the store
+ def _fetch_version(self, key):
+ versioned = self.get(key)
+ if versioned:
+ version = versioned[0][1]
+ else:
+ version = protocol.VectorClock()
+ version.timestamp = int(time.time() * 1000)
+ return version
+
+
+ def _put(self, key, value, version):
+ req = protocol.VoldemortRequest()
+ req.should_route = True
+ req.store = self.store_name
+ req.type = protocol.PUT
+ req.put.key = key
+ req.put.versioned.value = value
+ req.put.versioned.version.MergeFrom(version)
+
+ # send request
+ self._send_request(self.connection, req.SerializeToString())
+
+ # read and parse response
+ resp_str = self._receive_response(self.connection)
+ resp = protocol.PutResponse()
+ resp.ParseFromString(resp_str)
+ self._check_error(resp)
+ return self._increment(version)
+
+
+ def put(self, key, value, version = None):
+ """Execute a put request using the given key and value. If no version is specified a get(key) request
+ will be done to get the current version. The updated version is returned."""
+
+ raw_key = self.key_serializer.writes(key)
+ raw_value = self.value_serializer.writes(value)
+
+ # if we don't have a version, fetch one
+ if not version:
+ version = self._fetch_version(key)
+ return self._execute_request(self._put, [raw_key, raw_value, version])
+
+
+ def maybe_put(self, key, value, version = None):
+ """Execute a put request using the given key and value. If the version being put is obsolete,
+ no modification will be made and this function will return None. Otherwise it will return the new version."""
+ try:
+ return self.put(key, value, version)
+ except:
+ return None
+
+ def _delete(self, key, version):
+ req = protocol.VoldemortRequest()
+ req.should_route = True
+ req.store = self.store_name
+ req.type = protocol.DELETE
+ req.delete.key = key
+ req.delete.version.MergeFrom(version)
+
+ # send request
+ self._send_request(self.connection, req.SerializeToString())
+
+ # read and parse response
+ resp_str = self._receive_response(self.connection)
+ resp = protocol.DeleteResponse()
+ resp.ParseFromString(resp_str)
+ self._check_error(resp)
+
+ return resp.success
+
+
+ def delete(self, key, version = None):
+ """Execute a delete request, deleting all keys up to and including the given version.
+ If no version is given a get(key) request will be done to find the latest version."""
+
+ raw_key = self.key_serializer.writes(key)
+
+ # if we don't have a version, fetch one
+ if version == None:
+ version = self._fetch_version(key)
+ return self._execute_request(self._delete, [raw_key, version])
+
+
+ def close(self):
+ """Close the connection this store maintains."""
+ self.open = False
+ self.connection.close()
+
+
+
+
View
0  clients/python/voldemort/protocol/__init__.py
No changes.
View
88 clients/python/voldemort/protocol/slop_pb2.py
@@ -0,0 +1,88 @@
+# Generated by the protocol buffer compiler. DO NOT EDIT!
+
+from google.protobuf import descriptor
+from google.protobuf import message
+from google.protobuf import reflection
+from google.protobuf import descriptor_pb2
+# @@protoc_insertion_point(imports)
+
+
+DESCRIPTOR = descriptor.FileDescriptor(
+ name='slop.proto',
+ package='voldemort',
+ serialized_pb='\n\nslop.proto\x12\tvoldemort\"f\n\x04Slop\x12\r\n\x05store\x18\x01 \x01(\t\x12\x11\n\toperation\x18\x02 \x01(\t\x12\x0b\n\x03key\x18\x03 \x01(\x0c\x12\r\n\x05value\x18\x04 \x01(\x0c\x12\x0f\n\x07node_id\x18\x05 \x01(\x05\x12\x0f\n\x07\x61rrived\x18\x06 \x01(\x03\x42\'\n\x17voldemort.serializationB\nVSlopProtoH\x01')
+
+
+
+
+_SLOP = descriptor.Descriptor(
+ name='Slop',
+ full_name='voldemort.Slop',
+ filename=None,
+ file=DESCRIPTOR,
+ containing_type=None,
+ fields=[
+ descriptor.FieldDescriptor(
+ name='store', full_name='voldemort.Slop.store', index=0,
+ number=1, type=9, cpp_type=9, label=1,
+ has_default_value=False, default_value=unicode("", "utf-8"),
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ descriptor.FieldDescriptor(
+ name='operation', full_name='voldemort.Slop.operation', index=1,
+ number=2, type=9, cpp_type=9, label=1,
+ has_default_value=False, default_value=unicode("", "utf-8"),
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ descriptor.FieldDescriptor(
+ name='key', full_name='voldemort.Slop.key', index=2,
+ number=3, type=12, cpp_type=9, label=1,
+ has_default_value=False, default_value="",
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ descriptor.FieldDescriptor(
+ name='value', full_name='voldemort.Slop.value', index=3,
+ number=4, type=12, cpp_type=9, label=1,
+ has_default_value=False, default_value="",
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ descriptor.FieldDescriptor(
+ name='node_id', full_name='voldemort.Slop.node_id', index=4,
+ number=5, type=5, cpp_type=1, label=1,
+ has_default_value=False, default_value=0,
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ descriptor.FieldDescriptor(
+ name='arrived', full_name='voldemort.Slop.arrived', index=5,
+ number=6, type=3, cpp_type=2, label=1,
+ has_default_value=False, default_value=0,
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ ],
+ extensions=[