Permalink
Browse files

Merge branch 'master' of github.com:icefury71/voldemort into ro_bw_op…

…timize
  • Loading branch information...
2 parents af20132 + 9f69898 commit 8e088b98d72838606656cd543dfe74f157fe0d7b Chinmay Soman committed Jan 25, 2012
Showing with 7,065 additions and 1,009 deletions.
  1. +7 −2 .classpath
  2. +2 −0 .gitignore
  3. +5 −2 .settings/org.eclipse.jdt.core.prefs
  4. +1 −0 CONTRIBUTORS
  5. +2 −2 bin/voldemort-server.sh
  6. +73 −5 clients/python/voldemort/protocol/voldemort_admin_pb2.py
  7. +183 −0 contrib/collections/src/java/voldemort/collections/AddNodeAction.java
  8. +113 −0 contrib/collections/src/java/voldemort/collections/MappedListIterator.java
  9. +63 −0 contrib/collections/src/java/voldemort/collections/UpdateElementById.java
  10. +51 −0 contrib/collections/src/java/voldemort/collections/UpdatePageIndex.java
  11. +28 −0 contrib/collections/src/java/voldemort/collections/VLinkedPagedKey.java
  12. +224 −0 contrib/collections/src/java/voldemort/collections/VLinkedPagedList.java
  13. +206 −0 contrib/collections/src/java/voldemort/collections/VLinkedPagedListIterator.java
  14. +264 −0 contrib/collections/src/java/voldemort/collections/VListIterator.java
  15. +90 −0 contrib/collections/src/java/voldemort/collections/VListIteratorValues.java
  16. +33 −0 contrib/collections/src/java/voldemort/collections/VListKey.java
  17. +70 −0 contrib/collections/src/java/voldemort/collections/VListNode.java
  18. +60 −0 contrib/collections/src/java/voldemort/collections/VPageIndexEntry.java
  19. +354 −0 contrib/collections/src/java/voldemort/collections/VStack.java
  20. +12 −0 contrib/collections/src/java/voldemort/utils/Identifiable.java
  21. +33 −0 contrib/collections/src/java/voldemort/utils/Identified.java
  22. +260 −0 contrib/collections/test/voldemort/collections/TestVLinkedPagedList.java
  23. +168 −0 contrib/collections/test/voldemort/collections/TestVStack.java
  24. BIN lib/commons-httpclient-3.1.jar
  25. BIN lib/compress-lzf-0.9.1.jar
  26. BIN lib/httpclient-4.1.2.jar
  27. BIN lib/httpcore-4.1.2.jar
  28. BIN lib/snappy-0.2.jar
  29. +44 −5 src/java/voldemort/VoldemortAdminTool.java
  30. +18 −0 src/java/voldemort/client/AbstractStoreClientFactory.java
  31. +34 −29 src/java/voldemort/client/HttpStoreClientFactory.java
  32. +60 −0 src/java/voldemort/client/protocol/admin/AdminClient.java
  33. +565 −67 src/java/voldemort/client/protocol/pb/VAdminProto.java
  34. +19 −0 src/java/voldemort/cluster/Cluster.java
  35. +2 −1 src/java/voldemort/cluster/Node.java
  36. +3 −0 src/java/voldemort/cluster/failuredetector/AbstractFailureDetector.java
  37. +1 −2 src/java/voldemort/cluster/failuredetector/BasicStoreVerifier.java
  38. +12 −4 src/java/voldemort/cluster/failuredetector/FailureDetectorConfig.java
  39. +25 −3 src/java/voldemort/routing/ConsistentRoutingStrategy.java
  40. +10 −1 src/java/voldemort/routing/ZoneRoutingStrategy.java
  41. +41 −3 src/java/voldemort/server/VoldemortConfig.java
  42. +3 −1 src/java/voldemort/server/VoldemortJsvcDaemon.java
  43. +42 −1 src/java/voldemort/server/VoldemortServer.java
  44. +55 −0 src/java/voldemort/server/protocol/admin/AdminServiceRequestHandler.java
  45. +51 −0 src/java/voldemort/store/InsufficientOperationalNodesException.java
  46. +1 −0 src/java/voldemort/store/StoreDefinition.java
  47. +27 −0 src/java/voldemort/store/backup/NativeBackupable.java
  48. +297 −0 src/java/voldemort/store/bdb/BdbNativeBackup.java
  49. +4 −0 src/java/voldemort/store/bdb/BdbStorageConfiguration.java
  50. +12 −4 src/java/voldemort/store/bdb/BdbStorageEngine.java
  51. +2 −0 src/java/voldemort/store/compress/CompressionStrategyFactory.java
  52. +2 −2 src/java/voldemort/store/compress/LzfCompressionStrategy.java
  53. +34 −0 src/java/voldemort/store/compress/SnappyCompressionStrategy.java
  54. +0 −188 src/java/voldemort/store/compress/lzf/ChunkEncoder.java
  55. +0 −88 src/java/voldemort/store/compress/lzf/LZF.java
  56. +0 −96 src/java/voldemort/store/compress/lzf/LZFChunk.java
  57. +0 −163 src/java/voldemort/store/compress/lzf/LZFDecoder.java
  58. +0 −70 src/java/voldemort/store/compress/lzf/LZFEncoder.java
  59. +0 −5 src/java/voldemort/store/compress/lzf/package.html
  60. +39 −38 src/java/voldemort/store/http/HttpStore.java
  61. +97 −42 src/java/voldemort/store/readonly/swapper/HttpStoreSwapper.java
  62. +16 −12 src/java/voldemort/store/readonly/swapper/StoreSwapper.java
  63. +10 −0 src/java/voldemort/store/routed/PipelineData.java
  64. +28 −0 src/java/voldemort/store/routed/PipelineRoutedStore.java
  65. +32 −8 src/java/voldemort/store/routed/action/AbstractConfigureNodes.java
  66. +11 −1 src/java/voldemort/store/routed/action/ConfigureNodes.java
  67. +4 −0 src/java/voldemort/store/routed/action/PerformParallelDeleteRequests.java
  68. +4 −0 src/java/voldemort/store/routed/action/PerformParallelPutRequests.java
  69. +10 −0 src/java/voldemort/store/routed/action/PerformParallelRequests.java
  70. +5 −1 src/java/voldemort/store/routed/action/PerformSerialPutRequests.java
  71. +5 −1 src/java/voldemort/store/routed/action/PerformSerialRequests.java
  72. +1 −1 src/java/voldemort/store/slop/HintedHandoff.java
  73. +108 −0 src/java/voldemort/utils/VoldemortIOUtils.java
  74. +9 −0 src/proto/voldemort-admin.proto
  75. +1 −0 test/common/voldemort/FailureDetectorTestUtils.java
  76. +13 −11 test/common/voldemort/ServerTestUtils.java
  77. +7 −0 test/common/voldemort/VoldemortTestConstants.java
  78. +38 −30 test/common/voldemort/{ → cluster/failuredetector}/MutableStoreVerifier.java
  79. +86 −0 test/common/voldemort/config/ten-node-cluster.xml
  80. +1 −2 test/integration/voldemort/cluster/failuredetector/FailureDetectorPerformanceTest.java
  81. +46 −38 test/integration/voldemort/performance/HttpClientBench.java
  82. +35 −24 test/integration/voldemort/performance/RemoteStoreComparisonTest.java
  83. +35 −19 test/integration/voldemort/performance/RequestFileFilter.java
  84. +1 −1 test/integration/voldemort/performance/RoutedStoreParallelismTest.java
  85. +2 −2 test/integration/voldemort/rebalance/metadata/voldemort-server.sh
  86. +10 −2 test/unit/voldemort/client/HttpStoreClientFactoryTest.java
  87. +1 −1 test/unit/voldemort/cluster/TestCluster.java
  88. +1 −1 test/unit/voldemort/cluster/failuredetector/BannagePeriodFailureDetectorTest.java
  89. +20 −3 test/unit/voldemort/cluster/failuredetector/ThresholdFailureDetectorTest.java
  90. +45 −1 test/unit/voldemort/store/bdb/BdbStorageEngineTest.java
  91. +1 −1 test/unit/voldemort/store/compress/CompressingStoreTest.java
  92. +2 −0 test/unit/voldemort/store/compress/lzf/TestLZF.java
  93. +28 −5 test/unit/voldemort/store/http/HttpStoreTest.java
  94. +22 −16 test/unit/voldemort/store/readonly/swapper/StoreSwapperTest.java
  95. +1 −1 test/unit/voldemort/store/routed/HintedHandoffTest.java
  96. +1 −1 test/unit/voldemort/store/routed/ReadRepairerTest.java
  97. +2 −2 test/unit/voldemort/store/routed/RoutedStoreTest.java
  98. +52 −0 test/unit/voldemort/utils/VoldemortIOUtilsTest.java
  99. +2,562 −0 test/unit/voldemort/utils/Xtranslcl.c.input
  100. +7 −0 test/unit/voldemort/utils/maze.c.input
View
9 .classpath
@@ -12,11 +12,11 @@
<classpathentry kind="src" path="example/java"/>
<classpathentry kind="src" path="contrib/krati/src/java"/>
<classpathentry kind="src" path="contrib/krati/test"/>
- <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
+ <classpathentry kind="src" path="contrib/collections/src/java"/>
+ <classpathentry kind="src" path="contrib/collections/test"/>
<classpathentry kind="lib" path="lib/catalina-ant.jar"/>
<classpathentry kind="lib" path="lib/commons-codec-1.3.jar"/>
<classpathentry kind="lib" path="lib/commons-dbcp-1.2.2.jar"/>
- <classpathentry kind="lib" path="lib/commons-httpclient-3.1.jar"/>
<classpathentry kind="lib" path="lib/colt-1.2.0.jar"/>
<classpathentry kind="lib" path="contrib/hadoop-store-builder/lib/commons-cli-2.0-SNAPSHOT.jar"/>
<classpathentry kind="lib" path="contrib/hadoop-store-builder/lib/hadoop-0.20.2-core.jar"/>
@@ -49,5 +49,10 @@
<classpathentry kind="lib" path="lib/mockito-all-1.8.5.jar"/>
<classpathentry kind="lib" path="lib/avro-1.4.0.jar"/>
<classpathentry kind="lib" path="lib/libthrift-0.5.0.jar"/>
+ <classpathentry kind="lib" path="lib/compress-lzf-0.9.1.jar"/>
+ <classpathentry kind="lib" path="lib/snappy-0.2.jar"/>
+ <classpathentry kind="lib" path="lib/httpclient-4.1.2.jar" />
+ <classpathentry kind="lib" path="lib/httpcore-4.1.2.jar" />
+ <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
<classpathentry kind="output" path="classes"/>
</classpath>
View
2 .gitignore
@@ -6,8 +6,10 @@ dist
*.iws
*#
.#*
+test-output
node.id
rebalancing.slave.list
server.state
.version
.temp
+.idea
View
7 .settings/org.eclipse.jdt.core.prefs
@@ -1,4 +1,4 @@
-#Fri Sep 18 15:20:52 BST 2009
+#Fri Dec 30 14:37:10 PST 2011
eclipse.preferences.version=1
org.eclipse.jdt.core.codeComplete.argumentPrefixes=
org.eclipse.jdt.core.codeComplete.argumentSuffixes=
@@ -149,9 +149,12 @@ org.eclipse.jdt.core.formatter.indent_statements_compare_to_body=true
org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_cases=true
org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_switch=true
org.eclipse.jdt.core.formatter.indentation.size=4
+org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_field=insert
org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_local_variable=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_member=insert
+org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_method=insert
+org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_package=insert
org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_parameter=do not insert
+org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_type=insert
org.eclipse.jdt.core.formatter.insert_new_line_after_opening_brace_in_array_initializer=do not insert
org.eclipse.jdt.core.formatter.insert_new_line_at_end_of_file_if_missing=do not insert
org.eclipse.jdt.core.formatter.insert_new_line_before_catch_in_try_statement=do not insert
View
1 CONTRIBUTORS
@@ -12,6 +12,7 @@ Eric Evans
Geir Magnusson Jr.
Ismael Juma
Jakob Homan
+James Roper
Janne Hietamäki
Jay Kreps
Jonathan Traupman
View
4 bin/voldemort-server.sh
@@ -16,9 +16,9 @@
# limitations under the License.
#
-if [ $# -gt 1 ];
+if [ $# -gt 2 ];
then
- echo 'USAGE: bin/voldemort-server.sh [voldemort_home]'
+ echo 'USAGE: bin/voldemort-server.sh [voldemort_home] [voldemort_config_dir]'
exit 1
fi
View
78 clients/python/voldemort/protocol/voldemort_admin_pb2.py
@@ -10,7 +10,7 @@
DESCRIPTOR = descriptor.FileDescriptor(
name='voldemort-admin.proto',
package='voldemort',
- serialized_pb='\n\x15voldemort-admin.proto\x12\tvoldemort\x1a\x16voldemort-client.proto\"!\n\x12GetMetadataRequest\x12\x0b\n\x03key\x18\x01 \x02(\x0c\"]\n\x13GetMetadataResponse\x12%\n\x07version\x18\x01 \x01(\x0b\x32\x14.voldemort.Versioned\x12\x1f\n\x05\x65rror\x18\x02 \x01(\x0b\x32\x10.voldemort.Error\"M\n\x15UpdateMetadataRequest\x12\x0b\n\x03key\x18\x01 \x02(\x0c\x12\'\n\tversioned\x18\x02 \x02(\x0b\x32\x14.voldemort.Versioned\"9\n\x16UpdateMetadataResponse\x12\x1f\n\x05\x65rror\x18\x01 \x01(\x0b\x32\x10.voldemort.Error\"7\n\tFileEntry\x12\x11\n\tfile_name\x18\x01 \x02(\t\x12\x17\n\x0f\x66ile_size_bytes\x18\x02 \x02(\x03\"F\n\x0ePartitionEntry\x12\x0b\n\x03key\x18\x01 \x02(\x0c\x12\'\n\tversioned\x18\x02 \x02(\x0b\x32\x14.voldemort.Versioned\"\x8e\x01\n\x1dUpdatePartitionEntriesRequest\x12\r\n\x05store\x18\x01 \x02(\t\x12\x32\n\x0fpartition_entry\x18\x02 \x02(\x0b\x32\x19.voldemort.PartitionEntry\x12*\n\x06\x66ilter\x18\x03 \x01(\x0b\x32\x1a.voldemort.VoldemortFilter\"A\n\x1eUpdatePartitionEntriesResponse\x12\x1f\n\x05\x65rror\x18\x01 \x01(\x0b\x32\x10.voldemort.Error\"-\n\x0fVoldemortFilter\x12\x0c\n\x04name\x18\x01 \x02(\t\x12\x0c\n\x04\x64\x61ta\x18\x02 \x02(\x0c\"\xaf\x01\n\x18UpdateSlopEntriesRequest\x12\r\n\x05store\x18\x01 \x02(\t\x12\x0b\n\x03key\x18\x02 \x02(\x0c\x12\'\n\x07version\x18\x03 \x02(\x0b\x32\x16.voldemort.VectorClock\x12,\n\x0crequest_type\x18\x04 \x02(\x0e\x32\x16.voldemort.RequestType\x12\r\n\x05value\x18\x05 \x01(\x0c\x12\x11\n\ttransform\x18\x06 \x01(\x0c\"<\n\x19UpdateSlopEntriesResponse\x12\x1f\n\x05\x65rror\x18\x01 \x01(\x0b\x32\x10.voldemort.Error\"d\n\x1a\x46\x65tchPartitionFilesRequest\x12\r\n\x05store\x18\x01 \x02(\t\x12\x37\n\x14replica_to_partition\x18\x02 \x03(\x0b\x32\x19.voldemort.PartitionTuple\"\xd7\x01\n\x1c\x46\x65tchPartitionEntriesRequest\x12\x37\n\x14replica_to_partition\x18\x01 \x03(\x0b\x32\x19.voldemort.PartitionTuple\x12\r\n\x05store\x18\x02 \x02(\t\x12*\n\x06\x66ilter\x18\x03 \x01(\x0b\x32\x1a.voldemort.VoldemortFilter\x12\x14\n\x0c\x66\x65tch_values\x18\x04 \x01(\x08\x12\x14\n\x0cskip_records\x18\x05 \x01(\x03\x12\x17\n\x0finitial_cluster\x18\x06 \x01(\t\"\x81\x01\n\x1d\x46\x65tchPartitionEntriesResponse\x12\x32\n\x0fpartition_entry\x18\x01 \x01(\x0b\x32\x19.voldemort.PartitionEntry\x12\x0b\n\x03key\x18\x02 \x01(\x0c\x12\x1f\n\x05\x65rror\x18\x03 \x01(\x0b\x32\x10.voldemort.Error\"\xac\x01\n\x1d\x44\x65letePartitionEntriesRequest\x12\r\n\x05store\x18\x01 \x02(\t\x12\x37\n\x14replica_to_partition\x18\x02 \x03(\x0b\x32\x19.voldemort.PartitionTuple\x12*\n\x06\x66ilter\x18\x03 \x01(\x0b\x32\x1a.voldemort.VoldemortFilter\x12\x17\n\x0finitial_cluster\x18\x04 \x01(\t\"P\n\x1e\x44\x65letePartitionEntriesResponse\x12\r\n\x05\x63ount\x18\x01 \x01(\x03\x12\x1f\n\x05\x65rror\x18\x02 \x01(\x0b\x32\x10.voldemort.Error\"\xcf\x01\n\x1dInitiateFetchAndUpdateRequest\x12\x0f\n\x07node_id\x18\x01 \x02(\x05\x12\r\n\x05store\x18\x02 \x02(\t\x12*\n\x06\x66ilter\x18\x03 \x01(\x0b\x32\x1a.voldemort.VoldemortFilter\x12\x37\n\x14replica_to_partition\x18\x04 \x03(\x0b\x32\x19.voldemort.PartitionTuple\x12\x17\n\x0finitial_cluster\x18\x05 \x01(\t\x12\x10\n\x08optimize\x18\x06 \x01(\x08\"1\n\x1b\x41syncOperationStatusRequest\x12\x12\n\nrequest_id\x18\x01 \x02(\x05\"/\n\x19\x41syncOperationStopRequest\x12\x12\n\nrequest_id\x18\x01 \x02(\x05\"=\n\x1a\x41syncOperationStopResponse\x12\x1f\n\x05\x65rror\x18\x01 \x01(\x0b\x32\x10.voldemort.Error\"2\n\x19\x41syncOperationListRequest\x12\x15\n\rshow_complete\x18\x02 \x02(\x08\"R\n\x1a\x41syncOperationListResponse\x12\x13\n\x0brequest_ids\x18\x01 \x03(\x05\x12\x1f\n\x05\x65rror\x18\x02 \x01(\x0b\x32\x10.voldemort.Error\":\n\x0ePartitionTuple\x12\x14\n\x0creplica_type\x18\x01 \x02(\x05\x12\x12\n\npartitions\x18\x02 \x03(\x05\"e\n\x16PerStorePartitionTuple\x12\x12\n\nstore_name\x18\x01 \x02(\t\x12\x37\n\x14replica_to_partition\x18\x02 \x03(\x0b\x32\x19.voldemort.PartitionTuple\"\xf8\x01\n\x19RebalancePartitionInfoMap\x12\x12\n\nstealer_id\x18\x01 \x02(\x05\x12\x10\n\x08\x64onor_id\x18\x02 \x02(\x05\x12\x0f\n\x07\x61ttempt\x18\x03 \x02(\x05\x12\x43\n\x18replica_to_add_partition\x18\x04 \x03(\x0b\x32!.voldemort.PerStorePartitionTuple\x12\x46\n\x1breplica_to_delete_partition\x18\x05 \x03(\x0b\x32!.voldemort.PerStorePartitionTuple\x12\x17\n\x0finitial_cluster\x18\x06 \x02(\t\"f\n\x1cInitiateRebalanceNodeRequest\x12\x46\n\x18rebalance_partition_info\x18\x01 \x02(\x0b\x32$.voldemort.RebalancePartitionInfoMap\"m\n#InitiateRebalanceNodeOnDonorRequest\x12\x46\n\x18rebalance_partition_info\x18\x01 \x03(\x0b\x32$.voldemort.RebalancePartitionInfoMap\"\x8a\x01\n\x1c\x41syncOperationStatusResponse\x12\x12\n\nrequest_id\x18\x01 \x01(\x05\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x0e\n\x06status\x18\x03 \x01(\t\x12\x10\n\x08\x63omplete\x18\x04 \x01(\x08\x12\x1f\n\x05\x65rror\x18\x05 \x01(\x0b\x32\x10.voldemort.Error\"\'\n\x16TruncateEntriesRequest\x12\r\n\x05store\x18\x01 \x02(\t\":\n\x17TruncateEntriesResponse\x12\x1f\n\x05\x65rror\x18\x01 \x01(\x0b\x32\x10.voldemort.Error\"*\n\x0f\x41\x64\x64StoreRequest\x12\x17\n\x0fstoreDefinition\x18\x01 \x02(\t\"3\n\x10\x41\x64\x64StoreResponse\x12\x1f\n\x05\x65rror\x18\x01 \x01(\x0b\x32\x10.voldemort.Error\"\'\n\x12\x44\x65leteStoreRequest\x12\x11\n\tstoreName\x18\x01 \x02(\t\"6\n\x13\x44\x65leteStoreResponse\x12\x1f\n\x05\x65rror\x18\x01 \x01(\x0b\x32\x10.voldemort.Error\"P\n\x11\x46\x65tchStoreRequest\x12\x12\n\nstore_name\x18\x01 \x02(\t\x12\x11\n\tstore_dir\x18\x02 \x02(\t\x12\x14\n\x0cpush_version\x18\x03 \x01(\x03\"9\n\x10SwapStoreRequest\x12\x12\n\nstore_name\x18\x01 \x02(\t\x12\x11\n\tstore_dir\x18\x02 \x02(\t\"P\n\x11SwapStoreResponse\x12\x1f\n\x05\x65rror\x18\x01 \x01(\x0b\x32\x10.voldemort.Error\x12\x1a\n\x12previous_store_dir\x18\x02 \x01(\t\"@\n\x14RollbackStoreRequest\x12\x12\n\nstore_name\x18\x01 \x02(\t\x12\x14\n\x0cpush_version\x18\x02 \x02(\x03\"8\n\x15RollbackStoreResponse\x12\x1f\n\x05\x65rror\x18\x01 \x01(\x0b\x32\x10.voldemort.Error\"&\n\x10RepairJobRequest\x12\x12\n\nstore_name\x18\x01 \x01(\t\"4\n\x11RepairJobResponse\x12\x1f\n\x05\x65rror\x18\x01 \x01(\x0b\x32\x10.voldemort.Error\"=\n\x14ROStoreVersionDirMap\x12\x12\n\nstore_name\x18\x01 \x02(\t\x12\x11\n\tstore_dir\x18\x02 \x02(\t\"/\n\x19GetROMaxVersionDirRequest\x12\x12\n\nstore_name\x18\x01 \x03(\t\"y\n\x1aGetROMaxVersionDirResponse\x12:\n\x11ro_store_versions\x18\x01 \x03(\x0b\x32\x1f.voldemort.ROStoreVersionDirMap\x12\x1f\n\x05\x65rror\x18\x02 \x01(\x0b\x32\x10.voldemort.Error\"3\n\x1dGetROCurrentVersionDirRequest\x12\x12\n\nstore_name\x18\x01 \x03(\t\"}\n\x1eGetROCurrentVersionDirResponse\x12:\n\x11ro_store_versions\x18\x01 \x03(\x0b\x32\x1f.voldemort.ROStoreVersionDirMap\x12\x1f\n\x05\x65rror\x18\x02 \x01(\x0b\x32\x10.voldemort.Error\"/\n\x19GetROStorageFormatRequest\x12\x12\n\nstore_name\x18\x01 \x03(\t\"y\n\x1aGetROStorageFormatResponse\x12:\n\x11ro_store_versions\x18\x01 \x03(\x0b\x32\x1f.voldemort.ROStoreVersionDirMap\x12\x1f\n\x05\x65rror\x18\x02 \x01(\x0b\x32\x10.voldemort.Error\"@\n\x17\x46\x61iledFetchStoreRequest\x12\x12\n\nstore_name\x18\x01 \x02(\t\x12\x11\n\tstore_dir\x18\x02 \x02(\t\";\n\x18\x46\x61iledFetchStoreResponse\x12\x1f\n\x05\x65rror\x18\x01 \x01(\x0b\x32\x10.voldemort.Error\"\xe6\x01\n\x1bRebalanceStateChangeRequest\x12K\n\x1drebalance_partition_info_list\x18\x01 \x03(\x0b\x32$.voldemort.RebalancePartitionInfoMap\x12\x16\n\x0e\x63luster_string\x18\x02 \x02(\t\x12\x0f\n\x07swap_ro\x18\x03 \x02(\x08\x12\x1f\n\x17\x63hange_cluster_metadata\x18\x04 \x02(\x08\x12\x1e\n\x16\x63hange_rebalance_state\x18\x05 \x02(\x08\x12\x10\n\x08rollback\x18\x06 \x02(\x08\"?\n\x1cRebalanceStateChangeResponse\x12\x1f\n\x05\x65rror\x18\x01 \x01(\x0b\x32\x10.voldemort.Error\"G\n DeleteStoreRebalanceStateRequest\x12\x12\n\nstore_name\x18\x01 \x02(\t\x12\x0f\n\x07node_id\x18\x02 \x02(\x05\"D\n!DeleteStoreRebalanceStateResponse\x12\x1f\n\x05\x65rror\x18\x01 \x01(\x0b\x32\x10.voldemort.Error\"\x80\x0e\n\x15VoldemortAdminRequest\x12)\n\x04type\x18\x01 \x02(\x0e\x32\x1b.voldemort.AdminRequestType\x12\x33\n\x0cget_metadata\x18\x02 \x01(\x0b\x32\x1d.voldemort.GetMetadataRequest\x12\x39\n\x0fupdate_metadata\x18\x03 \x01(\x0b\x32 .voldemort.UpdateMetadataRequest\x12J\n\x18update_partition_entries\x18\x04 \x01(\x0b\x32(.voldemort.UpdatePartitionEntriesRequest\x12H\n\x17\x66\x65tch_partition_entries\x18\x05 \x01(\x0b\x32\'.voldemort.FetchPartitionEntriesRequest\x12J\n\x18\x64\x65lete_partition_entries\x18\x06 \x01(\x0b\x32(.voldemort.DeletePartitionEntriesRequest\x12K\n\x19initiate_fetch_and_update\x18\x07 \x01(\x0b\x32(.voldemort.InitiateFetchAndUpdateRequest\x12\x46\n\x16\x61sync_operation_status\x18\x08 \x01(\x0b\x32&.voldemort.AsyncOperationStatusRequest\x12H\n\x17initiate_rebalance_node\x18\t \x01(\x0b\x32\'.voldemort.InitiateRebalanceNodeRequest\x12\x42\n\x14\x61sync_operation_stop\x18\n \x01(\x0b\x32$.voldemort.AsyncOperationStopRequest\x12\x42\n\x14\x61sync_operation_list\x18\x0b \x01(\x0b\x32$.voldemort.AsyncOperationListRequest\x12;\n\x10truncate_entries\x18\x0c \x01(\x0b\x32!.voldemort.TruncateEntriesRequest\x12-\n\tadd_store\x18\r \x01(\x0b\x32\x1a.voldemort.AddStoreRequest\x12\x33\n\x0c\x64\x65lete_store\x18\x0e \x01(\x0b\x32\x1d.voldemort.DeleteStoreRequest\x12\x31\n\x0b\x66\x65tch_store\x18\x0f \x01(\x0b\x32\x1c.voldemort.FetchStoreRequest\x12/\n\nswap_store\x18\x10 \x01(\x0b\x32\x1b.voldemort.SwapStoreRequest\x12\x37\n\x0erollback_store\x18\x11 \x01(\x0b\x32\x1f.voldemort.RollbackStoreRequest\x12\x44\n\x16get_ro_max_version_dir\x18\x12 \x01(\x0b\x32$.voldemort.GetROMaxVersionDirRequest\x12L\n\x1aget_ro_current_version_dir\x18\x13 \x01(\x0b\x32(.voldemort.GetROCurrentVersionDirRequest\x12\x44\n\x15\x66\x65tch_partition_files\x18\x14 \x01(\x0b\x32%.voldemort.FetchPartitionFilesRequest\x12@\n\x13update_slop_entries\x18\x16 \x01(\x0b\x32#.voldemort.UpdateSlopEntriesRequest\x12>\n\x12\x66\x61iled_fetch_store\x18\x18 \x01(\x0b\x32\".voldemort.FailedFetchStoreRequest\x12\x43\n\x15get_ro_storage_format\x18\x19 \x01(\x0b\x32$.voldemort.GetROStorageFormatRequest\x12\x46\n\x16rebalance_state_change\x18\x1a \x01(\x0b\x32&.voldemort.RebalanceStateChangeRequest\x12/\n\nrepair_job\x18\x1b \x01(\x0b\x32\x1b.voldemort.RepairJobRequest\x12X\n initiate_rebalance_node_on_donor\x18\x1c \x01(\x0b\x32..voldemort.InitiateRebalanceNodeOnDonorRequest\x12Q\n\x1c\x64\x65lete_store_rebalance_state\x18\x1d \x01(\x0b\x32+.voldemort.DeleteStoreRebalanceStateRequest*\xa1\x05\n\x10\x41\x64minRequestType\x12\x10\n\x0cGET_METADATA\x10\x00\x12\x13\n\x0fUPDATE_METADATA\x10\x01\x12\x1c\n\x18UPDATE_PARTITION_ENTRIES\x10\x02\x12\x1b\n\x17\x46\x45TCH_PARTITION_ENTRIES\x10\x03\x12\x1c\n\x18\x44\x45LETE_PARTITION_ENTRIES\x10\x04\x12\x1d\n\x19INITIATE_FETCH_AND_UPDATE\x10\x05\x12\x1a\n\x16\x41SYNC_OPERATION_STATUS\x10\x06\x12\x1b\n\x17INITIATE_REBALANCE_NODE\x10\x07\x12\x18\n\x14\x41SYNC_OPERATION_STOP\x10\x08\x12\x18\n\x14\x41SYNC_OPERATION_LIST\x10\t\x12\x14\n\x10TRUNCATE_ENTRIES\x10\n\x12\r\n\tADD_STORE\x10\x0b\x12\x10\n\x0c\x44\x45LETE_STORE\x10\x0c\x12\x0f\n\x0b\x46\x45TCH_STORE\x10\r\x12\x0e\n\nSWAP_STORE\x10\x0e\x12\x12\n\x0eROLLBACK_STORE\x10\x0f\x12\x1a\n\x16GET_RO_MAX_VERSION_DIR\x10\x10\x12\x1e\n\x1aGET_RO_CURRENT_VERSION_DIR\x10\x11\x12\x19\n\x15\x46\x45TCH_PARTITION_FILES\x10\x12\x12\x17\n\x13UPDATE_SLOP_ENTRIES\x10\x14\x12\x16\n\x12\x46\x41ILED_FETCH_STORE\x10\x16\x12\x19\n\x15GET_RO_STORAGE_FORMAT\x10\x17\x12\x1a\n\x16REBALANCE_STATE_CHANGE\x10\x18\x12\x0e\n\nREPAIR_JOB\x10\x19\x12$\n INITIATE_REBALANCE_NODE_ON_DONOR\x10\x1a\x12 \n\x1c\x44\x45LETE_STORE_REBALANCE_STATE\x10\x1b\x42-\n\x1cvoldemort.client.protocol.pbB\x0bVAdminProtoH\x01')
+ serialized_pb='\n\x15voldemort-admin.proto\x12\tvoldemort\x1a\x16voldemort-client.proto\"!\n\x12GetMetadataRequest\x12\x0b\n\x03key\x18\x01 \x02(\x0c\"]\n\x13GetMetadataResponse\x12%\n\x07version\x18\x01 \x01(\x0b\x32\x14.voldemort.Versioned\x12\x1f\n\x05\x65rror\x18\x02 \x01(\x0b\x32\x10.voldemort.Error\"M\n\x15UpdateMetadataRequest\x12\x0b\n\x03key\x18\x01 \x02(\x0c\x12\'\n\tversioned\x18\x02 \x02(\x0b\x32\x14.voldemort.Versioned\"9\n\x16UpdateMetadataResponse\x12\x1f\n\x05\x65rror\x18\x01 \x01(\x0b\x32\x10.voldemort.Error\"7\n\tFileEntry\x12\x11\n\tfile_name\x18\x01 \x02(\t\x12\x17\n\x0f\x66ile_size_bytes\x18\x02 \x02(\x03\"F\n\x0ePartitionEntry\x12\x0b\n\x03key\x18\x01 \x02(\x0c\x12\'\n\tversioned\x18\x02 \x02(\x0b\x32\x14.voldemort.Versioned\"\x8e\x01\n\x1dUpdatePartitionEntriesRequest\x12\r\n\x05store\x18\x01 \x02(\t\x12\x32\n\x0fpartition_entry\x18\x02 \x02(\x0b\x32\x19.voldemort.PartitionEntry\x12*\n\x06\x66ilter\x18\x03 \x01(\x0b\x32\x1a.voldemort.VoldemortFilter\"A\n\x1eUpdatePartitionEntriesResponse\x12\x1f\n\x05\x65rror\x18\x01 \x01(\x0b\x32\x10.voldemort.Error\"-\n\x0fVoldemortFilter\x12\x0c\n\x04name\x18\x01 \x02(\t\x12\x0c\n\x04\x64\x61ta\x18\x02 \x02(\x0c\"\xaf\x01\n\x18UpdateSlopEntriesRequest\x12\r\n\x05store\x18\x01 \x02(\t\x12\x0b\n\x03key\x18\x02 \x02(\x0c\x12\'\n\x07version\x18\x03 \x02(\x0b\x32\x16.voldemort.VectorClock\x12,\n\x0crequest_type\x18\x04 \x02(\x0e\x32\x16.voldemort.RequestType\x12\r\n\x05value\x18\x05 \x01(\x0c\x12\x11\n\ttransform\x18\x06 \x01(\x0c\"<\n\x19UpdateSlopEntriesResponse\x12\x1f\n\x05\x65rror\x18\x01 \x01(\x0b\x32\x10.voldemort.Error\"d\n\x1a\x46\x65tchPartitionFilesRequest\x12\r\n\x05store\x18\x01 \x02(\t\x12\x37\n\x14replica_to_partition\x18\x02 \x03(\x0b\x32\x19.voldemort.PartitionTuple\"\xd7\x01\n\x1c\x46\x65tchPartitionEntriesRequest\x12\x37\n\x14replica_to_partition\x18\x01 \x03(\x0b\x32\x19.voldemort.PartitionTuple\x12\r\n\x05store\x18\x02 \x02(\t\x12*\n\x06\x66ilter\x18\x03 \x01(\x0b\x32\x1a.voldemort.VoldemortFilter\x12\x14\n\x0c\x66\x65tch_values\x18\x04 \x01(\x08\x12\x14\n\x0cskip_records\x18\x05 \x01(\x03\x12\x17\n\x0finitial_cluster\x18\x06 \x01(\t\"\x81\x01\n\x1d\x46\x65tchPartitionEntriesResponse\x12\x32\n\x0fpartition_entry\x18\x01 \x01(\x0b\x32\x19.voldemort.PartitionEntry\x12\x0b\n\x03key\x18\x02 \x01(\x0c\x12\x1f\n\x05\x65rror\x18\x03 \x01(\x0b\x32\x10.voldemort.Error\"\xac\x01\n\x1d\x44\x65letePartitionEntriesRequest\x12\r\n\x05store\x18\x01 \x02(\t\x12\x37\n\x14replica_to_partition\x18\x02 \x03(\x0b\x32\x19.voldemort.PartitionTuple\x12*\n\x06\x66ilter\x18\x03 \x01(\x0b\x32\x1a.voldemort.VoldemortFilter\x12\x17\n\x0finitial_cluster\x18\x04 \x01(\t\"P\n\x1e\x44\x65letePartitionEntriesResponse\x12\r\n\x05\x63ount\x18\x01 \x01(\x03\x12\x1f\n\x05\x65rror\x18\x02 \x01(\x0b\x32\x10.voldemort.Error\"\xcf\x01\n\x1dInitiateFetchAndUpdateRequest\x12\x0f\n\x07node_id\x18\x01 \x02(\x05\x12\r\n\x05store\x18\x02 \x02(\t\x12*\n\x06\x66ilter\x18\x03 \x01(\x0b\x32\x1a.voldemort.VoldemortFilter\x12\x37\n\x14replica_to_partition\x18\x04 \x03(\x0b\x32\x19.voldemort.PartitionTuple\x12\x17\n\x0finitial_cluster\x18\x05 \x01(\t\x12\x10\n\x08optimize\x18\x06 \x01(\x08\"1\n\x1b\x41syncOperationStatusRequest\x12\x12\n\nrequest_id\x18\x01 \x02(\x05\"/\n\x19\x41syncOperationStopRequest\x12\x12\n\nrequest_id\x18\x01 \x02(\x05\"=\n\x1a\x41syncOperationStopResponse\x12\x1f\n\x05\x65rror\x18\x01 \x01(\x0b\x32\x10.voldemort.Error\"2\n\x19\x41syncOperationListRequest\x12\x15\n\rshow_complete\x18\x02 \x02(\x08\"R\n\x1a\x41syncOperationListResponse\x12\x13\n\x0brequest_ids\x18\x01 \x03(\x05\x12\x1f\n\x05\x65rror\x18\x02 \x01(\x0b\x32\x10.voldemort.Error\":\n\x0ePartitionTuple\x12\x14\n\x0creplica_type\x18\x01 \x02(\x05\x12\x12\n\npartitions\x18\x02 \x03(\x05\"e\n\x16PerStorePartitionTuple\x12\x12\n\nstore_name\x18\x01 \x02(\t\x12\x37\n\x14replica_to_partition\x18\x02 \x03(\x0b\x32\x19.voldemort.PartitionTuple\"\xf8\x01\n\x19RebalancePartitionInfoMap\x12\x12\n\nstealer_id\x18\x01 \x02(\x05\x12\x10\n\x08\x64onor_id\x18\x02 \x02(\x05\x12\x0f\n\x07\x61ttempt\x18\x03 \x02(\x05\x12\x43\n\x18replica_to_add_partition\x18\x04 \x03(\x0b\x32!.voldemort.PerStorePartitionTuple\x12\x46\n\x1breplica_to_delete_partition\x18\x05 \x03(\x0b\x32!.voldemort.PerStorePartitionTuple\x12\x17\n\x0finitial_cluster\x18\x06 \x02(\t\"f\n\x1cInitiateRebalanceNodeRequest\x12\x46\n\x18rebalance_partition_info\x18\x01 \x02(\x0b\x32$.voldemort.RebalancePartitionInfoMap\"m\n#InitiateRebalanceNodeOnDonorRequest\x12\x46\n\x18rebalance_partition_info\x18\x01 \x03(\x0b\x32$.voldemort.RebalancePartitionInfoMap\"\x8a\x01\n\x1c\x41syncOperationStatusResponse\x12\x12\n\nrequest_id\x18\x01 \x01(\x05\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x0e\n\x06status\x18\x03 \x01(\t\x12\x10\n\x08\x63omplete\x18\x04 \x01(\x08\x12\x1f\n\x05\x65rror\x18\x05 \x01(\x0b\x32\x10.voldemort.Error\"\'\n\x16TruncateEntriesRequest\x12\r\n\x05store\x18\x01 \x02(\t\":\n\x17TruncateEntriesResponse\x12\x1f\n\x05\x65rror\x18\x01 \x01(\x0b\x32\x10.voldemort.Error\"*\n\x0f\x41\x64\x64StoreRequest\x12\x17\n\x0fstoreDefinition\x18\x01 \x02(\t\"3\n\x10\x41\x64\x64StoreResponse\x12\x1f\n\x05\x65rror\x18\x01 \x01(\x0b\x32\x10.voldemort.Error\"\'\n\x12\x44\x65leteStoreRequest\x12\x11\n\tstoreName\x18\x01 \x02(\t\"6\n\x13\x44\x65leteStoreResponse\x12\x1f\n\x05\x65rror\x18\x01 \x01(\x0b\x32\x10.voldemort.Error\"P\n\x11\x46\x65tchStoreRequest\x12\x12\n\nstore_name\x18\x01 \x02(\t\x12\x11\n\tstore_dir\x18\x02 \x02(\t\x12\x14\n\x0cpush_version\x18\x03 \x01(\x03\"9\n\x10SwapStoreRequest\x12\x12\n\nstore_name\x18\x01 \x02(\t\x12\x11\n\tstore_dir\x18\x02 \x02(\t\"P\n\x11SwapStoreResponse\x12\x1f\n\x05\x65rror\x18\x01 \x01(\x0b\x32\x10.voldemort.Error\x12\x1a\n\x12previous_store_dir\x18\x02 \x01(\t\"@\n\x14RollbackStoreRequest\x12\x12\n\nstore_name\x18\x01 \x02(\t\x12\x14\n\x0cpush_version\x18\x02 \x02(\x03\"8\n\x15RollbackStoreResponse\x12\x1f\n\x05\x65rror\x18\x01 \x01(\x0b\x32\x10.voldemort.Error\"&\n\x10RepairJobRequest\x12\x12\n\nstore_name\x18\x01 \x01(\t\"4\n\x11RepairJobResponse\x12\x1f\n\x05\x65rror\x18\x01 \x01(\x0b\x32\x10.voldemort.Error\"=\n\x14ROStoreVersionDirMap\x12\x12\n\nstore_name\x18\x01 \x02(\t\x12\x11\n\tstore_dir\x18\x02 \x02(\t\"/\n\x19GetROMaxVersionDirRequest\x12\x12\n\nstore_name\x18\x01 \x03(\t\"y\n\x1aGetROMaxVersionDirResponse\x12:\n\x11ro_store_versions\x18\x01 \x03(\x0b\x32\x1f.voldemort.ROStoreVersionDirMap\x12\x1f\n\x05\x65rror\x18\x02 \x01(\x0b\x32\x10.voldemort.Error\"3\n\x1dGetROCurrentVersionDirRequest\x12\x12\n\nstore_name\x18\x01 \x03(\t\"}\n\x1eGetROCurrentVersionDirResponse\x12:\n\x11ro_store_versions\x18\x01 \x03(\x0b\x32\x1f.voldemort.ROStoreVersionDirMap\x12\x1f\n\x05\x65rror\x18\x02 \x01(\x0b\x32\x10.voldemort.Error\"/\n\x19GetROStorageFormatRequest\x12\x12\n\nstore_name\x18\x01 \x03(\t\"y\n\x1aGetROStorageFormatResponse\x12:\n\x11ro_store_versions\x18\x01 \x03(\x0b\x32\x1f.voldemort.ROStoreVersionDirMap\x12\x1f\n\x05\x65rror\x18\x02 \x01(\x0b\x32\x10.voldemort.Error\"@\n\x17\x46\x61iledFetchStoreRequest\x12\x12\n\nstore_name\x18\x01 \x02(\t\x12\x11\n\tstore_dir\x18\x02 \x02(\t\";\n\x18\x46\x61iledFetchStoreResponse\x12\x1f\n\x05\x65rror\x18\x01 \x01(\x0b\x32\x10.voldemort.Error\"\xe6\x01\n\x1bRebalanceStateChangeRequest\x12K\n\x1drebalance_partition_info_list\x18\x01 \x03(\x0b\x32$.voldemort.RebalancePartitionInfoMap\x12\x16\n\x0e\x63luster_string\x18\x02 \x02(\t\x12\x0f\n\x07swap_ro\x18\x03 \x02(\x08\x12\x1f\n\x17\x63hange_cluster_metadata\x18\x04 \x02(\x08\x12\x1e\n\x16\x63hange_rebalance_state\x18\x05 \x02(\x08\x12\x10\n\x08rollback\x18\x06 \x02(\x08\"?\n\x1cRebalanceStateChangeResponse\x12\x1f\n\x05\x65rror\x18\x01 \x01(\x0b\x32\x10.voldemort.Error\"G\n DeleteStoreRebalanceStateRequest\x12\x12\n\nstore_name\x18\x01 \x02(\t\x12\x0f\n\x07node_id\x18\x02 \x02(\x05\"D\n!DeleteStoreRebalanceStateResponse\x12\x1f\n\x05\x65rror\x18\x01 \x01(\x0b\x32\x10.voldemort.Error\"h\n\x13NativeBackupRequest\x12\x12\n\nstore_name\x18\x01 \x02(\t\x12\x12\n\nbackup_dir\x18\x02 \x02(\t\x12\x14\n\x0cverify_files\x18\x03 \x02(\x08\x12\x13\n\x0bincremental\x18\x04 \x02(\x08\"\xb7\x0e\n\x15VoldemortAdminRequest\x12)\n\x04type\x18\x01 \x02(\x0e\x32\x1b.voldemort.AdminRequestType\x12\x33\n\x0cget_metadata\x18\x02 \x01(\x0b\x32\x1d.voldemort.GetMetadataRequest\x12\x39\n\x0fupdate_metadata\x18\x03 \x01(\x0b\x32 .voldemort.UpdateMetadataRequest\x12J\n\x18update_partition_entries\x18\x04 \x01(\x0b\x32(.voldemort.UpdatePartitionEntriesRequest\x12H\n\x17\x66\x65tch_partition_entries\x18\x05 \x01(\x0b\x32\'.voldemort.FetchPartitionEntriesRequest\x12J\n\x18\x64\x65lete_partition_entries\x18\x06 \x01(\x0b\x32(.voldemort.DeletePartitionEntriesRequest\x12K\n\x19initiate_fetch_and_update\x18\x07 \x01(\x0b\x32(.voldemort.InitiateFetchAndUpdateRequest\x12\x46\n\x16\x61sync_operation_status\x18\x08 \x01(\x0b\x32&.voldemort.AsyncOperationStatusRequest\x12H\n\x17initiate_rebalance_node\x18\t \x01(\x0b\x32\'.voldemort.InitiateRebalanceNodeRequest\x12\x42\n\x14\x61sync_operation_stop\x18\n \x01(\x0b\x32$.voldemort.AsyncOperationStopRequest\x12\x42\n\x14\x61sync_operation_list\x18\x0b \x01(\x0b\x32$.voldemort.AsyncOperationListRequest\x12;\n\x10truncate_entries\x18\x0c \x01(\x0b\x32!.voldemort.TruncateEntriesRequest\x12-\n\tadd_store\x18\r \x01(\x0b\x32\x1a.voldemort.AddStoreRequest\x12\x33\n\x0c\x64\x65lete_store\x18\x0e \x01(\x0b\x32\x1d.voldemort.DeleteStoreRequest\x12\x31\n\x0b\x66\x65tch_store\x18\x0f \x01(\x0b\x32\x1c.voldemort.FetchStoreRequest\x12/\n\nswap_store\x18\x10 \x01(\x0b\x32\x1b.voldemort.SwapStoreRequest\x12\x37\n\x0erollback_store\x18\x11 \x01(\x0b\x32\x1f.voldemort.RollbackStoreRequest\x12\x44\n\x16get_ro_max_version_dir\x18\x12 \x01(\x0b\x32$.voldemort.GetROMaxVersionDirRequest\x12L\n\x1aget_ro_current_version_dir\x18\x13 \x01(\x0b\x32(.voldemort.GetROCurrentVersionDirRequest\x12\x44\n\x15\x66\x65tch_partition_files\x18\x14 \x01(\x0b\x32%.voldemort.FetchPartitionFilesRequest\x12@\n\x13update_slop_entries\x18\x16 \x01(\x0b\x32#.voldemort.UpdateSlopEntriesRequest\x12>\n\x12\x66\x61iled_fetch_store\x18\x18 \x01(\x0b\x32\".voldemort.FailedFetchStoreRequest\x12\x43\n\x15get_ro_storage_format\x18\x19 \x01(\x0b\x32$.voldemort.GetROStorageFormatRequest\x12\x46\n\x16rebalance_state_change\x18\x1a \x01(\x0b\x32&.voldemort.RebalanceStateChangeRequest\x12/\n\nrepair_job\x18\x1b \x01(\x0b\x32\x1b.voldemort.RepairJobRequest\x12X\n initiate_rebalance_node_on_donor\x18\x1c \x01(\x0b\x32..voldemort.InitiateRebalanceNodeOnDonorRequest\x12Q\n\x1c\x64\x65lete_store_rebalance_state\x18\x1d \x01(\x0b\x32+.voldemort.DeleteStoreRebalanceStateRequest\x12\x35\n\rnative_backup\x18\x1e \x01(\x0b\x32\x1e.voldemort.NativeBackupRequest*\xb4\x05\n\x10\x41\x64minRequestType\x12\x10\n\x0cGET_METADATA\x10\x00\x12\x13\n\x0fUPDATE_METADATA\x10\x01\x12\x1c\n\x18UPDATE_PARTITION_ENTRIES\x10\x02\x12\x1b\n\x17\x46\x45TCH_PARTITION_ENTRIES\x10\x03\x12\x1c\n\x18\x44\x45LETE_PARTITION_ENTRIES\x10\x04\x12\x1d\n\x19INITIATE_FETCH_AND_UPDATE\x10\x05\x12\x1a\n\x16\x41SYNC_OPERATION_STATUS\x10\x06\x12\x1b\n\x17INITIATE_REBALANCE_NODE\x10\x07\x12\x18\n\x14\x41SYNC_OPERATION_STOP\x10\x08\x12\x18\n\x14\x41SYNC_OPERATION_LIST\x10\t\x12\x14\n\x10TRUNCATE_ENTRIES\x10\n\x12\r\n\tADD_STORE\x10\x0b\x12\x10\n\x0c\x44\x45LETE_STORE\x10\x0c\x12\x0f\n\x0b\x46\x45TCH_STORE\x10\r\x12\x0e\n\nSWAP_STORE\x10\x0e\x12\x12\n\x0eROLLBACK_STORE\x10\x0f\x12\x1a\n\x16GET_RO_MAX_VERSION_DIR\x10\x10\x12\x1e\n\x1aGET_RO_CURRENT_VERSION_DIR\x10\x11\x12\x19\n\x15\x46\x45TCH_PARTITION_FILES\x10\x12\x12\x17\n\x13UPDATE_SLOP_ENTRIES\x10\x14\x12\x16\n\x12\x46\x41ILED_FETCH_STORE\x10\x16\x12\x19\n\x15GET_RO_STORAGE_FORMAT\x10\x17\x12\x1a\n\x16REBALANCE_STATE_CHANGE\x10\x18\x12\x0e\n\nREPAIR_JOB\x10\x19\x12$\n INITIATE_REBALANCE_NODE_ON_DONOR\x10\x1a\x12 \n\x1c\x44\x45LETE_STORE_REBALANCE_STATE\x10\x1b\x12\x11\n\rNATIVE_BACKUP\x10\x1c\x42-\n\x1cvoldemort.client.protocol.pbB\x0bVAdminProtoH\x01')
_ADMINREQUESTTYPE = descriptor.EnumDescriptor(
name='AdminRequestType',
@@ -122,11 +122,15 @@
name='DELETE_STORE_REBALANCE_STATE', index=25, number=27,
options=None,
type=None),
+ descriptor.EnumValueDescriptor(
+ name='NATIVE_BACKUP', index=26, number=28,
+ options=None,
+ type=None),
],
containing_type=None,
options=None,
- serialized_start=6631,
- serialized_end=7304,
+ serialized_start=6792,
+ serialized_end=7484,
)
@@ -156,6 +160,7 @@
REPAIR_JOB = 25
INITIATE_REBALANCE_NODE_ON_DONOR = 26
DELETE_STORE_REBALANCE_STATE = 27
+NATIVE_BACKUP = 28
@@ -2070,6 +2075,55 @@
)
+_NATIVEBACKUPREQUEST = descriptor.Descriptor(
+ name='NativeBackupRequest',
+ full_name='voldemort.NativeBackupRequest',
+ filename=None,
+ file=DESCRIPTOR,
+ containing_type=None,
+ fields=[
+ descriptor.FieldDescriptor(
+ name='store_name', full_name='voldemort.NativeBackupRequest.store_name', index=0,
+ number=1, type=9, cpp_type=9, label=2,
+ has_default_value=False, default_value=unicode("", "utf-8"),
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ descriptor.FieldDescriptor(
+ name='backup_dir', full_name='voldemort.NativeBackupRequest.backup_dir', index=1,
+ number=2, type=9, cpp_type=9, label=2,
+ has_default_value=False, default_value=unicode("", "utf-8"),
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ descriptor.FieldDescriptor(
+ name='verify_files', full_name='voldemort.NativeBackupRequest.verify_files', index=2,
+ number=3, type=8, cpp_type=7, label=2,
+ has_default_value=False, default_value=False,
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ descriptor.FieldDescriptor(
+ name='incremental', full_name='voldemort.NativeBackupRequest.incremental', index=3,
+ number=4, type=8, cpp_type=7, label=2,
+ has_default_value=False, default_value=False,
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ ],
+ extensions=[
+ ],
+ nested_types=[],
+ enum_types=[
+ ],
+ options=None,
+ is_extendable=False,
+ extension_ranges=[],
+ serialized_start=4835,
+ serialized_end=4939,
+)
+
+
_VOLDEMORTADMINREQUEST = descriptor.Descriptor(
name='VoldemortAdminRequest',
full_name='voldemort.VoldemortAdminRequest',
@@ -2266,6 +2320,13 @@
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
+ descriptor.FieldDescriptor(
+ name='native_backup', full_name='voldemort.VoldemortAdminRequest.native_backup', index=27,
+ number=30, type=11, cpp_type=10, label=1,
+ has_default_value=False, default_value=None,
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
],
extensions=[
],
@@ -2275,8 +2336,8 @@
options=None,
is_extendable=False,
extension_ranges=[],
- serialized_start=4836,
- serialized_end=6628,
+ serialized_start=4942,
+ serialized_end=6789,
)
import voldemort_client_pb2
@@ -2353,6 +2414,7 @@
_VOLDEMORTADMINREQUEST.fields_by_name['repair_job'].message_type = _REPAIRJOBREQUEST
_VOLDEMORTADMINREQUEST.fields_by_name['initiate_rebalance_node_on_donor'].message_type = _INITIATEREBALANCENODEONDONORREQUEST
_VOLDEMORTADMINREQUEST.fields_by_name['delete_store_rebalance_state'].message_type = _DELETESTOREREBALANCESTATEREQUEST
+_VOLDEMORTADMINREQUEST.fields_by_name['native_backup'].message_type = _NATIVEBACKUPREQUEST
class GetMetadataRequest(message.Message):
__metaclass__ = reflection.GeneratedProtocolMessageType
@@ -2678,6 +2740,12 @@ class DeleteStoreRebalanceStateResponse(message.Message):
# @@protoc_insertion_point(class_scope:voldemort.DeleteStoreRebalanceStateResponse)
+class NativeBackupRequest(message.Message):
+ __metaclass__ = reflection.GeneratedProtocolMessageType
+ DESCRIPTOR = _NATIVEBACKUPREQUEST
+
+ # @@protoc_insertion_point(class_scope:voldemort.NativeBackupRequest)
+
class VoldemortAdminRequest(message.Message):
__metaclass__ = reflection.GeneratedProtocolMessageType
DESCRIPTOR = _VOLDEMORTADMINREQUEST
View
183 contrib/collections/src/java/voldemort/collections/AddNodeAction.java
@@ -0,0 +1,183 @@
+package voldemort.collections;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import voldemort.client.StoreClient;
+import voldemort.client.UpdateAction;
+import voldemort.versioning.ObsoleteVersionException;
+import voldemort.versioning.Versioned;
+
+/**
+ * Adds a node E to the front of the list identified by K. Inserts the node to
+ * the front of the list, with id 0, and then modifies the pointers for the
+ * first node and next node accordingly.
+ *
+ * @param <E> the type of the value
+ * @param <K> the type of the key identifying the list
+ */
+public class AddNodeAction<K, E> extends UpdateAction<Map<String, Object>, Map<String, Object>> {
+
+ private final K _key;
+ private final E _value;
+
+ @SuppressWarnings("unchecked")
+ private StoreClient<Map<String, Object>, Map<String, Object>> _storeClient = null;
+ private Map<Integer, Map<String, Object>> _rollback = new HashMap<Integer, Map<String, Object>>();
+
+ public AddNodeAction(K key, E value) {
+ _key = key;
+ _value = value;
+ }
+
+ @Override
+ public void rollback() {
+ if(_storeClient == null)
+ return;
+
+ for(Entry<Integer, Map<String, Object>> entry: _rollback.entrySet()) {
+ VListKey<K> key = new VListKey<K>(_key, entry.getKey());
+ _storeClient.put(key.mapValue(), entry.getValue());
+ }
+ }
+
+ /**
+ * @throws ObsoleteVersionException if a concurrent modification (remove or
+ * another add) has not completed modifying the structure.
+ * @throws ArrayIndexOutOfBoundsException if no more ids left to identify
+ * object
+ */
+ @Override
+ public void update(StoreClient<Map<String, Object>, Map<String, Object>> storeClient) {
+ _storeClient = storeClient;
+ VListKey<K> newKey = new VListKey<K>(_key, 0);
+ Versioned<Map<String, Object>> firstNodeMap = storeClient.get(newKey.mapValue());
+ // adding first node of the list
+ if(firstNodeMap == null) {
+ Versioned<Map<String, Object>> newNode = new Versioned<Map<String, Object>>((new VListNode<E>(_value,
+ 0,
+ VStack.NULL_ID,
+ VStack.NULL_ID,
+ true)).mapValue());
+ // throws ObsoleteVersionException if another process has created a
+ // new node already
+ storeClient.put(newKey.mapValue(), newNode);
+ } else // add to front of list
+ {
+ Versioned<VListNode<E>> firstNode = new Versioned<VListNode<E>>(VListNode.<E> valueOf(firstNodeMap.getValue()),
+ firstNodeMap.getVersion());
+
+ if(!firstNode.getValue().isStable()) {
+ throw new ObsoleteVersionException("cannot add when list node is not stable");
+ }
+
+ // set stable flag to false
+ Map<String, Object> tmpMap = new HashMap<String, Object>(firstNodeMap.getValue());
+ tmpMap.put(VListNode.STABLE, false);
+ storeClient.put(newKey.mapValue(),
+ new Versioned<Map<String, Object>>(tmpMap, firstNodeMap.getVersion()));
+ _rollback.put(0, firstNodeMap.getValue());
+
+ int newId;
+ int nextId = firstNode.getValue().getNextId();
+ newId = (nextId == VStack.NULL_ID) ? 1 : nextId + 1;
+ if(newId == Integer.MAX_VALUE)
+ throw new ArrayIndexOutOfBoundsException(newId + " out of bounds");
+
+ Versioned<VListNode<E>> nextNode = null;
+ VListKey<K> nextKey = new VListKey<K>(_key, nextId);
+ if(nextId != VStack.NULL_ID) {
+ Versioned<Map<String, Object>> nextNodeMap = storeClient.get(nextKey.mapValue());
+ if(nextNodeMap == null)
+ throw new ObsoleteVersionException("possible concurrent modification");
+ nextNode = new Versioned<VListNode<E>>(VListNode.<E> valueOf(nextNodeMap.getValue()),
+ nextNodeMap.getVersion());
+ if(!nextNode.getValue().isStable()) {
+ throw new ObsoleteVersionException("cannot add when list node is not stable");
+ }
+
+ // set stable flag to false
+ tmpMap = new HashMap<String, Object>(nextNode.getValue().mapValue());
+ tmpMap.put(VListNode.STABLE, false);
+ storeClient.put(nextKey.mapValue(),
+ new Versioned<Map<String, Object>>(tmpMap, nextNode.getVersion()));
+ _rollback.put(nextId, nextNode.getValue().mapValue());
+ }
+
+ // insert new node
+ Map<String, Object> newNode = (new VListNode<E>(_value, 0, VStack.NULL_ID, newId, true)).mapValue();
+ // don't need to specify versioned because node is already "locked"
+ storeClient.put(newKey.mapValue(), newNode);
+
+ // move first node to next index
+ VListKey<K> firstKey = new VListKey<K>(_key, newId);
+ firstNode = new Versioned<VListNode<E>>(new VListNode<E>(firstNode.getValue()
+ .getValue(),
+ newId,
+ 0,
+ firstNode.getValue()
+ .getNextId(),
+ true));
+ // don't need to specify versioned because node is already "locked"
+ storeClient.put(firstKey.mapValue(), firstNode.getValue().mapValue());
+
+ // redefine previous pointer on next node
+ if(nextNode != null) {
+ if(!storeClient.applyUpdate(new UpdateNextNode<K, E>(nextNode, nextKey, newId)))
+ throw new ObsoleteVersionException("unable to update node");
+ }
+ }
+ }
+
+ /**
+ * Updates a the previous pointer of a node specified by key to a newId.
+ *
+ * @param <K>
+ * @param <E>
+ */
+ private static class UpdateNextNode<K, E> extends
+ UpdateAction<Map<String, Object>, Map<String, Object>> {
+
+ private final VListKey<K> _key;
+ private final int _newId;
+
+ private Versioned<VListNode<E>> _listNode;
+ private int numCalls = 0;
+
+ /**
+ * @param listNode
+ * @param key
+ * @param newId
+ */
+ public UpdateNextNode(Versioned<VListNode<E>> listNode, VListKey<K> key, int newId) {
+ _listNode = listNode;
+ _key = key;
+ _newId = newId;
+ }
+
+ @Override
+ public void update(StoreClient<Map<String, Object>, Map<String, Object>> storeClient) {
+ if(numCalls > 0) {
+ // TODO jko maybe delete this if unnecessary
+ Versioned<Map<String, Object>> nextNodeMap = storeClient.get(_key.mapValue());
+ if(nextNodeMap == null)
+ throw new ObsoleteVersionException("possible concurrent modification");
+ _listNode = new Versioned<VListNode<E>>(VListNode.<E> valueOf(nextNodeMap.getValue()),
+ nextNodeMap.getVersion());
+ }
+
+ VListNode<E> nodeValue = _listNode.getValue();
+ _listNode.setObject(new VListNode<E>(nodeValue.getValue(),
+ nodeValue.getId(),
+ _newId,
+ nodeValue.getNextId(),
+ true));
+ Map<String, Object> nextNodeMap = _listNode.getValue().mapValue();
+ storeClient.put(_key.mapValue(), nextNodeMap);
+
+ numCalls++;
+ }
+
+ }
+}
View
113 contrib/collections/src/java/voldemort/collections/MappedListIterator.java
@@ -0,0 +1,113 @@
+package voldemort.collections;
+
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+/**
+ * Analogous to ListIterator, except for use with a map-backed list. The main
+ * difference is that these lists are not indexed by sequential number, but
+ * instead referred to by an id whose order has nothing to do with the position
+ * of the element in the list.
+ *
+ * @param <K> the type of the key used by the underlying map
+ * @param <E> the type of the elements being stored
+ */
+public interface MappedListIterator<K, E> extends Iterator<E> {
+
+ /**
+ * Inserts the specified element into the list (optional operation).
+ */
+ void add(E e);
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.util.Iterator#hasNext()
+ */
+ boolean hasNext();
+
+ /**
+ * Returns true if this list iterator has more elements when traversing the
+ * list in the reverse direction. (In other words, returns true if previous
+ * would return an element rather than throwing an exception.)
+ *
+ * @return true if the list iterator has more elements when traversing the
+ * list in the reverse direction
+ */
+ boolean hasPrevious();
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.util.Iterator#next()
+ */
+ E next();
+
+ /**
+ * Returns the id associated with the next element in the list, or null if
+ * the list iterator is at the end of the list.
+ *
+ * @return the id associated with the next element in the list
+ */
+ K nextId();
+
+ /**
+ * Returns the previous element in the list. This method may be called
+ * repeatedly to iterate through the list backwards, or intermixed with
+ * calls to next to go back and forth. (Note that alternating calls to next
+ * and previous will return the same element repeatedly.)
+ *
+ * @return the previous element in the list
+ *
+ * @throws NoSuchElementException - if the iteration has no previous
+ * element.
+ */
+ E previous();
+
+ /**
+ * Returns the id associated with the previous element in the list, or null
+ * if the list iterator is at the beginning of the list.
+ *
+ * @return the id associated with the previous element in the list
+ */
+ K previousId();
+
+ /**
+ * Returns the id for the last element that was returned by next or
+ * previous.
+ *
+ * @return the id for the last element that was returned by next or previous
+ * @throws IllegalStateException if neither next nor previous has been
+ * called, or remove or add has been called after the last call to
+ * next or previous
+ */
+ K lastId();
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.util.Iterator#remove()
+ */
+ void remove();
+
+ /**
+ * Replaces the last element returned by next or previous with the specified
+ * element (optional operation). This call can be made only if neither
+ * ListIterator.remove nor ListIterator.add have been called after the last
+ * call to next or previous.
+ *
+ * @param e the element with which to replace the last element returned by
+ * next or previous.
+ *
+ * @throws UnsupportedOperationException if the set operation is not
+ * supported by this list iterator.
+ * @throws ClassCastException if the class of the specified element prevents
+ * it from being added to this list.
+ * @throws IllegalArgumentException if some aspect of the specified element
+ * prevents it from being added to this list.
+ * @throws IllegalStateException if neither next nor previous have been
+ * called, or remove or add have been called after the last call to
+ * next or previous.
+ */
+ void set(E e);
+}
View
63 contrib/collections/src/java/voldemort/collections/UpdateElementById.java
@@ -0,0 +1,63 @@
+package voldemort.collections;
+
+import java.util.Map;
+
+import voldemort.client.StoreClient;
+import voldemort.client.UpdateAction;
+import voldemort.versioning.ObsoleteVersionException;
+import voldemort.versioning.Version;
+import voldemort.versioning.Versioned;
+
+/**
+ * Put the given value to the appropriate id in the stack, using the version of
+ * the current list node identified by that id.
+ *
+ * @param id
+ * @param element element to set
+ * @return element that was replaced by the new element
+ */
+public class UpdateElementById<K, E> extends UpdateAction<Map<String, Object>, Map<String, Object>> {
+
+ private final VListKey<K> _key;
+ private final E _element;
+ private final Version _version;
+ private E _result = null;
+
+ public UpdateElementById(VListKey<K> key, E element) {
+ _key = key;
+ _element = element;
+ _version = null;
+ }
+
+ public UpdateElementById(VListKey<K> key, Versioned<E> element) {
+ _key = key;
+ _element = element.getValue();
+ _version = element.getVersion();
+ }
+
+ @Override
+ public void update(StoreClient<Map<String, Object>, Map<String, Object>> storeClient) {
+ Versioned<Map<String, Object>> nodeMap = storeClient.get(_key.mapValue());
+ if(nodeMap == null)
+ throw new IndexOutOfBoundsException("invalid id " + _key.getId());
+ Version version = (_version != null) ? _version : nodeMap.getVersion();
+ VListNode<E> listNode = VListNode.valueOf(nodeMap.getValue());
+ if(!listNode.isStable()) {
+ throw new ObsoleteVersionException("node " + _key.getId() + " not stable.");
+ }
+ _result = listNode.getValue();
+ VListNode<E> newNode = new VListNode<E>(_element,
+ listNode.getId(),
+ listNode.getPreviousId(),
+ listNode.getNextId(),
+ true);
+ storeClient.put(_key.mapValue(), new Versioned<Map<String, Object>>(newNode.mapValue(),
+ version));
+
+ }
+
+ public E getResult() {
+ return _result;
+ }
+
+}
View
51 contrib/collections/src/java/voldemort/collections/UpdatePageIndex.java
@@ -0,0 +1,51 @@
+package voldemort.collections;
+
+import java.util.List;
+import java.util.Map;
+
+import voldemort.client.StoreClient;
+import voldemort.client.UpdateAction;
+import voldemort.serialization.Serializer;
+import voldemort.versioning.Versioned;
+
+/**
+ * Update the VLinkedPagedList index with a new node.
+ */
+public class UpdatePageIndex<I, LK extends Comparable<LK>> extends
+ UpdateAction<I, List<Map<String, Object>>> {
+
+ private final I _identifier;
+ private final LK _newIndex;
+ private final Serializer<LK> _serializer;
+
+ public UpdatePageIndex(I identifier, LK newKey, Serializer<LK> serializer) {
+ _identifier = identifier;
+ _newIndex = newKey;
+ _serializer = serializer;
+ }
+
+ @Override
+ public void update(StoreClient<I, List<Map<String, Object>>> storeClient) {
+ Versioned<List<Map<String, Object>>> versionedIndex = storeClient.get(_identifier);
+
+ List<Map<String, Object>> pageIndex = versionedIndex.getValue();
+ VPageIndexEntry<LK> oldIndexEntry = VPageIndexEntry.valueOf(pageIndex.remove(0),
+ _serializer);
+ int updatedPageId = 0;
+ if(pageIndex.size() >= 1) {
+ updatedPageId = ((Integer) pageIndex.get(0).get("pageId")) + 1;
+ } else {
+ updatedPageId = 1;
+ }
+ oldIndexEntry = new VPageIndexEntry<LK>(updatedPageId,
+ oldIndexEntry.getLastIndex(),
+ _serializer);
+ VPageIndexEntry<LK> newIndexEntry = new VPageIndexEntry<LK>(0, _newIndex, _serializer);
+ pageIndex.add(0, oldIndexEntry.mapValue());
+ pageIndex.add(0, newIndexEntry.mapValue());
+ storeClient.put(_identifier,
+ new Versioned<List<Map<String, Object>>>(pageIndex,
+ versionedIndex.getVersion()));
+ }
+
+}
View
28 contrib/collections/src/java/voldemort/collections/VLinkedPagedKey.java
@@ -0,0 +1,28 @@
+package voldemort.collections;
+
+/**
+ * Key into a VLinkedPagedList. Specifies the page to look into, and the index
+ * of the array within the page.
+ */
+public class VLinkedPagedKey {
+
+ private final int _pageId;
+ private final int _arrayIndex;
+
+ public VLinkedPagedKey(int pageId, int arrayIndex) {
+ _pageId = pageId;
+ _arrayIndex = arrayIndex;
+ }
+
+ public int getPageId() {
+ return _pageId;
+ }
+
+ public int getArrayIndex() {
+ return _arrayIndex;
+ }
+
+ public String toString() {
+ return "PageId " + _pageId + ", arrayIndex " + _arrayIndex;
+ }
+}
View
224 contrib/collections/src/java/voldemort/collections/VLinkedPagedList.java
@@ -0,0 +1,224 @@
+package voldemort.collections;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import voldemort.client.StoreClient;
+import voldemort.serialization.Serializer;
+import voldemort.versioning.Versioned;
+
+/**
+ * VLinkedPagedList is a voldemort-store backed list that keeps track of a value
+ * in relation to its position in a sorted key. Items are placed in the
+ * LinkedPagedList in descending order. New elements are added to the front of
+ * the list.
+ *
+ * The voldemort store configuration must use JSON serialization. For the
+ * stackClient, the key must be defined as a JSON object: {"value" : <JSON
+ * serialization format of I>, "id" : "int32"} The value must be defined as the
+ * following: {"id" : "int32", "nextId" : "int32", "previousId" : "int32",
+ * "stable" : "boolean", "value" : ["bytes"]}
+ *
+ * For the pageIndex, the JSON serialization format of the key is the same as
+ * the JSON serialization format of I. The JSON serialization format of the
+ * value is: [{"pageId" : "int32", "lastIndex" : "bytes"}]
+ *
+ * @param <I> type used to identify this list
+ * @param <LK> value to be stored
+ */
+public class VLinkedPagedList<I, LK extends Comparable<LK>> implements Iterable<LK> {
+
+ private final I _identifier;
+ // stack of an array of LK
+ private final VStack<I, List<byte[]>> _index;
+
+ private final StoreClient<I, List<Map<String, Object>>> _pageIndex;
+ private final IndexEntryComparator<LK> _pageIndexComparator;
+
+ private final Serializer<LK> _serializer;
+ private final int _maxSize;
+ private final int _fixedSize;
+
+ /**
+ *
+ * @param identifier name of the list
+ * @param indexStack VStack to store the list of LK entries
+ * @param pageIndexClient voldemort store for the page index
+ * @param serializer serializes LK to/from bytes
+ * @param maxSize maximum size of each list in the VStack
+ * @param fixedSize once the maxSize is reached, truncate down to a standard
+ * fixedSize
+ * @param iteratorSize maximum size of list to be stored within an iterator.
+ * Should comfortably fit in memory
+ */
+ public VLinkedPagedList(I identifier,
+ StoreClient<Map<String, Object>, Map<String, Object>> stackClient,
+ StoreClient<I, List<Map<String, Object>>> pageIndexClient,
+ Serializer<LK> serializer,
+ int maxSize,
+ int fixedSize) {
+ if(identifier == null || stackClient == null || pageIndexClient == null)
+ throw new NullPointerException();
+ if(maxSize == 0 || fixedSize == 0)
+ throw new IllegalArgumentException("maxSize and fixedSize must be greater than 0");
+ if(fixedSize > maxSize)
+ throw new IllegalArgumentException("fixedSize must be less than or equal to maxSize");
+
+ _identifier = identifier;
+ _index = new VStack<I, List<byte[]>>(identifier, stackClient);
+ _pageIndex = pageIndexClient;
+ _serializer = serializer;
+ _pageIndexComparator = new IndexEntryComparator<LK>(serializer);
+ _maxSize = maxSize;
+ _fixedSize = fixedSize;
+ }
+
+ public void add(LK linearKey) {
+ if(linearKey == null)
+ throw new NullPointerException();
+
+ Versioned<List<byte[]>> keyList = _index.peekVersioned();
+ if(keyList == null) {
+ // start a brand new list
+ List<byte[]> keyListValue = new ArrayList<byte[]>(1);
+ keyListValue.add(0, _serializer.toBytes(linearKey));
+ _index.add(keyListValue);
+ VPageIndexEntry<LK> indexEntry = new VPageIndexEntry<LK>(0, linearKey, _serializer);
+ ArrayList<Map<String, Object>> pageIndexList = new ArrayList<Map<String, Object>>(1);
+ pageIndexList.add(indexEntry.mapValue());
+ _pageIndex.put(_identifier, pageIndexList);
+ } else {
+ List<byte[]> keyListValue = keyList.getValue();
+ if(keyListValue.size() >= _maxSize) {
+ // split up the list into 2 nodes
+ keyListValue.add(0, _serializer.toBytes(linearKey));
+ List<byte[]> fixedList = keyListValue.subList(_maxSize - _fixedSize + 1,
+ keyListValue.size());
+ List<byte[]> truncList = keyListValue.subList(0, _maxSize - _fixedSize + 1);
+
+ _index.setById(0, new Versioned<List<byte[]>>(fixedList, keyList.getVersion()));
+ _index.add(truncList);
+
+ LK lastIndex = _serializer.toObject(truncList.get(truncList.size() - 1));
+ _pageIndex.applyUpdate(new UpdatePageIndex<I, LK>(_identifier,
+ lastIndex,
+ _serializer));
+ } else {
+ // add to existing node
+ keyListValue.add(0, _serializer.toBytes(linearKey));
+ _index.setById(0, keyList);
+ }
+ }
+ }
+
+ public LK peek() {
+ List<byte[]> indexList = _index.peek();
+ LK key = _serializer.toObject(indexList.get(0));
+ return key;
+ }
+
+ /**
+ * Return a ListIterator<K> that starts from the beginning of the list.
+ *
+ * @return
+ */
+ public MappedListIterator<VLinkedPagedKey, LK> listIterator() {
+ return listIterator(null, true);
+ }
+
+ /**
+ * Return a listIterator<K>. Uses the page index to figure out what page to
+ * start iterating from.
+ *
+ * Runtime: 2 - 3 gets.
+ *
+ * @param linearKey key to position the cursor.
+ * @param next true if the cursor is to be placed directly before the first
+ * key that is less than linearKey false if the cursor is to be
+ * placed directly after the last key that is greater than linearKey
+ * @return a ListIterator of <K> ids.
+ */
+ public MappedListIterator<VLinkedPagedKey, LK> listIterator(LK linearKey, boolean next) {
+ int indexPage = 0;
+ if(linearKey == null) {
+ if(next) {
+ indexPage = 0;
+ } else {
+ List<Map<String, Object>> indexList = _pageIndex.getValue(_identifier);
+ if(indexList != null) {
+ VPageIndexEntry<LK> entry = VPageIndexEntry.valueOf(indexList.get(indexList.size()),
+ _serializer);
+ indexPage = entry.getPageId();
+ }
+ }
+ } else {
+ List<Map<String, Object>> indexList = _pageIndex.getValue(_identifier);
+ if(indexList != null) {
+ Map<String, Object> searchKey = new VPageIndexEntry<LK>(0, linearKey, _serializer).mapValue();
+ int position = Collections.binarySearch(indexList, searchKey, _pageIndexComparator);
+ if(position < 0) {
+ position = -1 * (position + 1);
+ }
+ indexPage = VPageIndexEntry.valueOf(indexList.get(position), _serializer)
+ .getPageId();
+ }
+ }
+
+ return listIterator(linearKey, next, indexPage);
+ }
+
+ /**
+ * Similar to listIterator(LK linearKey, boolean next) except with the
+ * addition of the hint "pageId" which specifies the first page to look for
+ * the key.
+ *
+ * Runtime: up to O(n) gets, unless a good hint page is provided, then it
+ * could be 1 - 2 gets.
+ *
+ * @param linearKey
+ * @param next
+ * @param pageId
+ * @return
+ */
+ public MappedListIterator<VLinkedPagedKey, LK> listIterator(LK linearKey,
+ boolean next,
+ Integer pageId) {
+ return new VLinkedPagedListIterator<I, LK>(_index, _serializer, linearKey, next, pageId);
+ }
+
+ public LK getById(VLinkedPagedKey key) {
+ byte[] bytes = _index.getById(key.getPageId()).get(key.getArrayIndex());
+ return _serializer.toObject(bytes);
+ }
+
+ public Iterator<LK> iterator() {
+ return listIterator();
+ }
+
+ /**
+ * Reverse order comparison of VPageIndexEntry objects. Only compares the
+ * lastIndex component.
+ *
+ * @param <LK>
+ */
+ private static class IndexEntryComparator<LK extends Comparable<LK>> implements
+ Comparator<Map<String, Object>> {
+
+ private final Serializer<LK> _serializer;
+
+ public IndexEntryComparator(Serializer<LK> serializer) {
+ _serializer = serializer;
+ }
+
+ public int compare(Map<String, Object> arg0, Map<String, Object> arg1) {
+ VPageIndexEntry<LK> entry0 = VPageIndexEntry.valueOf(arg0, _serializer);
+ VPageIndexEntry<LK> entry1 = VPageIndexEntry.valueOf(arg1, _serializer);
+
+ return entry1.getLastIndex().compareTo(entry0.getLastIndex());
+ }
+ }
+}
View
206 contrib/collections/src/java/voldemort/collections/VLinkedPagedListIterator.java
@@ -0,0 +1,206 @@
+package voldemort.collections;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.NoSuchElementException;
+
+import voldemort.serialization.Serializer;
+
+/**
+ *
+ */
+public class VLinkedPagedListIterator<I, LK extends Comparable<LK>> implements
+ MappedListIterator<VLinkedPagedKey, LK> {
+
+ private final Serializer<LK> _serializer;
+
+ /*
+ * Internal iterator to track the position inside the index
+ */
+ private ListIterator<byte[]> _keyIterator;
+ private VListIterator<List<byte[]>> _indexIterator;
+ private int _currentIndexId = VStack.NULL_ID;
+ private VLinkedPagedKey _lastId = null;
+
+ VLinkedPagedListIterator(VStack<I, List<byte[]>> index,
+ Serializer<LK> serializer,
+ LK linearKey,
+ boolean next,
+ int pageId) {
+ _serializer = serializer;
+ _currentIndexId = pageId;
+ if(next) {
+ _indexIterator = index.listIterator(pageId);
+ if(!_indexIterator.hasNext()) {
+ _keyIterator = new ArrayList<byte[]>(0).listIterator();
+ } else {
+ List<byte[]> byteList = _indexIterator.next().getValue();
+ if(linearKey == null) {
+ _keyIterator = byteList.listIterator();
+ } else {
+ boolean found = false;
+ int position = 0;
+ while(!found) {
+ position = Collections.binarySearch(byteList,
+ _serializer.toBytes(linearKey),
+ new LKByteReverseComparator<LK>(_serializer));
+ if(position < 0) {
+ position = -1 * (position + 1);
+ }
+ if(position < byteList.size()) {
+ found = true;
+ } else {
+ byteList = _indexIterator.next().getValue();
+ }
+ }
+ _keyIterator = byteList.listIterator(position);
+ }
+ }
+ } else {
+ _indexIterator = index.previousListIterator(pageId);
+ if(!_indexIterator.hasPrevious()) {
+ _keyIterator = new ArrayList<byte[]>(0).listIterator();
+ } else {
+ List<byte[]> byteList = _indexIterator.previous().getValue();
+ if(linearKey == null) {
+ _keyIterator = byteList.listIterator(byteList.size());
+ } else {
+ boolean found = false;
+ int position = 0;
+ while(!found) {
+ position = Collections.binarySearch(byteList,
+ serializer.toBytes(linearKey),
+ new LKByteReverseComparator<LK>(_serializer));
+ if(position < 0) {
+ position = -1 * (position + 1);
+ }
+ if(position > 0) {
+ found = true;
+ } else {
+ byteList = _indexIterator.previous().getValue();
+ }
+ }
+ // position should be no larger than byteList.size()
+ _keyIterator = byteList.listIterator(Math.min(position + 1, byteList.size()));
+ }
+ }
+ }
+ }
+
+ public void add(LK e) {
+ throw new UnsupportedOperationException();
+ }
+
+ public boolean hasNext() {
+ // put the cursor in the right position
+ if(_indexIterator.nextId() != null && _indexIterator.lastId() != null) {
+ if(_indexIterator.nextId().equals(_indexIterator.lastId())) {
+ _indexIterator.next();
+ }
+ }
+
+ while(!_keyIterator.hasNext()) {
+ if(_indexIterator.hasNext()) {
+ _currentIndexId = _indexIterator.nextId();
+ _keyIterator = _indexIterator.next().getValue().listIterator();
+ } else {
+ break;
+ }
+ }
+
+ return _keyIterator.hasNext();
+ }
+
+ public boolean hasPrevious() {
+ // put the cursor in the right position
+ if(_indexIterator.previousId() != null && _indexIterator.lastId() != null) {
+ if(_indexIterator.previousId().equals(_indexIterator.lastId())) {
+ _indexIterator.previous();
+ }
+ }
+
+ while(!_keyIterator.hasPrevious()) {
+ if(_indexIterator.hasPrevious()) {
+ _currentIndexId = _indexIterator.previousId();
+ List<byte[]> byteList = _indexIterator.previous().getValue();
+ _keyIterator = byteList.listIterator(byteList.size());
+ } else {
+ break;
+ }
+ }
+
+ return _keyIterator.hasPrevious();
+ }
+
+ public LK next() {
+ if(hasNext()) {
+ _lastId = new VLinkedPagedKey(_currentIndexId, _keyIterator.nextIndex());
+ return _serializer.toObject(_keyIterator.next());
+ } else {
+ throw new NoSuchElementException();
+ }
+ }
+
+ public VLinkedPagedKey nextId() {
+ if(hasNext())
+ return new VLinkedPagedKey(_currentIndexId, _keyIterator.nextIndex());
+ else
+ return null;
+ }
+
+ public LK previous() {
+ if(hasPrevious()) {
+ _lastId = new VLinkedPagedKey(_currentIndexId, _keyIterator.previousIndex());
+ return _serializer.toObject((_keyIterator.previous()));
+ } else {
+ throw new NoSuchElementException();
+ }
+ }
+
+ public VLinkedPagedKey previousId() {
+ if(hasPrevious())
+ return new VLinkedPagedKey(_currentIndexId, _keyIterator.previousIndex());
+ else
+ return null;
+ }
+
+ public VLinkedPagedKey lastId() {
+ return _lastId;
+ }
+
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ public void set(LK e) {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Comparator that does a reverse comparison of serialized LK objects. Used
+ * because elements in the List<byte[]> in the index are sorted in
+ * descending order.
+ *
+ * @param <LK>
+ */
+ private static class LKByteReverseComparator<LK extends Comparable<LK>> implements
+ Comparator<byte[]> {
+
+ private final Serializer<LK> _serializer;
+
+ private LKByteReverseComparator(Serializer<LK> serializer) {
+ _serializer = serializer;
+ }
+
+ public int compare(byte[] arg0, byte[] arg1) {
+ LK lk0 = _serializer.toObject(arg0);
+ LK lk1 = _serializer.toObject(arg1);
+ return lk1.compareTo(lk0);
+ }
+
+ }
+
+}
View
264 contrib/collections/src/java/voldemort/collections/VListIterator.java
@@ -0,0 +1,264 @@
+package voldemort.collections;
+
+import java.util.NoSuchElementException;
+
+import voldemort.versioning.Versioned;
+
+/**
+ *
+ */
+public class VListIterator<E> implements MappedListIterator<Integer, Versioned<E>> {
+
+ private final VStack<?, E> _stack;
+
+ private Versioned<VListNode<E>> _nextNode = null;
+ private boolean _isNextSet = false;
+
+ private Versioned<VListNode<E>> _previousNode = null;
+ private boolean _isPreviousSet = false;
+
+ // last item returned by next or previous
+ private int _lastId = VStack.NULL_ID;
+ private LastCall _lastCall = null;
+
+ /**
+ * package private -- we don't want outsiders constructing this
+ *
+ * Creates a ListIterator with a cursor position specified by id and next.
+ *
+ * @param id - the id of the node to position the cursor
+ * @param next - true if the cursor is to be placed directly before the
+ * node, false if the cursor is to be placed directly after the node
+ */
+ VListIterator(VStack<?, E> stack, int id, boolean next) {
+ _stack = stack;
+
+ if(next) {
+ _nextNode = stack.getListNode(id);
+ if(_nextNode == null) {
+ if(id == 0)
+ _isPreviousSet = true;
+ else
+ throw new IndexOutOfBoundsException();
+ }
+ _isNextSet = true;
+ } else {
+ _previousNode = stack.getListNode(id);
+ if(_previousNode == null) {
+ if(id == 0)
+ _isNextSet = true;
+ else
+ throw new IndexOutOfBoundsException();
+ }
+ _isPreviousSet = true;
+ }
+ }
+
+ VListIterator(VStack<?, E> stack, int id) {
+ this(stack, id, true);
+ }
+
+ private void setPreviousNode() {
+ if(!_isPreviousSet) {
+ if(!_isNextSet)
+ throw new IllegalStateException("either next or previous must be set at any given time");
+ Integer previousId = _nextNode.getValue().getPreviousId();
+ if(previousId != VStack.NULL_ID) {
+ _previousNode = _stack.getListNode(previousId);
+ } else {
+ _previousNode = null;
+ }
+
+ _isPreviousSet = true;
+ }
+ }
+
+ private void setNextNode() {
+ if(!_isNextSet) {
+ if(!_isPreviousSet)
+ throw new IllegalStateException("either next or previous must be set at any given time");
+
+ Integer nextId = _previousNode.getValue().getNextId();
+ if(nextId != VStack.NULL_ID) {
+ _nextNode = _stack.getListNode(nextId);
+ } else {
+ _nextNode = null;
+ }
+
+ _isNextSet = true;
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.util.ListIterator#add(java.lang.Object)
+ */
+ public void add(Versioned<E> arg0) {
+ _lastId = VStack.NULL_ID;
+ _lastCall = LastCall.ADD;
+ throw new UnsupportedOperationException();
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.util.ListIterator#hasNext()
+ */
+ public boolean hasNext() {
+ setNextNode();
+ return _nextNode != null;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.util.ListIterator#hasPrevious()
+ */
+ public boolean hasPrevious() {
+ setPreviousNode();
+ return _previousNode != null;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.util.ListIterator#next()
+ */
+ public Versioned<E> next() {
+ if(!hasNext())
+ throw new NoSuchElementException();
+
+ Versioned<VListNode<E>> tmpNode = _nextNode;
+ VListNode<E> tmpNodeValue = _nextNode.getValue();
+ _lastId = tmpNodeValue.getId();
+ _lastCall = LastCall.NEXT;
+
+ if(tmpNodeValue.getNextId() != VStack.NULL_ID) {
+ _nextNode = _stack.getListNode(tmpNodeValue.getNextId());
+ } else {
+ _nextNode = null;
+ }
+
+ _previousNode = tmpNode;
+ _isPreviousSet = true;
+
+ return new Versioned<E>(tmpNodeValue.getValue(), tmpNode.getVersion());
+ }
+
+ /**
+ * Returns the id of the element that would be returned by a subsequent call
+ * to next. Returns null if the list iterator is at the end of the list.
+ */
+ public Integer nextId() {
+ if(hasNext())
+ return _nextNode.getValue().getId();
+ else
+ return null;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.util.ListIterator#previous()
+ */
+ public Versioned<E> previous() {
+ if(hasPrevious()) {
+ E resultValue = _previousNode.getValue().getValue();
+ _lastId = _previousNode.getValue().getId();
+ _lastCall = LastCall.PREVIOUS;
+ _nextNode = _previousNode;
+ _isNextSet = true;
+ _isPreviousSet = false;
+ return new Versioned<E>(resultValue, _previousNode.getVersion());
+ } else {
+ throw new NoSuchElementException();
+ }
+ }
+
+ /**
+ * Returns the id of the element that would be returned by a subsequent call
+ * to next. Returns null if the list iterator is at the end of the list.
+ */
+ public Integer previousId() {
+ if(hasPrevious()) {
+ return _previousNode.getValue().getId();
+ } else {
+ return null;
+ }
+ }
+
+ public Integer lastId() {
+ if(_lastCall != LastCall.NEXT && _lastCall != LastCall.PREVIOUS)
+ throw new IllegalStateException("neither next() nor previous() has been called");
+ return _lastId;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.util.ListIterator#remove()
+ */
+ public void remove() {
+ _lastId = VStack.NULL_ID;
+ _lastCall = LastCall.REMOVE;
+ throw new UnsupportedOperationException();
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.util.ListIterator#set(java.lang.Object)
+ *
+ * Replaces the last element returned by next or previous with the specified
+ * element (optional operation). This call can be made only if neither
+ * ListIterator.remove nor ListIterator.add have been called after the last
+ * call to next or previous.
+ */
+ public void set(Versioned<E> element) {
+ if(element == null)
+ throw new NullPointerException("cannot set a null element");
+ if(_lastCall != LastCall.NEXT && _lastCall != LastCall.PREVIOUS)
+ throw new IllegalStateException("neither next() nor previous() has been called");
+
+ _stack.setById(_lastId, element);
+
+ afterSet(element.getValue());
+ }
+
+ /*
+ * Set the value of the last element returned by next or previous,
+ * discarding any version information
+ */
+ public void setValue(E element) {
+ if(element == null)
+ throw new NullPointerException("cannot set a null element");
+ if(_lastId == VStack.NULL_ID)
+ throw new IllegalStateException("neither next() nor previous() has been called");
+
+ _stack.setById(_lastId, element);
+
+ afterSet(element);
+ }
+
+ // helper function to be called after set() and setValue()
+ private void afterSet(E element) {
+ switch(_lastCall) {
+ case NEXT:
+ _previousNode = _stack.getListNode(_lastId);
+ break;
+ case PREVIOUS:
+ _nextNode = _stack.getListNode(_lastId);
+ break;
+ default:
+ throw new IllegalStateException();
+ }
+ }
+
+ private enum LastCall {
+ NEXT,
+ PREVIOUS,
+ ADD,
+ REMOVE
+ }
+}
View
90 contrib/collections/src/java/voldemort/collections/VListIteratorValues.java
@@ -0,0 +1,90 @@
+package voldemort.collections;
+
+/**
+ * Wrapper around VListIterator that strips out the version information. Only
+ * useful when no further storage operations will be done on these values.
+ */
+public class VListIteratorValues<E> implements MappedListIterator<Integer, E> {
+
+ private final VListIterator<E> _listIterator;
+
+ public VListIteratorValues(VListIterator<E> vListIterator) {
+ _listIterator = vListIterator;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.util.ListIterator#add(java.lang.Object)
+ */
+ public void add(E arg0) {
+ throw new UnsupportedOperationException();
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.util.ListIterator#hasNext()
+ */
+ public boolean hasNext() {
+ return _listIterator.hasNext();
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.util.ListIterator#hasPrevious()
+ */
+ public boolean hasPrevious() {
+ return _listIterator.hasPrevious();
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.util.ListIterator#next()
+ */
+ public E next() {
+ return _listIterator.next().getValue();
+ }
+
+ public Integer nextId() {
+ return _listIterator.nextId();
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.util.ListIterator#previous()
+ */
+ public E previous() {
+ return _listIterator.previous().getValue();
+ }
+
+ public Integer previousId() {
+ return _listIterator.previousId();
+ }
+
+ public Integer lastId() {
+ return _listIterator.lastId();
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.util.ListIterator#remove()
+ */
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.util.ListIterator#set(java.lang.Object)
+ */
+ public void set(E arg0) {
+ throw new UnsupportedOperationException();
+ }
+
+}
View
33 contrib/collections/src/java/voldemort/collections/VListKey.java
@@ -0,0 +1,33 @@
+package voldemort.collections;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import voldemort.utils.Identified;
+
+/**
+ * Pointer to a VListNode.
+ */
+public class VListKey<T> extends Identified<T, Integer> {
+
+ private final Map<String, Object> _map;
+
+ public VListKey(T object, Integer id) {
+ super(object, id);
+
+ Map<String, Object> tempMap = new HashMap<String, Object>();
+ tempMap.put("id", id);
+ tempMap.put("value", object);
+ _map = Collections.unmodifiableMap(tempMap);
+ }
+
+ public Map<String, Object> mapValue() {
+ return _map;
+ }
+
+ @Override
+ public String toString() {
+ return _map.toString();
+ }
+}
View
70 contrib/collections/src/java/voldemort/collections/VListNode.java
@@ -0,0 +1,70 @@
+package voldemort.collections;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import voldemort.utils.Identified;
+
+/**
+ * Immutable wrapper around an object that adds a Integer id, nextId,
+ * previousId, and stable. Used for implementing larger list-based structures in
+ * voldemort.
+ */
+public class VListNode<T> extends Identified<T, Integer> {
+
+ /*
+ * specifies if this node is participating in a structural change (i.e.,
+ * add/remove)
+ */
+ public static final String STABLE = "stable";
+
+ private static final String NEXT_ID = "nextId";
+ private static final String PREVIOUS_ID = "previousId";
+ private static final String ID = "id";
+ private static final String VALUE = "value";
+
+ private final Map<String, Object> _map;
+
+ public VListNode(T object, Integer id, Integer prev, Integer next, Boolean stable) {
+ super(object, id);
+
+ Map<String, Object> tempMap = new HashMap<String, Object>();
+ tempMap.put(ID, id);
+ tempMap.put(VALUE, object);
+ tempMap.put(NEXT_ID, next);
+ tempMap.put(PREVIOUS_ID, prev);
+ tempMap.put(STABLE, stable);
+ _map = Collections.unmodifiableMap(tempMap);
+ }
+
+ public static <T> VListNode<T> valueOf(Map<String, Object> map) {
+ @SuppressWarnings("unchecked")
+ T value = (T) map.get(VALUE);
+ return new VListNode<T>(value,
+ (Integer) map.get(ID),
+ (Integer) map.get(PREVIOUS_ID),
+ (Integer) map.get(NEXT_ID),
+ (Boolean) map.get(STABLE));
+ }
+
+ public Integer getNextId() {
+ return (Integer) _map.get(NEXT_ID);
+ }
+
+ public Integer getPreviousId() {
+ return (Integer) _map.get(PREVIOUS_ID);
+ }
+
+ public Map<String, Object> mapValue() {
+ return _map;
+ }
+
+ public Boolean isStable() {
+ return (Boolean) _map.get(STABLE);
+ }
+
+ public String toString() {
+ return _map.toString();
+ }
+}
View
60 contrib/collections/src/java/voldemort/collections/VPageIndexEntry.java
@@ -0,0 +1,60 @@
+package voldemort.collections;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import voldemort.serialization.Serializer;
+
+/**
+ * Keeps tracking of a single page and its associated first entry.
+ */
+public class VPageIndexEntry<LK extends Comparable<LK>> {
+
+ private static final String PAGE_ID = "pageId";
+ private static final String LAST_INDEX = "lastIndex";
+
+ private final Map<String, Object> _map;
+
+ private final Serializer<LK> _serializer;
+
+ public VPageIndexEntry(int pageId, LK lastIndex, Serializer<LK> serializer) {
+ this(pageId, serializer.toBytes(lastIndex), serializer);
+ }
+
+ public VPageIndexEntry(int pageId, byte[] lastIndexBytes, Serializer<LK> serializer) {
+ Map<String, Object> tempMap = new HashMap<String, Object>();
+ tempMap.put(PAGE_ID, pageId);
+ tempMap.put(LAST_INDEX, lastIndexBytes);
+ _map = Collections.unmodifiableMap(tempMap);
+ _serializer = serializer;
+ }
+
+ public static <LK extends Comparable<LK>> VPageIndexEntry<LK> valueOf(Map<String, Object> map,
+ Serializer<LK> serializer) {
+ byte[] lastIndexBytes = (byte[]) map.get(LAST_INDEX);
+ LK lastIndex = serializer.toObject(lastIndexBytes);
+ return new VPageIndexEntry<LK>((Integer) map.get(PAGE_ID), lastIndex, serializer);
+ }
+
+ public Integer getPageId() {
+ return (Integer) _map.get(PAGE_ID);
+ }
+
+ public LK getLastIndex() {
+ return _serializer.toObject((byte[]) _map.get(LAST_INDEX));
+ }
+
+ public Map<String, Object> mapValue() {
+ return _map;
+ }
+
+ // copy to tmpMap to get the pretty version of LK instead of bytes
+ public String toString() {
+ Map<String, Object> tmpMap = new HashMap<String, Object>();
+ tmpMap.put(PAGE_ID, getPageId());
+ tmpMap.put(LAST_INDEX, getLastIndex().toString());
+ return tmpMap.toString();
+ }
+
+}
View
354 contrib/collections/src/java/voldemort/collections/VStack.java
@@ -0,0 +1,354 @@
+package voldemort.collections;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Queue;
+
+import voldemort.client.StoreClient;
+import voldemort.versioning.ObsoleteVersionException;
+import voldemort.versioning.Versioned;
+
+/**
+ * Linked list stack implementation on top of voldemort. Nodes within the stack
+ * are identified by their id. Only adding and setting nodes are allowed.
+ * Removing is unsupported due to possible stack corruption.
+ *
+ * The voldemort store configuration must use JSON serialization. The key must
+ * be defined as a JSON object as the following: {"value" : <JSON serialization
+ * format of K>, "id" : "int32"} The value must be defined as a JSON object as
+ * the following: {"id" : "int32", "nextId" : "int32", "previousId" : "int32",
+ * "stable" : "boolean", "value" : <JSON serialization format of E>}
+ *
+ * @param <K> the type of key used to identify this stack. Must conform to valid
+ * voldemort JSON formats: http://project-voldemort.com/design.php
+ * @param <E> the type of element being stored
+ */
+public class VStack<K, E> implements Queue<E> {
+
+ public static final Integer NULL_ID = Integer.MIN_VALUE + 1;
+
+ // could be used to store the size of the list
+ public static final Integer SIZE_ID = Integer.MIN_VALUE + 2;
+
+ private final K _key;
+ private final StoreClient<Map<String, Object>, Map<String, Object>> _storeClient;
+
+ /**
+ *
+ * @param key
+ * @param storeClient
+ */
+ public VStack(K key, StoreClient<Map<String, Object>, Map<String, Object>> storeClient) {
+ if(key == null)
+ throw new NullPointerException("key must not be null");
+
+ _key = key;
+ _storeClient = storeClient;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.util.Collection#clear()
+ */
+ public void clear() {
+ // TODO Auto-generated method stub
+ throw new UnsupportedOperationException();
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.util.Collection#contains(java.lang.Object)
+ */
+ public boolean contains(Object arg0) {
+ throw new UnsupportedOperationException();
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.util.Collection#containsAll(java.util.Collection)
+ */
+ public boolean containsAll(Collection<?> arg0) {
+ throw new UnsupportedOperationException();
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.util.Collection#isEmpty()
+ *
+ * Runtime: 1 voldemort get operation
+ */
+ public boolean isEmpty() {
+ VListKey<K> newKey = new VListKey<K>(_key, 0);
+ Versioned<Map<String, Object>> firstNode = _storeClient.get(newKey.mapValue());
+ return firstNode == null;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.util.Collection#iterator()
+ */
+ public Iterator<E> iterator() {
+ return new VListIteratorValues<E>(listIterator(0));
+ }
+
+ /*
+ * Returns a versioned iterator that starts at a specified id.
+ */
+ public Iterator<Versioned<E>> iterator(int id) {
+ return listIterator(id);
+ }
+
+ public MappedListIterator<Integer, Versioned<E>> listIterator() {
+ return listIterator(0);
+ }
+
+ /*
+ * Returns a VListIterator<E> that starts at a specified id.
+ *
+ * @throws IndexOutOfBoundsException if the id does not exist in the list
+ */
+ public VListIterator<E> listIterator(int id) {
+ return new VListIterator<E>(this, id);
+ }
+
+ /**
+ * Returns a ListIterator that ends at a specified id. The cursor position
+ * is just after the id.
+ *
+ * @throws IndexOutOfBoundsException if the id does not exist
+ */
+ public VListIterator<E> previousListIterator(int id) {
+ return new VListIterator<E>(this, id, false);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.util.Collection#remove(java.lang.Object)
+ */
+ public boolean remove(Object arg0) {
+ throw new UnsupportedOperationException();
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.util.Collection#removeAll(java.util.Collection)
+ */
+ public boolean removeAll(Collection<?> arg0) {
+ throw new UnsupportedOperationException();
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.util.Collection#retainAll(java.util.Collection)
+ */
+ public boolean retainAll(Collection<?> arg0) {
+ throw new UnsupportedOperationException();
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.util.Collection#size()
+ *
+ * Runtime: O(n) gets (WARNING -- very very slow. Only use for debugging
+ * purposes)
+ */
+ public int size() {
+ int result = 0;
+ for(@SuppressWarnings("unused")
+ E element: this) {
+ result++;
+ }
+ return result;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.util.Collection#toArray()
+ */
+ public Object[] toArray() {
+ throw new UnsupportedOperationException();
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.util.Collection#toArray(T[])
+ */
+ public <T> T[] toArray(T[] arg0) {
+ throw new UnsupportedOperationException();
+ }
+
+ /*
+ * (non-Javadoc) Adds a new element into the VStack with a new version.
+ */
+ public boolean add(E e) {
+ if(!offer(e))
+ throw new IllegalStateException();
+ return true;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.util.Collection#addAll(java.util.Collection)
+ */
+ public boolean addAll(Collection<? extends E> elements) {
+ throw new UnsupportedOperationException();
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.util.Queue#element()
+ */
+ public E element() {
+ E result = peek();
+ if(result == null)
+ throw new NoSuchElementException();
+
+ return result;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.util.Queue#offer(java.lang.Object)
+ *
+ * Runtime: worst case: 2 gets, 5 puts
+ */
+ public boolean offer(E e) {
+ if(e == null)
+ throw new NullPointerException("null objects are not allowed");
+ AddNodeAction<K, E> addAction = new AddNodeAction<K, E>(_key, e);
+ try {
+ return _storeClient.applyUpdate(addAction);
+ } catch(IndexOutOfBoundsException ex) {
+ // over-capacity
+ return false;
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.util.Queue#peek()
+ */
+ public E peek() {
+ Versioned<E> versioned = peekVersioned();
+ if(versioned == null) {
+ return null;
+ } else {
+ return versioned.getValue();
+ }
+ }
+
+ public Versioned<E> peekVersioned() {
+ Versioned<VListNode<E>> node = getListNode(0);
+ if(node == null) {
+ return null;
+ } else {
+ return new Versioned<E>(node.getValue().getValue(), node.getVersion());
+ }
+ }
+
+ /**
+ * Get the ver
+ *
+ * @param id
+ * @return
+ */
+ public Versioned<E> getVersionedById(int id) {
+ Versioned<VListNode<E>> listNode = getListNode(id);
+ if(listNode == null)
+ throw new IndexOutOfBoundsException();
+ return new Versioned<E>(listNode.getValue().getValue(), listNode.getVersion());
+ }
+
+ /**
+ * Get the value associated with the id. This method strips off all version
+ * information and is only useful when no further storage operations will be
+ * done on this value.
+ *
+ * @param id
+ * @return
+ */
+ public E getById(int id) {
+ return getVersionedById(id).getValue();
+ }
+
+ /**
+ * Put the given value to the appropriate id in the stack, using the version
+ * of the current list node identified by that id.
+ *
+ * @param id
+ * @param element element to set
+ * @return element that was replaced by the new element
+ * @throws ObsoleteVersionException when an update fails
+ */
+ public E setById(final int id, final E element) {
+ VListKey<K> key = new VListKey<K>(_key, id);
+ UpdateElementById<K, E> updateElementAction = new UpdateElementById<K, E>(key, element);
+
+ if(!_storeClient.applyUpdate(updateElementAction))
+ throw new ObsoleteVersionException("update failed");
+
+ return updateElementAction.getResult();
+ }
+
+ public E setById(int id, Versioned<E> element) {
+ VListKey<K> key = new VListKey<K>(_key, id);
+ UpdateElementById<K, E> updateElementAction = new UpdateElementById<K, E>(key, element);
+
+ if(!_storeClient.applyUpdate(updateElementAction))
+ throw new ObsoleteVersionException("update failed");
+
+ return updateElementAction.getResult();
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.util.Queue#poll()
+ */
+ public E poll() {
+ throw new UnsupportedOperationException();
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.util.Queue#remove()
+ */
+ public E remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ *
+ * @param id
+ * @return the VListNode identified by id. If it doesn't exist, return null
+ */
+ Versioned<VListNode<E>> getListNode(int id) {
+ VListKey<K> key = new VListKey<K>(_key, id);
+ Versioned<Map<String, Object>> resultMap = _storeClient.get(key.mapValue());
+ if(resultMap == null)
+ return null;
+
+ VListNode<E> listNode = VListNode.<E> valueOf(resultMap.getValue());
+ Versioned<VListNode<E>> versionedNode = new Versioned<VListNode<E>>(listNode,
+ resultMap.getVersion());
+ return versionedNode;
+ }
+}
View
12 contrib/collections/src/java/voldemort/utils/Identifiable.java
@@ -0,0 +1,12 @@
+package voldemort.utils;
+
+import java.io.Serializable;
+
+/**
+ * Interface providing the identifier of an entity
+ */
+public interface Identifiable<K extends Serializable>
+{
+ public K getId();
+}
+
View
33 contrib/collections/src/java/voldemort/utils/Identified.java
@@ -0,0 +1,33 @@
+package voldemort.utils;
+
+import java.io.Serializable;
+
+/**
+ * @param T the type of object that is being wrapped
+ * @param K the type of value used to identify the object
+ *
+ * Immutable wrapper around an object that adds an id.
+ */
+public class Identified<T, K extends Serializable> implements Identifiable<K> {
+
+ private final T _value;
+ private final K _id;
+
+ public Identified(T object, K id) {
+ _value = object;
+ _id = id;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see com.linkedin.common.entity.Identifiable#getId()
+ */
+ public K getId() {
+ return _id;
+ }
+
+ public T getValue() {
+ return _value;
+ }
+}
View
260 contrib/collections/test/voldemort/collections/TestVLinkedPagedList.java
@@ -0,0 +1,260 @@
+package voldemort.collections;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+
+import junit.framework.TestCase;
+import voldemort.client.MockStoreClientFactory;
+import voldemort.client.StoreClient;
+import voldemort.client.StoreClientFactory;
+import voldemort.serialization.Serializer;
+import voldemort.serialization.json.JsonTypeSerializer;
+import voldemort.utils.Identifiable;</