Permalink
Browse files

Merge remote branch 'upstream/master'

Conflicts:
	clients/python/voldemort/protocol/voldemort_admin_pb2.py
	src/java/voldemort/client/protocol/pb/VAdminProto.java
	src/java/voldemort/server/protocol/admin/AdminServiceRequestHandler.java
	src/java/voldemort/utils/RebalanceUtils.java
	src/proto/voldemort-admin.proto
	test/unit/voldemort/client/rebalance/AbstractRebalanceTest.java
  • Loading branch information...
2 parents c94ab05 + e491023 commit f3cd4f9fef530b4f10cbad1b4c884be2b551ee9b Lei Gao committed Sep 7, 2011
Showing with 1,631 additions and 608 deletions.
  1. +3 −3 NOTES
  2. +1 −1 build.properties
  3. +119 −37 clients/python/voldemort/protocol/voldemort_admin_pb2.py
  4. +9 −9 clients/ruby/lib/connection/connection.rb
  5. +4 −4 clients/ruby/lib/connection/tcp_connection.rb
  6. +1 −1 clients/ruby/lib/voldemort-serializer.rb
  7. +1 −1 clients/ruby/voldemort-rb.gemspec
  8. +1 −0 config/test_config1/config/server.properties
  9. +1 −0 config/test_config2/config/server.properties
  10. +5 −5 contrib/krati/src/java/voldemort/store/krati/KratiStorageEngine.java
  11. +62 −0 release_notes.txt
  12. +23 −3 src/java/voldemort/VoldemortAdminTool.java
  13. +27 −4 src/java/voldemort/client/protocol/admin/AdminClient.java
  14. +1 −1 src/java/voldemort/client/protocol/admin/AdminClientConfig.java
  15. +817 −122 src/java/voldemort/client/protocol/pb/VAdminProto.java
  16. +4 −4 src/java/voldemort/cluster/failuredetector/FailureDetectorConfig.java
  17. +14 −0 src/java/voldemort/server/StoreRepository.java
  18. +26 −0 src/java/voldemort/server/protocol/admin/AdminServiceRequestHandler.java
  19. +27 −51 src/java/voldemort/server/scheduler/slop/RepairJob.java
  20. +7 −13 src/java/voldemort/server/storage/StorageService.java
  21. +9 −4 src/java/voldemort/store/bdb/BdbStorageConfiguration.java
  22. +14 −8 src/java/voldemort/store/bdb/BdbStorageEngine.java
  23. +4 −4 src/java/voldemort/store/configuration/ConfigurationStorageEngine.java
  24. +5 −5 src/java/voldemort/store/memory/InMemoryStorageEngine.java
  25. +5 −5 src/java/voldemort/store/mysql/MysqlStorageEngine.java
  26. +4 −4 src/java/voldemort/store/readonly/swapper/AdminStoreSwapper.java
  27. +5 −5 src/java/voldemort/store/readonly/swapper/HttpStoreSwapper.java
  28. +3 −3 src/java/voldemort/store/readonly/swapper/StoreSwapper.java
  29. +4 −4 src/java/voldemort/store/routed/ReadRepairer.java
  30. +103 −23 src/java/voldemort/utils/Entropy.java
  31. +64 −0 src/java/voldemort/utils/KeyLocationValidation.java
  32. +14 −7 src/java/voldemort/utils/RebalanceUtils.java
  33. +1 −1 src/java/voldemort/versioning/{Occured.java → Occurred.java}
  34. +6 −6 src/java/voldemort/versioning/VectorClock.java
  35. +3 −3 src/java/voldemort/versioning/VectorClockInconsistencyResolver.java
  36. +1 −1 src/java/voldemort/versioning/Version.java
  37. +3 −3 src/java/voldemort/versioning/Versioned.java
  38. +14 −4 src/proto/voldemort-admin.proto
  39. +20 −1 test/unit/voldemort/client/AdminServiceBasicTest.java
  40. +175 −23 test/unit/voldemort/client/rebalance/AbstractRebalanceTest.java
  41. +0 −218 test/unit/voldemort/scheduled/RepairTest.java
  42. +10 −6 test/unit/voldemort/server/EndToEndTest.java
  43. +4 −4 test/unit/voldemort/store/routed/RoutedStoreTest.java
  44. +7 −7 test/unit/voldemort/versioning/VectorClockTest.java
View
@@ -17,13 +17,13 @@ Commands:
put key value -- Associate the given value with the key.
get key -- Retrieve the value associated with the key.
delete key -- Remove all values associated with the key.
-locate key -- Determine which servers host the give key.
+preflist key -- Get node preference list for given key.
help -- Print this message.
exit -- Exit from this shell.
> put "hello" "there"
> get "hello"
version(0:1): "there"
-> locate "hello"
+> preflist "hello"
Node 0
host: localhost
port: 6666
@@ -71,4 +71,4 @@ Socket servers share threadpool?
A WireProtocol takes bytes and creates a voldemort request, and takes objects to create a voldemort response.
Likewise the client does the opposite
-Connector.handleRequest()
+Connector.handleRequest()
View
@@ -34,4 +34,4 @@ tomcat.manager.password=tomcat
tomcat.context=/voldemort
## Release
-curr.release=0.81
+curr.release=0.90
Oops, something went wrong.
@@ -85,12 +85,12 @@ def parse_nodes_from(doc)
nodes = []
doc.xpath("/cluster/server").each do |n|
node = VoldemortNode.new
- node.id = n.xpath("//id").text
- node.host = n.xpath("//host").text
- node.port = n.xpath("//socket-port").text
- node.http_port = n.xpath("//http_port").text
- node.admin_port = n.xpath("//admin-port").text
- node.partitions = n.xpath("//partitions").text
+ node.id = n.xpath("id").text
+ node.host = n.xpath("host").text
+ node.port = n.xpath("socket-port").text
+ node.http_port = n.xpath("http_port").text
+ node.admin_port = n.xpath("admin-port").text
+ node.partitions = n.xpath("partitions").text
nodes << node
end
nodes
@@ -128,15 +128,15 @@ def rebalance_connection_if_needed
def get(key)
self.rebalance_connection_if_needed
- self.get_from(self.db_name, key, true)
+ self.get_from(self.db_name, key, false)
end
def get_all(keys)
self.rebalance_connection_if_needed
- self.get_all_from(self.db_name, keys, true)
+ self.get_all_from(self.db_name, keys, false)
end
- def put(key, value, version = nil, route = true)
+ def put(key, value, version = nil, route = false)
self.rebalance_connection_if_needed
self.put_from(self.db_name, key, value, version, route)
end
@@ -28,7 +28,7 @@ def connect_to(host, port)
end
end
- def get_from(db_name, key, route = true)
+ def get_from(db_name, key, route = false)
request = VoldemortRequest.new
request.should_route = route
request.store = db_name
@@ -43,7 +43,7 @@ def get_from(db_name, key, route = true)
response
end
- def get_all_from(db_name, keys, route = true)
+ def get_all_from(db_name, keys, route = false)
request = VoldemortRequest.new
request.should_route = route
request.store = db_name
@@ -58,7 +58,7 @@ def get_all_from(db_name, keys, route = true)
response
end
- def put_from(db_name, key, value, version = nil, route = true)
+ def put_from(db_name, key, value, version = nil, route = false)
version = get_version(key) unless version
request = VoldemortRequest.new
request.should_route = route
@@ -80,7 +80,7 @@ def put_from(db_name, key, value, version = nil, route = true)
version
end
- def delete_from(db_name, key, version = nil, route = true)
+ def delete_from(db_name, key, version = nil, route = false)
version = get_version(key) unless version
request = VoldemortRequest.new
request.should_route = route
@@ -54,7 +54,7 @@ def read_slice(length, bytes)
def to_bytes(object)
bytes = ''
- newest_version = 0 # TODO get highest number from map
+ newest_version = type_def_versions.size > 0 ? (type_def_versions.size-1) : 0
type_def = @type_def_versions[newest_version]
if(@has_version)
@@ -1,6 +1,6 @@
Gem::Specification.new do |s|
s.name = 'voldemort-rb'
- s.version = '0.1.5'
+ s.version = '0.1.8'
s.summary = %{A Ruby client for the Voldemort distributed key value store}
s.description = %Q{voldemort-rb allows you to connect to the Voldemort descentralized key value store.}
s.authors = ["Alejandro Crosa"]
@@ -2,6 +2,7 @@
node.id=0
max.threads=100
+enable.repair=true
############### DB options ######################
@@ -4,6 +4,7 @@ bdb.flush.transactions=false
bdb.cache.size=100MB
max.threads=100
+enable.repair=true
http.enable=true
socket.enable=true
@@ -30,7 +30,7 @@
import voldemort.utils.StripedLock;
import voldemort.utils.Utils;
import voldemort.versioning.ObsoleteVersionException;
-import voldemort.versioning.Occured;
+import voldemort.versioning.Occurred;
import voldemort.versioning.VectorClock;
import voldemort.versioning.Version;
import voldemort.versioning.Versioned;
@@ -171,7 +171,7 @@ public boolean delete(ByteArray key, Version maxVersion) throws VoldemortExcepti
while(iter.hasNext()) {
Versioned<byte[]> currentValue = iter.next();
Version currentVersion = currentValue.getVersion();
- if(currentVersion.compare(maxVersion) == Occured.BEFORE) {
+ if(currentVersion.compare(maxVersion) == Occurred.BEFORE) {
iter.remove();
}
}
@@ -206,11 +206,11 @@ public void put(ByteArray key, Versioned<byte[]> value, byte[] transforms)
// Update the value
List<Versioned<byte[]>> removedValueList = new ArrayList<Versioned<byte[]>>();
for(Versioned<byte[]> versioned: existingValuesList) {
- Occured occured = value.getVersion().compare(versioned.getVersion());
- if(occured == Occured.BEFORE)
+ Occurred occurred = value.getVersion().compare(versioned.getVersion());
+ if(occurred == Occurred.BEFORE)
throw new ObsoleteVersionException("Obsolete version for key '" + key
+ "': " + value.getVersion());
- else if(occured == Occured.AFTER)
+ else if(occurred == Occurred.AFTER)
removedValueList.add(versioned);
}
existingValuesList.removeAll(removedValueList);
View
@@ -1,3 +1,65 @@
+Release 0.90 on 7/10/2011
+
+Changes made since 0.81
+ * All upgrading instructions can be found here - https://github.com/voldemort/voldemort/wiki/Upgrading-from-0.81
+ * Tooling
+ - Single consolidated administrative tool - Got rid of the admin client
+ shell and have a better Voldemort admin tool. More documentation -
+ https://github.com/voldemort/voldemort/wiki/Voldemort-Admin-tool
+
+ * Web manager
+ - Basic GUI in JRuby ( and Sinatra ), capable of querying for store metadata.
+ - Read contrib/web-manager/README.md for more details
+
+ * Rebalancing
+ - New better rebalancing support with (a) checkpointing (b) progress bar (c) support for RO store rebalancing
+ - Documentation - https://github.com/voldemort/voldemort/wiki/Voldemort-Rebalancing
+ - In the process solves Issue 203, 288, 305, 307, 311
+
+ * Client side changes
+ - Pipeline routed store ( with support for topology awareness ) - Changed the default client side routing
+ to use a state machine like system. More documentation - https://github.com/voldemort/voldemort/wiki/RoutedStore-redesign
+ - Lazy store client - Some of our clients are very inactive and don't really do many requests. For such
+ clients it doesn't make sense to bootstrap the metadata at startup. Hence from now onwards clients instantiated
+ from SocketStoreClientFactory give a LazyStoreClient which will not bootstrap till the first request is made
+ - Caching store client factory - We have also checked-in a new store client factory which we use internally
+ at LinkedIn to cache store clients for stores which we repeatedly.
+ - Failure detector - We have fixed some bugs in the ThresholdFailureDetector. This is an improvement over the naive
+ BannageFailureDetector which would aggressively mark nodes down after a single failure. More details -
+ https://github.com/voldemort/voldemort/wiki/Client-side-failure-detector-implementations
+ - Issue 219: StoreClient::put returns a Version - You may have to upgrade your clients since now the StoreClient returns
+ a version to be used in successive requests.
+
+ * Read-only store changes
+ - Admin based store swapper - Till 0.81 we relied on a servlet based fetcher / swapper.
+ Now we support an admin based store swapper which reports progress.
+ - Other changes - (a) New directory format (b) Two new data format. Support for iterating over read-only data
+ (c) Checksum of data in .metadata file (d) Some monitoring changes of RO stores
+
+ * Jar upgrades
+ - Introduction of JNA 3.2.7 - We have introduced JNA 3.2.7 in this release for supporting symbolic links in read-only stores.
+ More details about the new RO format and changes can be found here - https://github.com/voldemort/voldemort/wiki/Upgrading-from-0.81
+ - Protocol Buffers 2.3.0 - Upgrading protocol buffers from 2.2.0 to 2.3.0
+ - Avro 1.4.0 - Upgraded Avro from 1.3.0 to 1.4.0
+
+ * Storage engine
+ - Added support for Krati 0.3.6 as a storage engine ( http://sna-projects.com/krati/ ) -
+ https://github.com/voldemort/voldemort/tree/release-090/contrib/krati
+
+ * Core features
+ - Hinted handoff - https://github.com/voldemort/voldemort/wiki/Hinted-Handoff
+ - Experimental support for server side transforms - https://github.com/voldemort/voldemort/wiki/Server-side-transforms-in-Voldemort
+ - Topology awareness ( i.e. datacenter / rack ) - https://github.com/voldemort/voldemort/wiki/Topology-awareness-capability
+ - Repair Job ( Job which will delete data if the node is not responsible for it )
+
+ * Better monitoring
+ - Tons of JMX changes ( average bytes, etc ) - More details ( https://github.com/voldemort/voldemort/wiki/JMX-Monitoring )
+ - Key distribution generator - Ability to estimate skew of your cluster ( ./bin/run-class.sh voldemort.utils.KeyDistributionGenerator )
+
+ * Clients
+ - Python - Updated Python client with support for Binary JSON serialized stores ( ./clients/python/README )
+ - Ruby - Updated Ruby client ( ./clients/ruby/README.md )
+
Release 0.81 on 6/15/2010
Changes made since 0.80.2:
@@ -190,6 +190,7 @@ public static void main(String[] args) throws Exception {
.describedAs("job-ids")
.withValuesSeparatedBy(',')
.ofType(Integer.class);
+ parser.accepts("repair-job", "Clean after rebalancing is done");
OptionSet options = parser.parse(args);
@@ -264,10 +265,13 @@ public static void main(String[] args) throws Exception {
if(options.has("async")) {
ops += "b";
}
+ if(options.has("repair-job")) {
+ ops += "l";
+ }
if(ops.length() < 1) {
Utils.croak("At least one of (delete-partitions, restore, add-node, fetch-entries, "
+ "fetch-keys, add-stores, delete-store, update-entries, get-metadata, ro-metadata, "
- + "set-metadata, check-metadata, key-distribution, clear-rebalancing-metadata, async) "
+ + "set-metadata, check-metadata, key-distribution, clear-rebalancing-metadata, async, repair-job) "
+ "must be specified");
}
@@ -414,12 +418,25 @@ public static void main(String[] args) throws Exception {
asyncIds = (List<Integer>) options.valuesOf("async-id");
executeAsync(nodeId, adminClient, asyncKey, asyncIds);
}
+ if(ops.contains("l")) {
+ executeRepairJob(nodeId, adminClient);
+ }
} catch(Exception e) {
e.printStackTrace();
Utils.croak(e.getMessage());
}
}
+ private static void executeRepairJob(Integer nodeId, AdminClient adminClient) {
+ if(nodeId < 0) {
+ for(Node node: adminClient.getAdminClientCluster().getNodes()) {
+ adminClient.repairJob(node.getId());
+ }
+ } else {
+ adminClient.repairJob(nodeId);
+ }
+ }
+
public static void printHelp(PrintStream stream, OptionParser parser) throws IOException {
stream.println("Commands supported");
stream.println("------------------");
@@ -516,6 +533,8 @@ public static void printHelp(PrintStream stream, OptionParser parser) throws IOE
stream.println("\t\t./bin/voldemort-admin-tool.sh --restore 10 --url [url] --node [node-id]");
stream.println("\t3) Generates the key distribution on a per node basis [ both store wise and overall ]");
stream.println("\t\t./bin/voldemort-admin-tool.sh --key-distribution --url [url]");
+ stream.println("\t4) Clean a node after rebalancing is done");
+ stream.println("\t\t./bin/voldemort-admin-tool.sh --repair-job --url [url] --node [node-id]");
parser.printHelpOn(stream);
}
@@ -665,8 +684,9 @@ private static void executeSetMetadata(Integer nodeId,
+ adminClient.getAdminClientCluster()
.getNodeById(currentNodeId)
.getId());
- adminClient.updateRemoteMetadata(currentNodeId, key, Versioned.value(value.toString(),
- updatedVersion));
+ adminClient.updateRemoteMetadata(currentNodeId,
+ key,
+ Versioned.value(value.toString(), updatedVersion));
}
}
@@ -31,8 +31,8 @@
import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.Map.Entry;
+import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
@@ -49,8 +49,8 @@
import voldemort.client.protocol.VoldemortFilter;
import voldemort.client.protocol.pb.ProtoUtils;
import voldemort.client.protocol.pb.VAdminProto;
-import voldemort.client.protocol.pb.VProto;
import voldemort.client.protocol.pb.VAdminProto.RebalancePartitionInfoMap;
+import voldemort.client.protocol.pb.VProto;
import voldemort.client.protocol.pb.VProto.RequestType;
import voldemort.client.rebalance.RebalancePartitionsInfo;
import voldemort.cluster.Cluster;
@@ -715,8 +715,7 @@ public Thread newThread(Runnable r) {
}
}
for(StoreDefinition def: writableStores) {
- if(def.getName().compareTo("test-recovery-data") == 0)
- restoreStoreFromReplication(nodeId, cluster, def, executors);
+ restoreStoreFromReplication(nodeId, cluster, def, executors);
}
} finally {
executors.shutdown();
@@ -1702,6 +1701,30 @@ public void rollbackStore(int nodeId, String storeName, long pushVersion) {
}
/**
+ * Repair the stores on a rebalanced node 'nodeId'
+ * <p>
+ *
+ * @param nodeId The id of the node on which to do the repair
+ */
+ public void repairJob(int nodeId) {
+ VAdminProto.RepairJobRequest.Builder repairJobRequest = VAdminProto.RepairJobRequest.newBuilder();
+
+ VAdminProto.VoldemortAdminRequest adminRequest = VAdminProto.VoldemortAdminRequest.newBuilder()
+ .setRepairJob(repairJobRequest)
+ .setType(VAdminProto.AdminRequestType.REPAIR_JOB)
+ .build();
+ VAdminProto.AsyncOperationStatusResponse.Builder response = sendAndReceive(nodeId,
+ adminRequest,
+ VAdminProto.AsyncOperationStatusResponse.newBuilder());
+
+ if(response.hasError()) {
+ throwException(response.getError());
+ }
+
+ return;
+ }
+
+ /**
* Fetch data from directory 'storeDir' on node id
* <p>
*
@@ -18,7 +18,7 @@
private volatile long adminSocketTimeoutSec = 24 * 60 * 60;
private volatile int adminSocketBufferSize = 16 * 1024 * 1024;
private volatile boolean adminSocketKeepAlive = false;
- private volatile int restoreDataTimeoutSec = 24 * 60 * 60;
+ private volatile int restoreDataTimeoutSec = 365 * 24 * 60 * 60;
private volatile int maxBackoffDelayMs = 60 * 1000;
public static final String MAX_CONNECTIONS_PER_NODE_PROPERTY = "max_connections";
Oops, something went wrong.

0 comments on commit f3cd4f9

Please sign in to comment.