Merge branch 'master' of github.com:voldemort/voldemort

btu · Oct 25, 2011 · 5c76a5a · 5c76a5a
2 parents 67e2bcf + f8d4cc2
commit 5c76a5a
Show file tree

Hide file tree

Showing 16 changed files with 220 additions and 42 deletions.
diff --git a/build.properties b/build.properties
@@ -34,4 +34,4 @@ tomcat.manager.password=tomcat
 tomcat.context=/voldemort
 
 ## Release
-curr.release=0.90
+curr.release=0.90.1
diff --git a/release_notes.txt b/release_notes.txt
@@ -1,3 +1,24 @@
+Release 0.90.1 on 10/10/2011
+
+Changes made since 0.90
+
+ * Updated the documentation for Voldemort shell tool in NOTES
+ * Added Admin API to perform Bdb data cleanup (repairJob) 
+   and corresponding unit tests
+ * Fixes in restore from replication, store creation code
+ * Improved failure detector configuration. ThresholdFailureDetector 
+   is now the default option
+ * Multiple Fixes to the Voldemort ruby client
+ * Added additional Jmx metrics to expose Bdb environment statistics, 
+   caching statistics and Voldemort batch operation statistics
+ * Updated default timeout for restore 'from replica' to 365 days
+ * New feature in the performance tool: '--use-sample' option enables 
+   'read, write back unmodified' transactions in place of writes in 
+   the workload (allows for testing read-write transactions on stores 
+   with complex schemas)
+ * Added the ability to dynamically update cluster.xml and 
+   reinitialize the scheduler
+
 Release 0.90 on 7/10/2011
 
 Changes made since 0.81

diff --git a/src/java/voldemort/server/VoldemortConfig.java b/src/java/voldemort/server/VoldemortConfig.java
@@ -132,6 +132,7 @@ public class VoldemortConfig implements Serializable {
     private boolean enableNetworkClassLoader;
     private boolean enableGossip;
     private boolean enableRebalanceService;
+    private boolean enableJmxClusterName;
 
     private List<String> storageConfigurations;
 
@@ -276,6 +277,7 @@ public VoldemortConfig(Props props) {
         this.enableGossip = props.getBoolean("enable.gossip", false);
         this.enableRebalanceService = props.getBoolean("enable.rebalancing", true);
         this.enableRepair = props.getBoolean("enable.repair", false);
+        this.enableJmxClusterName = props.getBoolean("enable.jmx.clustername", false);
 
         this.gossipInterval = props.getInt("gossip.interval.ms", 30 * 1000);
 
@@ -1385,4 +1387,12 @@ public void setMaxParallelStoresRebalancing(boolean rebalancingOptimization) {
         this.rebalancingOptimization = rebalancingOptimization;
     }
 
+    public boolean isEnableJmxClusterName() {
+        return enableJmxClusterName;
+    }
+
+    public void setEnableJmxClusterName(boolean enableJmxClusterName) {
+        this.enableJmxClusterName = enableJmxClusterName;
+    }
+
 }
diff --git a/src/java/voldemort/server/jmx/JmxService.java b/src/java/voldemort/server/jmx/JmxService.java
@@ -78,9 +78,16 @@ protected void startInner() {
         for(VoldemortService service: services)
             registerBean(service, JmxUtils.createObjectName(service.getClass()));
         for(Store<ByteArray, byte[], byte[]> store: this.storeRepository.getAllStorageEngines()) {
-            registerBean(store,
-                         JmxUtils.createObjectName(JmxUtils.getPackageName(store.getClass()),
-                                                   store.getName()));
+            if(server.getVoldemortConfig().isEnableJmxClusterName())
+                registerBean(store,
+                             JmxUtils.createObjectName(this.cluster.getName()
+                                                               + "."
+                                                               + JmxUtils.getPackageName(store.getClass()),
+                                                       store.getName()));
+            else
+                registerBean(store,
+                             JmxUtils.createObjectName(JmxUtils.getPackageName(store.getClass()),
+                                                       store.getName()));
             if(store instanceof BdbStorageEngine) {
                 // Temp hack for now
                 BdbStorageEngine bdbStore = (BdbStorageEngine) store;

diff --git a/src/java/voldemort/server/protocol/SocketRequestHandlerFactory.java b/src/java/voldemort/server/protocol/SocketRequestHandlerFactory.java
@@ -46,7 +46,15 @@ public SocketRequestHandlerFactory(StorageService storageService,
         this.rebalancer = rebalancer;
         this.stats = new StreamStats();
         if(null != voldemortConfig && voldemortConfig.isJmxEnabled())
-            JmxUtils.registerMbean("admin-streaming", new StreamStatsJmx(stats));
+            if(this.voldemortConfig.isEnableJmxClusterName())
+                JmxUtils.registerMbean(new StreamStatsJmx(stats),
+                                       JmxUtils.createObjectName(metadata.getCluster().getName()
+                                                                         + ".voldemort.store.stats",
+                                                                 "admin-streaming"));
+            else
+                JmxUtils.registerMbean(new StreamStatsJmx(stats),
+                                       JmxUtils.createObjectName("voldemort.store.stats",
+                                                                 "admin-streaming"));
     }
 
     public RequestHandler getRequestHandler(RequestFormatType type) {

diff --git a/src/java/voldemort/server/storage/StorageService.java b/src/java/voldemort/server/storage/StorageService.java
@@ -255,9 +255,15 @@ protected void startInner() {
 
         // enable aggregate jmx statistics
         if(voldemortConfig.isStatTrackingEnabled())
-            JmxUtils.registerMbean(new StoreStatsJmx(this.storeStats),
-                                   JmxUtils.createObjectName("voldemort.store.stats.aggregate",
-                                                             "aggregate-perf"));
+            if(this.voldemortConfig.isEnableJmxClusterName())
+                JmxUtils.registerMbean(new StoreStatsJmx(this.storeStats),
+                                       JmxUtils.createObjectName(metadata.getCluster().getName()
+                                                                         + ".voldemort.store.stats.aggregate",
+                                                                 "aggregate-perf"));
+            else
+                JmxUtils.registerMbean(new StoreStatsJmx(this.storeStats),
+                                       JmxUtils.createObjectName("voldemort.store.stats.aggregate",
+                                                                 "aggregate-perf"));
 
         logger.info("All stores initialized.");
     }
@@ -329,8 +335,15 @@ public void unregisterEngine(StorageEngine<ByteArray, byte[], byte[]> engine,
                 if(!isSlop && voldemortConfig.isEnableRebalanceService() && !isReadOnly
                    && !isMetadata && !isView) {
 
-                    ObjectName name = JmxUtils.createObjectName(JmxUtils.getPackageName(RedirectingStore.class),
-                                                                store.getName());
+                    ObjectName name = null;
+                    if(this.voldemortConfig.isEnableJmxClusterName())
+                        name = JmxUtils.createObjectName(metadata.getCluster().getName()
+                                                                 + "."
+                                                                 + JmxUtils.getPackageName(RedirectingStore.class),
+                                                         store.getName());
+                    else
+                        name = JmxUtils.createObjectName(JmxUtils.getPackageName(RedirectingStore.class),
+                                                         store.getName());
 
                     synchronized(mbeanServer) {
                         if(mbeanServer.isRegistered(name))
@@ -340,9 +353,15 @@ public void unregisterEngine(StorageEngine<ByteArray, byte[], byte[]> engine,
                 }
 
                 if(voldemortConfig.isStatTrackingEnabled()) {
-
-                    ObjectName name = JmxUtils.createObjectName(JmxUtils.getPackageName(store.getClass()),
-                                                                store.getName());
+                    ObjectName name = null;
+                    if(this.voldemortConfig.isEnableJmxClusterName())
+                        name = JmxUtils.createObjectName(metadata.getCluster().getName()
+                                                                 + "."
+                                                                 + JmxUtils.getPackageName(store.getClass()),
+                                                         store.getName());
+                    else
+                        name = JmxUtils.createObjectName(JmxUtils.getPackageName(store.getClass()),
+                                                         store.getName());
 
                     synchronized(mbeanServer) {
                         if(mbeanServer.isRegistered(name))
@@ -397,8 +416,16 @@ public void registerEngine(StorageEngine<ByteArray, byte[], byte[]> engine,
                                              storeFactory);
                 if(voldemortConfig.isJmxEnabled()) {
                     MBeanServer mbeanServer = ManagementFactory.getPlatformMBeanServer();
-                    ObjectName name = JmxUtils.createObjectName(JmxUtils.getPackageName(RedirectingStore.class),
-                                                                store.getName());
+                    ObjectName name = null;
+                    if(this.voldemortConfig.isEnableJmxClusterName())
+                        name = JmxUtils.createObjectName(cluster.getName()
+                                                                 + "."
+                                                                 + JmxUtils.getPackageName(RedirectingStore.class),
+                                                         store.getName());
+                    else
+                        name = JmxUtils.createObjectName(JmxUtils.getPackageName(RedirectingStore.class),
+                                                         store.getName());
+
                     synchronized(mbeanServer) {
                         if(mbeanServer.isRegistered(name))
                             JmxUtils.unregisterMbean(mbeanServer, name);
@@ -419,8 +446,15 @@ public void registerEngine(StorageEngine<ByteArray, byte[], byte[]> engine,
             if(voldemortConfig.isJmxEnabled()) {
 
                 MBeanServer mbeanServer = ManagementFactory.getPlatformMBeanServer();
-                ObjectName name = JmxUtils.createObjectName(JmxUtils.getPackageName(store.getClass()),
-                                                            store.getName());
+                ObjectName name = null;
+                if(this.voldemortConfig.isEnableJmxClusterName())
+                    name = JmxUtils.createObjectName(metadata.getCluster().getName()
+                                                             + "."
+                                                             + JmxUtils.getPackageName(store.getClass()),
+                                                     store.getName());
+                else
+                    name = JmxUtils.createObjectName(JmxUtils.getPackageName(store.getClass()),
+                                                     store.getName());
 
                 synchronized(mbeanServer) {
                     if(mbeanServer.isRegistered(name))

diff --git a/src/java/voldemort/store/routed/action/PerformParallelDeleteRequests.java b/src/java/voldemort/store/routed/action/PerformParallelDeleteRequests.java
@@ -20,8 +20,8 @@
 import voldemort.store.nonblockingstore.NonblockingStoreCallback;
 import voldemort.store.routed.BasicPipelineData;
 import voldemort.store.routed.Pipeline;
-import voldemort.store.routed.Response;
 import voldemort.store.routed.Pipeline.Event;
+import voldemort.store.routed.Response;
 import voldemort.store.slop.HintedHandoff;
 import voldemort.store.slop.Slop;
 import voldemort.utils.ByteArray;

diff --git a/src/java/voldemort/store/routed/action/PerformParallelGetAllRequests.java b/src/java/voldemort/store/routed/action/PerformParallelGetAllRequests.java
@@ -34,8 +34,8 @@
 import voldemort.store.nonblockingstore.NonblockingStoreCallback;
 import voldemort.store.routed.GetAllPipelineData;
 import voldemort.store.routed.Pipeline;
-import voldemort.store.routed.Response;
 import voldemort.store.routed.Pipeline.Event;
+import voldemort.store.routed.Response;
 import voldemort.utils.ByteArray;
 import voldemort.versioning.Versioned;
 

diff --git a/src/java/voldemort/store/routed/action/PerformParallelPutRequests.java b/src/java/voldemort/store/routed/action/PerformParallelPutRequests.java
@@ -35,9 +35,9 @@
 import voldemort.store.nonblockingstore.NonblockingStore;
 import voldemort.store.nonblockingstore.NonblockingStoreCallback;
 import voldemort.store.routed.Pipeline;
+import voldemort.store.routed.Pipeline.Event;
 import voldemort.store.routed.PutPipelineData;
 import voldemort.store.routed.Response;
-import voldemort.store.routed.Pipeline.Event;
 import voldemort.store.slop.HintedHandoff;
 import voldemort.store.slop.Slop;
 import voldemort.utils.ByteArray;

diff --git a/src/java/voldemort/store/routed/action/PerformParallelRequests.java b/src/java/voldemort/store/routed/action/PerformParallelRequests.java
@@ -33,9 +33,9 @@
 import voldemort.store.nonblockingstore.NonblockingStoreCallback;
 import voldemort.store.routed.BasicPipelineData;
 import voldemort.store.routed.Pipeline;
-import voldemort.store.routed.Response;
 import voldemort.store.routed.Pipeline.Event;
 import voldemort.store.routed.Pipeline.Operation;
+import voldemort.store.routed.Response;
 import voldemort.utils.ByteArray;
 import voldemort.utils.Utils;
 

diff --git a/src/java/voldemort/store/socket/SocketStore.java b/src/java/voldemort/store/socket/SocketStore.java
@@ -219,7 +219,7 @@ public String getName() {
     }
 
     public void close() throws VoldemortException {
-    // don't close the socket pool, it is shared
+        // don't close the socket pool, it is shared
     }
 
     /**
@@ -328,29 +328,25 @@ public NonblockingStoreCallbackClientRequest(ClientRequest<T> clientRequest,
             this.startNs = System.nanoTime();
         }
 
+        private void invokeCallback(Object o, long requestTime) {
+            if(callback != null) {
+                try {
+                    callback.requestComplete(o, requestTime);
+                } catch(Exception e) {
+                    if(logger.isEnabledFor(Level.WARN))
+                        logger.warn(e, e);
+                }
+            }
+        }
+
         public void complete() {
             try {
                 clientRequest.complete();
                 Object result = clientRequest.getResult();
 
-                if(callback != null) {
-                    try {
-                        callback.requestComplete(result, (System.nanoTime() - startNs)
-                                                         / Time.NS_PER_MS);
-                    } catch(Exception e) {
-                        if(logger.isEnabledFor(Level.WARN))
-                            logger.warn(e, e);
-                    }
-                }
+                invokeCallback(result, (System.nanoTime() - startNs) / Time.NS_PER_MS);
             } catch(Exception e) {
-                if(callback != null) {
-                    try {
-                        callback.requestComplete(e, (System.nanoTime() - startNs) / Time.NS_PER_MS);
-                    } catch(Exception ex) {
-                        if(logger.isEnabledFor(Level.WARN))
-                            logger.warn(ex, ex);
-                    }
-                }
+                invokeCallback(e, (System.nanoTime() - startNs) / Time.NS_PER_MS);
             } finally {
                 pool.checkin(destination, clientRequestExecutor);
                 isComplete = true;
@@ -377,6 +373,16 @@ public void parseResponse(DataInputStream inputStream) {
             clientRequest.parseResponse(inputStream);
         }
 
+        public void timeOut() {
+            clientRequest.timeOut();
+            invokeCallback(new UnreachableStoreException("ClientRequestExecutor timed out. Cannot complete request."),
+                           (System.nanoTime() - startNs) / Time.NS_PER_MS);
+            pool.checkin(destination, clientRequestExecutor);
+        }
+
+        public boolean isTimedOut() {
+            return clientRequest.isTimedOut();
+        }
     }
 
 }
diff --git a/src/java/voldemort/store/socket/clientrequest/AbstractClientRequest.java b/src/java/voldemort/store/socket/clientrequest/AbstractClientRequest.java
@@ -40,6 +40,8 @@ public abstract class AbstractClientRequest<T> implements ClientRequest<T> {
 
     private volatile boolean isParsed = false;
 
+    private volatile boolean isTimedOut = false;
+
     protected abstract void formatRequestInternal(DataOutputStream outputStream) throws IOException;
 
     protected abstract T parseResponseInternal(DataInputStream inputStream) throws IOException;
@@ -93,4 +95,12 @@ public boolean isComplete() {
         return isComplete;
     }
 
+    public final void timeOut() {
+        isTimedOut = true;
+    }
+
+    public boolean isTimedOut() {
+        return isTimedOut;
+    }
+
 }
diff --git a/src/java/voldemort/store/socket/clientrequest/BlockingClientRequest.java b/src/java/voldemort/store/socket/clientrequest/BlockingClientRequest.java
@@ -77,4 +77,12 @@ public boolean formatRequest(DataOutputStream outputStream) {
         return delegate.formatRequest(outputStream);
     }
 
+    public void timeOut() {
+        delegate.timeOut();
+    }
+
+    public boolean isTimedOut() {
+        return delegate.isTimedOut();
+    }
+
 }
diff --git a/src/java/voldemort/store/socket/clientrequest/ClientRequest.java b/src/java/voldemort/store/socket/clientrequest/ClientRequest.java
@@ -120,4 +120,23 @@ public interface ClientRequest<T> {
 
     public boolean isComplete();
 
+    /**
+     * Called by the {@link ClientRequestExecutor} after it has timed out. This
+     * is different from the complete call, since the timeout event needs to be
+     * notified to the caller in a special way.
+     * <p/>
+     * 
+     * This is used internally by the {@link ClientRequest} logic and should not
+     * be invoked by users of the sub-system.
+     */
+
+    public void timeOut();
+
+    /**
+     * Returns <code>true</code> if {@link ClientRequestExecutor} timed out.
+     * 
+     * @return <code>true</code> if timed out, <code>false</code> otherwise
+     */
+
+    public boolean isTimedOut();
 }