MONDRIAN-PACINO: Make external segment cache faster, safer, more scal…

…able by adopting the actor model more strictly. Requests to query/update the segment index are handled by an actor (dedicated thread). But requests to execute SQL or to query an external cache -- which can take a long time -- cannot be executed from within that thread. They are either farmed out to a worker (and a future returned to the caller if necessary), or thrown back to the calling thread to execute. SegmentConverter exists as a moderately-acceptable hack to deal with the fact that a lot of context from the RolapStar or CellRequest is required to convert a SegmentHeader to a Segment. See my notes in SegmentCacheManager for what should be done about this in the longer term. Implement PartiallySortedSet.remove, and fix a couple of bugs. Make the monitor aware of the external cache. Add validation-api.jar (package javax.validation.constraints). We can start using annotations to declare that parameters and results are never-null, etc. [git-p4: depot-paths = "//open/mondrian-release/pacino/": change = 14868]
pentaho · Jan 3, 2012 · e37e448 · e37e448
1 parent baee022
commit e37e448
Show file tree

Hide file tree

Showing 57 changed files with 3,805 additions and 2,165 deletions.
diff --git a/build.xml b/build.xml
@@ -4,7 +4,7 @@
   == Agreement, available at the following URL:
   == http://www.eclipse.org/legal/epl-v10.html.
   == Copyright (C) 2001-2003 Kana Software, Inc.
-  == Copyright (C) 2001-2011 Julian Hyde and others
+  == Copyright (C) 2001-2012 Julian Hyde and others
   == All Rights Reserved.
   == You must accept the terms of that agreement to use this software.
   -->
@@ -169,6 +169,7 @@ demo/access/MondrianFoodMart.mdb"/>
     <pathelement location="${lib.dir}/olap4j.jar"/>
     <pathelement location="${lib.dir}/xalan.jar"/>
     <pathelement location="${lib.dir}/xercesImpl.jar"/>
+    <pathelement location="${lib.dir}/validation-api.jar"/>
     <pathelement path="${env.CLASSPATH}"/>
     <pathelement location="${ant.home}/lib/ant.jar"/>
     <pathelement location="${ant.home}/lib/optional.jar"/>

diff --git a/ivy.xml b/ivy.xml
@@ -50,6 +50,7 @@
         <dependency org="javacup" name="javacup" rev="10k"/>
         <dependency org="net.java.dev.javacc" name="javacc" rev="5.0"/>
         <dependency org="dom4j" name="dom4j" rev="1.6.1"/>
+        <dependency org="javax.validation" name="validation-api" rev="1.0.0.GA"/>
         <dependency org="eigenbase" name="eigenbase-xom" rev="1.3.0.13768"/>
         <dependency org="eigenbase" name="eigenbase-properties" rev="1.1.0.10924"/>
         <dependency org="eigenbase" name="eigenbase-resgen" rev="1.3.0.13768"/>

diff --git a/src/main/mondrian/calc/impl/DelegatingTupleList.java b/src/main/mondrian/calc/impl/DelegatingTupleList.java
@@ -3,7 +3,7 @@
 // This software is subject to the terms of the Eclipse Public License v1.0
 // Agreement, available at the following URL:
 // http://www.eclipse.org/legal/epl-v10.html.
-// Copyright (C) 2011-2011 Julian Hyde
+// Copyright (C) 2011-2012 Julian Hyde
 // All Rights Reserved.
 // You must accept the terms of that agreement to use this software.
 */
@@ -80,9 +80,8 @@ public Member set(int index, Member element) {
                     // The sub list is probably a singleton list.
                     // calling set() on it will fail. We have to
                     // create a new singleton list.
-                    return 
-                        list.set(index, Collections.singletonList(element))
-                            .get(0);
+                    return list.set(index, Collections.singletonList(element))
+                        .get(0);
                 }
                 return subList.set(column, element);
             };

diff --git a/src/main/mondrian/olap/Aggregator.java b/src/main/mondrian/olap/Aggregator.java
@@ -3,7 +3,7 @@
 // This software is subject to the terms of the Eclipse Public License v1.0
 // Agreement, available at the following URL:
 // http://www.eclipse.org/legal/epl-v10.html.
-// Copyright (C) 2003-2011 Julian Hyde
+// Copyright (C) 2003-2012 Julian Hyde
 // All Rights Reserved.
 // You must accept the terms of that agreement to use this software.
 */
@@ -29,6 +29,8 @@
 public interface Aggregator {
     /**
      * Returns the aggregator used to combine sub-totals into a grand-total.
+     *
+     * @return aggregator used to combine sub-totals into a grand-total
      */
     Aggregator getRollup();
 
@@ -39,19 +41,21 @@ public interface Aggregator {
      * @param evaluator Evaluation context
      * @param members List of members, not null
      * @param calc Expression to evaluate
+     *
+     * @return result of applying this aggregator to a set of members/tuples
      */
     Object aggregate(Evaluator evaluator, TupleList members, Calc calc);
 
     /**
      * Tells Mondrian if this aggregator can perform fast aggregation
      * using only the raw data of a given object type. This will
      * determine if Mondrian will attempt to perform in-memory rollups
-     * on raw segment data by invoking {@link Aggregator#aggregate(Object[])}.
+     * on raw segment data by invoking {@link #aggregate(java.util.List)}.
      *
      * <p>This is only invoked for rollup operations.
      *
      * @param datatype The datatype of the object we would like to rollup.
-     * @return True or false, depending on the support status.
+     * @return Whether this aggregator supports fast aggregation
      */
     boolean supportsFastAggregates(Datatype datatype);
 
@@ -60,8 +64,8 @@ public interface Aggregator {
      * operation. This is useful when the values are already resolved
      * and we are dealing with a raw {@link SegmentBody} object.
      *
-     * <p>Only gets called if {@link Aggregator#supportsFastAggregates()}
-     * is true.
+     * <p>Only gets called if
+     * {@link #supportsFastAggregates(mondrian.spi.Dialect.Datatype)} is true.
      *
      * <p>This is only invoked for rollup operations.
      *

diff --git a/src/main/mondrian/olap/Connection.java b/src/main/mondrian/olap/Connection.java
@@ -4,7 +4,7 @@
 // Agreement, available at the following URL:
 // http://www.eclipse.org/legal/epl-v10.html.
 // Copyright (C) 2000-2002 Kana Software, Inc.
-// Copyright (C) 2001-2011 Julian Hyde and others
+// Copyright (C) 2001-2012 Julian Hyde and others
 // All Rights Reserved.
 // You must accept the terms of that agreement to use this software.
 //
@@ -16,6 +16,8 @@
 import java.util.Locale;
 import javax.sql.DataSource;
 
+import javax.validation.constraints.*;
+
 /**
  * Connection to a multi-dimensional database.
  *
@@ -126,7 +128,7 @@ public interface Connection {
      *
      * @param pw Writer to which to write logging information; may be null
      */
-    CacheControl getCacheControl(PrintWriter pw);
+    CacheControl getCacheControl(@Null PrintWriter pw);
 
     /**
      * Returns the data source this connection uses to create connections

diff --git a/src/main/mondrian/olap/MondrianProperties.xml b/src/main/mondrian/olap/MondrianProperties.xml
@@ -4,7 +4,7 @@
   == Agreement, available at the following URL:
   == http://www.eclipse.org/legal/epl-v10.html.
   == You must accept the terms of that agreement to use this software.
-  == Copyright (C) 2011-2011 Julian Hyde
+  == Copyright (C) 2011-2012 Julian Hyde
   == All Rights Reserved.
   ==
   == Definitions of configuration properties used by Mondrian. From this file,
@@ -304,66 +304,6 @@ is an implementation of {@link mondrian.spi.SegmentCache}.
         </Description>
         <Type>String</Type>
     </PropertyDefinition>
-    <PropertyDefinition>
-        <Name>SegmentCacheReadTimeout</Name>
-        <Path>mondrian.rolap.SegmentCacheReadTimeout</Path>
-        <Description>
-<p>Property which defines the timeout for
-{@link mondrian.spi.SegmentCache#get(mondrian.spi.SegmentHeader)}
-in milliseconds. Defaults to 5000.</p>
-
-<p>This is an internal control property. The timeout value
-won't be passed to the underlying
-{@link mondrian.spi.SegmentCache} SPI.</p>
-        </Description>
-        <Type>int</Type>
-        <Default>5000</Default>
-    </PropertyDefinition>
-    <PropertyDefinition>
-        <Name>SegmentCacheWriteTimeout</Name>
-        <Path>mondrian.rolap.SegmentCacheWriteTimeout</Path>
-        <Description>
-<p>Property which defines the timeout for
-{@link mondrian.spi.SegmentCache#put(mondrian.spi.SegmentHeader, mondrian.spi.SegmentBody)}
-in milliseconds. Defaults to 5000.</p>
-
-<p>This is an internal control property. The timeout value
-won't be passed to the underlying
-{@link mondrian.spi.SegmentCache} SPI.</p>
-        </Description>
-        <Type>int</Type>
-        <Default>5000</Default>
-    </PropertyDefinition>
-    <PropertyDefinition>
-        <Name>SegmentCacheLookupTimeout</Name>
-        <Path>mondrian.rolap.SegmentCacheLookupTimeout</Path>
-        <Description>
-<p>Property which defines the timeout for
-{@link mondrian.spi.SegmentCache#contains(mondrian.spi.SegmentHeader)}
-in milliseconds. Defaults to 5000.</p>
-
-<p>This is an internal control property. The timeout value
-won't be passed to the underlying
-{@link mondrian.spi.SegmentCache} SPI.</p>
-        </Description>
-        <Type>int</Type>
-        <Default>5000</Default>
-    </PropertyDefinition>
-    <PropertyDefinition>
-        <Name>SegmentCacheScanTimeout</Name>
-        <Path>mondrian.rolap.SegmentCacheScanTimeout</Path>
-        <Description>
-<p>Property which defines the timeout for
-{@link mondrian.spi.SegmentCache#getSegmentHeaders()}
-in milliseconds. Defaults to 5000.</p>
-
-<p>This is an internal control property. The timeout value
-won't be passed to the underlying
-{@link mondrian.spi.SegmentCache} SPI.</p>
-        </Description>
-        <Type>int</Type>
-        <Default>5000</Default>
-    </PropertyDefinition>
     <PropertyDefinition>
         <Name>SparseSegmentCountThreshold</Name>
         <Path>mondrian.rolap.SparseSegmentValueThreshold</Path>

diff --git a/src/main/mondrian/resource/MondrianResource.xml b/src/main/mondrian/resource/MondrianResource.xml
@@ -9,7 +9,7 @@
   == Agreement, available at the following URL:
   == http://www.eclipse.org/legal/epl-v10.html.
   == (C) Copyright 2000-2002 Kana Software, Inc.
-  == (C) Copyright 2002-2010 Julian Hyde and others
+  == (C) Copyright 2002-2012 Julian Hyde and others
   == All Rights Reserved.
   == You must accept the terms of that agreement to use this software.
   ==
@@ -832,12 +832,6 @@
     </text>
 </exception>
 
-<exception id="7000653" name="SegmentCacheScanTimeout">
-    <text>
-        Timeout reached while getting a list of segment headers from the SegmentCache.
-    </text>
-</exception>
-
 <!-- ====================================================================== -->
 <!-- Optimizer -->
 <exception id="8000000" name="NativeEvaluationUnsupported"

diff --git a/src/main/mondrian/rolap/CacheControlImpl.java b/src/main/mondrian/rolap/CacheControlImpl.java
@@ -3,7 +3,7 @@
 // This software is subject to the terms of the Eclipse Public License v1.0
 // Agreement, available at the following URL:
 // http://www.eclipse.org/legal/epl-v10.html.
-// Copyright (C) 2006-2011 Julian Hyde and others
+// Copyright (C) 2006-2012 Julian Hyde and others
 // All Rights Reserved.
 // You must accept the terms of that agreement to use this software.
 */
@@ -13,12 +13,10 @@
 import mondrian.olap.Id.Quoting;
 import mondrian.resource.MondrianResource;
 import mondrian.rolap.agg.SegmentCacheManager;
-import mondrian.rolap.cache.SegmentCacheIndexImpl;
 import mondrian.rolap.sql.MemberChildrenConstraint;
 import mondrian.server.Execution;
 import mondrian.server.Locus;
 import mondrian.spi.SegmentColumn;
-import mondrian.spi.SegmentHeader;
 import mondrian.util.ArraySortedSet;
 
 import org.eigenbase.util.property.BooleanProperty;
@@ -175,7 +173,19 @@ public CellRegion createMeasuresRegion(Cube cube) {
         return new MemberCellRegion(measures, false);
     }
 
-    public void flush(CellRegion region) {
+    public void flush(final CellRegion region) {
+        Locus.execute(
+            connection,
+            "Flush",
+            new Locus.Action<Void>() {
+                public Void execute() {
+                    flushInternal(region);
+                    return null;
+                }
+            });
+    }
+
+    private void flushInternal(CellRegion region) {
         if (region instanceof EmptyCellRegion) {
             return;
         }
@@ -519,11 +529,13 @@ public void printCacheState(
         final SegmentCacheManager manager =
             MondrianServer.forConnection(connection)
                 .getAggregationManager().cacheMgr;
-        manager.execute(
-            new SegmentCacheManager.Command<Void>() {
-                public Void call() throws Exception {
-                    manager.segmentIndex.printCacheState(pw);
-                    return null;
+        Locus.execute(
+            Execution.NONE,
+            "CacheControlImpl.printCacheState",
+            new Locus.Action<Object>() {
+                public Object execute() {
+                    return manager.execute(
+                        new PrintCacheStateCommand(manager, pw, Locus.peek()));
                 }
             });
     }
@@ -1519,6 +1531,33 @@ public void commit() {
         }
     }
 
+    private static class PrintCacheStateCommand
+        implements SegmentCacheManager.Command<Void>
+    {
+        private final SegmentCacheManager manager;
+        private final PrintWriter pw;
+        private final Locus locus;
+
+        public PrintCacheStateCommand(
+            SegmentCacheManager manager,
+            PrintWriter pw,
+            Locus locus)
+        {
+            this.manager = manager;
+            this.pw = pw;
+            this.locus = locus;
+        }
+
+        public Void call() {
+            manager.segmentIndex.printCacheState(pw);
+            return null;
+        }
+
+        public Locus getLocus() {
+            return locus;
+        }
+    }
+
     /**
      * Command that deletes a member and its descendants from the cache.
      */

diff --git a/src/main/mondrian/rolap/CellKey.java b/src/main/mondrian/rolap/CellKey.java
@@ -4,7 +4,7 @@
 // Agreement, available at the following URL:
 // http://www.eclipse.org/legal/epl-v10.html.
 // Copyright (C) 2001-2002 Kana Software, Inc.
-// Copyright (C) 2001-2010 Julian Hyde and others
+// Copyright (C) 2001-2012 Julian Hyde and others
 // All Rights Reserved.
 // You must accept the terms of that agreement to use this software.
 //
@@ -25,8 +25,8 @@
  *
  * <p>It is important that CellKey is memory-efficient, and that the
  * {@link Object#hashCode} and {@link Object#equals} methods are extremely
- * efficient. There are particular implementations for the
- * most likely cases where the number of axes is 1, 2, 3 and 4 as well as a general
+ * efficient. There are particular implementations for the most likely cases
+ * where the number of axes is 1, 2, 3 and 4 as well as a general
  * implementation.
  *
  * <p>To create a key, call the
@@ -502,8 +502,10 @@ class Four implements CellKey {
          * Creates a Four.
          */
         private Four(
-                int ordinal0, int ordinal1,
-                int ordinal2, int ordinal3)
+            int ordinal0,
+            int ordinal1,
+            int ordinal2,
+            int ordinal3)
         {
             this.ordinal0 = ordinal0;
             this.ordinal1 = ordinal1;