MONDRIAN: Fix bug 1533677, "LastNonEmpty requires too many iterations".

[git-p4: depot-paths = "//open/mondrian/": change = 7320]

src/main/mondrian/udf/LastNonEmptyUdf.java

@@ -60,14 +60,35 @@ public Object execute(Evaluator evaluator, Argument[] arguments) {
         final Argument memberListExp = arguments[0];
         final List memberList = (List) memberListExp.evaluate(evaluator);
         final Argument exp = arguments[1];
+        int nullCount = 0;
+        int missCount = 0;
         for (int i = memberList.size() - 1; i >= 0; --i) {
             Member member = (Member) memberList.get(i);
             // Create an evaluator with the member as its context.
             Evaluator subEvaluator = evaluator.push(member);
+            int missCountBefore = subEvaluator.getMissCount();
             final Object o = exp.evaluateScalar(subEvaluator);
+            int missCountAfter = subEvaluator.getMissCount();
             if (Util.isNull(o)) {
+                ++nullCount;
                 continue;
             }
+            if (missCountAfter > missCountBefore) {
+                // There was a cache miss while evaluating the expression, so
+                // the result is bogus. It would be a mistake to give up after
+                // one cache miss, because then it would take us N
+                // evaluate/fetch passes to move back through N members, which
+                // is way too many.
+                //
+                // Carry on until we have seen as many misses as we have seen
+                // null cells. The effect of this policy is that each pass
+                // examines twice as many cells as the previous pass. Thus
+                // we can move back through N members in log2(N) passes.
+                ++missCount;
+                if (missCount < nullCount) {
+                    continue;
+                }
+            }
             if (o instanceof RuntimeException) {
                 RuntimeException runtimeException = (RuntimeException) o;
                 if (o == RolapUtil.valueNotReadyException) {

testsrc/main/mondrian/test/UdfTest.java

@@ -158,6 +158,33 @@ public void testLastNonEmpty() {
                     ""}));
     }
 
+    /**
+     * Tests a performance issue with LastNonEmpty (bug 1533677). The naive
+     * implementation of LastNonEmpty crawls backward one period at a time,
+     * generates a cache miss, and the next iteration reads precisely one cell.
+     * So the query soon exceeds the {@link MondrianProperties#MaxEvalDepth}
+     * property.
+     */
+    public void testLastNonEmptyBig() {
+        assertQueryReturns(
+            "with\n" +
+            "     member\n" +
+            "     [Measures].[Last Sale] as ([Measures].[Unit Sales],\n" +
+            "         LastNonEmpty(Descendants([Time].CurrentMember, [Time].[Month]),\n" +
+            "         [Measures].[Unit Sales]))\n" +
+            "select\n" +
+            "     NON EMPTY {[Measures].[Last Sale]} ON columns,\n" +
+            "     NON EMPTY Order([Store].[All Stores].Children,\n" +
+            "         [Measures].[Last Sale], DESC) ON rows\n" +
+            "from [Sales]\n" +
+            "where [Time].LastSibling",
+            fold(
+                "Axis #0:\n" +
+                "{[Time].[1998]}\n" +
+                "Axis #1:\n" +
+                "Axis #2:\n"));
+    }
+
     public void testBadFun() {
         final TestContext tc = new TestContext() {
                     public synchronized Connection getFoodMartConnection(boolean fresh) {
@@ -224,7 +251,7 @@ public void testException() {
                 Pattern.compile(".*Invalid value for inverse normal distribution: 1.4435.*"),
                 cell.getValue().toString());
     }
-    
+
     public void testCurrentDateString()
     {
         String actual = executeExpr("CurrentDateString(\"Ddd mmm dd yyyy\")");
@@ -251,7 +278,7 @@ public void testCurrentDateMemberBefore()
                 ""
             }));
     }
-    
+
     public void testCurrentDateMemberAfter()
     {
         // CurrentDateMember will return null since the latest date in
@@ -267,7 +294,7 @@ public void testCurrentDateMemberAfter()
                 ""
             }));
     }
-    
+
     public void testCurrentDateMemberExact()
     {
         // CurrentDateMember will return null since the latest date in
@@ -283,7 +310,7 @@ public void testCurrentDateMemberExact()
                 ""
             }));
     }
-    
+
     public void testCurrentDateMemberNoFindArg()
     {
         // CurrentDateMember will return null since the latest date in
@@ -299,7 +326,7 @@ public void testCurrentDateMemberNoFindArg()
                 ""
             }));
     }
-    
+
     public void testCurrentDateMemberHierarchy()
     {
         assertQueryReturns(
@@ -315,7 +342,7 @@ public void testCurrentDateMemberHierarchy()
                 ""
             }));
     }
-    
+
     public void testCurrentDateMemberRealAfter()
     {
         // omit formatting characters from the format so the current date
@@ -334,7 +361,7 @@ public void testCurrentDateMemberRealAfter()
                 ""
             }));
     }
-    
+
     public void testCurrentDateMemberRealExact1()
     {
         // omit formatting characters from the format so the current date
@@ -353,7 +380,7 @@ public void testCurrentDateMemberRealExact1()
                 ""
             }));
     }
-    
+
     public void testCurrentDateMemberRealExact2()
     {
         // omit formatting characters from the format so the current date
@@ -372,7 +399,7 @@ public void testCurrentDateMemberRealExact2()
                 ""
             }));
     }
-    
+
     public void testCurrentDateMemberPrev()
     {
         // apply a function on the result of the UDF
@@ -389,7 +416,7 @@ public void testCurrentDateMemberPrev()
                 ""
             }));
     }
-    
+
     /**
      * Dynamic schema which adds two user-defined functions to any given
      * schema.