MONDRIAN: [MONDRIAN-944] Adds the first version of native support for…

… Filter matching operations. It still needs to support aggregate tables.

It can't use native matching natively for any arguments of filter. Currently, only the following syntax gets transformed into a native filter match:

    Filter([Dimension].[Level].Members, [Dimension].CurrentMember.Caption Matches ( [ regexp here ]  )
    Filter([Dimension].[Level].Members, [Dimension].CurrentMember.Name Matches ( [ regexp here ]  )

As of now, it passes the regexp directly from MDX to SQL. There is no conversion made since there is no standard way to convert regular expressions from Java to SQL. Next step is to implement a conversion helper inside of RolapNativeSql.MatchingSqlCompiler. Maybe this needs to be in JdbcDialectImpl.

Only the Oracle dialect currently supports regular expression matching for native filters. We will add new implementations for the different dialects as we go along and do the research on each of those.

[git-p4: depot-paths = "//open/mondrian/": change = 14305]

src/main/mondrian/olap/MondrianProperties.xml

@@ -672,6 +672,21 @@ lists of members.
         <Type>boolean</Type>
         <Default>false</Default>
     </PropertyDefinition>
+    <PropertyDefinition>
+        <Name>EnableNativeRegexpFilter</Name>
+        <Path>mondrian.native.EnableNativeRegexpFilter</Path>
+        <Category>SQL generation</Category>
+        <Description>
+<p>If this property is true, when evaluating a filter which using a
+regular expression, Mondrian will try to nativize the operation.
+It must be implemented in the database dialect used, and only a
+subset of Filter expressions are supported.</p>
+<p>An example of a supported filter expression is:</p>
+<p>Filter([Dimension].[Level].Members, [Dimension].CurrentMember.Caption Matches ("(^i).*Foo.*")</p>
+        </Description>
+        <Type>boolean</Type>
+        <Default>false</Default>
+    </PropertyDefinition>
     <PropertyDefinition>
         <Name>CompareSiblingsByOrderKey</Name>
         <Path>mondrian.rolap.compareSiblingsByOrderKey</Path>

src/main/mondrian/rolap/RolapNativeFilter.java

@@ -16,7 +16,10 @@
 import javax.sql.DataSource;
 
 import mondrian.olap.*;
+import mondrian.olap.type.SetType;
+import mondrian.olap.type.StringType;
 import mondrian.rolap.aggmatcher.AggStar;
+import mondrian.rolap.sql.DescendantsCrossJoinArg;
 import mondrian.rolap.sql.SqlQuery;
 import mondrian.rolap.sql.TupleConstraint;
 import mondrian.rolap.sql.CrossJoinArg;
@@ -63,7 +66,9 @@ public void addConstraint(
             AggStar aggStar)
         {
             // Use aggregate table to generate filter condition
-            RolapNativeSql sql = new RolapNativeSql(sqlQuery, aggStar);
+            RolapNativeSql sql =
+                new RolapNativeSql(
+                    sqlQuery, aggStar, getEvaluator(), args[0].getLevel());
             String filterSql =  sql.generateFilterCondition(filterExpr);
             sqlQuery.addHaving(filterSql);
             super.addConstraint(sqlQuery, baseCube, aggStar);
@@ -132,9 +137,11 @@ evaluator, restrictMemberTypes()))
         // generate the WHERE condition
         // Need to generate where condition here to determine whether
         // or not the filter condition can be created. The filter
-        // condition could change to use an aggregate table later in evaulation
+        // condition could change to use an aggregate table later in evaluation
         SqlQuery sqlQuery = SqlQuery.newQuery(ds, "NativeFilter");
-        RolapNativeSql sql = new RolapNativeSql(sqlQuery, null);
+        RolapNativeSql sql =
+            new RolapNativeSql(
+                sqlQuery, null, evaluator, cjArgs[0].getLevel());
         final Exp filterExpr = args[1];
         String filterExprStr = sql.generateFilterCondition(filterExpr);
         if (filterExprStr == null) {

src/main/mondrian/rolap/RolapNativeSql.java

@@ -13,9 +13,13 @@
 import java.util.List;
 
 import mondrian.olap.*;
+import mondrian.olap.type.MemberType;
+import mondrian.olap.type.StringType;
 import mondrian.rolap.aggmatcher.AggStar;
 import mondrian.rolap.sql.SqlQuery;
+import mondrian.mdx.DimensionExpr;
 import mondrian.mdx.MemberExpr;
+import mondrian.mdx.ResolvedFunCall;
 import mondrian.spi.Dialect;
 
 /**
@@ -35,7 +39,9 @@ public class RolapNativeSql {
     CompositeSqlCompiler booleanCompiler;
 
     RolapStoredMeasure storedMeasure;
-    AggStar aggStar;
+    final AggStar aggStar;
+    final Evaluator evaluator;
+    final RolapLevel rolapLevel;
 
     /**
      * We remember one of the measures so we can generate
@@ -187,6 +193,111 @@ public String toString() {
         }
     }
 
+    /**
+     * Compiles a MATCHES MDX operator into SQL regular
+     * expression match.
+     */
+    class MatchingSqlCompiler extends FunCallSqlCompilerBase {
+
+        protected MatchingSqlCompiler()
+        {
+            super(Category.Logical, "MATCHES", 2);
+        }
+
+        @Override
+        public String compile(Exp exp) {
+            if (!match(exp)) {
+                return null;
+            }
+            if (!MondrianProperties.instance()
+                .EnableNativeRegexpFilter.get())
+            {
+                return null;
+            }
+            if (!dialect.allowsRegularExpressionInWhereClause()
+                || !(exp instanceof ResolvedFunCall)
+                || evaluator == null)
+            {
+                return null;
+            }
+
+            final Exp arg0 = ((ResolvedFunCall)exp).getArg(0);
+            final Exp arg1 = ((ResolvedFunCall)exp).getArg(1);
+
+            // Must finish by ".Caption" or ".Name"
+            if (!(arg0 instanceof ResolvedFunCall)
+                || ((ResolvedFunCall)arg0).getArgCount() != 1
+                || !(arg0.getType() instanceof StringType)
+                || (!((ResolvedFunCall)arg0).getFunName().equals("Name")
+                    && !((ResolvedFunCall)arg0)
+                            .getFunName().equals("Caption")))
+            {
+                return null;
+            }
+
+            final boolean useCaption;
+            if (((ResolvedFunCall)arg0).getFunName().equals("Name")) {
+                useCaption = false;
+            } else {
+                useCaption = true;
+            }
+
+            // Must be ".CurrentMember"
+            final Exp currMemberExpr = ((ResolvedFunCall)arg0).getArg(0);
+            if (!(currMemberExpr instanceof ResolvedFunCall)
+                || ((ResolvedFunCall)currMemberExpr).getArgCount() != 1
+                || !(currMemberExpr.getType() instanceof MemberType)
+                || !((ResolvedFunCall)currMemberExpr)
+                        .getFunName().equals("CurrentMember"))
+            {
+                return null;
+            }
+
+            // Must be a dimension
+            RolapCubeDimension dimension;
+            final Exp dimExpr = ((ResolvedFunCall)currMemberExpr).getArg(0);
+            if (!(dimExpr instanceof DimensionExpr)) {
+                return null;
+            } else {
+                dimension =
+                    (RolapCubeDimension) evaluator.getCachedResult(
+                        new ExpCacheDescriptor(dimExpr, evaluator));
+            }
+
+            // TODO Add support for aggregate tables when using
+            // matching native filter.
+
+            if (rolapLevel != null
+                && dimension.equals(rolapLevel.getDimension()))
+            {
+                // We can't use the evaluator because the filter is filtering
+                // a set which is uses same dimension as the predicate.
+                // We must use, in order of priority,
+                //  caption requested: caption->name->key
+                //  name requested: name->key
+                return
+                dialect.generateRegularExpression(
+                    useCaption
+                        ? rolapLevel.captionExp == null
+                            ? rolapLevel.nameExp == null
+                                ? rolapLevel.keyExp.getExpression(sqlQuery)
+                                : rolapLevel.nameExp.getExpression(sqlQuery)
+                            : rolapLevel.captionExp.getExpression(sqlQuery)
+                        : rolapLevel.nameExp == null
+                            ? rolapLevel.keyExp.getExpression(sqlQuery)
+                            : rolapLevel.nameExp.getExpression(sqlQuery),
+                    String.valueOf(
+                        evaluator.getCachedResult(
+                            new ExpCacheDescriptor(arg1, evaluator))));
+            } else {
+                return null;
+            }
+        }
+        public String toString() {
+            return "MatchingSqlCompiler";
+        }
+    }
+
     /**
      * Compiles the underlying expression of a calculated member.
      */
@@ -435,11 +546,18 @@ public String compile(Exp exp) {
     /**
      * Creates a RolapNativeSql.
      *
-     * @param sqlQuery the query which is needed for different SQL dialects - it
-     * is not modified
+     * @param sqlQuery the query which is needed for different SQL dialects -
+     * it is not modified
      */
-    RolapNativeSql(SqlQuery sqlQuery, AggStar aggStar) {
+    public RolapNativeSql(
+        SqlQuery sqlQuery,
+        AggStar aggStar,
+        Evaluator evaluator,
+        RolapLevel rolapLevel)
+    {
         this.sqlQuery = sqlQuery;
+        this.rolapLevel = rolapLevel;
+        this.evaluator = evaluator;
         this.dialect = sqlQuery.getDialect();
         this.aggStar = aggStar;
 
@@ -497,6 +615,8 @@ public String compile(Exp exp) {
         booleanCompiler.add(
             new UnaryOpSqlCompiler(
                 Category.Logical, "not", "NOT", booleanCompiler));
+        booleanCompiler.add(
+            new MatchingSqlCompiler());
         booleanCompiler.add(
             new ParenthesisSqlCompiler(Category.Logical, booleanCompiler));
         booleanCompiler.add(

src/main/mondrian/rolap/RolapNativeTopCount.java

@@ -68,7 +68,9 @@ public void addConstraint(
             AggStar aggStar)
         {
             if (orderByExpr != null) {
-                RolapNativeSql sql = new RolapNativeSql(sqlQuery, aggStar);
+                RolapNativeSql sql =
+                    new RolapNativeSql(
+                        sqlQuery, aggStar, getEvaluator(), null);
                 String orderBySql = sql.generateTopCountOrderBy(orderByExpr);
                 Dialect dialect = sqlQuery.getDialect();
                 boolean nullable = deduceNullability(orderByExpr);
@@ -175,7 +177,9 @@ evaluator, restrictMemberTypes()))
         // or not it can be created. The top count
         // could change to use an aggregate table later in evaulation
         SqlQuery sqlQuery = SqlQuery.newQuery(ds, "NativeTopCount");
-        RolapNativeSql sql = new RolapNativeSql(sqlQuery, null);
+        RolapNativeSql sql =
+            new RolapNativeSql(
+                sqlQuery, null, evaluator, null);
         Exp orderByExpr = null;
         if (args.length == 3) {
             orderByExpr = args[2];

src/main/mondrian/spi/Dialect.java

@@ -705,6 +705,10 @@ void appendHintsAfterFromClause(
      */
     boolean allowsJoinOn();
 
+    boolean allowsRegularExpressionInWhereClause();
+
+    String generateRegularExpression(String source, String javaRegExp);
+
     /**
      * Enumeration of common database types.
      *

src/main/mondrian/spi/impl/JdbcDialectImpl.java

@@ -840,6 +840,17 @@ public int getMaxColumnNameLength() {
         return maxColumnNameLength;
     }
 
+    public boolean allowsRegularExpressionInWhereClause() {
+        return false;
+    }
+
+    public String generateRegularExpression(
+        String source,
+        String javaRegExp)
+    {
+        throw new UnsupportedOperationException();
+    }
+
     /**
      * Converts a product name and version (per the JDBC driver) into a product
      * enumeration.

src/main/mondrian/spi/impl/OracleDialect.java

@@ -62,6 +62,19 @@ public String generateOrderByNullsLast(String expr, boolean ascending) {
     public boolean allowsJoinOn() {
         return false;
     }
+
+    @Override
+    public boolean allowsRegularExpressionInWhereClause() {
+        return true;
+    }
+
+    @Override
+    public String generateRegularExpression(
+            String source,
+            String javaRegExp)
+    {
+        return "REGEXP_LIKE(" + source + ", '" + javaRegExp + "')";
+    }
 }
 
 // End OracleDialect.java

testsrc/main/mondrian/rolap/NativeFilterMatchingTest.java

@@ -0,0 +1,74 @@
+/*
+// $Id$
+// This software is subject to the terms of the Eclipse Public License v1.0
+// Agreement, available at the following URL:
+// http://www.eclipse.org/legal/epl-v10.html.
+// Copyright (C) 2011 Julian Hyde
+// All Rights Reserved.
+// You must accept the terms of that agreement to use this software.
+*/
+package mondrian.rolap;
+
+import mondrian.olap.MondrianProperties;
+import mondrian.spi.Dialect;
+import mondrian.test.SqlPattern;
+
+public class NativeFilterMatchingTest extends BatchTestCase {
+    public void testBugMondrian944() throws Exception {
+        propSaver.set(
+            MondrianProperties.instance().EnableNativeRegexpFilter,
+            true);
+        final String sqlOracle =
+            "select \"customer\".\"country\" as \"c0\", \"customer\".\"state_province\" as \"c1\", \"customer\".\"city\" as \"c2\", \"customer\".\"customer_id\" as \"c3\", \"fname\" || ' ' || \"lname\" as \"c4\", \"fname\" || ' ' || \"lname\" as \"c5\", \"customer\".\"gender\" as \"c6\", \"customer\".\"marital_status\" as \"c7\", \"customer\".\"education\" as \"c8\", \"customer\".\"yearly_income\" as \"c9\" from \"customer\" \"customer\", \"sales_fact_1997\" \"sales_fact_1997\", \"time_by_day\" \"time_by_day\" where \"sales_fact_1997\".\"customer_id\" = \"customer\".\"customer_id\" and \"sales_fact_1997\".\"time_id\" = \"time_by_day\".\"time_id\" and \"time_by_day\".\"the_year\" = 1997 group by \"customer\".\"country\", \"customer\".\"state_province\", \"customer\".\"city\", \"customer\".\"customer_id\", \"fname\" || ' ' || \"lname\", \"customer\".\"gender\", \"customer\".\"marital_status\", \"customer\".\"education\", \"customer\".\"yearly_income\" having REGEXP_LIKE(\"fname\" || ' ' || \"lname\", '.*Jeanne.*') order by \"customer\".\"country\" ASC, \"customer\".\"state_province\" ASC, \"customer\".\"city\" ASC, \"fname\" || ' ' || \"lname\" ASC";
+        SqlPattern[] patterns = {
+                new SqlPattern(
+                    Dialect.DatabaseProduct.ORACLE,
+                    sqlOracle,
+                    sqlOracle.length())
+            };
+        final String query =
+            "With\n"
+            + "Set [*NATIVE_CJ_SET] as 'Filter([*BASE_MEMBERS_Customers], Not IsEmpty ([Measures].[Unit Sales]))'\n"
+            + "Set [*SORTED_COL_AXIS] as 'Order([*CJ_COL_AXIS],[Customers].CurrentMember.OrderKey,BASC,Ancestor([Customers].CurrentMember,[Customers].[City]).OrderKey,BASC)'\n"
+            + "Set [*BASE_MEMBERS_Customers] as 'Filter([Customers].[Name].Members,[Customers].CurrentMember.Caption Matches (\".*Jeanne.*\"))'\n"
+            + "Set [*BASE_MEMBERS_Measures] as '{[Measures].[*FORMATTED_MEASURE_0]}'\n"
+            + "Set [*CJ_COL_AXIS] as 'Generate([*NATIVE_CJ_SET], {([Customers].currentMember)})'\n"
+            + "Member [Measures].[*FORMATTED_MEASURE_0] as '[Measures].[Unit Sales]', FORMAT_STRING = 'Standard', SOLVE_ORDER=400\n"
+            + "Select\n"
+            + "CrossJoin([*SORTED_COL_AXIS],[*BASE_MEMBERS_Measures]) on columns\n"
+            + "From [Sales]";
+        assertQuerySqlOrNot(
+            getTestContext(),
+            query,
+            patterns,
+            false,
+            true,
+            true);
+        assertQueryReturns(
+            query,
+            "Axis #0:\n"
+            + "{}\n"
+            + "Axis #1:\n"
+            + "{[Customers].[USA].[WA].[Issaquah].[Jeanne Derry], [Measures].[*FORMATTED_MEASURE_0]}\n"
+            + "{[Customers].[USA].[CA].[Los Angeles].[Jeannette Eldridge], [Measures].[*FORMATTED_MEASURE_0]}\n"
+            + "{[Customers].[USA].[CA].[Burbank].[Jeanne Bohrnstedt], [Measures].[*FORMATTED_MEASURE_0]}\n"
+            + "{[Customers].[USA].[OR].[Portland].[Jeanne Zysko], [Measures].[*FORMATTED_MEASURE_0]}\n"
+            + "{[Customers].[USA].[WA].[Everett].[Jeanne McDill], [Measures].[*FORMATTED_MEASURE_0]}\n"
+            + "{[Customers].[USA].[CA].[West Covina].[Jeanne Whitaker], [Measures].[*FORMATTED_MEASURE_0]}\n"
+            + "{[Customers].[USA].[WA].[Everett].[Jeanne Turner], [Measures].[*FORMATTED_MEASURE_0]}\n"
+            + "{[Customers].[USA].[WA].[Puyallup].[Jeanne Wentz], [Measures].[*FORMATTED_MEASURE_0]}\n"
+            + "{[Customers].[USA].[OR].[Albany].[Jeannette Bura], [Measures].[*FORMATTED_MEASURE_0]}\n"
+            + "{[Customers].[USA].[WA].[Lynnwood].[Jeanne Ibarra], [Measures].[*FORMATTED_MEASURE_0]}\n"
+            + "Row #0: 50\n"
+            + "Row #0: 21\n"
+            + "Row #0: 31\n"
+            + "Row #0: 42\n"
+            + "Row #0: 110\n"
+            + "Row #0: 59\n"
+            + "Row #0: 42\n"
+            + "Row #0: 157\n"
+            + "Row #0: 146\n"
+            + "Row #0: 78\n");
+    }
+}
+// End NativeFilterMatchingTest.java

testsrc/main/mondrian/test/Main.java

@@ -208,6 +208,7 @@ public static Test suite() throws Exception {
                 addTest(suite, Base64Test.class);
                 return suite;
             }
+            addTest(suite, NativeFilterMatchingTest.class);
             addTest(suite, RolapConnectionTest.class);
             addTest(suite, FilteredIterableTest.class);
             addTest(suite, HighDimensionsTest.class);