MONDRIAN: Make drill-through deterministic by adding ORDER BY to query;

	Add Cell.getDrillThroughCount().

[git-p4: depot-paths = "//open/mondrian/": change = 8143]

build.xml

@@ -322,6 +322,7 @@ META-INF/**"/>
     <javacup
         srcdir="${java.dir}"
         input="${java.dir}/mondrian/olap/Parser.cup"
+        expect="61"
         interface="true"/>
   </target>
 

src/main/mondrian/olap/Cell.java

@@ -85,6 +85,11 @@ public interface Cell {
      */
     boolean canDrillThrough();
 
+    /**
+     * Returns the number of fact table rows which contributed to this Cell.
+     */
+    int getDrillThroughCount();
+
     /**
      * Returns the value of a property.
      */

src/main/mondrian/rolap/RolapAggregationManager.java

@@ -185,7 +185,17 @@ public Object get(Evaluator evaluator) {
         return getCell(rolapEvaluator.getCurrentMembers());
     }
 
-    public abstract String getDrillThroughSQL(CellRequest request);
+    /**
+     * Generates a SQL statement which will return the rows which contribute to
+     * this request.
+     *
+     * @param request Cell request
+     * @param countOnly If true, return a statment which returns only the count
+     * @return SQL statement
+     */
+    public abstract String getDrillThroughSql(
+        CellRequest request,
+        boolean countOnly);
 
     public int getMissCount() {
         return 0; // never lies

src/main/mondrian/rolap/RolapCell.java

@@ -13,6 +13,10 @@
 import mondrian.rolap.agg.AggregationManager;
 import mondrian.rolap.agg.CellRequest;
 
+import java.sql.Statement;
+import java.sql.SQLException;
+import java.sql.ResultSet;
+
 /**
  * <code>RolapCell</code> implements {@link mondrian.olap.Cell} within a
  * {@link RolapResult}.
@@ -90,7 +94,51 @@ public String getDrillThroughSQL(boolean extendedContext) {
                 currentMembers, extendedContext, true);
         return (cellRequest == null)
             ? null
-            : aggMan.getDrillThroughSQL(cellRequest);
+            : aggMan.getDrillThroughSql(cellRequest, false);
+    }
+
+
+    public int getDrillThroughCount() {
+        RolapAggregationManager aggMan = AggregationManager.instance();
+
+        final RolapEvaluator evaluator = getEvaluator();
+        final Member[] currentMembers = evaluator.getCurrentMembers();
+        CellRequest cellRequest = RolapAggregationManager.makeRequest(
+            currentMembers, false, true);
+        if (cellRequest == null) {
+            return -1;
+        }
+        RolapConnection connection =
+            (RolapConnection) evaluator.getQuery().getConnection();
+        java.sql.Connection jdbcConnection = null;
+        ResultSet rs = null;
+        final String sql = aggMan.getDrillThroughSql(cellRequest, true);
+        try {
+            jdbcConnection = connection.getDataSource().getConnection();
+            rs = RolapUtil.executeQuery(
+                jdbcConnection, sql, "RolapCell.getDrillThroughCount");
+            rs.next();
+            int count = rs.getInt(1);
+            rs.close();
+            return count;
+        } catch (SQLException e) {
+            throw Util.newError(
+                e,
+                "Error while counting drill-through, SQL ='" + sql + "'");
+        } finally {
+            try {
+                if (rs != null) {
+                    rs.close();
+                }
+            } catch (SQLException ignored) {
+            }
+            try {
+                if (jdbcConnection != null && !jdbcConnection.isClosed()) {
+                    jdbcConnection.close();
+                }
+            } catch (SQLException ignored) {
+            }
+        }
     }
 
     /**

src/main/mondrian/rolap/RolapUtil.java

@@ -243,7 +243,9 @@ public static ResultSet executeQuery(
         }
         try {
             final long start = System.currentTimeMillis();
-            statement = jdbcConnection.createStatement();
+            statement = jdbcConnection.createStatement(
+                ResultSet.TYPE_SCROLL_INSENSITIVE,
+                ResultSet.CONCUR_READ_ONLY);
             if (maxRows > 0) {
                 statement.setMaxRows(maxRows);
             }

src/main/mondrian/rolap/agg/AbstractQuerySpec.java

@@ -13,6 +13,7 @@
 
 import mondrian.rolap.RolapStar;
 import mondrian.rolap.sql.SqlQuery;
+import mondrian.util.Bug;
 
 /**
  * Base class for {@link QuerySpec} implementations.
@@ -47,10 +48,15 @@ public RolapStar getStar() {
     protected abstract void addMeasure(final int i, final SqlQuery sqlQuery);
     protected abstract boolean isAggregate();
 
-    protected void nonDistinctGenerateSQL(final SqlQuery sqlQuery) {
+    protected void nonDistinctGenerateSql(
+        final SqlQuery sqlQuery, boolean ordered, boolean countOnly)
+    {
         // add constraining dimensions
         RolapStar.Column[] columns = getColumns();
         int arity = columns.length;
+        if (countOnly) {
+            sqlQuery.addSelect("count(*)");
+        }
         for (int i = 0; i < arity; i++) {
             RolapStar.Column column = columns[i];
             RolapStar.Table table = column.getTable();
@@ -69,9 +75,14 @@ protected void nonDistinctGenerateSQL(final SqlQuery sqlQuery) {
                                                column.isNumeric()));
             }
 
+            if (countOnly) {
+                continue;
+            }
+
             // some DB2 (AS400) versions throw an error, if a column alias is
             // there and *not* used in a subsequent order by/group by
-            if (sqlQuery.getDialect().isAS400()) {
+            final SqlQuery.Dialect dialect = sqlQuery.getDialect();
+            if (dialect.isAS400()) {
                 sqlQuery.addSelect(expr, null);
             } else {
                 sqlQuery.addSelect(expr, getColumnAlias(i));
@@ -80,11 +91,19 @@ protected void nonDistinctGenerateSQL(final SqlQuery sqlQuery) {
             if (isAggregate()) {
                 sqlQuery.addGroupBy(expr);
             }
+
+            // Add ORDER BY clause to make the results deterministic.
+            // Derby has a bug with ORDER BY, so ignore it.
+            if (ordered) {
+                sqlQuery.addOrderBy(expr, true, false, false);
+            }
         }
 
         // add measures
-        for (int i = 0, count = getMeasureCount(); i < count; i++) {
-            addMeasure(i, sqlQuery);
+        if (!countOnly) {
+            for (int i = 0, count = getMeasureCount(); i < count; i++) {
+                addMeasure(i, sqlQuery);
+            }
         }
     }
 }

src/main/mondrian/rolap/agg/AggregationManager.java

@@ -117,8 +117,14 @@ public Object getCellFromCache(CellRequest request, Set pinSet) {
         }
     }
 
-    public String getDrillThroughSQL(final CellRequest request) {
-        DrillThroughQuerySpec spec = new DrillThroughQuerySpec(request);
+    public String getDrillThroughSql(
+        final CellRequest request,
+        boolean countOnly)
+    {
+        DrillThroughQuerySpec spec =
+            new DrillThroughQuerySpec(
+                request,
+                countOnly);
         String sql = spec.generateSqlQuery();
 
         if (getLogger().isDebugEnabled()) {

src/main/mondrian/rolap/agg/DrillThroughQuerySpec.java

@@ -29,11 +29,16 @@
  */
 class DrillThroughQuerySpec extends AbstractQuerySpec {
     private final CellRequest request;
+    private final boolean countOnly;
     private final String[] columnNames;
 
-    public DrillThroughQuerySpec(final CellRequest request) {
+    public DrillThroughQuerySpec(
+        final CellRequest request,
+        boolean countOnly)
+    {
         super(request.getMeasure().getStar());
         this.request = request;
+        this.countOnly = countOnly;
         this.columnNames = computeDistinctColumnNames();
     }
 
@@ -52,9 +57,11 @@ private String[] computeDistinctColumnNames() {
         return (String[]) columnNames.toArray(new String[columnNames.size()]);
     }
 
-    private void addColumnName(final RolapStar.Column column, 
-                               final List columnNames, 
-                               final Set columnNameSet) {
+    private void addColumnName(
+        final RolapStar.Column column,
+        final List columnNames,
+        final Set columnNameSet)
+    {
         String columnName = column.getName();
         if (columnName != null) {
             // nothing
@@ -106,10 +113,11 @@ public ColumnConstraint[] getConstraints(final int i) {
     public String generateSqlQuery() {
         SqlQuery sqlQuery = newSqlQuery();
 
-        nonDistinctGenerateSQL(sqlQuery);
+        nonDistinctGenerateSql(sqlQuery, true, countOnly);
 
         return sqlQuery.toString();
     }
+
     protected void addMeasure(final int i, final SqlQuery sqlQuery) {
         RolapStar.Measure measure = getMeasure(i);
 
@@ -119,6 +127,7 @@ protected void addMeasure(final int i, final SqlQuery sqlQuery) {
         String expr = measure.generateExprString(sqlQuery);
         sqlQuery.addSelect(expr, getMeasureAlias(i));
     }
+
     protected boolean isAggregate() {
         return false;
     }

src/main/mondrian/rolap/agg/SegmentArrayQuerySpec.java

@@ -81,7 +81,7 @@ public String generateSqlQuery() {
         if (!sqlQuery.getDialect().allowsCountDistinct() && hasDistinct()) {
             distinctGenerateSql(sqlQuery);
         } else {
-            nonDistinctGenerateSQL(sqlQuery);
+            nonDistinctGenerateSql(sqlQuery, false, false);
         }
 
         return sqlQuery.toString();

src/main/mondrian/rolap/sql/SqlQuery.java

@@ -1169,6 +1169,17 @@ public String forceNullsCollateLast(String expr) {
             }
             return expr;
         }
+
+        /**
+         * Returns whether this Dialect supports expressions in the GROUP BY
+         * clause. Derby/Cloudscape do not.
+         *
+         * @return Whether this Dialect allows expressions in the GROUP BY
+         *   clause
+         */
+        public boolean supportsGroupByExpressions() {
+            return !(isDerby() || isCloudscape());
+        }
     }
 
     /**

src/main/mondrian/xmla/XmlaHandler.java

@@ -13,6 +13,7 @@
 import mondrian.olap.Connection;
 import mondrian.olap.DriverManager;
 import mondrian.rolap.RolapConnection;
+import mondrian.rolap.RolapUtil;
 import mondrian.spi.CatalogLocator;
 import mondrian.xmla.impl.DefaultSaxWriter;
 
@@ -796,7 +797,8 @@ private QueryResult executeDrillThroughQuery(XmlaRequest request)
         DataSourcesConfig.Catalog dsCatalog = getCatalog(request, ds);
         String role = request.getRole();
 
-        final Connection connection = getConnection(dsCatalog, role);
+        final RolapConnection connection =
+            (RolapConnection) getConnection(dsCatalog, role);
 
         final String statement = request.getStatement();
         final Query query = connection.parseQuery(statement);
@@ -813,37 +815,21 @@ private QueryResult executeDrillThroughQuery(XmlaRequest request)
 
         String dtSql = dtCell.getDrillThroughSQL(true);
         TabularRowSet rowset = null;
-        java.sql.Connection conn = null;
-        Statement stmt = null;
+        java.sql.Connection sqlConn = null;
         ResultSet rs = null;
 
         try {
-            conn = ((RolapConnection) connection).getDataSource().getConnection();
-            stmt = conn.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE,
-                                        ResultSet.CONCUR_READ_ONLY);
-
             int count = -1;
             if (MondrianProperties.instance().EnableTotalCount.booleanValue()) {
-                String temp = dtSql.toUpperCase();
-                int fromOff = temp.indexOf("FROM");
-                StringBuffer buf = new StringBuffer();
-                buf.append("select count(*) ");
-                buf.append(dtSql.substring(fromOff));
-
-                String countSql = buf.toString();
-                if (LOGGER.isDebugEnabled()) {
-                    LOGGER.debug("drill through counting sql: " + countSql);
-                }
-                rs = stmt.executeQuery(countSql);
-                rs.next();
-                count = rs.getInt(1);
-                rs.close();
+                count = dtCell.getDrillThroughCount();
             }
 
+            sqlConn = connection.getDataSource().getConnection();
             if (LOGGER.isDebugEnabled()) {
                 LOGGER.debug("drill through sql: " + dtSql);
             }
-            rs = stmt.executeQuery(dtSql);
+            rs = RolapUtil.executeQuery(
+                sqlConn, dtSql, "XmlaHandler.executeDrillThroughQuery");
             rowset = new TabularRowSet(
                     rs, request.drillThroughMaxRows(),
                     request.drillThroughFirstRowset(), count);
@@ -854,19 +840,20 @@ private QueryResult executeDrillThroughQuery(XmlaRequest request)
                 SERVER_FAULT_FC,
                 HSB_DRILL_THROUGH_SQL_CODE,
                 HSB_DRILL_THROUGH_SQL_FAULT_FS,
-                Util.newError(sqle,
-                    "Errors when executing DrillThrough sql '" + dtSql + "'"));
+                Util.newError(sqle, "Error in drill through"));
+        } catch (RuntimeException e) {
+            throw new XmlaException(
+                SERVER_FAULT_FC,
+                HSB_DRILL_THROUGH_SQL_CODE,
+                HSB_DRILL_THROUGH_SQL_FAULT_FS,
+                Util.newError(e, "Error in drill through"));
         } finally {
             try {
                 if (rs != null) rs.close();
             } catch (SQLException ignored) {
             }
             try {
-                if (stmt != null) stmt.close();
-            } catch (SQLException ignored) {
-            }
-            try {
-                if (conn != null && !conn.isClosed()) conn.close();
+                if (sqlConn != null && !sqlConn.isClosed()) sqlConn.close();
             } catch (SQLException ignored) {
             }
         }

testsrc/main/mondrian/olap/fun/FunctionTest.java

@@ -312,14 +312,14 @@ public void testIsEmpty()
         // Not a runtime exception.
         assertBooleanExprReturns("[Gender].CurrentMember.Parent.NextMember IS NULL", true);
 
+        if (!Bug.Bug1530543Fixed) return;
+
         // When resolving a tuple's value in the cube, if there is
         // at least one NULL member in the tuple should return a
         // NULL cell value.
         assertBooleanExprReturns("IsEmpty( ([Time].currentMember.Parent, [Measures].[Unit Sales]) )", false);
         assertBooleanExprReturns("IsEmpty( ([Time].currentMember, [Measures].[Unit Sales]) )", false);
 
-        if (!Bug.Bug1530543Fixed) return;
-
         // EMPTY refers to a genuine cell value that exists in the cube space,
         // and has no NULL members in the tuple,
         // but has no fact data at that crossing,

testsrc/main/mondrian/test/CompatibilityTest.java

@@ -310,7 +310,7 @@ public void testNullNameColumn() {
      * generate a modified ORDER BY clause.
       */
     public void testNullCollation() {
-        if (getTestContext().getDialect().isDerby()) {
+        if (!getTestContext().getDialect().supportsGroupByExpressions()) {
             // Derby does not support expressions in the GROUP BY clause,
             // therefore this testing strategy of using an expression for the
             // store key won't work. Give the test a bye.