Skip to content

Commit

Permalink
MONDRIAN: rolling back removeoverlappingtuples optimization for disti…
Browse files Browse the repository at this point in the history 
…nct count.refer http://forums.pentaho.org/showthread.php?t=59308.By removing this tuple list optimization we get a boost in performance at the cost of losing capability to handle elimination of duplicates and duplicates occur in rare scenarios.

optimizechildren will still rollup children to their parent member thereby reducing the number of entries in the sql IN clause.

[git-p4: depot-paths = "//open/mondrian/": change = 10846]
  • Loading branch information
Ajit Joglekar committed Apr 9, 2008
1 parent 06b76b0 commit ffac3f3
Show file tree
Hide file tree
Showing 4 changed files with 49 additions and 208 deletions.
86 changes: 6 additions & 80 deletions src/main/mondrian/olap/fun/AggregateFunDef.java
View file
Expand Up @@ -17,8 +17,8 @@
import mondrian.calc.impl.ValueCalc;
import mondrian.mdx.ResolvedFunCall;
import mondrian.olap.*;
import mondrian.rolap.*;
import mondrian.rolap.sql.*;
import mondrian.rolap.RolapAggregator;
import mondrian.rolap.RolapEvaluator;

import java.util.*;

Expand Down Expand Up @@ -110,7 +110,6 @@ public double evaluateDouble(Evaluator evaluator) {
// very slow. May want to revisit this if someone
// improves the algorithm.
} else {
list = removeOverlappingTupleEntries(list);
list = optimizeChildren(list,
evaluator.getSchemaReader(),evaluator.getMeasureCube());
checkIfAggregationSizeIsTooLarge(list);
Expand All @@ -131,49 +130,6 @@ public double evaluateDouble(Evaluator evaluator) {
return (Double) rollup.aggregate(evaluator.push(), list, calc);
}

/**
* In case of distinct count aggregation if a tuple which is a super
* set of other tuples in the set exists then the child tuples can be
* ignored.
*
* <p>
* E.g.
* List consists of:
* (Gender.[All Gender], [Product].[All Products]),
* (Gender.[All Gender].[F], [Product].[All Products].[Drink]),
* (Gender.[All Gender].[M], [Product].[All Products].[Food])
* Can be optimized to:
* (Gender.[All Gender], [Product].[All Products])
*
* @param list
*/

public static List<Member[]> removeOverlappingTupleEntries(List<Member[]> list) {
List<Member[]> trimmedList = new ArrayList<Member[]>();
for (Member[] tuple1 : list) {
if (trimmedList.isEmpty()) {
trimmedList.add(tuple1);
} else {
boolean ignore = false;
final Iterator<Member[]> iterator = trimmedList.iterator();
while (iterator.hasNext()) {
Member[] tuple2 = iterator.next();
if (isSuperSet(tuple1, tuple2)) {
iterator.remove();
} else if (isSuperSet(tuple2, tuple1) ||
isEqual(tuple1, tuple2)) {
ignore = true;
break;
}
}
if (!ignore) {
trimmedList.add(tuple1);
}
}
}
return trimmedList;
}

/**
* Forms a list tuples from a list of members
* @param list of members
Expand All @@ -187,28 +143,6 @@ public static List<Member[]> makeTupleList(List<Member> list) {
return tupleList;
}

/**
* Returns whether tuple1 is a superset of tuple2
* @param tuple1
* @param tuple2
* @return boolean
*/
public static boolean isSuperSet(Member[] tuple1, Member[] tuple2) {
int parentLevelCount = 0;
for (int i = 0; i < tuple1.length; i++) {
Member member1 = tuple1[i];
Member member2 = tuple2[i];

if (!member2.isChildOrEqualTo(member1)) {
return false;
}
if (member1.getLevel().getDepth() < member2.getLevel().getDepth()) {
parentLevelCount++;
}
}
return parentLevelCount > 0;
}

private void checkIfAggregationSizeIsTooLarge(List list) {
if (list.size() > MondrianProperties.instance().MaxConstraints.get()) {
throw newEvalException(
Expand Down Expand Up @@ -343,8 +277,10 @@ private static Set optimizeMemberSet(
while (iterator.hasNext()) {
Member current = iterator.next();
Member currentParentMember = current.getParentMember();
if (firstParentMember == null && currentParentMember == null ||
firstParentMember.equals(currentParentMember)) {
if (firstParentMember == null &&
currentParentMember == null ||
(firstParentMember!= null &&
firstParentMember.equals(currentParentMember))) {
membersToBeOptimized.add(current);
iterator.remove();
}
Expand Down Expand Up @@ -375,16 +311,6 @@ private static Set optimizeMemberSet(
return optimizedMembers;
}

private static boolean isEqual(Member[] tuple1, Member[] tuple2) {
for (int i = 0; i < tuple1.length; i++) {
if (!tuple1[i].getUniqueName().
equals(tuple2[i].getUniqueName())) {
return false;
}
}
return true;
}

private static boolean canOptimize(
Member parentMember,
Cube baseCube)
Expand Down
70 changes: 35 additions & 35 deletions testsrc/main/mondrian/rolap/FastBatchingCellReaderTest.java
View file
Expand Up @@ -1280,34 +1280,31 @@ public void testAggregateDistinctCount4() {
" {[Store].[CA plus USA]} * {[Time].[Q1 plus July]} ON ROWS\n" +
"FROM Sales";

String accessSql = "select count(`m0`) as `c0` from (" +
"select distinct `sales_fact_1997`.`customer_id` as `m0` " +
"from `store` as `store`," +
" `sales_fact_1997` as `sales_fact_1997`," +
" `time_by_day` as `time_by_day` " +
"where `sales_fact_1997`.`store_id` = `store`.`store_id` " +
"and `store`.`store_country` = 'USA' " +
"and `sales_fact_1997`.`time_id` = `time_by_day`.`time_id` " +
"and ((`time_by_day`.`quarter` = 'Q1' and `time_by_day`.`the_year` = 1997)" +
" or (`time_by_day`.`month_of_year` = 7 and `time_by_day`.`quarter` = 'Q3' and `time_by_day`.`the_year` = 1997))" +
") as `dummyname`";
String accessSql = "select count(`m0`) as `c0` " +
"from (select distinct `sales_fact_1997`.`customer_id` as `m0` from `store` as `store`, " +
"`sales_fact_1997` as `sales_fact_1997`, `time_by_day` as `time_by_day` " +
"where `sales_fact_1997`.`store_id` = `store`.`store_id` and (`store`.`store_state` = 'CA' " +
"or `store`.`store_country` = 'USA') and `sales_fact_1997`.`time_id` = `time_by_day`.`time_id` " +
"and ((`time_by_day`.`quarter` = 'Q1' and `time_by_day`.`the_year` = 1997) " +
"or (`time_by_day`.`month_of_year` = 7 and `time_by_day`.`quarter` = 'Q3' " +
"and `time_by_day`.`the_year` = 1997))) as `dummyname`";

String derbySql = "select count(distinct \"sales_fact_1997\".\"customer_id\") as \"m0\" " +
"from \"store\" as \"store\"," +
" \"sales_fact_1997\" as \"sales_fact_1997\"," +
" \"time_by_day\" as \"time_by_day\" " +
"where \"sales_fact_1997\".\"store_id\" = \"store\".\"store_id\" " +
"and \"store\".\"store_country\" = 'USA' " +
"from \"store\" as \"store\", \"sales_fact_1997\" as \"sales_fact_1997\", " +
"\"time_by_day\" as \"time_by_day\" where \"sales_fact_1997\".\"store_id\" = \"store\".\"store_id\" " +
"and (\"store\".\"store_state\" = 'CA' or \"store\".\"store_country\" = 'USA') " +
"and \"sales_fact_1997\".\"time_id\" = \"time_by_day\".\"time_id\" " +
"and ((\"time_by_day\".\"quarter\" = 'Q1' and \"time_by_day\".\"the_year\" = 1997)" +
" or (\"time_by_day\".\"month_of_year\" = 7 " +
"and \"time_by_day\".\"quarter\" = 'Q3' and \"time_by_day\".\"the_year\" = 1997))";
"and ((\"time_by_day\".\"quarter\" = 'Q1' and \"time_by_day\".\"the_year\" = 1997) " +
"or (\"time_by_day\".\"month_of_year\" = 7 and \"time_by_day\".\"quarter\" = 'Q3' " +
"and \"time_by_day\".\"the_year\" = 1997))";

final String mysqlSql = "select count(distinct `sales_fact_1997`.`customer_id`) as `m0` " +
"from `store` as `store`, `sales_fact_1997` as `sales_fact_1997`, `time_by_day` as `time_by_day` " +
"where `sales_fact_1997`.`store_id` = `store`.`store_id` " +
"and `store`.`store_country` = 'USA' and `sales_fact_1997`.`time_id` = `time_by_day`.`time_id` " +
"and ((`time_by_day`.`month_of_year` = 7 and `time_by_day`.`quarter` = 'Q3' and `time_by_day`.`the_year` = 1997) " +
"from `store` as `store`, `sales_fact_1997` as `sales_fact_1997`, " +
"`time_by_day` as `time_by_day` where `sales_fact_1997`.`store_id` = `store`.`store_id` " +
"and (`store`.`store_state` = 'CA' or `store`.`store_country` = 'USA') " +
"and `sales_fact_1997`.`time_id` = `time_by_day`.`time_id` " +
"and ((`time_by_day`.`month_of_year` = 7 and `time_by_day`.`quarter` = 'Q3' " +
"and `time_by_day`.`the_year` = 1997) " +
"or (`time_by_day`.`quarter` = 'Q1' and `time_by_day`.`the_year` = 1997))";

assertQuerySql(mdxQuery, new SqlPattern[] {
Expand Down Expand Up @@ -1392,20 +1389,23 @@ public void testAggregateDistinctCount6() {
"FROM Sales";

String derbySql = "select count(distinct \"sales_fact_1997\".\"customer_id\") as \"m0\" " +
"from \"store\" as \"store\", \"sales_fact_1997\" as \"sales_fact_1997\", " +
"\"time_by_day\" as \"time_by_day\" where " +
"\"sales_fact_1997\".\"store_id\" = \"store\".\"store_id\" " +
"and (\"store\".\"store_state\" in ('CA', 'OR') or " +
"\"store\".\"store_country\" in ('Mexico', 'Canada')) " +
"and \"sales_fact_1997\".\"time_id\" = \"time_by_day\".\"time_id\" " +
"and \"time_by_day\".\"the_year\" = 1997";
"from \"store\" as \"store\", \"sales_fact_1997\" as \"sales_fact_1997\", \"time_by_day\" " +
"as \"time_by_day\" where \"sales_fact_1997\".\"store_id\" = \"store\".\"store_id\" " +
"and (\"store\".\"store_state\" in ('CA', 'OR') or \"store\".\"store_country\" in ('Mexico', 'Canada')) " +
"and \"sales_fact_1997\".\"time_id\" = \"time_by_day\".\"time_id\" and (((\"time_by_day\".\"quarter\" = 'Q1' " +
"and \"time_by_day\".\"the_year\" = 1997) or (\"time_by_day\".\"quarter\" = 'Q4' " +
"and \"time_by_day\".\"the_year\" = 1997)) or (\"time_by_day\".\"month_of_year\" = 7 " +
"and \"time_by_day\".\"quarter\" = 'Q3' and \"time_by_day\".\"the_year\" = 1997) " +
"or \"time_by_day\".\"the_year\" = 1997)";

final String mysqlSql = "select count(distinct `sales_fact_1997`.`customer_id`) as `m0` " +
"from `store` as `store`, `sales_fact_1997` as `sales_fact_1997`, `time_by_day` as `time_by_day` " +
"where `sales_fact_1997`.`store_id` = `store`.`store_id` " +
"and (`store`.`store_state` in ('CA', 'OR') " +
"or `store`.`store_country` in ('Mexico', 'Canada')) and " +
"`sales_fact_1997`.`time_id` = `time_by_day`.`time_id` and `time_by_day`.`the_year` = 1997";
"from `store` as `store`, `sales_fact_1997` as `sales_fact_1997`, " +
"`time_by_day` as `time_by_day` where `sales_fact_1997`.`store_id` = `store`.`store_id` " +
"and (`store`.`store_state` in ('CA', 'OR') or `store`.`store_country` in ('Mexico', 'Canada')) " +
"and `sales_fact_1997`.`time_id` = `time_by_day`.`time_id` and ((`time_by_day`.`month_of_year` = 7 " +
"and `time_by_day`.`quarter` = 'Q3' and `time_by_day`.`the_year` = 1997) " +
"or (((`time_by_day`.`the_year`, `time_by_day`.`quarter`) in ((1997, 'Q1'), (1997, 'Q4')))) " +
"or `time_by_day`.`the_year` = 1997)";

assertQuerySql(mdxQuery, new SqlPattern[] {
new SqlPattern(SqlPattern.Dialect.DERBY, derbySql, derbySql),
Expand Down
99 changes: 7 additions & 92 deletions testsrc/main/mondrian/rolap/agg/AggregationOnDistinctCountMeasuresTest.java
View file
Expand Up @@ -926,61 +926,6 @@ public void testShouldConvertListOfMembersToTuples() {
tuples.get(0)[0].getUniqueName());
}

public void testMemberIsSuperSetOfAnotherMember() {
List <Member[]> tuples = tupleList(
genderMembersIncludingAll(true, salesCubeSchemaReader, salesCube));
assertTrue(AggregateFunDef.AggregateCalc.
isSuperSet(tuples.get(0), tuples.get(1)));
assertFalse(AggregateFunDef.AggregateCalc.
isSuperSet(tuples.get(1), tuples.get(2)));
}

public void testRemoveOverlappingTuplesForSameDimension() {
List <Member[]> tuples = tupleList(
genderMembersIncludingAll(true, salesCubeSchemaReader, salesCube));
tuples = removeOverlappingTuples(tuples);
assertEquals(1, tuples.size());
assertEquals(allMember("Gender", salesCube), tuples.get(0)[0]);
}

public void testShouldRemoveOverlappingTuplesFromDifferentDimensions() {
List<Member[]> memberList = CrossJoinFunDef.crossJoin(
genderMembersIncludingAll(true, salesCubeSchemaReader, salesCube),
storeMembersUsaAndCanada(true, salesCubeSchemaReader, salesCube));
List <Member[]>tuples = removeOverlappingTuples(memberList);
assertEquals(1, tuples.size());
assertEquals(allMember("Gender", salesCube), tuples.get(0)[0]);
assertEquals(allMember("Store", salesCube), tuples.get(0)[1]);
}

public void testShouldRemoveOverlappingTuplesWithoutAllLevelTuple() {
List<Member[]> memberList =
CrossJoinFunDef.crossJoin(
genderMembersIncludingAll(true, salesCubeSchemaReader, salesCube),
storeMembersUsaAndCanada(false, salesCubeSchemaReader, salesCube));

Member maleChild =
member(Id.Segment.toList("Gender","All Gender","M"), salesCubeSchemaReader);
Member femaleChild =
member(Id.Segment.toList("Gender","All Gender","F"), salesCubeSchemaReader);
Member storeAllMember = allMember("Store", salesCube);

memberList.add(new Member[]{maleChild, storeAllMember});
memberList.add(new Member[]{femaleChild, storeAllMember});

List tuples = removeOverlappingTuples(memberList);
assertEquals(4, tuples.size());
}


public void testShouldNotRemoveNonOverlappingTuplesAtSameLevels() {
List<Member[]> memberList =
CrossJoinFunDef.crossJoin(
genderMembersIncludingAll(false, salesCubeSchemaReader, salesCube),
storeMembersUsaAndCanada(false, salesCubeSchemaReader, salesCube));
List tuples = removeOverlappingTuples(memberList);
assertEquals(4, tuples.size());
}

public void testOptimizeChildren() {
String query =
Expand Down Expand Up @@ -1156,9 +1101,7 @@ public void testOptimizeChildrenForTuplesWithLength1() {
.makeTupleList(
productMembersPotScrubbersPotsAndPans(salesCubeSchemaReader));

List tuples = removeOverlappingTuples(memberList);
assertEquals(8, tuples.size());
tuples = optimizeChildren(memberList);
List tuples = optimizeChildren(memberList);
assertTrue(tuppleListContains(tuples,
member(Id.Segment.toList("Product", "All Products", "Non-Consumable",
"Household", "Kitchen Products", "Pot Scrubbers", "Cormorant"),
Expand All @@ -1185,9 +1128,7 @@ public void testOptimizeChildrenForTuplesWithLength3() {
productMembersPotScrubbersPotsAndPans(salesCubeSchemaReader));
memberList =
CrossJoinFunDef.crossJoin(memberList, storeMembersCAAndOR(salesCubeSchemaReader));
List tuples = removeOverlappingTuples(memberList);
assertEquals(80, tuples.size());
tuples = optimizeChildren(memberList);
List tuples = optimizeChildren(memberList);
assertFalse(tuppleListContains(tuples,
member(
Id.Segment.toList("Store","All Stores","USA","OR","Portland"),
Expand All @@ -1204,10 +1145,7 @@ public void testOptimizeChildrenWhenTuplesAreFormedWithDifferentLevels() {
CrossJoinFunDef.crossJoin(
genderMembersIncludingAll(false, salesCubeSchemaReader, salesCube),
productMembersPotScrubbersPotsAndPans(salesCubeSchemaReader));
List tuples = removeOverlappingTuples(memberList);
assertEquals(16, tuples.size());

tuples = optimizeChildren(memberList);
List tuples = optimizeChildren(memberList);
assertEquals(4, tuples.size());

assertFalse(tuppleListContains(tuples,
Expand All @@ -1233,14 +1171,11 @@ public void testWhetherCJOfChildren() {
genderMembersIncludingAll(false, salesCubeSchemaReader, salesCube),
storeMembersUsaAndCanada(false, salesCubeSchemaReader, salesCube));

List tuples = removeOverlappingTuples(memberList);
assertEquals(4, tuples.size());

tuples = optimizeChildren(memberList);
List tuples = optimizeChildren(memberList);
assertEquals(2, tuples.size());
}

public void testShouldRemoveDuplicateTuples() {
public void testShouldNotRemoveDuplicateTuples() {

Member maleChildMember = member(
Id.Segment.toList("Gender","All Gender","M"), salesCubeSchemaReader);
Expand All @@ -1252,26 +1187,10 @@ public void testShouldRemoveDuplicateTuples() {
memberList.add(maleChildMember);
memberList.add(femaleChildMember);
List<Member[]> tuples = tupleList(memberList);
tuples = removeOverlappingTuples(tuples);
assertEquals(2, tuples.size());
tuples = optimizeChildren(tuples);
assertEquals(3, tuples.size());
}

public void testShouldNotRemoveNonOverlappingTuplesAtDifferentLevels() {

Member genderMaleChild =
member(Id.Segment.toList("Gender","All Gender","M"),
salesCubeSchemaReader);
Member storeUsaChild =
member(Id.Segment.toList("Store","All Stores","USA"),
salesCubeSchemaReader);

List<Member[]> memberList = new ArrayList<Member[]>();
memberList.add(new Member[]{genderMaleChild, allMember("Store", salesCube)});
memberList.add(new Member[]{allMember("Gender", salesCube),storeUsaChild});
List tuples = removeOverlappingTuples(memberList);
assertEquals(2, tuples.size());
}

public void testMemberCountIsSameForAllMembersInTuple() {

List <Member[]>memberList =
Expand Down Expand Up @@ -1381,10 +1300,6 @@ private boolean tuppleListContains(
}
return false;
}

private List removeOverlappingTuples(List<Member[]> tuples) {
return AggregateFunDef.AggregateCalc.removeOverlappingTupleEntries(tuples);
}

private List<Member[]> optimizeChildren(List<Member[]> memberList) {
return AggregateFunDef.AggregateCalc.
Expand Down
2 changes: 1 addition & 1 deletion testsrc/main/mondrian/test/DrillThroughTest.java
View file
Expand Up @@ -675,7 +675,7 @@ public void testTruncateLevelName() throws Exception {
} else {
assertEquals(6, columnCount);
}
final String columnName = resultSet.getMetaData().getColumnName(5);
final String columnName = resultSet.getMetaData().getColumnLabel(5);
assertTrue(
columnName,
columnName.startsWith("Education Level but with a"));
Expand Down

0 comments on commit ffac3f3

Please sign in to comment.