Skip to content

Commit

Permalink
优化多字段的DistinctQuery
Browse files Browse the repository at this point in the history
  • Loading branch information
@codefollower
codefollower committed Jun 19, 2016
1 parent 605f530 commit 72528ad
Show file tree
Hide file tree
Showing 5 changed files with 138 additions and 29 deletions.
7 changes: 7 additions & 0 deletions lealone-db/src/main/java/org/lealone/db/index/Index.java
View file
Expand Up @@ -198,6 +198,13 @@ public interface Index extends SchemaObject {
*/
Column[] getColumns();

/**
* Get the indexed column ids.
*
* @return the column ids
*/
int[] getColumnIds();

/**
* Get the index type.
*
Expand Down
5 changes: 5 additions & 0 deletions lealone-db/src/main/java/org/lealone/db/index/IndexBase.java
View file
Expand Up @@ -89,6 +89,11 @@ public Column[] getColumns() {
return columns;
}

@Override
public int[] getColumnIds() {
return columnIds;
}

/**
* Create a duplicate key exception with a message that contains the index name.
*
Expand Down
135 changes: 109 additions & 26 deletions lealone-sql/src/main/java/org/lealone/sql/dml/Select.java
View file
Expand Up @@ -82,7 +82,7 @@ public class Select extends Query implements org.lealone.db.expression.Select {
private boolean isGroupQuery, isGroupSortedQuery;
private boolean isForUpdate, isForUpdateMvcc;
private double cost;
private boolean isQuickAggregateQuery, isDistinctQuery;
private boolean isQuickAggregateQuery, isDistinctQuery, isDistinctQueryForMultiFields;
private boolean isPrepared, checkInit;
private boolean sortUsingIndex;
private SortOrder sort;
Expand Down Expand Up @@ -370,30 +370,8 @@ public PreparedStatement prepare() {
// 以下3个if为特殊的distinct、sort、group by选择更合适的索引
// 1. distinct
if (distinct && session.getDatabase().getSettings().optimizeDistinct && !isGroupQuery && filters.size() == 1
&& expressions.size() == 1 && condition == null) {
Expression expr = expressions.get(0);
expr = expr.getNonAliasExpression();
if (expr instanceof ExpressionColumn) {
Column column = ((ExpressionColumn) expr).getColumn();
int selectivity = column.getSelectivity();
Index columnIndex = topTableFilter.getTable().getIndexForColumn(column);
if (columnIndex != null && selectivity != Constants.SELECTIVITY_DEFAULT && selectivity < 20) {
// the first column must be ascending
boolean ascending = columnIndex.getIndexColumns()[0].sortType == SortOrder.ASCENDING;
Index current = topTableFilter.getIndex();
// if another index is faster
if (columnIndex.supportsDistinctQuery() && ascending
&& (current == null || current.getIndexType().isScan() || columnIndex == current)) {
IndexType type = columnIndex.getIndexType();
// hash indexes don't work, and unique single column
// indexes don't work
if (!type.isHash() && (!type.isUnique() || columnIndex.getColumns().length > 1)) {
topTableFilter.setIndex(columnIndex);
isDistinctQuery = true;
}
}
}
}
&& condition == null) {
optimizeDistinct();
}
// 2. sort
if (sort != null && !isQuickAggregateQuery && !isGroupQuery) {
Expand Down Expand Up @@ -445,6 +423,73 @@ public PreparedStatement prepare() {
return this;
}

private void optimizeDistinct() {
// 1.1. distinct 单字段
if (expressions.size() == 1) {
Expression expr = expressions.get(0);
expr = expr.getNonAliasExpression();
if (expr instanceof ExpressionColumn) {
Column column = ((ExpressionColumn) expr).getColumn();
int selectivity = column.getSelectivity();
Index columnIndex = topTableFilter.getTable().getIndexForColumn(column);
if (columnIndex != null && selectivity != Constants.SELECTIVITY_DEFAULT && selectivity < 20) {
// the first column must be ascending
boolean ascending = columnIndex.getIndexColumns()[0].sortType == SortOrder.ASCENDING;
Index current = topTableFilter.getIndex();
// if another index is faster
if (columnIndex.supportsDistinctQuery() && ascending
&& (current == null || current.getIndexType().isScan() || columnIndex == current)) {
IndexType type = columnIndex.getIndexType();
// hash indexes don't work, and unique single column
// indexes don't work
if (!type.isHash() && (!type.isUnique() || columnIndex.getColumns().length > 1)) {
topTableFilter.setIndex(columnIndex);
isDistinctQuery = true;
}
}
}
}
}
// 1.2. distinct 多字段
else {
Index current = topTableFilter.getIndex();
if (current == null || current.getIndexType().isScan()) {
boolean isExpressionColumn = true;
int size = expressions.size();
Column[] columns = new Column[size];
for (int i = 0; isExpressionColumn && i < size; i++) {
Expression expr = expressions.get(i);
expr = expr.getNonAliasExpression();
isExpressionColumn &= (expr instanceof ExpressionColumn);
if (isExpressionColumn)
columns[i] = ((ExpressionColumn) expr).getColumn();
}
if (isExpressionColumn) {
for (Index index : topTableFilter.getTable().getIndexes()) {
IndexType type = index.getIndexType();
// hash indexes don't work, and unique single column
// indexes don't work
if (index.supportsDistinctQuery() && !type.isHash() && !type.isUnique()) {
Column[] indexColumns = index.getColumns();
if (indexColumns.length == size) {
boolean found = true;
for (int i = 0; found && i < size; i++) {
found &= (indexColumns[i] == columns[i])
&& index.getIndexColumns()[i].sortType == SortOrder.ASCENDING;
}
if (found) {
topTableFilter.setIndex(index);
isDistinctQueryForMultiFields = true;
break;
}
}
}
}
}
}
}
}

private double preparePlan() {
TableFilter[] topArray = topFilters.toArray(new TableFilter[topFilters.size()]);
for (TableFilter t : topArray) {
Expand Down Expand Up @@ -674,7 +719,7 @@ protected LocalResult queryWithoutCache(int maxRows, ResultTarget target) {
result = createLocalResult(result);
result.setSortOrder(sort);
}
if (distinct && !isDistinctQuery) {
if (distinct && (!isDistinctQuery && !isDistinctQueryForMultiFields)) {
result = createLocalResult(result);
result.setDistinct();
}
Expand Down Expand Up @@ -715,6 +760,8 @@ protected LocalResult queryWithoutCache(int maxRows, ResultTarget target) {
}
} else if (isDistinctQuery) {
queryDistinct(to, limitRows);
} else if (isDistinctQueryForMultiFields) {
queryDistinctForMultiFields(to, limitRows);
} else {
queryFlat(columnCount, to, limitRows);
}
Expand Down Expand Up @@ -743,6 +790,42 @@ private LocalResult createLocalResult(LocalResult old) {
return old != null ? old : new LocalResult(session, expressionArray, visibleColumnCount);
}

private void queryDistinctForMultiFields(ResultTarget result, long limitRows) {
// limitRows must be long, otherwise we get an int overflow
// if limitRows is at or near Integer.MAX_VALUE
// limitRows is never 0 here
if (limitRows > 0 && offsetExpr != null) {
int offset = offsetExpr.getValue(session).getInt();
if (offset > 0) {
limitRows += offset;
}
}
int rowNumber = 0;
setCurrentRowNumber(0);
Index index = topTableFilter.getIndex();
int[] columnIds = index.getColumnIds();
int size = columnIds.length;
int sampleSize = getSampleSizeValue(session);
Cursor cursor = index.findDistinct(session, null, null);
while (cursor.next()) {
setCurrentRowNumber(rowNumber + 1);
SearchRow found = cursor.getSearchRow();
Value[] row = new Value[size];
for (int i = 0; i < size; i++) {
row[i] = found.getValue(columnIds[i]);
}
result.addRow(row);
rowNumber++;
if ((sort == null || sortUsingIndex) && limitRows > 0 && rowNumber >= limitRows) {
break;
}
if (sampleSize > 0 && rowNumber >= sampleSize) {
break;
}
}
}

// 单字段distinct
private void queryDistinct(ResultTarget result, long limitRows) {
// limitRows must be long, otherwise we get an int overflow
// if limitRows is at or near Integer.MAX_VALUE
Expand Down
5 changes: 4 additions & 1 deletion lealone-test/src/test/java/org/lealone/test/sql/SqlTestBase.java
View file
Expand Up @@ -207,7 +207,8 @@ public boolean next() throws Exception {
return rs.next();
}

public void printResultSet() {
public int printResultSet() {
int count = 0;
try {
rs = stmt.executeQuery(sql);

Expand All @@ -216,6 +217,7 @@ public void printResultSet() {
for (int i = 1; i <= n; i++) {
System.out.print(rs.getString(i) + " ");
}
count++;
System.out.println();
}
rs.close();
Expand All @@ -224,5 +226,6 @@ public void printResultSet() {
} catch (SQLException e) {
e.printStackTrace();
}
return count;
}
}
15 changes: 13 additions & 2 deletions lealone-test/src/test/java/org/lealone/test/sql/misc/DistinctQueryTest.java
View file
Expand Up @@ -25,10 +25,11 @@ public class DistinctQueryTest extends SqlTestBase {
public void run() throws Exception {
stmt.executeUpdate("drop table IF EXISTS DistinctQueryTest");
stmt.executeUpdate("create table IF NOT EXISTS DistinctQueryTest(f1 int SELECTIVITY 10, f2 int, f3 int)");

stmt.executeUpdate("create index IF NOT EXISTS DistinctQueryTest_i1 on DistinctQueryTest(f1)");
stmt.executeUpdate("create index IF NOT EXISTS DistinctQueryTest_i1_2 on DistinctQueryTest(f1,f2)");

stmt.executeUpdate("insert into DistinctQueryTest(f1, f2, f3) values(1,2,3)");
stmt.executeUpdate("insert into DistinctQueryTest(f1, f2, f3) values(1,3,3)");
stmt.executeUpdate("insert into DistinctQueryTest(f1, f2, f3) values(5,2,3)");
stmt.executeUpdate("insert into DistinctQueryTest(f1, f2, f3) values(3,2,3)");
stmt.executeUpdate("insert into DistinctQueryTest(f1, f2, f3) values(8,2,3)");
Expand All @@ -38,9 +39,19 @@ public void run() throws Exception {
stmt.executeUpdate("insert into DistinctQueryTest(f1, f2, f3) values(8,2,3)");
stmt.executeUpdate("insert into DistinctQueryTest(f1, f2, f3) values(3,2,3)");
stmt.executeUpdate("insert into DistinctQueryTest(f1, f2, f3) values(8,2,3)");
stmt.executeUpdate("insert into DistinctQueryTest(f1, f2, f3) values(5,9,3)");

sql = "select distinct * from DistinctQueryTest where f1 > 3";
sql = "select distinct f1 from DistinctQueryTest";
printResultSet();
int count = printResultSet();
assertEquals(4, count);
sql = "select count(distinct f1) from DistinctQueryTest";
assertEquals(4, getIntValue(1, true));

sql = "select distinct f1, f2 from DistinctQueryTest";
count = printResultSet();
assertEquals(6, count);
// 不支持多个字段
// sql = "select count(distinct f1, f2) from DistinctQueryTest";
}
}

0 comments on commit 72528ad

Please sign in to comment.