Skip to content

Add aggregation+DateBin groupby support(aggregation table scan) and SumAccumulator support for table model. #13771

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JackieTien97 merged 6 commits into from
Oct 15, 2024
Merged

Add aggregation+DateBin groupby support(aggregation table scan) and SumAccumulator support for table model. #13771

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
8eb5428
add group by all id test
Beyyes Oct 14, 2024
0319dc6
add group by datebin impl
Beyyes Oct 14, 2024
d01ad83
refactor
Beyyes Oct 14, 2024
9c8d570
add it
Beyyes Oct 15, 2024
a08e5d9
merge with master
Beyyes Oct 15, 2024
e2f5a93
add more it
Beyyes Oct 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
173 changes: 172 additions & 1 deletion ...rc/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiIDsWithAttributesTableIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,8 +110,8 @@ public class IoTDBMultiIDsWithAttributesTableIT {
};

String[] expectedHeader;

String[] retArray;
String sql;

@BeforeClass
public static void setUp() throws Exception {
Expand Down Expand Up @@ -605,6 +605,177 @@ public void countStarTest() {
// TODO select count(*),count(t1) from (select avg(num+1) as t1 from table0) where time < 0
}

@Test
public void groupByAggregationTest() {
expectedHeader =
new String[] {
"device",
"level",
"count_num",
"count_star",
"count_device",
"count_date",
"count_attr1",
"count_attr2",
"count_time",
"sum_num"
};
retArray =
new String[] {
"d1,l1,3,3,3,0,3,3,3,20.0,",
"d1,l2,3,3,3,0,3,3,3,24.0,",
"d1,l3,3,3,3,0,3,3,3,19.0,",
"d1,l4,3,3,3,0,0,0,3,27.0,",
"d1,l5,3,3,3,1,0,0,3,30.0,",
"d2,l1,3,3,3,0,3,3,3,20.0,",
"d2,l2,3,3,3,0,3,0,3,24.0,",
"d2,l3,3,3,3,0,0,0,3,19.0,",
"d2,l4,3,3,3,0,0,0,3,27.0,",
"d2,l5,3,3,3,1,0,0,3,30.0,",
};
String sql =
"select device, level, "
+ "count(num) as count_num, count(*) as count_star, count(device) as count_device, count(date) as count_date, "
+ "count(attr1) as count_attr1, count(attr2) as count_attr2, count(time) as count_time, sum(num) as sum_num "
+ "from table0 group by device,level order by device, level";
tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME);
}

@Test
public void groupByDateBinTest() {
expectedHeader =
new String[] {
"device",
"level",
"bin",
"count_num",
"count_star",
"count_device",
"count_date",
"count_attr1",
"count_attr2",
"count_time",
"avg_num"
};
retArray =
new String[] {
"d1,l1,1970-01-01T00:00:00.000Z,1,1,1,0,1,1,1,3.0,",
"d1,l1,1971-01-01T00:00:00.000Z,2,2,2,0,2,2,2,8.5,",
"d1,l2,1970-01-01T00:00:00.000Z,1,1,1,0,1,1,1,2.0,",
"d1,l2,1971-01-01T00:00:00.000Z,2,2,2,0,2,2,2,11.0,",
"d1,l3,1970-01-01T00:00:00.000Z,1,1,1,0,1,1,1,1.0,",
"d1,l3,1971-01-01T00:00:00.000Z,2,2,2,0,2,2,2,9.0,",
"d1,l4,1970-01-01T00:00:00.000Z,1,1,1,0,0,0,1,9.0,",
"d1,l4,1971-01-01T00:00:00.000Z,2,2,2,0,0,0,2,9.0,",
"d1,l5,1970-01-01T00:00:00.000Z,1,1,1,0,0,0,1,8.0,",
"d1,l5,1971-01-01T00:00:00.000Z,2,2,2,1,0,0,2,11.0,",
"d2,l1,1970-01-01T00:00:00.000Z,1,1,1,0,1,1,1,3.0,",
"d2,l1,1971-01-01T00:00:00.000Z,2,2,2,0,2,2,2,8.5,",
"d2,l2,1970-01-01T00:00:00.000Z,1,1,1,0,1,0,1,2.0,",
"d2,l2,1971-01-01T00:00:00.000Z,2,2,2,0,2,0,2,11.0,",
"d2,l3,1970-01-01T00:00:00.000Z,1,1,1,0,0,0,1,1.0,",
"d2,l3,1971-01-01T00:00:00.000Z,2,2,2,0,0,0,2,9.0,",
"d2,l4,1970-01-01T00:00:00.000Z,1,1,1,0,0,0,1,9.0,",
"d2,l4,1971-01-01T00:00:00.000Z,2,2,2,0,0,0,2,9.0,",
"d2,l5,1970-01-01T00:00:00.000Z,1,1,1,0,0,0,1,8.0,",
"d2,l5,1971-01-01T00:00:00.000Z,2,2,2,1,0,0,2,11.0,",
};
sql =
"select device, level, date_bin(1y, time) as bin,"
+ "count(num) as count_num, count(*) as count_star, count(device) as count_device, count(date) as count_date, "
+ "count(attr1) as count_attr1, count(attr2) as count_attr2, count(time) as count_time, avg(num) as avg_num "
+ "from table0 group by 3, device, level order by device, level, bin";
tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME);

retArray =
new String[] {
"d1,l1,1970-01-01T00:00:00.000Z,1,1,1,0,1,1,1,3.0,",
"d1,l1,1971-01-01T00:00:00.000Z,2,2,2,0,2,2,2,8.5,",
"d1,l2,1970-01-01T00:00:00.000Z,1,1,1,0,1,1,1,2.0,",
"d1,l2,1971-01-01T00:00:00.000Z,1,1,1,0,1,1,1,10.0,",
"d1,l2,1971-04-26T00:00:00.000Z,1,1,1,0,1,1,1,12.0,",
"d1,l3,1970-01-01T00:00:00.000Z,1,1,1,0,1,1,1,1.0,",
"d1,l3,1971-01-01T00:00:00.000Z,1,1,1,0,1,1,1,4.0,",
"d1,l3,1971-04-26T00:00:00.000Z,1,1,1,0,1,1,1,14.0,",
"d1,l4,1970-01-01T00:00:00.000Z,1,1,1,0,0,0,1,9.0,",
"d1,l4,1971-01-01T00:00:00.000Z,1,1,1,0,0,0,1,5.0,",
"d1,l4,1971-04-26T00:00:00.000Z,1,1,1,0,0,0,1,13.0,",
"d1,l5,1970-01-01T00:00:00.000Z,1,1,1,0,0,0,1,8.0,",
"d1,l5,1971-01-01T00:00:00.000Z,1,1,1,0,0,0,1,7.0,",
"d1,l5,1971-08-20T00:00:00.000Z,1,1,1,1,0,0,1,15.0,",
"d2,l1,1970-01-01T00:00:00.000Z,1,1,1,0,1,1,1,3.0,",
"d2,l1,1971-01-01T00:00:00.000Z,2,2,2,0,2,2,2,8.5,",
"d2,l2,1970-01-01T00:00:00.000Z,1,1,1,0,1,0,1,2.0,",
"d2,l2,1971-01-01T00:00:00.000Z,1,1,1,0,1,0,1,10.0,",
"d2,l2,1971-04-26T00:00:00.000Z,1,1,1,0,1,0,1,12.0,",
"d2,l3,1970-01-01T00:00:00.000Z,1,1,1,0,0,0,1,1.0,",
"d2,l3,1971-01-01T00:00:00.000Z,1,1,1,0,0,0,1,4.0,",
"d2,l3,1971-04-26T00:00:00.000Z,1,1,1,0,0,0,1,14.0,",
"d2,l4,1970-01-01T00:00:00.000Z,1,1,1,0,0,0,1,9.0,",
"d2,l4,1971-01-01T00:00:00.000Z,1,1,1,0,0,0,1,5.0,",
"d2,l4,1971-04-26T00:00:00.000Z,1,1,1,0,0,0,1,13.0,",
"d2,l5,1970-01-01T00:00:00.000Z,1,1,1,0,0,0,1,8.0,",
"d2,l5,1971-01-01T00:00:00.000Z,1,1,1,0,0,0,1,7.0,",
"d2,l5,1971-08-20T00:00:00.000Z,1,1,1,1,0,0,1,15.0,",
};
sql =
"select device, level, date_bin(1d, time) as bin,"
+ "count(num) as count_num, count(*) as count_star, count(device) as count_device, count(date) as count_date, "
+ "count(attr1) as count_attr1, count(attr2) as count_attr2, count(time) as count_time, avg(num) as avg_num "
+ "from table0 group by 3, device, level order by device, level, bin";
tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME);

retArray =
new String[] {
"d1,l1,1970-01-01T00:00:00.000Z,1,1,1,0,1,1,1,3.0,",
"d1,l1,1971-01-01T00:00:00.000Z,1,1,1,0,1,1,1,6.0,",
"d1,l1,1971-01-01T00:01:40.000Z,1,1,1,0,1,1,1,11.0,",
"d1,l2,1970-01-01T00:00:00.000Z,1,1,1,0,1,1,1,2.0,",
"d1,l2,1971-01-01T00:00:00.000Z,1,1,1,0,1,1,1,10.0,",
"d1,l2,1971-04-26T17:46:40.000Z,1,1,1,0,1,1,1,12.0,",
"d1,l3,1970-01-01T00:00:00.000Z,1,1,1,0,1,1,1,1.0,",
"d1,l3,1971-01-01T00:00:00.000Z,1,1,1,0,1,1,1,4.0,",
"d1,l3,1971-04-26T17:46:40.000Z,1,1,1,0,1,1,1,14.0,",
"d1,l4,1970-01-01T00:00:00.000Z,1,1,1,0,0,0,1,9.0,",
"d1,l4,1971-01-01T00:00:01.000Z,1,1,1,0,0,0,1,5.0,",
"d1,l4,1971-04-26T18:01:40.000Z,1,1,1,0,0,0,1,13.0,",
"d1,l5,1970-01-01T00:00:00.000Z,1,1,1,0,0,0,1,8.0,",
"d1,l5,1971-01-01T00:00:10.000Z,1,1,1,0,0,0,1,7.0,",
"d1,l5,1971-08-20T11:33:20.000Z,1,1,1,1,0,0,1,15.0,",
"d2,l1,1970-01-01T00:00:00.000Z,1,1,1,0,1,1,1,3.0,",
"d2,l1,1971-01-01T00:00:00.000Z,1,1,1,0,1,1,1,6.0,",
"d2,l1,1971-01-01T00:01:40.000Z,1,1,1,0,1,1,1,11.0,",
"d2,l2,1970-01-01T00:00:00.000Z,1,1,1,0,1,0,1,2.0,",
"d2,l2,1971-01-01T00:00:00.000Z,1,1,1,0,1,0,1,10.0,",
"d2,l2,1971-04-26T17:46:40.000Z,1,1,1,0,1,0,1,12.0,",
"d2,l3,1970-01-01T00:00:00.000Z,1,1,1,0,0,0,1,1.0,",
"d2,l3,1971-01-01T00:00:00.000Z,1,1,1,0,0,0,1,4.0,",
"d2,l3,1971-04-26T17:46:40.000Z,1,1,1,0,0,0,1,14.0,",
"d2,l4,1970-01-01T00:00:00.000Z,1,1,1,0,0,0,1,9.0,",
"d2,l4,1971-01-01T00:00:01.000Z,1,1,1,0,0,0,1,5.0,",
"d2,l4,1971-04-26T18:01:40.000Z,1,1,1,0,0,0,1,13.0,",
"d2,l5,1970-01-01T00:00:00.000Z,1,1,1,0,0,0,1,8.0,",
"d2,l5,1971-01-01T00:00:10.000Z,1,1,1,0,0,0,1,7.0,",
"d2,l5,1971-08-20T11:33:20.000Z,1,1,1,1,0,0,1,15.0,",
};
sql =
"select device, level, date_bin(1s, time) as bin,"
+ "count(num) as count_num, count(*) as count_star, count(device) as count_device, count(date) as count_date, "
+ "count(attr1) as count_attr1, count(attr2) as count_attr2, count(time) as count_time, avg(num) as avg_num "
+ "from table0 group by 3, device, level order by device, level, bin";
tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME);

// TODO(beyyes) test below
// sql = "select count(*) from (\n" +
// "\tselect device, level, date_bin(1d, time) as bin, \n" +
// "\tcount(num) as count_num, count(*) as count_star, count(device) as count_device,
// count(date) as count_date, count(attr1) as count_attr1, count(attr2) as count_attr2,
// count(time) as count_time, avg(num) as avg_num \n" +
// "\tfrom table0 \n" +
// "\tgroup by 3, device, level order by device, level, bin\n" +
// ")\n";
}

// ============================ Join Test ===========================
// no filter
@Test
Expand Down
49 changes: 49 additions & 0 deletions ...iotdb/db/queryengine/execution/aggregation/timerangeiterator/ITableTimeRangeIterator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.iotdb.db.queryengine.execution.aggregation.timerangeiterator;

import org.apache.tsfile.read.common.TimeRange;

public interface ITableTimeRangeIterator {

TimeIteratorType getType();

/**
* @return whether current iterator has next time range.
*/
boolean hasNextTimeRange();

boolean hasCachedTimeRange();

TimeRange getCurTimeRange();

boolean canFinishCurrentTimeRange(long startTime);

void resetCurTimeRange();

void updateCurTimeRange(long startTime);

void setFinished();

enum TimeIteratorType {
DATE_BIN_TIME_ITERATOR,
SINGLE_TIME_ITERATOR
}
}
91 changes: 91 additions & 0 deletions ...db/queryengine/execution/aggregation/timerangeiterator/TableDateBinTimeRangeIterator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.iotdb.db.queryengine.execution.aggregation.timerangeiterator;

import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.DateBinFunctionColumnTransformer;

import org.apache.tsfile.read.common.TimeRange;

public class TableDateBinTimeRangeIterator implements ITableTimeRangeIterator {

private final DateBinFunctionColumnTransformer dateBinTransformer;

private boolean finished = false;

// left close, right close
private TimeRange curTimeRange;

public TableDateBinTimeRangeIterator(DateBinFunctionColumnTransformer dateBinTransformer) {
this.dateBinTransformer = dateBinTransformer;
}

@Override
public boolean canFinishCurrentTimeRange(long startTime) {
if (curTimeRange == null) {
return false;
}

return startTime > curTimeRange.getMax();
}

@Override
public void updateCurTimeRange(long startTime) {
long[] timeArray = dateBinTransformer.dateBinStartEnd(startTime);

if (curTimeRange != null) {
// meet new time range, remove old time range
if (timeArray[0] != curTimeRange.getMin()) {
this.curTimeRange = new TimeRange(timeArray[0], timeArray[1] - 1);
}
} else {
this.curTimeRange = new TimeRange(timeArray[0], timeArray[1] - 1);
}
}

@Override
public void setFinished() {
this.curTimeRange = null;
this.finished = true;
}

@Override
public TimeIteratorType getType() {
return TimeIteratorType.DATE_BIN_TIME_ITERATOR;
}

@Override
public boolean hasNextTimeRange() {
return !finished;
}

@Override
public boolean hasCachedTimeRange() {
return curTimeRange != null;
}

public TimeRange getCurTimeRange() {
return this.curTimeRange;
}

@Override
public void resetCurTimeRange() {
this.curTimeRange = null;
}
}
Loading