Fixing tpc ds queries

duckdb · Apr 1, 2020 · 525052a · 525052a
1 parent 67168f9
commit 525052a
Show file tree

Hide file tree

Showing 7 changed files with 74 additions and 4 deletions.
diff --git a/src/execution/operator/scan/physical_table_scan.cpp b/src/execution/operator/scan/physical_table_scan.cpp
@@ -51,6 +51,7 @@ void PhysicalTableScan::GetChunkInternal(ClientContext &context, DataChunk &chun
 	//! Get max value
     idx_t result_count = -1;
     do{
+        chunk.Reset();
         table.Scan(transaction, chunk, state->scan_offset, table_filters);
         if (expression){
             SelectionVector sel(STANDARD_VECTOR_SIZE);

diff --git a/src/execution/physical_plan/plan_get.cpp b/src/execution/physical_plan/plan_get.cpp
@@ -12,6 +12,14 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalGet &op) {
 	if (!op.table) {
 		return make_unique<PhysicalDummyScan>(op.types);
 	} else {
+	    for (auto& tableFilter: op.tableFilters){
+	        for (idx_t i = 0; i < op.column_ids.size(); i ++){
+	            if (tableFilter.column_index == op.column_ids[i]){
+                    tableFilter.column_index = i;
+                    break;
+	            }
+	        }
+	    }
 		dependencies.insert(op.table);
 		return make_unique<PhysicalTableScan>(op, *op.table, *op.table->storage, op.column_ids,move(op.expressions),move(op.tableFilters));
 	}

diff --git a/src/function/scalar/string/caseconvert.cpp b/src/function/scalar/string/caseconvert.cpp
@@ -27,6 +27,7 @@ template <class OP> static void caseconvert_function(Vector &input, Vector &resu
 
 		auto target = StringVector::EmptyString(result, input_length);
 		strcase<OP>(input_data, input_length, target.GetData());
+		target.Finalize();
 		return target;
 	});
 }

diff --git a/src/include/duckdb/storage/data_table.hpp b/src/include/duckdb/storage/data_table.hpp
@@ -35,7 +35,7 @@ typedef unique_ptr<vector<unique_ptr<PersistentSegment>>[]> persistent_data_t;
 class TableFilter{
 public:
     TableFilter(Value constant, ExpressionType comparison_type, idx_t column_index): constant(constant), comparison_type(comparison_type),
-    column_index(column_index){};
+                                                                                     column_index(column_index){};
 
     Value constant;
     ExpressionType comparison_type;

diff --git a/src/optimizer/filter_combiner.cpp b/src/optimizer/filter_combiner.cpp
@@ -175,8 +175,8 @@ vector<TableFilter> FilterCombiner::GenerateTableScanFilters(std::function<void(
                     auto filter_exp = equivalence_map.find(constant_value.first);
                     string rowid = "rowid";
                     if (filter_exp->second.size() == 1 && filter_exp->second[0]->type == ExpressionType::BOUND_COLUMN_REF && filter_exp->second[0]->alias != rowid){
-                        tableFilters.push_back(TableFilter(constant_value.second[i].constant,constant_value.second[i].comparison_type,
-						    static_cast<BoundColumnRefExpression*>(filter_exp->second[0])->binding.column_index));
+                        auto filter_col_exp =  static_cast<BoundColumnRefExpression*>(filter_exp->second[0]);
+                        tableFilters.push_back(TableFilter(constant_value.second[i].constant,constant_value.second[i].comparison_type,filter_col_exp->binding.column_index));
                         auto equivalence_set = filter_exp->first;
                         auto &entries = filter_exp->second;
                         auto &constant_list = constant_values.find(equivalence_set)->second;

diff --git a/src/optimizer/pushdown/pushdown_get.cpp b/src/optimizer/pushdown/pushdown_get.cpp
@@ -34,6 +34,9 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownGet(unique_ptr<LogicalOperat
       f->ExtractBindings();
       filtersToPushDown.push_back(move(f));
     });
+    for (auto&f : get.tableFilters){
+        f.column_index = get.column_ids[f.column_index];
+    }
 
     GenerateFilters();
     for (auto&f : filtersToPushDown){

diff --git a/test/optimizer/test_filter_pushdown_optimizer.cpp b/test/optimizer/test_filter_pushdown_optimizer.cpp
@@ -37,13 +37,44 @@ TEST_CASE("Test Table Filter Push Down", "[filterpushdown-optimizer]") {
     REQUIRE(plan->children[0]->expressions.size() == 1);
 }
 
+TEST_CASE("Test Table Filter Push Down Multiple Filters", "[filterpushdown-optimizer]") {
+    ExpressionHelper helper;
+    auto &con = helper.con;
+    Binder binder(*con.context);
+    Optimizer opt(binder,*con.context);
+    REQUIRE_NO_FAIL(con.Query("CREATE TABLE integers(i integer, j integer, k integer )"));
+    //! Checking if Optimizer push predicates down
+    auto tree = helper.ParseLogicalTree("SELECT k FROM integers where j = 5 and i = 10 ");
+    FilterPushdown predicatePushdown(opt);
+    //! The generated plan should be Projection ->Get (2)
+    auto plan = predicatePushdown.Rewrite(move(tree));
+    REQUIRE(plan->children[0]->type == LogicalOperatorType::GET);
+    REQUIRE(plan->children[0]->expressions.size()==2);
+}
+
+TEST_CASE("Test Table Filter All Data Types", "[filterpushdown-optimizer]") {
+    vector<string>data_types{"tinyint", "smallint", "integer", "bigint", "numeric", "real", "date", "timestamp"};
+    ExpressionHelper helper;
+    auto &con = helper.con;
+    Binder binder(*con.context);
+    Optimizer opt(binder,*con.context);
+    REQUIRE_NO_FAIL(con.Query("CREATE TABLE integers(i integer, j integer, k integer )"));
+    //! Checking if Optimizer push predicates down
+    auto tree = helper.ParseLogicalTree("SELECT k FROM integers where j = 5 and i = 10 ");
+    FilterPushdown predicatePushdown(opt);
+    //! The generated plan should be Projection ->Get (2)
+    auto plan = predicatePushdown.Rewrite(move(tree));
+    REQUIRE(plan->children[0]->type == LogicalOperatorType::GET);
+    REQUIRE(plan->children[0]->expressions.size()==2);
+}
+
 TEST_CASE("Test Table Filter Push Down Scan", "[filterpushdown-optimizer]") {
     unique_ptr<QueryResult> result;
     DuckDB db(nullptr);
     Connection con(db);
 
     vector<int> input;
-    idx_t input_size = 1000;
+    idx_t input_size = 100000;
     REQUIRE_NO_FAIL(con.Query("CREATE TABLE integers(i integer, j integer)"));
     for (idx_t i = 0; i < input_size; ++i){
         input.push_back(i);
@@ -56,4 +87,30 @@ TEST_CASE("Test Table Filter Push Down Scan", "[filterpushdown-optimizer]") {
     result = con.Query("SELECT i FROM integers where j = 99000 ");
     REQUIRE(CHECK_COLUMN(result, 0, {99000}));
 
+    result = con.Query("SELECT i FROM integers where j = 99000 and i = 20 ");
+    REQUIRE(CHECK_COLUMN(result, 0, {}));
+}
+
+
+TEST_CASE("Test Table Filter Push Down Scaaan", "[filterpushdown-optimizer]") {
+    unique_ptr<QueryResult> result;
+    DuckDB db(nullptr);
+    Connection con(db);
+
+    vector<int> input;
+    idx_t input_size = 100;
+    REQUIRE_NO_FAIL(con.Query("CREATE TABLE integers(i integer, j integer, k integer)"));
+    for (idx_t i = 0; i < input_size; ++i){
+        input.push_back(i);
+    }
+//    random_shuffle(input.begin(),input.end());
+    for (idx_t i = 0; i < input_size; ++i){
+        REQUIRE_NO_FAIL(con.Query("INSERT INTO integers VALUES("+ to_string(input[i])+ "," + to_string(input[i])+ "," + to_string(input[i])  +  ")"));
+    }
+
+    result = con.Query("SELECT i FROM integers where k = 99000 ");
+    REQUIRE(CHECK_COLUMN(result, 0, {}));
+
+    result = con.Query("SELECT i FROM integers where j = 99000 and i = 20 ");
+    REQUIRE(CHECK_COLUMN(result, 0, {}));
 }