Parse and represent window functions in the LQP (#2574)

Add the representation of SQL:2003 window functions in the LQP, including the translation in the SQLTranslator.
hyrise · Jul 26, 2023 · b685721 · b685721
1 parent 61e7c29
commit b685721
Show file tree

Hide file tree

Showing 58 changed files with 2,161 additions and 1,051 deletions.
diff --git a/src/benchmark/operators/aggregate_benchmark.cpp b/src/benchmark/operators/aggregate_benchmark.cpp
@@ -17,8 +17,8 @@ using namespace expression_functional;  // NOLINT(build/namespaces)
 BENCHMARK_F(MicroBenchmarkBasicFixture, BM_AggregateHash)(benchmark::State& state) {
   _clear_cache();
 
-  auto aggregates = std::vector<std::shared_ptr<AggregateExpression>>{
-      std::static_pointer_cast<AggregateExpression>(min_(pqp_column_(ColumnID{1}, DataType::Int, false, "b")))};
+  auto aggregates = std::vector<std::shared_ptr<WindowFunctionExpression>>{
+      std::static_pointer_cast<WindowFunctionExpression>(min_(pqp_column_(ColumnID{1}, DataType::Int, false, "b")))};
 
   std::vector<ColumnID> groupby = {ColumnID{0} /* "a" */};
 
@@ -33,8 +33,8 @@ BENCHMARK_F(MicroBenchmarkBasicFixture, BM_AggregateHash)(benchmark::State& stat
 BENCHMARK_F(MicroBenchmarkBasicFixture, BM_AggregateSortNotSortedNoGroupBy)(benchmark::State& state) {
   _clear_cache();
 
-  auto aggregates = std::vector<std::shared_ptr<AggregateExpression>>{
-      std::static_pointer_cast<AggregateExpression>(min_(pqp_column_(ColumnID{1}, DataType::Int, false, "b")))};
+  auto aggregates = std::vector<std::shared_ptr<WindowFunctionExpression>>{
+      std::static_pointer_cast<WindowFunctionExpression>(min_(pqp_column_(ColumnID{1}, DataType::Int, false, "b")))};
 
   std::vector<ColumnID> groupby = {};
 
@@ -49,8 +49,8 @@ BENCHMARK_F(MicroBenchmarkBasicFixture, BM_AggregateSortNotSortedNoGroupBy)(benc
 BENCHMARK_F(MicroBenchmarkBasicFixture, BM_AggregateSortSortedNoGroupBy)(benchmark::State& state) {
   _clear_cache();
 
-  auto aggregates = std::vector<std::shared_ptr<AggregateExpression>>{
-      std::static_pointer_cast<AggregateExpression>(min_(pqp_column_(ColumnID{1}, DataType::Int, false, "b")))};
+  auto aggregates = std::vector<std::shared_ptr<WindowFunctionExpression>>{
+      std::static_pointer_cast<WindowFunctionExpression>(min_(pqp_column_(ColumnID{1}, DataType::Int, false, "b")))};
 
   std::vector<ColumnID> groupby = {};
 
@@ -73,8 +73,8 @@ BENCHMARK_F(MicroBenchmarkBasicFixture, BM_AggregateSortSortedNoGroupBy)(benchma
 BENCHMARK_F(MicroBenchmarkBasicFixture, BM_AggregateSortNotSortedOneGroupBy)(benchmark::State& state) {
   _clear_cache();
 
-  auto aggregates = std::vector<std::shared_ptr<AggregateExpression>>{
-      std::static_pointer_cast<AggregateExpression>(min_(pqp_column_(ColumnID{1}, DataType::Int, false, "b")))};
+  auto aggregates = std::vector<std::shared_ptr<WindowFunctionExpression>>{
+      std::static_pointer_cast<WindowFunctionExpression>(min_(pqp_column_(ColumnID{1}, DataType::Int, false, "b")))};
 
   std::vector<ColumnID> groupby = {ColumnID{1}};
 
@@ -89,8 +89,8 @@ BENCHMARK_F(MicroBenchmarkBasicFixture, BM_AggregateSortNotSortedOneGroupBy)(ben
 BENCHMARK_F(MicroBenchmarkBasicFixture, BM_AggregateSortSortedOneGroupBy)(benchmark::State& state) {
   _clear_cache();
 
-  auto aggregates = std::vector<std::shared_ptr<AggregateExpression>>{
-      std::static_pointer_cast<AggregateExpression>(min_(pqp_column_(ColumnID{1}, DataType::Int, false, "b")))};
+  auto aggregates = std::vector<std::shared_ptr<WindowFunctionExpression>>{
+      std::static_pointer_cast<WindowFunctionExpression>(min_(pqp_column_(ColumnID{1}, DataType::Int, false, "b")))};
 
   std::vector<ColumnID> groupby = {ColumnID{1}};
 
@@ -113,8 +113,8 @@ BENCHMARK_F(MicroBenchmarkBasicFixture, BM_AggregateSortSortedOneGroupBy)(benchm
 BENCHMARK_F(MicroBenchmarkBasicFixture, BM_AggregateSortNotSortedMultipleGroupBy)(benchmark::State& state) {
   _clear_cache();
 
-  auto aggregates = std::vector<std::shared_ptr<AggregateExpression>>{
-      std::static_pointer_cast<AggregateExpression>(min_(pqp_column_(ColumnID{1}, DataType::Int, false, "b")))};
+  auto aggregates = std::vector<std::shared_ptr<WindowFunctionExpression>>{
+      std::static_pointer_cast<WindowFunctionExpression>(min_(pqp_column_(ColumnID{1}, DataType::Int, false, "b")))};
 
   std::vector<ColumnID> groupby = {ColumnID{0}, ColumnID{1}};
 
@@ -129,8 +129,8 @@ BENCHMARK_F(MicroBenchmarkBasicFixture, BM_AggregateSortNotSortedMultipleGroupBy
 BENCHMARK_F(MicroBenchmarkBasicFixture, BM_AggregateSortSortedMultipleGroupBy)(benchmark::State& state) {
   _clear_cache();
 
-  auto aggregates = std::vector<std::shared_ptr<AggregateExpression>>{
-      std::static_pointer_cast<AggregateExpression>(min_(pqp_column_(ColumnID{1}, DataType::Int, false, "b")))};
+  auto aggregates = std::vector<std::shared_ptr<WindowFunctionExpression>>{
+      std::static_pointer_cast<WindowFunctionExpression>(min_(pqp_column_(ColumnID{1}, DataType::Int, false, "b")))};
 
   std::vector<ColumnID> groupby = {ColumnID{0}, ColumnID{1}};
 

diff --git a/src/benchmark/operators/join_aggregate_benchmark.cpp b/src/benchmark/operators/join_aggregate_benchmark.cpp
@@ -158,8 +158,8 @@ void BM_Join_Aggregate(benchmark::State& state) {
   auto operator_join_predicate =
       OperatorJoinPredicate(std::make_pair(ColumnID{0}, ColumnID{0}), PredicateCondition::Equals);
 
-  auto aggregates = std::vector<std::shared_ptr<AggregateExpression>>{
-      std::static_pointer_cast<AggregateExpression>(avg_(pqp_column_(ColumnID{0}, DataType::Int, false, "b")))};
+  auto aggregates = std::vector<std::shared_ptr<WindowFunctionExpression>>{
+      std::static_pointer_cast<WindowFunctionExpression>(avg_(pqp_column_(ColumnID{0}, DataType::Int, false, "b")))};
 
   std::vector<ColumnID> groupby = {ColumnID{0}, ColumnID{2}};
 

diff --git a/src/benchmark/tpch_data_micro_benchmark.cpp b/src/benchmark/tpch_data_micro_benchmark.cpp
@@ -1,8 +1,8 @@
 #include "micro_benchmark_basic_fixture.hpp"
 
 #include "benchmark_config.hpp"
-#include "expression/aggregate_expression.hpp"
 #include "expression/expression_functional.hpp"
+#include "expression/window_function_expression.hpp"
 #include "hyrise.hpp"
 #include "logical_query_plan/join_node.hpp"
 #include "logical_query_plan/lqp_translator.hpp"
@@ -218,7 +218,7 @@ BENCHMARK_F(TPCHDataMicroBenchmarkFixture, BM_ScanAggregate)(benchmark::State& s
   const auto mocked_table_scan_output = sorted_lineitem->get_output();
   const ColumnID group_by_column = l_orderkey_id;
   const std::vector<ColumnID> group_by = {l_orderkey_id};
-  const auto aggregate_expressions = std::vector<std::shared_ptr<AggregateExpression>>{
+  const auto aggregate_expressions = std::vector<std::shared_ptr<WindowFunctionExpression>>{
       count_(pqp_column_(group_by_column, mocked_table_scan_output->column_data_type(group_by_column),
                          mocked_table_scan_output->column_is_nullable(group_by_column),
                          mocked_table_scan_output->column_name(group_by_column)))};

diff --git a/src/bin/console/console.cpp b/src/bin/console/console.cpp
@@ -767,64 +767,69 @@ int Console::_visualize(const std::string& input) {
   }
 
   if (no_execute && !sql.empty() && _sql_pipeline->requires_execution()) {
-    out("Error: We do not support the visualization of multiple dependant statements in 'noexec' mode.\n");
+    out("Error: We do not support the visualization of multiple dependent statements in 'noexec' mode.\n");
     return ReturnCode::Error;
   }
 
   const auto img_filename = plan_type_str + ".png";
 
-  switch (plan_type) {
-    case PlanType::LQP:
-    case PlanType::UnoptLQP: {
-      auto lqp_roots = std::vector<std::shared_ptr<AbstractLQPNode>>{};
+  try {
+    switch (plan_type) {
+      case PlanType::LQP:
+      case PlanType::UnoptLQP: {
+        auto lqp_roots = std::vector<std::shared_ptr<AbstractLQPNode>>{};
 
-      const auto& lqps = (plan_type == PlanType::LQP) ? _sql_pipeline->get_optimized_logical_plans()
-                                                      : _sql_pipeline->get_unoptimized_logical_plans();
+        const auto& lqps = (plan_type == PlanType::LQP) ? _sql_pipeline->get_optimized_logical_plans()
+                                                        : _sql_pipeline->get_unoptimized_logical_plans();
 
-      lqp_roots.reserve(lqps.size());
+        lqp_roots.reserve(lqps.size());
 
-      for (const auto& lqp : lqps) {
-        lqp_roots.emplace_back(lqp);
-      }
+        for (const auto& lqp : lqps) {
+          lqp_roots.emplace_back(lqp);
+        }
 
-      auto visualizer = LQPVisualizer{};
-      visualizer.visualize(lqp_roots, img_filename);
-    } break;
+        auto visualizer = LQPVisualizer{};
+        visualizer.visualize(lqp_roots, img_filename);
+      } break;
 
-    case PlanType::PQP: {
-      if (!no_execute) {
-        _sql_pipeline->get_result_table();
+      case PlanType::PQP: {
+        if (!no_execute) {
+          _sql_pipeline->get_result_table();
 
-        // Store the transaction context as potentially modified by the pipeline. It might be a new context if a
-        // transaction was started or nullptr if we are in auto-commit mode or the last transaction was finished.
-        _explicitly_created_transaction_context = _sql_pipeline->transaction_context();
-      }
+          // Store the transaction context as potentially modified by the pipeline. It might be a new context if a
+          // transaction was started or nullptr if we are in auto-commit mode or the last transaction was finished.
+          _explicitly_created_transaction_context = _sql_pipeline->transaction_context();
+        }
 
-      auto visualizer = PQPVisualizer{};
-      visualizer.visualize(_sql_pipeline->get_physical_plans(), img_filename);
-    } break;
+        auto visualizer = PQPVisualizer{};
+        visualizer.visualize(_sql_pipeline->get_physical_plans(), img_filename);
+      } break;
 
-    case PlanType::Joins: {
-      out("NOTE: Join graphs will show only Cross and Inner joins, not Semi, Left, Right, Full outer, "
-          "AntiNullAsTrue and AntiNullAsFalse joins.\n");
+      case PlanType::Joins: {
+        out("NOTE: Join graphs will show only Cross and Inner joins, not Semi, Left, Right, Full outer, "
+            "AntiNullAsTrue and AntiNullAsFalse joins.\n");
 
-      auto join_graphs = std::vector<JoinGraph>{};
+        auto join_graphs = std::vector<JoinGraph>{};
 
-      const auto& lqps = _sql_pipeline->get_optimized_logical_plans();
-      for (const auto& lqp : lqps) {
-        const auto sub_lqps = lqp_find_subplan_roots(lqp);
+        const auto& lqps = _sql_pipeline->get_optimized_logical_plans();
+        for (const auto& lqp : lqps) {
+          const auto sub_lqps = lqp_find_subplan_roots(lqp);
 
-        for (const auto& sub_lqp : sub_lqps) {
-          const auto sub_lqp_join_graphs = JoinGraph::build_all_in_lqp(sub_lqp);
-          for (const auto& sub_lqp_join_graph : sub_lqp_join_graphs) {
-            join_graphs.emplace_back(sub_lqp_join_graph);
+          for (const auto& sub_lqp : sub_lqps) {
+            const auto sub_lqp_join_graphs = JoinGraph::build_all_in_lqp(sub_lqp);
+            for (const auto& sub_lqp_join_graph : sub_lqp_join_graphs) {
+              join_graphs.emplace_back(sub_lqp_join_graph);
+            }
           }
         }
-      }
 
-      auto visualizer = JoinGraphVisualizer{};
-      visualizer.visualize(join_graphs, img_filename);
-    } break;
+        auto visualizer = JoinGraphVisualizer{};
+        visualizer.visualize(join_graphs, img_filename);
+      } break;
+    }
+  } catch (const InvalidInputException& exception) {
+    out(std::string(exception.what()) + '\n');
+    return false;
   }
 
   // NOLINTBEGIN(concurrency-mt-unsafe) - system() is not thread-safe, but it's not used concurrently here.

diff --git a/src/lib/CMakeLists.txt b/src/lib/CMakeLists.txt
@@ -25,8 +25,6 @@ set(
     expression/abstract_expression.hpp
     expression/abstract_predicate_expression.cpp
     expression/abstract_predicate_expression.hpp
-    expression/aggregate_expression.cpp
-    expression/aggregate_expression.hpp
     expression/arithmetic_expression.cpp
     expression/arithmetic_expression.hpp
     expression/between_expression.cpp
@@ -81,6 +79,10 @@ set(
     expression/unary_minus_expression.hpp
     expression/value_expression.cpp
     expression/value_expression.hpp
+    expression/window_expression.cpp
+    expression/window_expression.hpp
+    expression/window_function_expression.cpp
+    expression/window_function_expression.hpp
     hyrise.cpp
     hyrise.hpp
     import_export/binary/binary_parser.cpp
@@ -164,6 +166,8 @@ set(
     logical_query_plan/update_node.hpp
     logical_query_plan/validate_node.cpp
     logical_query_plan/validate_node.hpp
+    logical_query_plan/window_node.cpp
+    logical_query_plan/window_node.hpp
     lossless_cast.cpp
     lossless_cast.hpp
     lossy_cast.hpp
@@ -180,7 +184,7 @@ set(
     operators/abstract_read_only_operator.hpp
     operators/abstract_read_write_operator.cpp
     operators/abstract_read_write_operator.hpp
-    operators/aggregate/aggregate_traits.hpp
+    operators/aggregate/window_function_traits.hpp
     operators/aggregate_hash.cpp
     operators/aggregate_hash.hpp
     operators/aggregate_sort.cpp

diff --git a/src/lib/expression/abstract_expression.cpp b/src/lib/expression/abstract_expression.cpp
@@ -8,10 +8,10 @@
 #include "expression_utils.hpp"
 #include "utils/assert.hpp"
 
-using namespace std::string_literals;  // NOLINT
-
 namespace hyrise {
 
+using namespace std::string_literals;  // NOLINT(build/namespaces)
+
 AbstractExpression::AbstractExpression(const ExpressionType init_type,
                                        const std::vector<std::shared_ptr<AbstractExpression>>& init_arguments)
     : type(init_type), arguments(init_arguments) {}

diff --git a/src/lib/expression/abstract_expression.hpp b/src/lib/expression/abstract_expression.hpp
@@ -15,7 +15,6 @@ class AbstractLQPNode;
 class AbstractOperator;
 
 enum class ExpressionType {
-  Aggregate,
   Arithmetic,
   Cast,
   Case,
@@ -33,7 +32,9 @@ enum class ExpressionType {
   PQPSubquery,
   LQPSubquery,
   UnaryMinus,
-  Value
+  Value,
+  Window,
+  WindowFunction
 };
 
 /**