[DYOD] Support window function queries

Dockerfile

@@ -16,9 +16,9 @@ RUN apt-get update \
         cmake \
         curl \
         dos2unix \
-        g++-9 \
+        g++-10 \
         g++-11 \
-        gcc-9 \
+        gcc-10 \
         gcc-11 \
         gcovr \
         git \

Jenkinsfile

@@ -90,7 +90,7 @@ try {
             clang = '-DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++'
             clang11 = '-DCMAKE_C_COMPILER=clang-11 -DCMAKE_CXX_COMPILER=clang++-11'
             gcc = '-DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++'
-            gcc9 = '-DCMAKE_C_COMPILER=gcc-9 -DCMAKE_CXX_COMPILER=g++-9'
+            gcc10 = '-DCMAKE_C_COMPILER=gcc-10 -DCMAKE_CXX_COMPILER=g++-10'
 
             debug = '-DCMAKE_BUILD_TYPE=Debug'
             release = '-DCMAKE_BUILD_TYPE=Release'
@@ -114,7 +114,7 @@ try {
             mkdir gcc-debug && cd gcc-debug &&                                                           ${cmake} ${debug}          ${gcc}              .. &\
             mkdir gcc-release && cd gcc-release &&                                                       ${cmake} ${release}        ${gcc}              .. &\
             mkdir clang-11-debug && cd clang-11-debug &&                                                 ${cmake} ${debug}          ${clang11}          .. &\
-            mkdir gcc-9-debug && cd gcc-9-debug &&                                                       ${cmake} ${debug}          ${gcc9}             .. &\
+            mkdir gcc-10-debug && cd gcc-10-debug &&                                                       ${cmake} ${debug}          ${gcc10}             .. &\
             wait"
           }
 
@@ -133,10 +133,10 @@ try {
               sh "cd gcc-debug && make all -j \$(( \$(nproc) / 4))"
               sh "cd gcc-debug && ./hyriseTest"
             }
-          }, gcc9Debug: {
-            stage("gcc-9-debug") {
-              sh "cd gcc-9-debug && make all -j \$(( \$(nproc) / 4))"
-              sh "cd gcc-9-debug && ./hyriseTest"
+          }, gcc10Debug: {
+            stage("gcc-10-debug") {
+              sh "cd gcc-10-debug && make all -j \$(( \$(nproc) / 4))"
+              sh "cd gcc-10-debug && ./hyriseTest"
             }
           }, lint: {
             stage("Linting") {

install_dependencies.sh

@@ -50,7 +50,7 @@ if echo $REPLY | grep -E '^[Yy]$' > /dev/null; then
             echo "Installing dependencies (this may take a while)..."
             if sudo apt-get update >/dev/null; then
                 # Packages added here should also be added to the Dockerfile
-                sudo apt-get install --no-install-recommends -y autoconf bash-completion bc clang-11 clang-14 clang-format-14 clang-tidy-14 cmake curl dos2unix g++-9 gcc-9 g++-11 gcc-11 gcovr git graphviz libboost-all-dev libhwloc-dev libncurses5-dev libnuma-dev libnuma1 libpq-dev libreadline-dev libsqlite3-dev libtbb-dev lld man parallel postgresql-server-dev-all python3 python3-pip valgrind &
+                sudo apt-get install --no-install-recommends -y autoconf bash-completion bc clang-11 clang-14 clang-format-14 clang-tidy-14 cmake curl dos2unix g++-10 gcc-10 g++-11 gcc-11 gcovr git graphviz libboost-all-dev libhwloc-dev libncurses5-dev libnuma-dev libnuma1 libpq-dev libreadline-dev libsqlite3-dev libtbb-dev lld man parallel postgresql-server-dev-all python3 python3-pip valgrind &
 
                 if ! git submodule update --jobs 5 --init --recursive; then
                     echo "Error during git fetching submodules."

resources/benchmark/tpcds/query_blacklist.cfg

@@ -9,7 +9,7 @@
 #09.sql
 #10.sql
 11.sql
-12.sql
+#12.sql
 #13.sql
 14a.sql
 14b.sql
@@ -18,7 +18,7 @@
 #17.sql
 18.sql
 #19.sql
-20.sql
+#20.sql
 21.sql
 22.sql
 23a.sql
@@ -45,14 +45,14 @@
 #41.sql
 #42.sql
 #43.sql
-44.sql
+#44.sql
 #45.sql
 46.sql
 47.sql
 #48.sql
 49.sql
 #50.sql
-51.sql
+#51.sql
 #52.sql
 53.sql
 54.sql
@@ -99,5 +99,5 @@
 #95.sql
 #96.sql
 #97.sql
-98.sql
+#98.sql
 #99.sql

resources/test_data/sqlite_testrunner_queries.sql

@@ -233,6 +233,46 @@ SELECT * FROM id_int_int_int_100 WHERE NOT EXISTS (SELECT * FROM int_date WHERE
 -- exists to semi join reformulation: query not rewriteable
 SELECT * FROM id_int_int_int_100 WHERE EXISTS (SELECT * FROM int_date WHERE id_int_int_int_100.id = int_date.a) OR id < 20
 
+-- Window Functions
+SELECT id, a, b, c, RANK() OVER (PARTITION BY a ORDER BY c) AS r FROM mixed
+SELECT id, a, b, c, SUM(b) OVER (PARTITION BY a ORDER BY c ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS s FROM mixed
+SELECT a, b, c, SUM(b) OVER (PARTITION BY a ORDER BY c ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS s FROM mixed_null
+SELECT a, b, c, AVG(b) OVER (PARTITION BY a ORDER BY c ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS s FROM mixed_null
+-- multiple order by
+SELECT id, a, b, c, RANK() OVER (PARTITION BY b ORDER BY a, c) AS r FROM mixed
+SELECT id, a, b, c, RANK() OVER (PARTITION BY a, b ORDER BY c) AS r FROM mixed
+-- rows frame
+SELECT id, a, b, c, SUM(b) OVER (PARTITION BY a ORDER BY c ROWS BETWEEN 5 PRECEDING AND 5 FOLLOWING) as s FROM mixed
+SELECT id, a, b, c, SUM(b) OVER (PARTITION BY a ORDER BY c ROWS BETWEEN 3 PRECEDING AND UNBOUNDED FOLLOWING) as s FROM mixed
+SELECT id, a, b, c, SUM(b) OVER (PARTITION BY a ORDER BY c ROWS BETWEEN UNBOUNDED PRECEDING AND 10 FOLLOWING) as s FROM mixed
+-- min, max
+SELECT id, MIN(b) OVER (PARTITION BY a ORDER BY c ROWS BETWEEN 3 PRECEDING AND 1 FOLLOWING) AS m FROM mixed
+SELECT id, MAX(b) OVER (PARTITION BY a ORDER BY c ROWS BETWEEN 3 PRECEDING AND 1 FOLLOWING) AS m FROM mixed
+SELECT id, MIN(c) OVER (PARTITION BY a ORDER BY b ROWS BETWEEN 3 PRECEDING AND 1 FOLLOWING) AS m FROM mixed
+SELECT id, MAX(c) OVER (PARTITION BY a ORDER BY b ROWS BETWEEN 3 PRECEDING AND 1 FOLLOWING) AS m FROM mixed
+-- ranks
+SELECT id, RANK() OVER (PARTITION BY a ORDER BY b) AS r FROM mixed
+SELECT id, DENSE_RANK() OVER (PARTITION BY a ORDER BY b) AS r FROM mixed
+SELECT id, ROW_NUMBER() OVER (PARTITION BY a ORDER BY b) AS r FROM mixed
+-- partition on nullable column
+SELECT a, b, RANK() OVER (PARTITION BY b ORDER BY a) AS r FROM mixed_null
+SELECT a, b, RANK() OVER (PARTITION BY a ORDER BY b) AS r FROM mixed_null
+-- column expressions with offset
+SELECT id, RANK() OVER (PARTITION BY b + 2 ORDER BY c + 3) AS r FROM mixed
+SELECT id, a, b, c, SUM(b + 5) OVER (PARTITION BY a ORDER BY c + 5 ROWS BETWEEN UNBOUNDED PRECEDING AND 10 FOLLOWING) as s FROM mixed
+SELECT id, a, b, c, SUM(b + 5) OVER (PARTITION BY a ORDER BY c + 5 ROWS BETWEEN UNBOUNDED PRECEDING AND 10 FOLLOWING) as s FROM id_int_int_int_100
+-- sort modes
+SELECT id, RANK() OVER (ORDER BY b DESC, c) AS r from id_int_int_int_100;
+SELECT id, DENSE_RANK() OVER (ORDER BY b DESC, c ASC) AS r from id_int_int_int_100;
+SELECT id, ROW_NUMBER() OVER (ORDER BY b DESC, c DESC) AS r from id_int_int_int_100;
+SELECT id, AVG(a) OVER (ORDER BY b ASC, c DESC ROWS BETWEEN 3 PRECEDING AND 3 FOLLOWING) AS average from id_int_int_int_100;
+-- range mode
+SELECT id, AVG(a) OVER (ORDER BY b RANGE BETWEEN 3 PRECEDING AND 3 FOLLOWING) AS average from id_int_int_int_100;
+SELECT id, SUM(a) OVER (PARTITION BY b) AS average from id_int_int_int_100;
+-- count
+SELECT id, a, b, c, count(a) over (PARTITION BY b ORDER BY c) AS count from id_int_int_int_100;
+SELECT a, b, COUNT(d) OVER (PARTITION BY a ORDER BY b ROWS BETWEEN 3 PRECEDING AND 1 FOLLOWING) AS count FROM mixed_null
+
 -- Aggregates
 SELECT SUM(b + b) AS sum_b_b FROM mixed;
 SELECT SUM(b) + AVG(c) AS x FROM mixed GROUP BY id + 5;

resources/test_data/tbl/window_operator/avg.tbl

@@ -0,0 +1,10 @@
+a|b|AVG(b)
+int|int|double
+1|5|3.5
+1|6|4.333333333333333
+1|6|4.75
+1|7|5.2
+1|2|2
+2|1|1
+2|1|1
+2|2|1.333333333333333

resources/test_data/tbl/window_operator/dense_rank.tbl

@@ -0,0 +1,10 @@
+a|b|DENSE_RANK()
+int|int|long
+1|5|2
+1|6|3
+1|6|3
+1|7|4
+1|2|1
+2|1|1
+2|1|1
+2|2|2

resources/test_data/tbl/window_operator/input_not_unique.tbl

@@ -0,0 +1,10 @@
+a|b
+int|int
+1|5
+1|6
+1|6
+1|7
+1|2
+2|1
+2|1
+2|2

resources/test_data/tbl/window_operator/input_three_columns.tbl

@@ -0,0 +1,10 @@
+a|b|c
+int|int|int
+1|5|2
+1|6|3
+1|6|5
+1|7|6
+1|2|3
+2|1|0
+2|1|3
+2|2|2

resources/test_data/tbl/window_operator/max.tbl

@@ -0,0 +1,10 @@
+a|b|MAX(b)
+int|int|int
+1|5|5
+1|6|6
+1|6|6
+1|7|7
+1|2|2
+2|1|1
+2|1|1
+2|2|2

resources/test_data/tbl/window_operator/min.tbl

@@ -0,0 +1,10 @@
+a|b|MIN(b)
+int|int|int
+1|5|2
+1|6|2
+1|6|2
+1|7|2
+1|2|2
+2|1|1
+2|1|1
+2|2|1

resources/test_data/tbl/window_operator/prefix_sum.tbl

@@ -0,0 +1,10 @@
+a|b|SUM(b)
+int|int|long
+1|5|7
+1|6|19
+1|6|19
+1|7|26
+1|2|2
+2|1|2
+2|1|2
+2|2|4

resources/test_data/tbl/window_operator/range_avg.tbl

@@ -0,0 +1,10 @@
+a|b|AVG(b)
+int|int|double
+1|5|3.5
+1|6|5.66666666666667
+1|6|5.66666666666667
+1|7|6
+1|2|2
+2|1|1
+2|1|1
+2|2|1.33333333333333

resources/test_data/tbl/window_operator/range_max.tbl

@@ -0,0 +1,10 @@
+a|b|MAX(b)
+int|int|int
+1|5|6
+1|6|7
+1|6|7
+1|7|7
+1|2|2
+2|1|2
+2|1|2
+2|2|2

resources/test_data/tbl/window_operator/range_min.tbl

@@ -0,0 +1,10 @@
+a|b|MIN(b)
+int|int|int
+1|5|2
+1|6|5
+1|6|5
+1|7|5
+1|2|2
+2|1|1
+2|1|1
+2|2|1

resources/test_data/tbl/window_operator/rank.tbl

@@ -0,0 +1,10 @@
+a|b|RANK()
+int|int|long
+1|5|2
+1|6|3
+1|6|3
+1|7|5
+1|2|1
+2|1|1
+2|1|1
+2|2|3

resources/test_data/tbl/window_operator/rank_reverse.tbl

@@ -0,0 +1,10 @@
+a|b|RANK()
+int|int|long
+1|5|4
+1|6|2
+1|6|2
+1|7|1
+1|2|5
+2|1|2
+2|1|2
+2|2|1

resources/test_data/tbl/window_operator/row_number.tbl

@@ -0,0 +1,10 @@
+a|b|ROW_NUMBER()
+int|int|long
+1|5|2
+1|6|3
+1|6|4
+1|7|5
+1|2|1
+2|1|1
+2|1|2
+2|2|3

resources/test_data/tbl/window_operator/sum.tbl

@@ -0,0 +1,10 @@
+a|b|SUM(b)
+int|int|long
+1|5|7
+1|6|13
+1|6|19
+1|7|26
+1|2|2
+2|1|1
+2|1|2
+2|2|4

src/lib/CMakeLists.txt

@@ -281,6 +281,11 @@ set(
     operators/update.hpp
     operators/validate.cpp
     operators/validate.hpp
+    operators/window_function_evaluator.cpp
+    operators/window_function_evaluator.hpp
+    operators/window_function_evaluator/util.cpp
+    operators/window_function_evaluator/util.hpp
+    operators/window_function_evaluator/window_function_evaluator_traits.hpp
     optimizer/join_ordering/abstract_join_ordering_algorithm.cpp
     optimizer/join_ordering/abstract_join_ordering_algorithm.hpp
     optimizer/join_ordering/dp_ccp.cpp
@@ -650,6 +655,7 @@ set(
     utils/print_utils.hpp
     utils/pruning_utils.cpp
     utils/pruning_utils.hpp
+    utils/segment_tree.hpp
     utils/settings/abstract_setting.cpp
     utils/settings/abstract_setting.hpp
     utils/settings_manager.cpp

src/lib/all_type_variant.hpp

@@ -139,6 +139,14 @@ inline DataType data_type_from_all_type_variant(const AllTypeVariant& all_type_v
   return static_cast<DataType>(all_type_variant.which());
 }
 
+template <typename T>
+std::optional<T> as_optional(AllTypeVariant value) {
+  if (variant_is_null(value)) {
+    return std::nullopt;
+  }
+  return boost::get<T>(value);
+}
+
 }  // namespace hyrise
 
 namespace std {

src/lib/expression/window_expression.cpp

@@ -106,6 +106,14 @@ WindowExpression::WindowExpression(std::vector<std::shared_ptr<AbstractExpressio
             arguments.begin() + order_by_expressions_begin_idx);
 }
 
+std::span<const std::shared_ptr<AbstractExpression>> WindowExpression::partition_by_expressions() const {
+  return {arguments.begin(), order_by_expressions_begin_idx};
+}
+
+std::span<const std::shared_ptr<AbstractExpression>> WindowExpression::order_by_expressions() const {
+  return {arguments.begin() + static_cast<int64_t>(order_by_expressions_begin_idx), arguments.end()};
+}
+
 std::shared_ptr<AbstractExpression> WindowExpression::_on_deep_copy(
     std::unordered_map<const AbstractOperator*, std::shared_ptr<AbstractOperator>>& copied_ops) const {
   const auto expression_count = arguments.size();

src/lib/expression/window_expression.hpp

@@ -1,5 +1,7 @@
 #pragma once
 
+#include <span>
+
 #include "abstract_expression.hpp"
 
 namespace hyrise {
@@ -61,6 +63,9 @@ class WindowExpression : public AbstractExpression {
   FrameDescription frame_description;
   size_t order_by_expressions_begin_idx{0};
 
+  std::span<const std::shared_ptr<AbstractExpression>> partition_by_expressions() const;
+  std::span<const std::shared_ptr<AbstractExpression>> order_by_expressions() const;
+
  protected:
   std::shared_ptr<AbstractExpression> _on_deep_copy(
       std::unordered_map<const AbstractOperator*, std::shared_ptr<AbstractOperator>>& copied_ops) const override;