Use Predicates with Uncorrelated Subqueries for Dynamic Pruning

scripts/test/hyriseBenchmarkStarSchema_test.py

@@ -6,8 +6,8 @@
 def main():
     build_dir = initialize()
 
-    # Run SSB and validate its output using pexpect and check if all queries were successfully verified with sqlite.
-    arguments = dict()
+    # Run SSB, validate its output using pexpect, and check if all queries were successfully verified with sqlite.
+    arguments = {}
     arguments["--queries"] = "'1.1,1.2,2.2,3.3'"
     arguments["--scale"] = "0.01"
     arguments["--time"] = "10"

src/lib/CMakeLists.txt

@@ -611,6 +611,7 @@ set(
     utils/lossless_predicate_cast.cpp
     utils/lossless_predicate_cast.hpp
     utils/make_bimap.hpp
+    utils/map_prunable_subquery_predicates.hpp
     utils/meta_table_manager.cpp
     utils/meta_table_manager.hpp
     utils/meta_tables/abstract_meta_table.cpp

src/lib/expression/lqp_subquery_expression.cpp

@@ -82,8 +82,9 @@ bool LQPSubqueryExpression::_shallow_equals(const AbstractExpression& expression
 }
 
 size_t LQPSubqueryExpression::_shallow_hash() const {
-  // Return 0, thus forcing a hash collision for LQPSubqueryExpressions and triggering a full equality check. Since we
-  // often hash full query plans (that do not contain many LQPSubqueryExpressions), this should be fine.
+  // Return AbstractExpression::_shallow_hash() (a.k. a.0), thus forcing a hash collision for LQPSubqueryExpressions and
+  // triggering a full equality check. Though we often hash entire query plans, we expect most plans to contain only few
+  // LQPSubqueryExpressions. Thus, these hash collisions should be fine.
   return AbstractExpression::_shallow_hash();
 }
 

src/lib/logical_query_plan/abstract_lqp_node.cpp

@@ -17,6 +17,7 @@
 #include "predicate_node.hpp"
 #include "update_node.hpp"
 #include "utils/assert.hpp"
+#include "utils/map_prunable_subquery_predicates.hpp"
 #include "utils/print_utils.hpp"
 
 namespace {
@@ -242,34 +243,10 @@ size_t AbstractLQPNode::output_count() const {
 std::shared_ptr<AbstractLQPNode> AbstractLQPNode::deep_copy(LQPNodeMapping node_mapping) const {
   const auto copy = _deep_copy_impl(node_mapping);
 
-  // Predicates that contain uncorrelated subqueries cannot be used for chunk pruning in the optimization phase since we
-  // do not know the predicate value yet. However, the ChunkPruningRule attaches the corresponding PredicateNodes to the
-  // StoreTableNode of the table the predicates are performed on. We attach the translated Predicates (i.e., TableScans)
-  // to the GetTable operators so they can use them for pruning during execution, when the subqueries might have already
-  // been executed and the predicate value is known. During a deep_copy, we must set the copied PredicateNodes as
-  // prunable subquery predicates of the StoredTableNode after copying: Due to the recursion into the inputs of each
-  // LQP node, the PredicateNodes are copied after the StoredTableNodes.
-  for (const auto& [node, node_copy] : node_mapping) {
-    if (node->type != LQPNodeType::StoredTable) {
-      continue;
-    }
-
-    const auto& stored_table_node = static_cast<const StoredTableNode&>(*node);
-    const auto& prunable_subquery_predicates = stored_table_node.prunable_subquery_predicates();
-    if (prunable_subquery_predicates.empty()) {
-      continue;
-    }
-
-    // Find the copies of the original PredicateNodes and set them as prunable subquery predicates of the
-    // StoredTableNode copy.
-    auto prunable_subquery_predicates_copy = std::vector<std::weak_ptr<AbstractLQPNode>>{};
-    prunable_subquery_predicates_copy.reserve(prunable_subquery_predicates.size());
-    for (const auto& predicate_node : prunable_subquery_predicates) {
-      DebugAssert(node_mapping.contains(predicate_node), "Could not find referenced node. LQP is invalid.");
-      prunable_subquery_predicates_copy.emplace_back(node_mapping.at(predicate_node));
-    }
-    static_cast<StoredTableNode&>(*node_copy).set_prunable_subquery_predicates(prunable_subquery_predicates_copy);
-  }
+  // StoredTableNodes can store references to PredicateNodes as prunable subquery predicates (see get_table.hpp for
+  // details). We must assign the copies of these PredicateNodes after copying the entire LQP (see
+  // map_prunable_subquery_predicates.hpp).
+  map_prunable_subquery_predicates(node_mapping);
 
   return copy;
 }

src/lib/logical_query_plan/lqp_translator.cpp

@@ -67,39 +67,17 @@
 #include "union_node.hpp"
 #include "update_node.hpp"
 #include "utils/column_pruning_utils.hpp"
+#include "utils/map_prunable_subquery_predicates.hpp"
 
 namespace hyrise {
 
 std::shared_ptr<AbstractOperator> LQPTranslator::translate_node(const std::shared_ptr<AbstractLQPNode>& node) const {
   const auto pqp = _translate_node_recursively(node);
 
-  // Predicates that contain uncorrelated subqueries cannot be used for chunk pruning in the optimization phase since we
-  // do not know the predicate value yet. However, the ChunkPruningRule attaches the corresponding PredicateNodes to the
-  // StoreTableNode of the table the predicates are performed on. We attach the translated Predicates (i.e., TableScans)
-  // to the GetTable operators so they can use them for pruning during execution, when the subqueries might have already
-  // been executed and the predicate value is known. We must set the PredicateNodes after translation: Due to the
-  // recursion into the inputs of each LQP node, the PredicateNodes are translated after the StoredTableNodes.
-  for (const auto& [_, op] : _operator_by_lqp_node) {
-    if (op->type() != OperatorType::GetTable) {
-      continue;
-    }
-
-    DebugAssert(op->lqp_node->type == LQPNodeType::StoredTable, "Traslated GetTable operator from wrong LQP node.");
-    const auto& stored_table_node = static_cast<const StoredTableNode&>(*op->lqp_node);
-    const auto& prunable_lqp_predicates = stored_table_node.prunable_subquery_predicates();
-    if (prunable_lqp_predicates.empty()) {
-      continue;
-    }
-
-    auto prunable_pqp_predicates = std::vector<std::weak_ptr<const AbstractOperator>>{};
-    prunable_pqp_predicates.reserve(prunable_lqp_predicates.size());
-    for (const auto& predicate : prunable_lqp_predicates) {
-      DebugAssert(_operator_by_lqp_node.contains(predicate), "Could not find referenced node. LQP/PQP is invalid.");
-      prunable_pqp_predicates.emplace_back(_operator_by_lqp_node.at(predicate));
-    }
-
-    static_cast<GetTable&>(*op).set_prunable_subquery_scans(prunable_pqp_predicates);
-  }
+  // StoredTableNodes can store references to PredicateNodes as prunable subquery predicates (see get_table.hpp for
+  // details). We must assign the TableScans translated from these PredicateNodes after translating the entire LQP (see
+  // map_prunable_subquery_predicates.hpp).
+  map_prunable_subquery_predicates(_operator_by_lqp_node);
 
   return pqp;
 }

src/lib/logical_query_plan/stored_table_node.cpp

@@ -181,18 +181,18 @@ size_t StoredTableNode::_on_shallow_hash() const {
   for (const auto& pruned_column_id : _pruned_column_ids) {
     boost::hash_combine(hash, static_cast<size_t>(pruned_column_id));
   }
-  // We intentionally firce a hash collision for StoredTableNodes with the same number of (but different) prunable
-  // subquery predicates. Since we assume that (i) these predicates are not often set and (ii) we do hash LQPs often,
-  // this reduces the hash overhead, makes the code simpler, and triggers an in-depth equality check for the rare cases
-  // with prunable subquery predicates.
+  // We intentionally force a hash collision for StoredTableNodes with the same number of prunable subquery predicates
+  // even though these predicates are different. Since we assume that (i) these predicates are not often set and (ii) we
+  // hash LQPs often, this reduces the hash overhead, makes the code simpler, and triggers an in-depth equality check
+  // for the rare cases with (the same number of) prunable subquery predicates.
   boost::hash_combine(hash, _prunable_subquery_predicates.size());
   return hash;
 }
 
 std::shared_ptr<AbstractLQPNode> StoredTableNode::_on_shallow_copy(LQPNodeMapping& /*node_mapping*/) const {
   // We cannot copy _prunable_subquery_predicated here since deep_copy() recurses into the input nodes and the
   // StoredTableNodes are the first ones to be copied. Instead, AbstractLQPNode::deep_copy() sets the copied
-  // PredicateNodes after the whole LQP has been copied.
+  // PredicateNodes after the entire LQP has been copied.
   const auto copy = make(table_name);
   copy->set_pruned_chunk_ids(_pruned_chunk_ids);
   copy->set_pruned_column_ids(_pruned_column_ids);

src/lib/operators/abstract_operator.cpp

@@ -13,6 +13,7 @@
 #include "utils/assert.hpp"
 #include "utils/format_bytes.hpp"
 #include "utils/format_duration.hpp"
+#include "utils/map_prunable_subquery_predicates.hpp"
 #include "utils/print_utils.hpp"
 #include "utils/timer.hpp"
 
@@ -201,34 +202,10 @@ std::shared_ptr<AbstractOperator> AbstractOperator::deep_copy() const {
   auto copied_ops = std::unordered_map<const AbstractOperator*, std::shared_ptr<AbstractOperator>>{};
   const auto copy = deep_copy(copied_ops);
 
-  // Predicates that contain uncorrelated subqueries cannot be used for chunk pruning in the optimization phase since we
-  // do not know the predicate value yet. However, the ChunkPruningRule attaches the corresponding PredicateNodes to the
-  // StoreTableNode of the table the predicates are performed on. We attach the translated Predicates (i.e., TableScans)
-  // to the GetTable operators so they can use them for pruning during execution, when the subqueries might have already
-  // been executed and the predicate value is known. During a deep_copy, we must set the copied TableScan operators as
-  // prunable subquery scans of the GetTable operator after copying: Due to the recursion into the inputs of each
-  // operator, the TableSans are copied after the GetTable operators.
-  for (const auto& [op, op_copy] : copied_ops) {
-    if (op->type() != OperatorType::GetTable) {
-      continue;
-    }
-
-    const auto& get_table = static_cast<const GetTable&>(*op);
-    const auto& prunable_subquery_scans = get_table.prunable_subquery_scans();
-    if (prunable_subquery_scans.empty()) {
-      continue;
-    }
-
-    // Find the copies of the original TableScans and set them as prunable subquery scans of the GetTable copy.
-    auto prunable_subquery_scans_copy = std::vector<std::weak_ptr<const AbstractOperator>>{};
-    prunable_subquery_scans_copy.reserve(prunable_subquery_scans.size());
-    for (const auto& table_scan : prunable_subquery_scans) {
-      DebugAssert(copied_ops.contains(table_scan.get()), "Could not find referenced operator. PQP is invalid.");
-      prunable_subquery_scans_copy.emplace_back(copied_ops.at(table_scan.get()));
-    }
-
-    static_cast<GetTable&>(*op_copy).set_prunable_subquery_scans(prunable_subquery_scans_copy);
-  }
+  // GetTable operators can store references to TableScans as prunable subquery predicates (see get_table.hpp for
+  // details). We must assign the copies of these TableScans after copying the entire PQP (see
+  // map_prunable_subquery_predicates.hpp).
+  map_prunable_subquery_predicates(copied_ops);
 
   return copy;
 }

src/lib/operators/get_table.cpp

@@ -81,15 +81,16 @@ const std::vector<ColumnID>& GetTable::pruned_column_ids() const {
   return _pruned_column_ids;
 }
 
-void GetTable::set_prunable_subquery_scans(std::vector<std::weak_ptr<const AbstractOperator>> subquery_scans) const {
+void GetTable::set_prunable_subquery_predicates(
+    const std::vector<std::weak_ptr<const AbstractOperator>>& subquery_scans) const {
   DebugAssert(std::all_of(subquery_scans.cbegin(), subquery_scans.cend(),
                           [](const auto& op) { return op.lock() && op.lock()->type() == OperatorType::TableScan; }),
               "No TableScan set as prunable predicate.");
 
   _prunable_subquery_scans = subquery_scans;
 }
 
-std::vector<std::shared_ptr<const AbstractOperator>> GetTable::prunable_subquery_scans() const {
+std::vector<std::shared_ptr<const AbstractOperator>> GetTable::prunable_subquery_predicates() const {
   auto subquery_scans = std::vector<std::shared_ptr<const AbstractOperator>>{};
   subquery_scans.reserve(_prunable_subquery_scans.size());
   for (const auto& subquery_scan_ref : _prunable_subquery_scans) {
@@ -138,10 +139,10 @@ std::shared_ptr<const Table> GetTable::_on_execute() {
   // flag, too, it needs to be forwarded here; otherwise it would be completely invisible in the PQP.
   DebugAssert(stored_table->value_clustered_by().empty(), "GetTable does not forward value_clustered_by");
   auto overall_pruned_chunk_ids = _prune_chunks_dynamically();
-  auto excluded_chunk_ids = std::vector<ChunkID>{};
   overall_pruned_chunk_ids.insert(_pruned_chunk_ids.cbegin(), _pruned_chunk_ids.cend());
   auto pruned_chunk_ids_iter = overall_pruned_chunk_ids.begin();
-  for (ChunkID stored_chunk_id{0}; stored_chunk_id < chunk_count; ++stored_chunk_id) {
+  auto excluded_chunk_ids = std::vector<ChunkID>{};
+  for (auto stored_chunk_id = ChunkID{0}; stored_chunk_id < chunk_count; ++stored_chunk_id) {
     // Check whether the Chunk is pruned
     if (pruned_chunk_ids_iter != overall_pruned_chunk_ids.end() && *pruned_chunk_ids_iter == stored_chunk_id) {
       ++pruned_chunk_ids_iter;
@@ -315,18 +316,20 @@ std::set<ChunkID> GetTable::_prune_chunks_dynamically() {
     return {};
   }
 
+  // Create a dummy PredicateNode for each predicate containing a subquery that has already been executed. We do not use
+  // the original predicate to ignore all other nodes between the StoredTableNode and the PredicateNodes. Since the
+  // ChunkPruningRule already took care to add only predicates that are safe to prune with, we can act as if there were
+  // no other LQP nodes.
+  auto prunable_predicate_nodes = std::vector<std::shared_ptr<PredicateNode>>{};
+  prunable_predicate_nodes.reserve(_prunable_subquery_scans.size());
+
   // Create a dummy StoredTableNode from the table to retrieve. `compute_chunk_exclude_list` modifies the node's
   // statistics and we want to avoid that. We cannot use `deep_copy()` here since it would complain that the referenced
   // prunable PredicateNodes are not part of the LQP.
   const auto& stored_table_node = static_cast<const StoredTableNode&>(*lqp_node);
   const auto dummy_stored_table_node = StoredTableNode::make(_name);
 
-  // Create a dummy PredicateNode for each predicate containing a subquery that has already been executed. We do not use
-  // the original predicate to ignore any other nodes in between. Since the ChunkPruningRule already took care to add
-  // only predicates that are safe to prune with, we can act as if there were no other LQP nodes.
-  auto prunable_predicate_nodes = std::vector<std::shared_ptr<PredicateNode>>{};
-  prunable_predicate_nodes.reserve(_prunable_subquery_scans.size());
-  for (const auto& op : prunable_subquery_scans()) {
+  for (const auto& op : prunable_subquery_predicates()) {
     const auto& table_scan = static_cast<const TableScan&>(*op);
     const auto& operator_predicate_arguments = table_scan.predicate()->arguments;
     const auto& predicate_node = static_cast<const PredicateNode&>(*table_scan.lqp_node);
@@ -340,7 +343,7 @@ std::set<ChunkID> GetTable::_prune_chunks_dynamically() {
       // Replace any column with the respective column from our dummy StoredTableNode.
       if (const auto lqp_column = std::dynamic_pointer_cast<LQPColumnExpression>(argument)) {
         Assert(*lqp_column->original_node.lock() == stored_table_node,
-               "Predicate is performed on wrong StoredTableNode");
+               "Predicate is performed on wrong StoredTableNode.");
         argument = lqp_column_(dummy_stored_table_node, lqp_column->original_column_id);
         continue;
       }
@@ -350,15 +353,27 @@ std::set<ChunkID> GetTable::_prune_chunks_dynamically() {
         continue;
       }
       Assert(operator_predicate_arguments[expression_idx]->type == ExpressionType::PQPSubquery,
-             "Cannot resolve PQPSubqueryExpression");
+             "Cannot resolve PQPSubqueryExpression.");
       const auto& subquery = static_cast<PQPSubqueryExpression&>(*operator_predicate_arguments[expression_idx]);
       if (subquery.is_correlated()) {
         continue;
       }
 
       // Ignore the subquery if it has not been executed yet. A reason might be that scheduling the subquery before the
       // GetTable operator would create a cycle. For instance, this can happen for a query like this:
-      // SELECT ... FROM a_table WHERE x > (SELECT AVG(x) FROM a_table);
+      // SELECT * FROM a_table WHERE x > (SELECT AVG(x) FROM a_table);
+      // The PQP of the query looks like the following:
+      //
+      //     [TableScan] x > SUBQUERY
+      //          |             *
+      //          |             * uncorrelated subquery
+      //          |             *
+      //          |      [AggregateHash] AVG(x)
+      //          |       /
+      //         [GetTable] a_table
+      //
+      // We cannot schedule the AggregateHash operator before the GetTable operator to obtain the subquery result for
+      // pruning: the OperatorTasks wrapping both operators would be in a circular wait for each other.
       if (subquery.pqp->state() != OperatorState::ExecutedAndAvailable) {
         continue;
       }

src/lib/operators/get_table.hpp

@@ -28,11 +28,13 @@ class GetTable : public AbstractReadOnlyOperator {
   const std::vector<ChunkID>& pruned_chunk_ids() const;
   const std::vector<ColumnID>& pruned_column_ids() const;
 
-  // We cannot use predicates with uncorrelated subqueries to get pruned ChunkIDs during optimization. However, we can
-  // reference these predicates and keep track of them in the plan. Once we execute the plan, the subqueries might have
-  // already been executed, so we can use them for pruning during execution.
-  void set_prunable_subquery_scans(std::vector<std::weak_ptr<const AbstractOperator>> subquery_scans) const;
-  std::vector<std::shared_ptr<const AbstractOperator>> prunable_subquery_scans() const;
+  // Predicates that contain uncorrelated subqueries cannot be used for chunk pruning in the optimization phase since we
+  // do not know the predicate value yet. However, the ChunkPruningRule attaches the corresponding PredicateNodes to the
+  // StoredTableNode of the table the predicates are performed on. We attach the translated predicates (i.e.,
+  // TableScans) to the GetTable operators so they can use them for pruning during execution ("dynamic pruning"), when
+  // the subqueries might have already been executed and the predicate value is known.
+  void set_prunable_subquery_predicates(const std::vector<std::weak_ptr<const AbstractOperator>>& subquery_scans) const;
+  std::vector<std::shared_ptr<const AbstractOperator>> prunable_subquery_predicates() const;
 
  protected:
   std::shared_ptr<AbstractOperator> _on_deep_copy(