[DYOD] Partial Hash Index

src/CMakeLists.txt

@@ -84,6 +84,7 @@ include_directories(
     ${PROJECT_SOURCE_DIR}/third_party/flat_hash_map
     ${PROJECT_SOURCE_DIR}/third_party/json
     ${PROJECT_SOURCE_DIR}/third_party/lz4
+    ${PROJECT_SOURCE_DIR}/third_party/robin-map/include
     ${PROJECT_SOURCE_DIR}/third_party/zstd
     ${PROJECT_SOURCE_DIR}/third_party/tpcds-result-reproduction
 )

src/benchmarklib/abstract_table_generator.cpp

@@ -9,6 +9,7 @@
 #include "operators/table_wrapper.hpp"
 #include "storage/index/group_key/composite_group_key_index.hpp"
 #include "storage/index/group_key/group_key_index.hpp"
+#include "storage/index/partial_hash/partial_hash_index.hpp"
 #include "storage/segment_iterate.hpp"
 #include "utils/format_duration.hpp"
 #include "utils/list_directory.hpp"
@@ -22,7 +23,8 @@ void to_json(nlohmann::json& json, const TableGenerationMetrics& metrics) {
           {"binary_caching_duration", metrics.binary_caching_duration.count()},
           {"sort_duration", metrics.sort_duration.count()},
           {"store_duration", metrics.store_duration.count()},
-          {"index_duration", metrics.index_duration.count()}};
+          {"index_duration", metrics.index_duration.count()},
+          {"table_index_duration", metrics.table_index_duration.count()}};
 }
 
 BenchmarkTableInfo::BenchmarkTableInfo(const std::shared_ptr<Table>& init_table) : table(init_table) {}
@@ -323,6 +325,45 @@ void AbstractTableGenerator::generate_and_store() {
   } else {
     std::cout << "- No indexes created as --indexes was not specified or set to false" << std::endl;
   }
+
+  /**
+   * Create table indexes if requested by the user
+   */
+  if (_benchmark_config->table_indexes) {
+    std::cout << "- Creating table indexes" << std::endl;
+    const auto& indexes_by_table = _indexes_by_table();
+    if (indexes_by_table.empty()) {
+      std::cout << "-  No indexes defined by benchmark" << std::endl;
+    }
+    for (const auto& [table_name, indexes] : indexes_by_table) {
+      const auto& table = table_info_by_name[table_name].table;
+
+      auto chunk_ids = std::vector<ChunkID>{};
+      for (auto chunk_id = ChunkID{0}; chunk_id < table->chunk_count(); ++chunk_id) {
+        chunk_ids.emplace_back(chunk_id);
+      }
+      for (const auto& index_columns : indexes) {
+        for (const auto& index_column : index_columns) {
+          std::cout << "-  Creating table index on " << table_name << " [ ";
+          std::cout << index_column << " ";
+          std::cout << "with chunk size"
+                    << " ";
+          std::cout << chunk_ids.size() << " ";
+          std::cout << "] " << std::flush;
+
+          Timer per_table_index_timer;
+          table->create_table_index<PartialHashIndex>(table->column_id_by_name(index_column), chunk_ids);
+
+          std::cout << "(" << per_table_index_timer.lap_formatted() << ")" << std::endl;
+          std::cout << "(" << table->column_id_by_name(index_column) << ")" << std::endl;
+        }
+      }
+    }
+    metrics.table_index_duration = timer.lap();
+    std::cout << "- Creating table indexes done (" << format_duration(metrics.table_index_duration) << ")" << std::endl;
+  } else {
+    std::cout << "- No table indexes created as --table_indexes was not specified or set to false" << std::endl;
+  }
 }
 
 std::shared_ptr<BenchmarkConfig> AbstractTableGenerator::create_benchmark_config_with_chunk_size(

src/benchmarklib/abstract_table_generator.hpp

@@ -43,6 +43,7 @@ struct TableGenerationMetrics {
   std::chrono::nanoseconds sort_duration{};
   std::chrono::nanoseconds store_duration{};
   std::chrono::nanoseconds index_duration{};
+  std::chrono::nanoseconds table_index_duration{};
 };
 
 void to_json(nlohmann::json& json, const TableGenerationMetrics& metrics);

src/benchmarklib/benchmark_config.cpp

@@ -4,8 +4,8 @@ namespace opossum {
 
 BenchmarkConfig::BenchmarkConfig(const BenchmarkMode init_benchmark_mode, const ChunkOffset init_chunk_size,
                                  const EncodingConfig& init_encoding_config, const bool init_indexes,
-                                 const int64_t init_max_runs, const Duration& init_max_duration,
-                                 const Duration& init_warmup_duration,
+                                 const bool init_table_indexes, const int64_t init_max_runs,
+                                 const Duration& init_max_duration, const Duration& init_warmup_duration,
                                  const std::optional<std::string>& init_output_file_path,
                                  const bool init_enable_scheduler, const uint32_t init_cores,
                                  const uint32_t init_clients, const bool init_enable_visualization,
@@ -14,6 +14,7 @@ BenchmarkConfig::BenchmarkConfig(const BenchmarkMode init_benchmark_mode, const
       chunk_size(init_chunk_size),
       encoding_config(init_encoding_config),
       indexes(init_indexes),
+      table_indexes(init_table_indexes),
       max_runs(init_max_runs),
       max_duration(init_max_duration),
       warmup_duration(init_warmup_duration),

src/benchmarklib/benchmark_config.hpp

@@ -19,8 +19,8 @@ using TimePoint = std::chrono::high_resolution_clock::time_point;
 class BenchmarkConfig {
  public:
   BenchmarkConfig(const BenchmarkMode init_benchmark_mode, const ChunkOffset init_chunk_size,
-                  const EncodingConfig& init_encoding_config, const bool init_indexes, const int64_t init_max_runs,
-                  const Duration& init_max_duration, const Duration& init_warmup_duration,
+                  const EncodingConfig& init_encoding_config, const bool init_indexes, const bool init_table_indexes,
+                  const int64_t init_max_runs, const Duration& init_max_duration, const Duration& init_warmup_duration,
                   const std::optional<std::string>& init_output_file_path, const bool init_enable_scheduler,
                   const uint32_t init_cores, const uint32_t init_clients, const bool init_enable_visualization,
                   const bool init_verify, const bool init_cache_binary_tables, const bool init_metrics);
@@ -31,6 +31,7 @@ class BenchmarkConfig {
   ChunkOffset chunk_size = Chunk::DEFAULT_SIZE;
   EncodingConfig encoding_config = EncodingConfig{};
   bool indexes = false;
+  bool table_indexes = false;
   int64_t max_runs = -1;
   Duration max_duration = std::chrono::seconds(60);
   Duration warmup_duration = std::chrono::seconds(0);

src/benchmarklib/benchmark_runner.cpp

@@ -494,6 +494,7 @@ cxxopts::Options BenchmarkRunner::get_basic_cli_options(const std::string& bench
     ("e,encoding", "Specify Chunk encoding as a string or as a JSON config file (for more detailed configuration, see --full_help). String options: " + encoding_strings_option, cxxopts::value<std::string>()->default_value("Dictionary"))  // NOLINT
     ("compression", "Specify vector compression as a string. Options: " + compression_strings_option, cxxopts::value<std::string>()->default_value(""))  // NOLINT
     ("indexes", "Create indexes (where defined by benchmark)", cxxopts::value<bool>()->default_value("false"))  // NOLINT
+    ("table_indexes", "Create table indexes (where defined by benchmark)", cxxopts::value<bool>()->default_value("false"))  // NOLINT
     ("scheduler", "Enable or disable the scheduler", cxxopts::value<bool>()->default_value("false")) // NOLINT
     ("cores", "Specify the number of cores used by the scheduler (if active). 0 means all available cores", cxxopts::value<uint32_t>()->default_value("0")) // NOLINT
     ("clients", "Specify how many items should run in parallel if the scheduler is active", cxxopts::value<uint32_t>()->default_value("1")) // NOLINT
@@ -531,6 +532,7 @@ nlohmann::json BenchmarkRunner::create_context(const BenchmarkConfig& config) {
       {"build_type", HYRISE_DEBUG ? "debug" : "release"},
       {"encoding", config.encoding_config.to_json()},
       {"indexes", config.indexes},
+      {"table_indexes", config.table_indexes},
       {"benchmark_mode", magic_enum::enum_name(config.benchmark_mode)},
       {"max_runs", config.max_runs},
       {"max_duration", std::chrono::duration_cast<std::chrono::nanoseconds>(config.max_duration).count()},

src/benchmarklib/cli_config_parser.cpp

@@ -88,6 +88,11 @@ BenchmarkConfig CLIConfigParser::parse_cli_options(const cxxopts::ParseResult& p
     std::cout << "- Creating indexes (as defined by the benchmark)" << std::endl;
   }
 
+  const auto table_indexes = parse_result["table_indexes"].as<bool>();
+  if (table_indexes) {
+    std::cout << "- Creating table indexes (as defined by the benchmark)" << std::endl;
+  }
+
   // Get all other variables
   const auto chunk_size = parse_result["chunk_size"].as<ChunkOffset>();
   std::cout << "- Chunk size is " << chunk_size << std::endl;
@@ -132,10 +137,10 @@ BenchmarkConfig CLIConfigParser::parse_cli_options(const cxxopts::ParseResult& p
     std::cout << "- Not tracking SQL metrics" << std::endl;
   }
 
-  return BenchmarkConfig{
-      benchmark_mode,  chunk_size,          *encoding_config, indexes, max_runs, timeout_duration,
-      warmup_duration, output_file_path,    enable_scheduler, cores,   clients,  enable_visualization,
-      verify,          cache_binary_tables, metrics};
+  return BenchmarkConfig{benchmark_mode, chunk_size,       *encoding_config,     indexes,          table_indexes,
+                         max_runs,       timeout_duration, warmup_duration,      output_file_path, enable_scheduler,
+                         cores,          clients,          enable_visualization, verify,           cache_binary_tables,
+                         metrics};
 }
 
 EncodingConfig CLIConfigParser::parse_encoding_config(const std::string& encoding_file_str) {

src/lib/CMakeLists.txt

@@ -454,6 +454,8 @@ set(
     storage/frame_of_reference_segment/frame_of_reference_segment_iterable.hpp
     storage/index/abstract_index.cpp
     storage/index/abstract_index.hpp
+    storage/index/abstract_table_index.cpp
+    storage/index/abstract_table_index.hpp
     storage/index/adaptive_radix_tree/adaptive_radix_tree_index.cpp
     storage/index/adaptive_radix_tree/adaptive_radix_tree_index.hpp
     storage/index/adaptive_radix_tree/adaptive_radix_tree_nodes.cpp
@@ -476,6 +478,10 @@ set(
     storage/index/group_key/variable_length_key_store.hpp
     storage/index/index_statistics.cpp
     storage/index/index_statistics.hpp
+    storage/index/partial_hash/partial_hash_index.cpp
+    storage/index/partial_hash/partial_hash_index.hpp
+    storage/index/partial_hash/partial_hash_index_impl.cpp
+    storage/index/partial_hash/partial_hash_index_impl.hpp
     storage/index/segment_index_type.hpp
     storage/lqp_view.cpp
     storage/lqp_view.hpp

src/lib/logical_query_plan/lqp_translator.cpp

@@ -43,6 +43,7 @@
 #include "operators/index_scan.hpp"
 #include "operators/insert.hpp"
 #include "operators/join_hash.hpp"
+// #include "operators/join_index.hpp"
 #include "operators/join_nested_loop.hpp"
 #include "operators/join_sort_merge.hpp"
 #include "operators/limit.hpp"
@@ -353,6 +354,36 @@ std::shared_ptr<AbstractOperator> LQPTranslator::_translate_join_node(
   const auto left_data_type = join_node->join_predicates().front()->arguments[0]->data_type();
   const auto right_data_type = join_node->join_predicates().front()->arguments[1]->data_type();
 
+  // comment in if JoinIndex should be tested in benchmarks
+  /*
+  const auto& left_input_type = join_node->left_input()->type;
+  const auto& right_input_type = join_node->right_input()->type;
+
+  auto left_table_type = TableType::References;
+  auto right_table_type = TableType::References;
+
+  auto index_side = std::optional<IndexSide>{};
+  if (left_input_type == LQPNodeType::StoredTable) {
+    left_table_type = TableType::Data;
+    index_side = IndexSide::Left;
+  }
+
+  if (right_input_type == LQPNodeType::StoredTable) {
+    right_table_type = TableType::Data;
+    index_side = IndexSide::Right;
+  }
+
+  Assert(left_table_type != TableType::Data || right_table_type != TableType::Data,
+         "Both input tables are data tables. Add additional logic to make a decision about the IndexSide.");
+
+  if ((primary_join_predicate.predicate_condition == PredicateCondition::Equals || primary_join_predicate.predicate_condition == PredicateCondition::NotEquals) && index_side &&
+      JoinIndex::supports({join_node->join_mode, primary_join_predicate.predicate_condition, left_data_type,
+                           right_data_type, !secondary_join_predicates.empty(), left_table_type, right_table_type,
+                           index_side})) {
+    return std::make_shared<JoinIndex>(left_input_operator, right_input_operator, join_node->join_mode,
+                                       primary_join_predicate, std::move(secondary_join_predicates), *index_side);
+  }*/
+
   // Lacking a proper cost model, we assume JoinHash is always faster than JoinSortMerge, which is faster than
   // JoinNestedLoop and thus check for an operator compatible with the JoinNode in that order
   constexpr auto JOIN_OPERATOR_PREFERENCE_ORDER =

src/lib/operators/get_table.cpp

@@ -232,8 +232,9 @@ std::shared_ptr<const Table> GetTable::_on_execute() {
     ++output_chunks_iter;
   }
 
+  // Also forward table indexes here.
   return std::make_shared<Table>(pruned_column_definitions, TableType::Data, std::move(output_chunks),
-                                 stored_table->uses_mvcc());
+                                 stored_table->uses_mvcc(), stored_table->get_table_indexes());
 }
 
 }  // namespace opossum