Improve build time

.github/workflows/analysis_workflow.yml

@@ -120,7 +120,7 @@ jobs:
           cmake -P cpp/CMake/CpuCount.cmake | sed 's/^-- //' | tee -a $GITHUB_ENV
           echo "ARCTICDB_CODE_COVERAGE_BUILD=1" | tee -a $GITHUB_ENV
         env:
-          CMAKE_BUILD_PARALLEL_LEVEL: ${{vars.CMAKE_BUILD_PARALLEL_LEVEL}}
+          CMAKE_BUILD_PARALLEL_LEVEL: 8
 
       - name: Prepare C++ compilation env
         run: . build_tooling/prep_cpp_build.sh 

.github/workflows/benchmark_commits.yml

@@ -71,12 +71,16 @@ jobs:
         run: | 
           git config --global --add safe.directory .
           python -m asv run -v --show-stderr ${{ inputs.commit }}^!
+        env:
+          CMAKE_BUILD_PARALLEL_LEVEL: 8
 
       - name: Benchmark against master
         if: github.event_name == 'pull_request' && inputs.run_all_benchmarks == false
         shell: bash -el {0}
         run: |
           python -m asv continuous -v --show-stderr origin/master HEAD -f 1.15
+        env:
+          CMAKE_BUILD_PARALLEL_LEVEL: 8
 
       - name: Add results to ArcticDB database
         shell: bash -el {0}

cpp/arcticdb/CMakeLists.txt

@@ -433,10 +433,6 @@ set(arcticdb_srcs
         processing/operation_dispatch.cpp
         processing/operation_dispatch_unary.cpp
         processing/operation_dispatch_binary.cpp
-        processing/operation_dispatch_binary_eq.cpp
-        processing/operation_dispatch_binary_gt.cpp
-        processing/operation_dispatch_binary_lt.cpp
-        processing/operation_dispatch_binary_operator.cpp
         python/python_to_tensor_frame.cpp
         storage/config_resolvers.cpp
         storage/failure_simulation.cpp
@@ -797,7 +793,9 @@ if(${TEST})
             storage/test/test_storage_factory.cpp
             storage/test/test_storage_exceptions.cpp
             storage/test/test_azure_storage.cpp
+            stream/test/stream_test_common.hpp
             stream/test/stream_test_common.cpp
+            stream/test/test_store_common.hpp
             stream/test/test_aggregator.cpp
             stream/test/test_append_map.cpp
             stream/test/test_row_builder.cpp
@@ -917,13 +915,13 @@ if(${TEST})
 
     target_link_libraries(benchmarks
             PUBLIC
-            benchmark::benchmark
-            benchmark::benchmark_main
-            ${COMMON_PUBLIC_TEST_LIBRARIES}
+                benchmark::benchmark
+                benchmark::benchmark_main
             PRIVATE
-            ${AWSSDK_LINK_LIBRARIES}
-            arcticdb_core_static
-            )
+                arcticdb_core_static
+                Python::Python
+                ${AWSSDK_LINK_LIBRARIES}
+    )
 
     set(rapidcheck_srcs
             column_store/test/rapidcheck_column_store.cpp

cpp/arcticdb/async/async_store.hpp

@@ -10,11 +10,9 @@
 #include <arcticdb/async/task_scheduler.hpp>
 #include <arcticdb/util/clock.hpp>
 #include <arcticdb/storage/store.hpp>
-#include <arcticdb/entity/performance_tracing.hpp>
 #include <arcticdb/storage/library.hpp>
 #include <folly/futures/Future.h>
 #include <arcticdb/async/tasks.hpp>
-#include <arcticdb/stream/stream_utils.hpp>
 #include <arcticdb/processing/clause.hpp>
 #include <arcticdb/storage/key_segment_pair.hpp>
 
@@ -350,7 +348,7 @@ std::vector<folly::Future<bool>> batch_key_exists(
                 std::move(seg),
                 codec_,
                 encoding_version_}();
-            return std::pair<storage::KeySegmentPair, FrameSlice>(std::move(key_seg), std::move(slice));
+            return std::pair<storage::KeySegmentPair, pipelines::FrameSlice>(std::move(key_seg), std::move(slice));
         })
         .thenValue([de_dup_map](auto &&ks) -> std::pair<KeyOptSegment, pipelines::FrameSlice> {
             auto [key_seg, slice] = std::forward<decltype(ks)>(ks);

cpp/arcticdb/async/python_bindings.cpp

@@ -5,7 +5,6 @@
  * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0.
  */
 
-#include <arcticdb/python/python_utils.hpp>
 #include <arcticdb/async/python_bindings.hpp>
 #include <arcticdb/async/task_scheduler.hpp>
 

cpp/arcticdb/async/task_scheduler.hpp

@@ -7,12 +7,7 @@
 
 #pragma once
 
-#include <arcticdb/entity/atom_key.hpp>
-#include <arcticdb/entity/types.hpp>
-#include <arcticdb/util/hash.hpp>
-#include <arcticdb/util/exponential_backoff.hpp>
 #include <arcticdb/util/configs_map.hpp>
-#include <arcticdb/util/home_directory.hpp>
 #include <arcticdb/async/base_task.hpp>
 #include <arcticdb/entity/performance_tracing.hpp>
 
@@ -25,7 +20,6 @@
 #include <filesystem>
 #include <string>
 #include <fstream>
-#include <fmt/format.h>
 #include <type_traits>
 
 namespace arcticdb::async {

cpp/arcticdb/async/tasks.cpp

@@ -6,7 +6,6 @@
  */
 
 #include <arcticdb/async/tasks.hpp>
-#include <arcticdb/pipeline/read_frame.hpp>
 
 namespace arcticdb::async {
 

cpp/arcticdb/async/tasks.hpp

@@ -11,16 +11,13 @@
 #include <arcticdb/storage/library.hpp>
 #include <arcticdb/storage/storage_options.hpp>
 #include <arcticdb/entity/types.hpp>
-#include <arcticdb/util/hash.hpp>
 #include <arcticdb/stream/stream_utils.hpp>
-#include <arcticdb/stream/protobuf_mappings.hpp>
 #include <arcticdb/stream/stream_source.hpp>
 #include <arcticdb/column_store/memory_segment.hpp>
 #include <arcticdb/entity/variant_key.hpp>
 #include <arcticdb/stream/stream_sink.hpp>
 #include <arcticdb/async/base_task.hpp>
 #include <arcticdb/pipeline/frame_slice.hpp>
-#include <arcticdb/processing/processing_unit.hpp>
 #include <arcticdb/util/constructors.hpp>
 #include <arcticdb/codec/codec.hpp>
 

cpp/arcticdb/async/test/test_async.cpp

@@ -9,13 +9,10 @@
 #include <arcticdb/storage/common.hpp>
 #include <arcticdb/storage/config_resolvers.hpp>
 #include <arcticdb/storage/library_index.hpp>
-#include <arcticdb/storage/storage_factory.hpp>
 #include <arcticdb/async/async_store.hpp>
 #include <arcticdb/util/test/config_common.hpp>
 #include <arcticdb/pipeline/frame_slice.hpp>
-#include <arcticdb/util/random.h>
 
-#include <fmt/format.h>
 
 #include <string>
 #include <vector>
@@ -43,7 +40,7 @@ TEST(Async, SinkBasic) {
 
     auto seg = ac::SegmentInMemory();
     aa::EncodeAtomTask enc{
-        ac::entity::KeyType::GENERATION, ac::entity::VersionId{6}, ac::entity::NumericId{123}, ac::entity::NumericId{456}, ac::timestamp{457}, {ac::entity::NumericIndex{999}}, std::move(seg), codec_opt, ac::EncodingVersion::V2
+        ac::entity::KeyType::GENERATION, ac::entity::VersionId{6}, ac::entity::NumericId{123}, ac::entity::NumericId{456}, ac::timestamp{457}, ac::entity::NumericIndex{999}, std::move(seg), codec_opt, ac::EncodingVersion::V2
     };
 
     auto v = sched.submit_cpu_task(std::move(enc)).via(&aa::io_executor()).thenValue(aa::WriteSegmentTask{lib}).get();

cpp/arcticdb/codec/codec-inl.hpp

@@ -13,7 +13,6 @@
 #include <arcticdb/codec/passthrough.hpp>
 #include <arcticdb/codec/zstd.hpp>
 #include <arcticdb/codec/lz4.hpp>
-#include <arcticdb/codec/encoded_field.hpp>
 #include <arcticdb/codec/magic_words.hpp>
 
 #include <arcticdb/util/bitset.hpp>

cpp/arcticdb/codec/codec.cpp

@@ -6,16 +6,13 @@
  */
 #include <arcticdb/codec/codec.hpp>
 #include <arcticdb/column_store/string_pool.hpp>
-#include <arcticdb/stream/protobuf_mappings.hpp>
 #include <arcticdb/entity/performance_tracing.hpp>
-#include <arcticdb/codec/default_codecs.hpp>
 #include <arcticdb/codec/encoded_field.hpp>
 #include <arcticdb/codec/encoded_field_collection.hpp>
-
+#include <arcticdb/column_store/memory_segment.hpp>
+#include <arcticdb/codec/segment.hpp>
 
 #include <string>
-#include <google/protobuf/io/zero_copy_stream_impl.h>
-
 
 #include <arcticdb/codec/encode_common.hpp>
 

cpp/arcticdb/codec/codec.hpp

@@ -7,16 +7,16 @@
 
 #pragma once
 
-#include <arcticdb/codec/core.hpp>
-#include <arcticdb/codec/segment.hpp>
-#include <arcticdb/column_store/memory_segment.hpp>
 #include <arcticdb/entity/types.hpp>
 #include <arcticdb/codec/encode_common.hpp>
 
 namespace arcticdb {
 
 using ShapesBlockTDT = TypeDescriptorTag<DataTypeTag<DataType::INT64>, DimensionTag<Dimension::Dim0>>;
 
+class SegmentInMemory;
+class Segment;
+
 Segment encode_dispatch(
     SegmentInMemory&& in_mem_seg,
     const arcticdb::proto::encoding::VariantCodec &codec_opts,

cpp/arcticdb/codec/encoding_sizes.cpp

@@ -5,9 +5,8 @@
  * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0.
  */
 
-#include <arcticdb/codec/encoding_sizes.hpp>
-#include <arcticdb/entity/types.hpp>
-#include <arcticdb/stream/protobuf_mappings.hpp>
+#include <arcticdb/entity/protobufs.hpp>
+#include <arcticdb/entity/types_proto.hpp>
 
 namespace arcticdb::encoding_sizes {
 

cpp/arcticdb/codec/encoding_sizes.hpp

@@ -7,11 +7,11 @@
 
 #pragma once
 
-#include <arcticdb/entity/protobufs.hpp>
 #include <arcticdb/util/pb_util.hpp>
 #include <arcticdb/log/log.hpp>
 #include <arcticdb/util/preconditions.hpp>
 #include <numeric>
+#include <arcticdb/entity/protobufs.hpp>
 
 namespace arcticdb::encoding_sizes {
 

cpp/arcticdb/codec/python_bindings.cpp

@@ -13,16 +13,12 @@
 #include <arcticdb/column_store/memory_segment.hpp>
 #include <arcticdb/entity/protobufs.hpp>
 #include <arcticdb/python/python_utils.hpp>
-#include <arcticdb/util/flatten_utils.hpp>
 #include <arcticdb/util/cursored_buffer.hpp>
 #include <arcticdb/util/preconditions.hpp>
 #include <arcticdb/util/pybind_mutex.hpp>
 
-#include <fmt/format.h>
-
 #include <memory>
 #include <algorithm>
-#include <string>
 
 namespace py = pybind11;
 using namespace arcticdb::python_util;

cpp/arcticdb/codec/segment.cpp

@@ -7,12 +7,9 @@
 
 #include <arcticdb/codec/segment.hpp>
 #include <arcticdb/entity/performance_tracing.hpp>
-#include <arcticdb/stream/protobuf_mappings.hpp>
 
-#include <arcticdb/util/pb_util.hpp>
 #include <arcticdb/util/dump_bytes.hpp>
 #include <arcticdb/codec/codec.hpp>
-#include <arcticdb/codec/magic_words.hpp>
 
 namespace arcticdb {
 namespace segment_size {

cpp/arcticdb/codec/segment.hpp

@@ -7,12 +7,9 @@
 
 #pragma once
 
-#include "util/buffer.hpp"
+#include <arcticdb/util/buffer.hpp>
 #include <arcticdb/storage/common.hpp>
-#include <codec/encoding_sizes.hpp>
-#include <google/protobuf/io/zero_copy_stream_impl.h>
 #include <google/protobuf/arena.h>
-#include <util/buffer_pool.hpp>
 #include <arcticdb/entity/field_collection.hpp>
 
 #include <iostream>

cpp/arcticdb/codec/slice_data_sink.hpp

@@ -16,9 +16,9 @@ struct SliceDataSink {
     SliceDataSink(uint8_t *data, std::size_t size) : data_(data), shape_(0), current_size_(0), total_size_(size) {
     }
 
-    shape_t *allocate_shapes(std::size_t s) ARCTICDB_UNUSED {
+    entity::shape_t *allocate_shapes(std::size_t s) ARCTICDB_UNUSED {
         if (s == 0) return nullptr;
-        util::check_arg(s == 8, "expected exactly one shape, actual {}", s / sizeof(shape_t));
+        util::check_arg(s == 8, "expected exactly one shape, actual {}", s / sizeof(entity::shape_t));
         return &shape_;
     }
 
@@ -37,7 +37,7 @@ struct SliceDataSink {
 
 private:
     uint8_t *data_;
-    shape_t shape_;
+    entity::shape_t shape_;
     std::size_t current_size_;
     std::size_t total_size_;
 };

cpp/arcticdb/codec/test/test_codec.cpp

@@ -11,7 +11,6 @@
 #include <arcticdb/util/test/test_utils.hpp>
 #include <arcticdb/util/test/generators.hpp>
 #include <arcticdb/util/random.h>
-#include <arcticdb/stream/row_builder.hpp>
 #include <arcticdb/stream/aggregator.hpp>
 #include <arcticdb/codec/typed_block_encoder_impl.hpp>
 

cpp/arcticdb/column_store/chunked_buffer.hpp

@@ -9,15 +9,16 @@
 #pragma once
 
 #include <arcticdb/util/preconditions.hpp>
-#include <arcticdb/util/magic_num.hpp>
 #include <arcticdb/util/allocator.hpp>
 #include <arcticdb/util/constructors.hpp>
 #include <arcticdb/column_store/block.hpp>
 
-#include <boost/container/small_vector.hpp>
-
 #include <cstdint>
 
+#ifndef DEBUG_BUILD
+#include <boost/container/small_vector.hpp>
+#endif
+
 namespace arcticdb {
 
 /*

cpp/arcticdb/column_store/column.hpp

@@ -19,13 +19,10 @@
 #include <arcticdb/util/preconditions.hpp>
 #include <arcticdb/util/sparse_utils.hpp>
 
-#include <folly/container/Enumerate.h>
 // Compilation fails on Mac if cstdio is not included prior to folly/Function.h due to a missing definition of memalign in folly/Memory.h
 #ifdef __APPLE__
 #include <cstdio>
 #endif
-#include <folly/Function.h>
-#include <pybind11/pybind11.h>
 #include <pybind11/numpy.h>
 
 #include <concepts>

cpp/arcticdb/column_store/column_utils.hpp

@@ -9,7 +9,7 @@
 
 #include <arcticdb/column_store/memory_segment.hpp>
 #include <arcticdb/pipeline/frame_data_wrapper.hpp>
-
+#include <arcticdb/util/memory_tracing.hpp>
 #include <pybind11/pybind11.h>
 
 namespace arcticdb::detail {

cpp/arcticdb/column_store/memory_segment_impl.cpp

@@ -5,12 +5,12 @@
  * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0.
  */
 
+#include <arcticdb/column_store/column_map.hpp>
 #include <arcticdb/column_store/memory_segment_impl.hpp>
 #include <arcticdb/column_store/string_pool.hpp>
 #include <arcticdb/entity/type_utils.hpp>
 #include <arcticdb/pipeline/string_pool_utils.hpp>
 
-#include <google/protobuf/any.h>
 #include <google/protobuf/any.pb.h>
 
 

cpp/arcticdb/column_store/memory_segment_impl.hpp

@@ -11,12 +11,12 @@
 #include <arcticdb/column_store/column.hpp>
 #include <arcticdb/util/offset_string.hpp>
 #include <arcticdb/util/preconditions.hpp>
+#include <arcticdb/column_store/string_pool.hpp>
 
 #include <arcticdb/entity/timeseries_descriptor.hpp>
 #include <arcticdb/entity/performance_tracing.hpp>
 #include <arcticdb/util/magic_num.hpp>
 #include <arcticdb/util/constructors.hpp>
-#include <arcticdb/column_store/column_map.hpp>
 #include <arcticdb/entity/stream_descriptor.hpp>
 
 #include <boost/iterator/iterator_facade.hpp>
@@ -28,7 +28,7 @@ namespace google::protobuf
 }
 
 namespace arcticdb {
-
+class ColumnMap;
 class SegmentInMemoryImpl;
 
 namespace {

cpp/arcticdb/column_store/python_bindings.cpp

@@ -10,7 +10,6 @@
 #include <arcticdb/column_store/column.hpp>
 #include <arcticdb/column_store/column_data.hpp>
 #include <arcticdb/column_store/string_pool.hpp>
-#include <arcticdb/python/python_utils.hpp>
 
 namespace py = pybind11;