diff --git a/.github/workflows/analysis_workflow.yml b/.github/workflows/analysis_workflow.yml index 41f260be4d..fde7a29e43 100644 --- a/.github/workflows/analysis_workflow.yml +++ b/.github/workflows/analysis_workflow.yml @@ -120,7 +120,7 @@ jobs: cmake -P cpp/CMake/CpuCount.cmake | sed 's/^-- //' | tee -a $GITHUB_ENV echo "ARCTICDB_CODE_COVERAGE_BUILD=1" | tee -a $GITHUB_ENV env: - CMAKE_BUILD_PARALLEL_LEVEL: ${{vars.CMAKE_BUILD_PARALLEL_LEVEL}} + CMAKE_BUILD_PARALLEL_LEVEL: 8 - name: Prepare C++ compilation env run: . build_tooling/prep_cpp_build.sh diff --git a/.github/workflows/benchmark_commits.yml b/.github/workflows/benchmark_commits.yml index 8984a49051..68789ce581 100644 --- a/.github/workflows/benchmark_commits.yml +++ b/.github/workflows/benchmark_commits.yml @@ -71,12 +71,16 @@ jobs: run: | git config --global --add safe.directory . python -m asv run -v --show-stderr ${{ inputs.commit }}^! + env: + CMAKE_BUILD_PARALLEL_LEVEL: 8 - name: Benchmark against master if: github.event_name == 'pull_request' && inputs.run_all_benchmarks == false shell: bash -el {0} run: | python -m asv continuous -v --show-stderr origin/master HEAD -f 1.15 + env: + CMAKE_BUILD_PARALLEL_LEVEL: 8 - name: Add results to ArcticDB database shell: bash -el {0} diff --git a/cpp/arcticdb/CMakeLists.txt b/cpp/arcticdb/CMakeLists.txt index 9e1eb0daeb..c68636850d 100644 --- a/cpp/arcticdb/CMakeLists.txt +++ b/cpp/arcticdb/CMakeLists.txt @@ -433,10 +433,6 @@ set(arcticdb_srcs processing/operation_dispatch.cpp processing/operation_dispatch_unary.cpp processing/operation_dispatch_binary.cpp - processing/operation_dispatch_binary_eq.cpp - processing/operation_dispatch_binary_gt.cpp - processing/operation_dispatch_binary_lt.cpp - processing/operation_dispatch_binary_operator.cpp python/python_to_tensor_frame.cpp storage/config_resolvers.cpp storage/failure_simulation.cpp @@ -797,7 +793,9 @@ if(${TEST}) storage/test/test_storage_factory.cpp storage/test/test_storage_exceptions.cpp storage/test/test_azure_storage.cpp + stream/test/stream_test_common.hpp stream/test/stream_test_common.cpp + stream/test/test_store_common.hpp stream/test/test_aggregator.cpp stream/test/test_append_map.cpp stream/test/test_row_builder.cpp @@ -917,13 +915,13 @@ if(${TEST}) target_link_libraries(benchmarks PUBLIC - benchmark::benchmark - benchmark::benchmark_main - ${COMMON_PUBLIC_TEST_LIBRARIES} + benchmark::benchmark + benchmark::benchmark_main PRIVATE - ${AWSSDK_LINK_LIBRARIES} - arcticdb_core_static - ) + arcticdb_core_static + Python::Python + ${AWSSDK_LINK_LIBRARIES} + ) set(rapidcheck_srcs column_store/test/rapidcheck_column_store.cpp diff --git a/cpp/arcticdb/async/async_store.hpp b/cpp/arcticdb/async/async_store.hpp index 6f2d6c9413..b89a202b93 100644 --- a/cpp/arcticdb/async/async_store.hpp +++ b/cpp/arcticdb/async/async_store.hpp @@ -10,11 +10,9 @@ #include #include #include -#include #include #include #include -#include #include #include @@ -350,7 +348,7 @@ std::vector> batch_key_exists( std::move(seg), codec_, encoding_version_}(); - return std::pair(std::move(key_seg), std::move(slice)); + return std::pair(std::move(key_seg), std::move(slice)); }) .thenValue([de_dup_map](auto &&ks) -> std::pair { auto [key_seg, slice] = std::forward(ks); diff --git a/cpp/arcticdb/async/python_bindings.cpp b/cpp/arcticdb/async/python_bindings.cpp index 1dab925a2a..c0f6418d9d 100644 --- a/cpp/arcticdb/async/python_bindings.cpp +++ b/cpp/arcticdb/async/python_bindings.cpp @@ -5,7 +5,6 @@ * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0. */ -#include #include #include diff --git a/cpp/arcticdb/async/task_scheduler.hpp b/cpp/arcticdb/async/task_scheduler.hpp index 91a7a18222..50bab8b33f 100644 --- a/cpp/arcticdb/async/task_scheduler.hpp +++ b/cpp/arcticdb/async/task_scheduler.hpp @@ -7,12 +7,7 @@ #pragma once -#include -#include -#include -#include #include -#include #include #include @@ -25,7 +20,6 @@ #include #include #include -#include #include namespace arcticdb::async { diff --git a/cpp/arcticdb/async/tasks.cpp b/cpp/arcticdb/async/tasks.cpp index b99bcf91e6..779a9c219b 100644 --- a/cpp/arcticdb/async/tasks.cpp +++ b/cpp/arcticdb/async/tasks.cpp @@ -6,7 +6,6 @@ */ #include -#include namespace arcticdb::async { diff --git a/cpp/arcticdb/async/tasks.hpp b/cpp/arcticdb/async/tasks.hpp index 5e870a3eae..1536b98639 100644 --- a/cpp/arcticdb/async/tasks.hpp +++ b/cpp/arcticdb/async/tasks.hpp @@ -11,16 +11,13 @@ #include #include #include -#include #include -#include #include #include #include #include #include #include -#include #include #include diff --git a/cpp/arcticdb/async/test/test_async.cpp b/cpp/arcticdb/async/test/test_async.cpp index 50111632ce..9d5a2b5b0a 100644 --- a/cpp/arcticdb/async/test/test_async.cpp +++ b/cpp/arcticdb/async/test/test_async.cpp @@ -9,13 +9,10 @@ #include #include #include -#include #include #include #include -#include -#include #include #include @@ -43,7 +40,7 @@ TEST(Async, SinkBasic) { auto seg = ac::SegmentInMemory(); aa::EncodeAtomTask enc{ - ac::entity::KeyType::GENERATION, ac::entity::VersionId{6}, ac::entity::NumericId{123}, ac::entity::NumericId{456}, ac::timestamp{457}, {ac::entity::NumericIndex{999}}, std::move(seg), codec_opt, ac::EncodingVersion::V2 + ac::entity::KeyType::GENERATION, ac::entity::VersionId{6}, ac::entity::NumericId{123}, ac::entity::NumericId{456}, ac::timestamp{457}, ac::entity::NumericIndex{999}, std::move(seg), codec_opt, ac::EncodingVersion::V2 }; auto v = sched.submit_cpu_task(std::move(enc)).via(&aa::io_executor()).thenValue(aa::WriteSegmentTask{lib}).get(); diff --git a/cpp/arcticdb/codec/codec-inl.hpp b/cpp/arcticdb/codec/codec-inl.hpp index 4adcfc6f76..8e1e0776a8 100644 --- a/cpp/arcticdb/codec/codec-inl.hpp +++ b/cpp/arcticdb/codec/codec-inl.hpp @@ -13,7 +13,6 @@ #include #include #include -#include #include #include diff --git a/cpp/arcticdb/codec/codec.cpp b/cpp/arcticdb/codec/codec.cpp index 2c21c6ad34..b99334d245 100644 --- a/cpp/arcticdb/codec/codec.cpp +++ b/cpp/arcticdb/codec/codec.cpp @@ -6,16 +6,13 @@ */ #include #include -#include #include -#include #include #include - +#include +#include #include -#include - #include diff --git a/cpp/arcticdb/codec/codec.hpp b/cpp/arcticdb/codec/codec.hpp index 28bc60c806..2820a13493 100644 --- a/cpp/arcticdb/codec/codec.hpp +++ b/cpp/arcticdb/codec/codec.hpp @@ -7,9 +7,6 @@ #pragma once -#include -#include -#include #include #include @@ -17,6 +14,9 @@ namespace arcticdb { using ShapesBlockTDT = TypeDescriptorTag, DimensionTag>; +class SegmentInMemory; +class Segment; + Segment encode_dispatch( SegmentInMemory&& in_mem_seg, const arcticdb::proto::encoding::VariantCodec &codec_opts, diff --git a/cpp/arcticdb/codec/encoding_sizes.cpp b/cpp/arcticdb/codec/encoding_sizes.cpp index 393b3ba8f1..31cb9fca36 100644 --- a/cpp/arcticdb/codec/encoding_sizes.cpp +++ b/cpp/arcticdb/codec/encoding_sizes.cpp @@ -5,9 +5,8 @@ * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0. */ -#include -#include -#include +#include +#include namespace arcticdb::encoding_sizes { diff --git a/cpp/arcticdb/codec/encoding_sizes.hpp b/cpp/arcticdb/codec/encoding_sizes.hpp index ca29b6bf05..06cefb7816 100644 --- a/cpp/arcticdb/codec/encoding_sizes.hpp +++ b/cpp/arcticdb/codec/encoding_sizes.hpp @@ -7,11 +7,11 @@ #pragma once -#include #include #include #include #include +#include namespace arcticdb::encoding_sizes { diff --git a/cpp/arcticdb/codec/python_bindings.cpp b/cpp/arcticdb/codec/python_bindings.cpp index eeddaffd12..8474bd8bee 100644 --- a/cpp/arcticdb/codec/python_bindings.cpp +++ b/cpp/arcticdb/codec/python_bindings.cpp @@ -13,16 +13,12 @@ #include #include #include -#include #include #include #include -#include - #include #include -#include namespace py = pybind11; using namespace arcticdb::python_util; diff --git a/cpp/arcticdb/codec/segment.cpp b/cpp/arcticdb/codec/segment.cpp index 5eeab44b0d..ae3a0b01fd 100644 --- a/cpp/arcticdb/codec/segment.cpp +++ b/cpp/arcticdb/codec/segment.cpp @@ -7,12 +7,9 @@ #include #include -#include -#include #include #include -#include namespace arcticdb { namespace segment_size { diff --git a/cpp/arcticdb/codec/segment.hpp b/cpp/arcticdb/codec/segment.hpp index 2e8b344843..3d9fc8638d 100644 --- a/cpp/arcticdb/codec/segment.hpp +++ b/cpp/arcticdb/codec/segment.hpp @@ -7,12 +7,9 @@ #pragma once -#include "util/buffer.hpp" +#include #include -#include -#include #include -#include #include #include diff --git a/cpp/arcticdb/codec/slice_data_sink.hpp b/cpp/arcticdb/codec/slice_data_sink.hpp index 0cc9d4374f..b85772aa14 100644 --- a/cpp/arcticdb/codec/slice_data_sink.hpp +++ b/cpp/arcticdb/codec/slice_data_sink.hpp @@ -16,9 +16,9 @@ struct SliceDataSink { SliceDataSink(uint8_t *data, std::size_t size) : data_(data), shape_(0), current_size_(0), total_size_(size) { } - shape_t *allocate_shapes(std::size_t s) ARCTICDB_UNUSED { + entity::shape_t *allocate_shapes(std::size_t s) ARCTICDB_UNUSED { if (s == 0) return nullptr; - util::check_arg(s == 8, "expected exactly one shape, actual {}", s / sizeof(shape_t)); + util::check_arg(s == 8, "expected exactly one shape, actual {}", s / sizeof(entity::shape_t)); return &shape_; } @@ -37,7 +37,7 @@ struct SliceDataSink { private: uint8_t *data_; - shape_t shape_; + entity::shape_t shape_; std::size_t current_size_; std::size_t total_size_; }; diff --git a/cpp/arcticdb/codec/test/test_codec.cpp b/cpp/arcticdb/codec/test/test_codec.cpp index 5cd20bddd2..3e60023e5d 100644 --- a/cpp/arcticdb/codec/test/test_codec.cpp +++ b/cpp/arcticdb/codec/test/test_codec.cpp @@ -11,7 +11,6 @@ #include #include #include -#include #include #include diff --git a/cpp/arcticdb/column_store/chunked_buffer.hpp b/cpp/arcticdb/column_store/chunked_buffer.hpp index 4daa91b1ca..b153970054 100644 --- a/cpp/arcticdb/column_store/chunked_buffer.hpp +++ b/cpp/arcticdb/column_store/chunked_buffer.hpp @@ -9,15 +9,16 @@ #pragma once #include -#include #include #include #include -#include - #include +#ifndef DEBUG_BUILD +#include +#endif + namespace arcticdb { /* diff --git a/cpp/arcticdb/column_store/column.hpp b/cpp/arcticdb/column_store/column.hpp index 7d7a11cdbe..69b2996576 100644 --- a/cpp/arcticdb/column_store/column.hpp +++ b/cpp/arcticdb/column_store/column.hpp @@ -19,13 +19,10 @@ #include #include -#include // Compilation fails on Mac if cstdio is not included prior to folly/Function.h due to a missing definition of memalign in folly/Memory.h #ifdef __APPLE__ #include #endif -#include -#include #include #include diff --git a/cpp/arcticdb/column_store/column_utils.hpp b/cpp/arcticdb/column_store/column_utils.hpp index 90a35a1048..d17a6f57a2 100644 --- a/cpp/arcticdb/column_store/column_utils.hpp +++ b/cpp/arcticdb/column_store/column_utils.hpp @@ -9,7 +9,7 @@ #include #include - +#include #include namespace arcticdb::detail { diff --git a/cpp/arcticdb/column_store/memory_segment_impl.cpp b/cpp/arcticdb/column_store/memory_segment_impl.cpp index bf69e721d7..804f206f28 100644 --- a/cpp/arcticdb/column_store/memory_segment_impl.cpp +++ b/cpp/arcticdb/column_store/memory_segment_impl.cpp @@ -5,12 +5,12 @@ * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0. */ +#include #include #include #include #include -#include #include diff --git a/cpp/arcticdb/column_store/memory_segment_impl.hpp b/cpp/arcticdb/column_store/memory_segment_impl.hpp index 9747be880b..34e2616bb8 100644 --- a/cpp/arcticdb/column_store/memory_segment_impl.hpp +++ b/cpp/arcticdb/column_store/memory_segment_impl.hpp @@ -11,12 +11,12 @@ #include #include #include +#include #include #include #include #include -#include #include #include @@ -28,7 +28,7 @@ namespace google::protobuf } namespace arcticdb { - +class ColumnMap; class SegmentInMemoryImpl; namespace { diff --git a/cpp/arcticdb/column_store/python_bindings.cpp b/cpp/arcticdb/column_store/python_bindings.cpp index d35b3cb657..224a9b2070 100644 --- a/cpp/arcticdb/column_store/python_bindings.cpp +++ b/cpp/arcticdb/column_store/python_bindings.cpp @@ -10,7 +10,6 @@ #include #include #include -#include namespace py = pybind11; diff --git a/cpp/arcticdb/column_store/string_pool.cpp b/cpp/arcticdb/column_store/string_pool.cpp index c16d3bc61b..948c603650 100644 --- a/cpp/arcticdb/column_store/string_pool.cpp +++ b/cpp/arcticdb/column_store/string_pool.cpp @@ -10,8 +10,6 @@ #include #include -#include - namespace arcticdb { /***************** diff --git a/cpp/arcticdb/column_store/test/benchmark_memory_segment.cpp b/cpp/arcticdb/column_store/test/benchmark_memory_segment.cpp index 69613fc011..b7dc12ad51 100644 --- a/cpp/arcticdb/column_store/test/benchmark_memory_segment.cpp +++ b/cpp/arcticdb/column_store/test/benchmark_memory_segment.cpp @@ -9,7 +9,6 @@ #include #include -#include #include diff --git a/cpp/arcticdb/column_store/test/ingestion_stress_test.cpp b/cpp/arcticdb/column_store/test/ingestion_stress_test.cpp index 5204eafdf2..b50bda6c4e 100644 --- a/cpp/arcticdb/column_store/test/ingestion_stress_test.cpp +++ b/cpp/arcticdb/column_store/test/ingestion_stress_test.cpp @@ -7,16 +7,14 @@ #include #include -#include -#include -#include #include #include -#include #include #include #include +#include +#include using namespace arcticdb; namespace as = arcticdb::stream; diff --git a/cpp/arcticdb/column_store/test/rapidcheck_chunked_buffer.cpp b/cpp/arcticdb/column_store/test/rapidcheck_chunked_buffer.cpp index 0567f9102f..494b33864f 100644 --- a/cpp/arcticdb/column_store/test/rapidcheck_chunked_buffer.cpp +++ b/cpp/arcticdb/column_store/test/rapidcheck_chunked_buffer.cpp @@ -7,12 +7,10 @@ #include "gtest/gtest.h" #include - #include #include #include -#include TEST(ChunkedBuffer, Basic) { using namespace arcticdb; diff --git a/cpp/arcticdb/column_store/test/rapidcheck_column.cpp b/cpp/arcticdb/column_store/test/rapidcheck_column.cpp index 53257e15ed..50d2a9189b 100644 --- a/cpp/arcticdb/column_store/test/rapidcheck_column.cpp +++ b/cpp/arcticdb/column_store/test/rapidcheck_column.cpp @@ -7,13 +7,10 @@ #include #include -#include - -#include -#include +#include +#include #include -#include struct ColumnModel { std::vector data; @@ -89,7 +86,7 @@ struct ColumnRead : rc::state::Command { RC_GTEST_PROP(Column, Rapidcheck, ()) { ColumnModel initial_state; - arcticdb::Column sut(TypeDescriptor(DataType::UINT64, Dimension::Dim0), 0, false, false); + arcticdb::Column sut(arcticdb::entity::TypeDescriptor(arcticdb::entity::DataType::UINT64, arcticdb::entity::Dimension::Dim0), 0, false, false); rc::state::check(initial_state, sut, &rc::state::gen::execOneOf); diff --git a/cpp/arcticdb/column_store/test/rapidcheck_column_data_random_accessor.cpp b/cpp/arcticdb/column_store/test/rapidcheck_column_data_random_accessor.cpp index 8f5c378e6a..4e26a7c338 100644 --- a/cpp/arcticdb/column_store/test/rapidcheck_column_data_random_accessor.cpp +++ b/cpp/arcticdb/column_store/test/rapidcheck_column_data_random_accessor.cpp @@ -6,10 +6,10 @@ */ #include +#include #include #include -#include // Tricky to construct columns using a ChunkedBuffer with the testing size of 64 bytes, so only use rapidcheck for // single block tests, and test regualr and irregular accessors in test_column_data_random_accessor.cpp diff --git a/cpp/arcticdb/column_store/test/rapidcheck_column_map.cpp b/cpp/arcticdb/column_store/test/rapidcheck_column_map.cpp index 9eb551b5a8..7cb8b9108b 100644 --- a/cpp/arcticdb/column_store/test/rapidcheck_column_map.cpp +++ b/cpp/arcticdb/column_store/test/rapidcheck_column_map.cpp @@ -5,14 +5,12 @@ * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0. */ -#include -#include #include - -#include -#include #include #include +#include +#include +#include #include using namespace arcticdb; diff --git a/cpp/arcticdb/column_store/test/test_column.cpp b/cpp/arcticdb/column_store/test/test_column.cpp index bd652428a5..8086f58b67 100644 --- a/cpp/arcticdb/column_store/test/test_column.cpp +++ b/cpp/arcticdb/column_store/test/test_column.cpp @@ -9,7 +9,6 @@ #include #include -#include #include #include diff --git a/cpp/arcticdb/column_store/test/test_index_filtering.cpp b/cpp/arcticdb/column_store/test/test_index_filtering.cpp index a2469d23e7..b647b40622 100644 --- a/cpp/arcticdb/column_store/test/test_index_filtering.cpp +++ b/cpp/arcticdb/column_store/test/test_index_filtering.cpp @@ -8,11 +8,11 @@ #include #include -#include #include #include #include #include +#include namespace arcticdb { using namespace arcticdb::pipelines; diff --git a/cpp/arcticdb/entity/atom_key.hpp b/cpp/arcticdb/entity/atom_key.hpp index a78b1d1997..ebfc29649a 100644 --- a/cpp/arcticdb/entity/atom_key.hpp +++ b/cpp/arcticdb/entity/atom_key.hpp @@ -7,6 +7,7 @@ #pragma once +#include #include #include #include diff --git a/cpp/arcticdb/entity/data_error.cpp b/cpp/arcticdb/entity/data_error.cpp index 0c06212020..326cfe2a44 100644 --- a/cpp/arcticdb/entity/data_error.cpp +++ b/cpp/arcticdb/entity/data_error.cpp @@ -7,8 +7,6 @@ #include -#include - namespace arcticdb::entity { DataError::DataError(StreamId symbol, diff --git a/cpp/arcticdb/entity/field_collection.hpp b/cpp/arcticdb/entity/field_collection.hpp index 891e5a975d..e0b10f5f56 100644 --- a/cpp/arcticdb/entity/field_collection.hpp +++ b/cpp/arcticdb/entity/field_collection.hpp @@ -8,9 +8,9 @@ #pragma once #include -#include -#include -#include +#include +#include +#include #include namespace arcticdb { diff --git a/cpp/arcticdb/entity/key.hpp b/cpp/arcticdb/entity/key.hpp index 106cd10239..6320613fb3 100644 --- a/cpp/arcticdb/entity/key.hpp +++ b/cpp/arcticdb/entity/key.hpp @@ -8,12 +8,7 @@ #pragma once #include -#include #include -#include -#include -#include -#include #include #include diff --git a/cpp/arcticdb/entity/merge_descriptors.hpp b/cpp/arcticdb/entity/merge_descriptors.hpp index be491c8f50..051e8716e2 100644 --- a/cpp/arcticdb/entity/merge_descriptors.hpp +++ b/cpp/arcticdb/entity/merge_descriptors.hpp @@ -6,9 +6,13 @@ */ #include -#include namespace arcticdb { +class Store; +namespace pipelines { + struct SliceAndKey; +} + StreamDescriptor merge_descriptors( const StreamDescriptor &original, const std::vector> &entries, diff --git a/cpp/arcticdb/entity/metrics.cpp b/cpp/arcticdb/entity/metrics.cpp index 1480e251a5..ac69330929 100644 --- a/cpp/arcticdb/entity/metrics.cpp +++ b/cpp/arcticdb/entity/metrics.cpp @@ -6,13 +6,16 @@ */ #include +#include +#include #include -#include -#include -#include +#include +#include #ifdef _WIN32 -# include // for gethostname +#include // for gethostname +#else +#include // for gethostname #endif using namespace prometheus; diff --git a/cpp/arcticdb/entity/metrics.hpp b/cpp/arcticdb/entity/metrics.hpp index b3366048b0..fecbb5c131 100644 --- a/cpp/arcticdb/entity/metrics.hpp +++ b/cpp/arcticdb/entity/metrics.hpp @@ -7,26 +7,21 @@ #pragma once -#include -#include -#include -#include -#include -#include #include -#ifndef _WIN32 -#include -#include -#endif +namespace prometheus { + class Exposer; + class Gateway; +} -#include -#include -#include #include #include #include -#include +#include + +namespace prometheus { + class Summary; +} namespace arcticdb { diff --git a/cpp/arcticdb/entity/ref_key.hpp b/cpp/arcticdb/entity/ref_key.hpp index 57b3fc877c..fd9bf61eea 100644 --- a/cpp/arcticdb/entity/ref_key.hpp +++ b/cpp/arcticdb/entity/ref_key.hpp @@ -10,6 +10,7 @@ #include #include #include +#include namespace arcticdb::entity { class RefKey { diff --git a/cpp/arcticdb/entity/types-inl.hpp b/cpp/arcticdb/entity/types-inl.hpp index 8c939a0aed..33d5e19ee9 100644 --- a/cpp/arcticdb/entity/types-inl.hpp +++ b/cpp/arcticdb/entity/types-inl.hpp @@ -9,7 +9,6 @@ #error "This should only be included by types.hpp" #endif -#include #include #include diff --git a/cpp/arcticdb/entity/types.hpp b/cpp/arcticdb/entity/types.hpp index b158ae91a4..141030fe85 100644 --- a/cpp/arcticdb/entity/types.hpp +++ b/cpp/arcticdb/entity/types.hpp @@ -13,7 +13,6 @@ #include #include #include -#include #include diff --git a/cpp/arcticdb/log/log.cpp b/cpp/arcticdb/log/log.cpp index 205e3a9129..079c5118c2 100644 --- a/cpp/arcticdb/log/log.cpp +++ b/cpp/arcticdb/log/log.cpp @@ -14,18 +14,14 @@ #include #include #include -#include #include #include #include #include -#include - #include -#include #include #include diff --git a/cpp/arcticdb/pipeline/column_stats.hpp b/cpp/arcticdb/pipeline/column_stats.hpp index 94ebf7a4e5..13a1d5d5ba 100644 --- a/cpp/arcticdb/pipeline/column_stats.hpp +++ b/cpp/arcticdb/pipeline/column_stats.hpp @@ -1,7 +1,6 @@ #pragma once #include -#include #include #include diff --git a/cpp/arcticdb/pipeline/frame_slice.hpp b/cpp/arcticdb/pipeline/frame_slice.hpp index 393ce7a862..68c79170ef 100644 --- a/cpp/arcticdb/pipeline/frame_slice.hpp +++ b/cpp/arcticdb/pipeline/frame_slice.hpp @@ -7,7 +7,6 @@ #pragma once -#include #include #include #include diff --git a/cpp/arcticdb/pipeline/frame_utils.cpp b/cpp/arcticdb/pipeline/frame_utils.cpp index b96fd3f515..a01a9afa29 100644 --- a/cpp/arcticdb/pipeline/frame_utils.cpp +++ b/cpp/arcticdb/pipeline/frame_utils.cpp @@ -6,8 +6,9 @@ */ #include -#include #include +#include +#include namespace arcticdb { diff --git a/cpp/arcticdb/pipeline/frame_utils.hpp b/cpp/arcticdb/pipeline/frame_utils.hpp index c303c67c3d..fc7f4add88 100644 --- a/cpp/arcticdb/pipeline/frame_utils.hpp +++ b/cpp/arcticdb/pipeline/frame_utils.hpp @@ -8,24 +8,24 @@ #pragma once #include -#include #include #include #include -#include -#include -#include #include #include #include #include #include -#include +#include #include #include namespace arcticdb { +namespace pipelines::index { + struct IndexSegmentReader; +} + inline size_t get_first_string_size(const pipelines::PipelineContextRow& context_row, ChunkedBuffer &src, std::size_t first_row_in_frame) { auto offset = first_context_row(context_row.slice_and_key(), first_row_in_frame); auto num_rows = context_row.slice_and_key().slice_.row_range.diff(); diff --git a/cpp/arcticdb/pipeline/index_segment_reader.cpp b/cpp/arcticdb/pipeline/index_segment_reader.cpp index b55b617144..2713496dc6 100644 --- a/cpp/arcticdb/pipeline/index_segment_reader.cpp +++ b/cpp/arcticdb/pipeline/index_segment_reader.cpp @@ -5,13 +5,10 @@ * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0. */ -#include -#include -#include #include -#include #include #include +#include using namespace arcticdb::entity; using namespace arcticdb::stream; diff --git a/cpp/arcticdb/pipeline/index_segment_reader.hpp b/cpp/arcticdb/pipeline/index_segment_reader.hpp index 63496e80fd..9fc31891b9 100644 --- a/cpp/arcticdb/pipeline/index_segment_reader.hpp +++ b/cpp/arcticdb/pipeline/index_segment_reader.hpp @@ -8,13 +8,7 @@ #pragma once #include -#include #include -#include - -#include - -#include namespace arcticdb { class Store; @@ -24,8 +18,10 @@ namespace arcticdb::pipelines { struct ReadQuery; } -namespace arcticdb::pipelines::index { + +namespace arcticdb::pipelines::index { +enum class Fields : uint32_t; struct IndexSegmentIterator; struct IndexSegmentReader { diff --git a/cpp/arcticdb/pipeline/pipeline_context.cpp b/cpp/arcticdb/pipeline/pipeline_context.cpp index 0b47842613..f66a44a6ff 100644 --- a/cpp/arcticdb/pipeline/pipeline_context.cpp +++ b/cpp/arcticdb/pipeline/pipeline_context.cpp @@ -5,9 +5,8 @@ * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0. */ +#include #include -#include -#include #include #include diff --git a/cpp/arcticdb/pipeline/pipeline_context.hpp b/cpp/arcticdb/pipeline/pipeline_context.hpp index e15f18dd76..cff5ff9d0e 100644 --- a/cpp/arcticdb/pipeline/pipeline_context.hpp +++ b/cpp/arcticdb/pipeline/pipeline_context.hpp @@ -7,14 +7,10 @@ #pragma once -#include #include #include -#include -#include -#include #include diff --git a/cpp/arcticdb/pipeline/python_output_frame.cpp b/cpp/arcticdb/pipeline/python_output_frame.cpp index 797ef7da36..3f09ec9ed9 100644 --- a/cpp/arcticdb/pipeline/python_output_frame.cpp +++ b/cpp/arcticdb/pipeline/python_output_frame.cpp @@ -7,7 +7,6 @@ #include #include -#include #include namespace arcticdb::pipelines { diff --git a/cpp/arcticdb/pipeline/python_output_frame.hpp b/cpp/arcticdb/pipeline/python_output_frame.hpp index c8164e713e..2a255b0174 100644 --- a/cpp/arcticdb/pipeline/python_output_frame.hpp +++ b/cpp/arcticdb/pipeline/python_output_frame.hpp @@ -8,7 +8,6 @@ #pragma once #include -#include namespace arcticdb::pipelines { diff --git a/cpp/arcticdb/pipeline/query.hpp b/cpp/arcticdb/pipeline/query.hpp index 5d20744500..9f09dd6004 100644 --- a/cpp/arcticdb/pipeline/query.hpp +++ b/cpp/arcticdb/pipeline/query.hpp @@ -9,16 +9,9 @@ #include #include -#include -#include -#include -#include #include -#include #include #include -#include -#include #include #include #include @@ -134,8 +127,8 @@ inline bool is_column_selected(size_t start_col, size_t end_col, const util::Bit inline FilterQuery create_static_col_filter(util::BitSet &&selected_columns) { return [sc = std::move(selected_columns)](const index::IndexSegmentReader &isr, std::unique_ptr&& input) mutable { auto res = std::make_unique(static_cast(isr.size())); - auto start_col = isr.column(index::Fields::start_col).begin(); - auto end_col = isr.column(index::Fields::end_col).begin(); + auto start_col = isr.column(arcticdb::pipelines::index::Fields::start_col).begin(); + auto end_col = isr.column(arcticdb::pipelines::index::Fields::end_col).begin(); if (input) { bm::bvector<>::enumerator en = input->first(); @@ -276,14 +269,14 @@ inline FilterQuery create_index_filter(const IndexRange &range, b switch (index_type.value()) { case IndexDescriptor::TIMESTAMP: { - return build_bitset_for_index(container, + return build_bitset_for_index(container, rg, dynamic_schema, column_groups, std::move(input)); } case IndexDescriptor::STRING: { - return build_bitset_for_index(container, rg, dynamic_schema, column_groups, std::move(input)); + return build_bitset_for_index(container, rg, dynamic_schema, column_groups, std::move(input)); } default:util::raise_rte("Unknown index type {} in create_index_filter", uint32_t(index_type.value())); } diff --git a/cpp/arcticdb/pipeline/read_frame.cpp b/cpp/arcticdb/pipeline/read_frame.cpp index 7fc44cd37c..84d8aa0083 100644 --- a/cpp/arcticdb/pipeline/read_frame.cpp +++ b/cpp/arcticdb/pipeline/read_frame.cpp @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include @@ -20,15 +19,15 @@ #include #include #include -#include #include #include #include #include #include -#include -#include +#include #include +#include +#include namespace arcticdb::pipelines { @@ -490,9 +489,9 @@ void decode_into_frame_dynamic( // decode_or_expand will invoke the empty type handler which will do backfilling with the default value depending on the // destination type. if (!trivially_compatible_types(m.source_type_desc_, m.dest_type_desc_) && !source_is_empty) { - m.dest_type_desc_.visit_tag([&buffer, &m, &data, encoded_field, buffers, encdoing_version] (auto dest_desc_tag) { + m.dest_type_desc_.visit_tag([&buffer, &m, buffers] (auto dest_desc_tag) { using DestinationType = typename decltype(dest_desc_tag)::DataTypeTag::raw_type; - m.source_type_desc_.visit_tag([&buffer, &m, &data, &encoded_field, &buffers, encdoing_version] (auto src_desc_tag ) { + m.source_type_desc_.visit_tag([&buffer, &m] (auto src_desc_tag ) { using SourceType = typename decltype(src_desc_tag)::DataTypeTag::raw_type; if constexpr(std::is_arithmetic_v && std::is_arithmetic_v) { // If the source and destination types are different, then sizeof(destination type) >= sizeof(source type) diff --git a/cpp/arcticdb/pipeline/read_frame.hpp b/cpp/arcticdb/pipeline/read_frame.hpp index d0c28348b8..d3cfededa6 100644 --- a/cpp/arcticdb/pipeline/read_frame.hpp +++ b/cpp/arcticdb/pipeline/read_frame.hpp @@ -7,21 +7,22 @@ #pragma once -#include -#include -#include #include - #include - +#include #include namespace arcticdb { struct BufferHolder; + struct ReadOptions; + namespace stream { + struct StreamSource; + } } namespace arcticdb::pipelines { - +struct PipelineContext; +struct PipelineContextRow; SegmentInMemory allocate_frame(const std::shared_ptr& context); template @@ -38,7 +39,7 @@ std::optional check_and_mark_slices( bool is_first = true; size_t count = 0u; - std::set row_ranges; + std::set row_ranges; for (auto[opt_seg, slice, key] : slice_and_keys) { is_first = row_ranges.insert(slice.row_range).second; if(return_bitset) { @@ -54,7 +55,7 @@ std::optional check_and_mark_slices( if(!row_ranges.empty()) { auto pos = row_ranges.begin(); - RowRange current = *pos; + arcticdb::pipelines::RowRange current = *pos; std::advance(pos, 1); for(; pos != row_ranges.end(); ++pos){ sorting::check(pos->start() == current.end(), "Non-contiguous rows, range search on unsorted data? {} {}", current, *pos); diff --git a/cpp/arcticdb/pipeline/slicing.hpp b/cpp/arcticdb/pipeline/slicing.hpp index 612b703676..d0f2d6357b 100644 --- a/cpp/arcticdb/pipeline/slicing.hpp +++ b/cpp/arcticdb/pipeline/slicing.hpp @@ -7,23 +7,19 @@ #pragma once -#include -#include #include #include -#include -#include #include -#include #include -#include - -#include #include #include #include +namespace arcticdb{ + struct WriteOptions; +} + namespace arcticdb::pipelines { class FixedSlicer { diff --git a/cpp/arcticdb/pipeline/string_pool_utils.cpp b/cpp/arcticdb/pipeline/string_pool_utils.cpp index ffbe509af7..9230c017b7 100644 --- a/cpp/arcticdb/pipeline/string_pool_utils.cpp +++ b/cpp/arcticdb/pipeline/string_pool_utils.cpp @@ -6,7 +6,7 @@ */ #include -#include +#include namespace arcticdb { size_t first_context_row(const pipelines::SliceAndKey& slice_and_key, size_t first_row_in_frame) { diff --git a/cpp/arcticdb/pipeline/test/test_pipeline.cpp b/cpp/arcticdb/pipeline/test/test_pipeline.cpp index 0c263c068d..ae7e2313ea 100644 --- a/cpp/arcticdb/pipeline/test/test_pipeline.cpp +++ b/cpp/arcticdb/pipeline/test/test_pipeline.cpp @@ -9,16 +9,10 @@ #include #include -#include #include -#include -#include #include -#include #include #include -#include -#include namespace arcticdb { diff --git a/cpp/arcticdb/pipeline/value.hpp b/cpp/arcticdb/pipeline/value.hpp index 7d9e20153c..fa61550ca0 100644 --- a/cpp/arcticdb/pipeline/value.hpp +++ b/cpp/arcticdb/pipeline/value.hpp @@ -8,6 +8,7 @@ #pragma once #include +#include namespace arcticdb { diff --git a/cpp/arcticdb/pipeline/write_frame.cpp b/cpp/arcticdb/pipeline/write_frame.cpp index 6b70204823..9d31074362 100644 --- a/cpp/arcticdb/pipeline/write_frame.cpp +++ b/cpp/arcticdb/pipeline/write_frame.cpp @@ -5,27 +5,18 @@ * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0. */ -#include #include -#include #include #include #include #include #include -#include #include #include #include -#include -#include #include -#include #include #include -#include -#include -#include #include #include #include diff --git a/cpp/arcticdb/pipeline/write_frame.hpp b/cpp/arcticdb/pipeline/write_frame.hpp index ea3cb4259f..1d3d3eae22 100644 --- a/cpp/arcticdb/pipeline/write_frame.hpp +++ b/cpp/arcticdb/pipeline/write_frame.hpp @@ -7,17 +7,13 @@ #pragma once -#include -#include #include #include #include #include #include -#include -#include namespace arcticdb::pipelines { diff --git a/cpp/arcticdb/processing/aggregation.cpp b/cpp/arcticdb/processing/aggregation.cpp index 2d27449960..d26c262e2b 100644 --- a/cpp/arcticdb/processing/aggregation.cpp +++ b/cpp/arcticdb/processing/aggregation.cpp @@ -6,7 +6,7 @@ */ #include - +#include #include namespace arcticdb @@ -72,13 +72,20 @@ namespace } } - inline util::BitMagic::enumerator::value_type deref(util::BitMagic::enumerator iter) { + #if defined(__clang__) + #pragma clang diagnostic push + #pragma clang diagnostic ignored "-Wunused-function" + #endif + static inline util::BitMagic::enumerator::value_type deref(util::BitMagic::enumerator iter) { return *iter; } - inline std::size_t deref(std::size_t index) { + static inline std::size_t deref(std::size_t index) { return index; } + #if defined(__clang__) + #pragma clang diagnostic pop + #endif template struct OutputType; diff --git a/cpp/arcticdb/processing/aggregation.hpp b/cpp/arcticdb/processing/aggregation.hpp index 468774f6b4..67b8b9c8da 100644 --- a/cpp/arcticdb/processing/aggregation.hpp +++ b/cpp/arcticdb/processing/aggregation.hpp @@ -8,8 +8,6 @@ #pragma once #include -#include -#include #include namespace arcticdb { diff --git a/cpp/arcticdb/processing/clause.cpp b/cpp/arcticdb/processing/clause.cpp index 8e4a050a0c..943eec5dbc 100644 --- a/cpp/arcticdb/processing/clause.cpp +++ b/cpp/arcticdb/processing/clause.cpp @@ -17,10 +17,12 @@ #include #include -#include #include -#include #include +#include +#include +#include +#include namespace arcticdb { @@ -33,9 +35,9 @@ std::vector> structure_by_row_slice(std::vector> res; - RowRange previous_row_range; + pipelines::RowRange previous_row_range; for (const auto& [idx, ranges_and_key]: folly::enumerate(ranges_and_keys)) { - RowRange current_row_range{ranges_and_key.row_range_}; + pipelines::RowRange current_row_range{ranges_and_key.row_range_}; if (current_row_range != previous_row_range) { res.emplace_back(); } @@ -50,9 +52,9 @@ std::vector> structure_by_column_slice(std::vector> res; - ColRange previous_col_range; + pipelines::ColRange previous_col_range; for (const auto& [idx, ranges_and_key]: folly::enumerate(ranges_and_keys)) { - ColRange current_col_range{ranges_and_key.col_range_}; + pipelines::ColRange current_col_range{ranges_and_key.col_range_}; if (current_col_range != previous_col_range) { res.emplace_back(); } @@ -77,15 +79,15 @@ Composite gather_entities(std::shared_ptr comp (const EntityIds& entity_ids) -> ProcessingUnit { ProcessingUnit res; std::vector> segments; - std::vector> row_ranges; - std::vector> col_ranges; + std::vector> row_ranges; + std::vector> col_ranges; segments.reserve(entity_ids.size()); row_ranges.reserve(entity_ids.size()); col_ranges.reserve(entity_ids.size()); for (auto entity_id: entity_ids) { segments.emplace_back(component_manager->get>(entity_id)); - row_ranges.emplace_back(component_manager->get>(entity_id)); - col_ranges.emplace_back(component_manager->get>(entity_id)); + row_ranges.emplace_back(component_manager->get>(entity_id)); + col_ranges.emplace_back(component_manager->get>(entity_id)); } res.set_segments(std::move(segments)); res.set_row_ranges(std::move(row_ranges)); @@ -556,8 +558,8 @@ Composite AggregationClause::process(Composite&& entity_id } if (output_seg.has_value()) { output.push_back(push_entities(component_manager_, ProcessingUnit(std::move(*output_seg), - RowRange{min_start_row, max_end_row}, - ColRange{min_start_col, max_end_col}))); + pipelines::RowRange{min_start_row, max_end_row}, + pipelines::ColRange{min_start_col, max_end_col}))); } }); return output; @@ -575,7 +577,7 @@ Composite SplitClause::process(Composite&& entity_ids) con for (auto&& split_seg : split_segs) { end_row = start_row + split_seg.row_count(); ret.push_back(push_entities(component_manager_, ProcessingUnit(std::move(split_seg), - RowRange(start_row, end_row), + pipelines::RowRange(start_row, end_row), std::move(*proc.col_ranges_->at(idx))))); start_row = end_row; } @@ -605,8 +607,8 @@ void merge_impl( QueueType &input_streams, bool add_symbol_column, StreamId stream_id, - const RowRange& row_range, - const ColRange& col_range, + const pipelines::RowRange& row_range, + const pipelines::ColRange& col_range, IndexType index, const StreamDescriptor& stream_descriptor) { auto num_segment_rows = ConfigsMap::instance()->get_int("Merge.SegmentSize", 100000); @@ -669,8 +671,8 @@ Composite MergeClause::process(Composite&& entity_ids) con input_streams.push(std::make_unique(std::move(*segment))); } }); - const RowRange row_range{min_start_row, max_end_row}; - const ColRange col_range{min_start_col, max_end_col}; + const pipelines::RowRange row_range{min_start_row, max_end_row}; + const pipelines::ColRange col_range{min_start_col, max_end_col}; Composite ret; std::visit( [this, &ret, &input_streams, &comp=compare, stream_id=stream_id_, &row_range, &col_range](auto idx, auto density) { diff --git a/cpp/arcticdb/processing/clause.hpp b/cpp/arcticdb/processing/clause.hpp index d646740fe8..cfba4ae1ed 100644 --- a/cpp/arcticdb/processing/clause.hpp +++ b/cpp/arcticdb/processing/clause.hpp @@ -7,32 +7,21 @@ #pragma once -#include -#include #include -#include #include #include #include -#include #include #include -#include #include -#include -#include -#include #include #include -#include #include #include #include -#include #include -#include namespace arcticdb { diff --git a/cpp/arcticdb/processing/component_manager.hpp b/cpp/arcticdb/processing/component_manager.hpp index 859d996e97..d4c405dd03 100644 --- a/cpp/arcticdb/processing/component_manager.hpp +++ b/cpp/arcticdb/processing/component_manager.hpp @@ -9,13 +9,25 @@ #include #include +#include +#include -#include +#include +#include #include namespace arcticdb { +class SegmentInMemory; -using namespace pipelines; +namespace pipelines { + struct RowRange; + struct ColRange; +} + +namespace entity { + class AtomKeyImpl; +} +using AtomKey = entity::AtomKeyImpl; using EntityId = uint64_t; using bucket_id = size_t; @@ -31,9 +43,9 @@ class ComponentManager { auto insertion_id = entity_id(id); if constexpr(std::is_same_v>) { segment_map_.add(insertion_id, std::move(component), expected_get_calls); - } else if constexpr(std::is_same_v>) { + } else if constexpr(std::is_same_v>) { row_range_map_.add(insertion_id, std::move(component)); - } else if constexpr(std::is_same_v>) { + } else if constexpr(std::is_same_v>) { col_range_map_.add(insertion_id, std::move(component)); } else if constexpr(std::is_same_v>) { atom_key_map_.add(insertion_id, std::move(component)); @@ -51,9 +63,9 @@ class ComponentManager { T get(EntityId id) { if constexpr(std::is_same_v>) { return segment_map_.get(id); - } else if constexpr(std::is_same_v>) { + } else if constexpr(std::is_same_v>) { return row_range_map_.get(id); - } else if constexpr(std::is_same_v>) { + } else if constexpr(std::is_same_v>) { return col_range_map_.get(id); } else if constexpr(std::is_same_v>) { return atom_key_map_.get(id); @@ -125,8 +137,8 @@ class ComponentManager { }; ComponentMap> segment_map_{"segment", true}; - ComponentMap> row_range_map_{"row range", false}; - ComponentMap> col_range_map_{"col range", false}; + ComponentMap> row_range_map_{"row range", false}; + ComponentMap> col_range_map_{"col range", false}; ComponentMap> atom_key_map_{"atom key", false}; ComponentMap bucket_map_{"bucket", false}; diff --git a/cpp/arcticdb/processing/operation_dispatch.cpp b/cpp/arcticdb/processing/operation_dispatch.cpp index 3dd0558732..edff103ae7 100644 --- a/cpp/arcticdb/processing/operation_dispatch.cpp +++ b/cpp/arcticdb/processing/operation_dispatch.cpp @@ -7,6 +7,7 @@ */ #include +#include namespace arcticdb { diff --git a/cpp/arcticdb/processing/operation_dispatch.hpp b/cpp/arcticdb/processing/operation_dispatch.hpp index 870b384958..53101e8345 100644 --- a/cpp/arcticdb/processing/operation_dispatch.hpp +++ b/cpp/arcticdb/processing/operation_dispatch.hpp @@ -7,23 +7,9 @@ */ #pragma once -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include -#include namespace arcticdb { diff --git a/cpp/arcticdb/processing/operation_dispatch_binary.cpp b/cpp/arcticdb/processing/operation_dispatch_binary.cpp index 41fd2d9d5c..5eadd1c2e6 100644 --- a/cpp/arcticdb/processing/operation_dispatch_binary.cpp +++ b/cpp/arcticdb/processing/operation_dispatch_binary.cpp @@ -9,6 +9,67 @@ #include namespace arcticdb { +template +VariantData visit_binary_comparator(const VariantData& left, const VariantData& right, Func&& func) { + if(std::holds_alternative(left) || std::holds_alternative(right)) + return EmptyResult{}; + + return std::visit(util::overload { + [&func] (const ColumnWithStrings& l, const std::shared_ptr& r) ->VariantData { + auto result = binary_comparator(l, *r, std::forward(func)); + return transform_to_placeholder(result); + }, + [&] (const ColumnWithStrings& l, const ColumnWithStrings& r) ->VariantData { + auto result = binary_comparator(l, r, std::forward(func)); + return transform_to_placeholder(result); + }, + [&](const std::shared_ptr& l, const ColumnWithStrings& r) ->VariantData { + auto result = binary_comparator(r, *l, std::forward(func)); + return transform_to_placeholder(result); + }, + [&] ([[maybe_unused]] const std::shared_ptr& l, [[maybe_unused]] const std::shared_ptr& r) ->VariantData { + util::raise_rte("Two value inputs not accepted to binary comparators"); + }, + [](const auto &, const auto&) -> VariantData { + util::raise_rte("Bitset/ValueSet inputs not accepted to binary comparators"); + } + }, left, right); +} +template VariantData visit_binary_comparator(const VariantData&, const VariantData&, EqualsOperator&&); +template VariantData visit_binary_comparator(const VariantData&, const VariantData&, NotEqualsOperator&&); +template VariantData visit_binary_comparator(const VariantData&, const VariantData&, LessThanOperator&&); +template VariantData visit_binary_comparator(const VariantData&, const VariantData&, LessThanEqualsOperator&&); +template VariantData visit_binary_comparator(const VariantData&, const VariantData&, GreaterThanOperator&&); +template VariantData visit_binary_comparator(const VariantData&, const VariantData&, GreaterThanEqualsOperator&&); + +template +VariantData visit_binary_operator(const VariantData& left, const VariantData& right, Func&& func) { + if(std::holds_alternative(left) || std::holds_alternative(right)) + return EmptyResult{}; + + return std::visit(util::overload { + [&] (const ColumnWithStrings& l, const std::shared_ptr& r) ->VariantData { + return binary_operator(*(l.column_), *r, std::forward(func)); + }, + [&] (const ColumnWithStrings& l, const ColumnWithStrings& r) ->VariantData { + return binary_operator(*(l.column_), *(r.column_), std::forward(func)); + }, + [&](const std::shared_ptr& l, const ColumnWithStrings& r) ->VariantData { + return binary_operator(*(r.column_), *l, std::forward(func)); + }, + [&] (const std::shared_ptr& l, const std::shared_ptr& r) -> VariantData { + return binary_operator(*l, *r, std::forward(func)); + }, + [](const auto &, const auto&) -> VariantData { + util::raise_rte("Bitset/ValueSet inputs not accepted to binary operators"); + } + }, left, right); +} + +template VariantData visit_binary_operator(const VariantData&, const VariantData&, PlusOperator&&); +template VariantData visit_binary_operator(const VariantData&, const VariantData&, MinusOperator&&); +template VariantData visit_binary_operator(const VariantData&, const VariantData&, TimesOperator&&); +template VariantData visit_binary_operator(const VariantData&, const VariantData&, DivideOperator&&); VariantData binary_boolean(const util::BitSet& left, const util::BitSet& right, OperationType operation) { util::check(left.size() == right.size(), "BitSets of different lengths ({} and {}) in binary comparator", left.size(), right.size()); diff --git a/cpp/arcticdb/processing/operation_dispatch_binary.hpp b/cpp/arcticdb/processing/operation_dispatch_binary.hpp index fa047931a5..86efba9777 100644 --- a/cpp/arcticdb/processing/operation_dispatch_binary.hpp +++ b/cpp/arcticdb/processing/operation_dispatch_binary.hpp @@ -8,24 +8,20 @@ #pragma once -#include #include #include #include -#include #include #include #include -#include #include #include -#include #include #include #include - +#include namespace arcticdb { VariantData binary_boolean(const util::BitSet& left, const util::BitSet& right, OperationType operation); @@ -247,31 +243,7 @@ VariantData binary_comparator(const ColumnWithStrings& column_with_strings, cons } template -VariantData visit_binary_comparator(const VariantData& left, const VariantData& right, Func&& func) { - if(std::holds_alternative(left) || std::holds_alternative(right)) - return EmptyResult{}; - - return std::visit(util::overload { - [&func] (const ColumnWithStrings& l, const std::shared_ptr& r) ->VariantData { - auto result = binary_comparator(l, *r, std::forward(func)); - return transform_to_placeholder(result); - }, - [&] (const ColumnWithStrings& l, const ColumnWithStrings& r) ->VariantData { - auto result = binary_comparator(l, r, std::forward(func)); - return transform_to_placeholder(result); - }, - [&](const std::shared_ptr& l, const ColumnWithStrings& r) ->VariantData { - auto result = binary_comparator(r, *l, std::forward(func)); - return transform_to_placeholder(result); - }, - [&] ([[maybe_unused]] const std::shared_ptr& l, [[maybe_unused]] const std::shared_ptr& r) ->VariantData { - util::raise_rte("Two value inputs not accepted to binary comparators"); - }, - [](const auto &, const auto&) -> VariantData { - util::raise_rte("Bitset/ValueSet inputs not accepted to binary comparators"); - } - }, left, right); -} +VariantData visit_binary_comparator(const VariantData& left, const VariantData& right, Func&& func); template VariantData binary_operator(const Value& left, const Value& right, Func&& func) { @@ -374,53 +346,8 @@ VariantData binary_operator(const Column& col, const Value& val, Func&& func) { } template -VariantData visit_binary_operator(const VariantData& left, const VariantData& right, Func&& func) { - if(std::holds_alternative(left) || std::holds_alternative(right)) - return EmptyResult{}; - - return std::visit(util::overload { - [&] (const ColumnWithStrings& l, const std::shared_ptr& r) ->VariantData { - return binary_operator(*(l.column_), *r, std::forward(func)); - }, - [&] (const ColumnWithStrings& l, const ColumnWithStrings& r) ->VariantData { - return binary_operator(*(l.column_), *(r.column_), std::forward(func)); - }, - [&](const std::shared_ptr& l, const ColumnWithStrings& r) ->VariantData { - return binary_operator(*(r.column_), *l, std::forward(func)); - }, - [&] (const std::shared_ptr& l, const std::shared_ptr& r) -> VariantData { - return binary_operator(*l, *r, std::forward(func)); - }, - [](const auto &, const auto&) -> VariantData { - util::raise_rte("Bitset/ValueSet inputs not accepted to binary operators"); - } - }, left, right); -} +VariantData visit_binary_operator(const VariantData& left, const VariantData& right, Func&& func); VariantData dispatch_binary(const VariantData& left, const VariantData& right, OperationType operation); -// instantiated in operation_dispatch_binary_operator.cpp to reduce compilation memory use -extern template -VariantData visit_binary_operator(const VariantData&, const VariantData&, PlusOperator&&); -extern template -VariantData visit_binary_operator(const VariantData&, const VariantData&, MinusOperator&&); -extern template -VariantData visit_binary_operator(const VariantData&, const VariantData&, TimesOperator&&); -extern template -VariantData visit_binary_operator(const VariantData&, const VariantData&, DivideOperator&&); - -// instantiated in operation_dispatch_binary_comparator.cpp to reduce compilation memory use -extern template -VariantData visit_binary_comparator(const VariantData&, const VariantData&, EqualsOperator&&); -extern template -VariantData visit_binary_comparator(const VariantData&, const VariantData&, NotEqualsOperator&&); -extern template -VariantData visit_binary_comparator(const VariantData&, const VariantData&, LessThanOperator&&); -extern template -VariantData visit_binary_comparator(const VariantData&, const VariantData&, LessThanEqualsOperator&&); -extern template -VariantData visit_binary_comparator(const VariantData&, const VariantData&, GreaterThanOperator&&); -extern template -VariantData visit_binary_comparator(const VariantData&, const VariantData&, GreaterThanEqualsOperator&&); - } diff --git a/cpp/arcticdb/processing/operation_dispatch_binary_eq.cpp b/cpp/arcticdb/processing/operation_dispatch_binary_eq.cpp deleted file mode 100644 index 7114ec36ab..0000000000 --- a/cpp/arcticdb/processing/operation_dispatch_binary_eq.cpp +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright 2023 Man Group Operations Limited - * - * Use of this software is governed by the Business Source License 1.1 included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0. - */ - -#include - -namespace arcticdb { - -template VariantData visit_binary_comparator(const VariantData&, const VariantData&, EqualsOperator&&); -template VariantData visit_binary_comparator(const VariantData&, const VariantData&, NotEqualsOperator&&); - -} diff --git a/cpp/arcticdb/processing/operation_dispatch_binary_gt.cpp b/cpp/arcticdb/processing/operation_dispatch_binary_gt.cpp deleted file mode 100644 index 6cf80de407..0000000000 --- a/cpp/arcticdb/processing/operation_dispatch_binary_gt.cpp +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright 2023 Man Group Operations Limited - * - * Use of this software is governed by the Business Source License 1.1 included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0. - */ - -#include - -namespace arcticdb { - -template VariantData visit_binary_comparator(const VariantData&, const VariantData&, GreaterThanOperator&&); -template VariantData visit_binary_comparator(const VariantData&, const VariantData&, GreaterThanEqualsOperator&&); - -} diff --git a/cpp/arcticdb/processing/operation_dispatch_binary_lt.cpp b/cpp/arcticdb/processing/operation_dispatch_binary_lt.cpp deleted file mode 100644 index 66cdb48cea..0000000000 --- a/cpp/arcticdb/processing/operation_dispatch_binary_lt.cpp +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright 2023 Man Group Operations Limited - * - * Use of this software is governed by the Business Source License 1.1 included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0. - */ - -#include - -namespace arcticdb { - -template VariantData visit_binary_comparator(const VariantData&, const VariantData&, LessThanOperator&&); -template VariantData visit_binary_comparator(const VariantData&, const VariantData&, LessThanEqualsOperator&&); - -} diff --git a/cpp/arcticdb/processing/operation_dispatch_binary_operator.cpp b/cpp/arcticdb/processing/operation_dispatch_binary_operator.cpp deleted file mode 100644 index ca69dd619b..0000000000 --- a/cpp/arcticdb/processing/operation_dispatch_binary_operator.cpp +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright 2023 Man Group Operations Limited - * - * Use of this software is governed by the Business Source License 1.1 included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0. - */ - -#include - -namespace arcticdb { - -template VariantData visit_binary_operator(const VariantData&, const VariantData&, PlusOperator&&); -template VariantData visit_binary_operator(const VariantData&, const VariantData&, MinusOperator&&); -template VariantData visit_binary_operator(const VariantData&, const VariantData&, TimesOperator&&); -template VariantData visit_binary_operator(const VariantData&, const VariantData&, DivideOperator&&); - -} \ No newline at end of file diff --git a/cpp/arcticdb/processing/operation_dispatch_unary.hpp b/cpp/arcticdb/processing/operation_dispatch_unary.hpp index c8018d7ba6..587e4e9ea7 100644 --- a/cpp/arcticdb/processing/operation_dispatch_unary.hpp +++ b/cpp/arcticdb/processing/operation_dispatch_unary.hpp @@ -8,23 +8,17 @@ #pragma once -#include #include #include #include -#include #include -#include #include -#include #include #include -#include #include #include -#include namespace arcticdb { @@ -115,7 +109,7 @@ VariantData unary_comparator(const Column& col, Func&& func) { constexpr auto sparse_missing_value_output = std::is_same_v; details::visit_type(col.type().data_type(), [&](auto col_tag) { using type_info = ScalarTypeInfo; - Column::transform(col, output_bitset, sparse_missing_value_output, [&func](auto input_value) -> bool { + Column::transform(col, output_bitset, sparse_missing_value_output, [&](auto input_value) -> bool { if constexpr (is_floating_point_type(type_info::data_type)) { return func.apply(input_value); } else if constexpr (is_sequence_type(type_info::data_type)) { diff --git a/cpp/arcticdb/processing/processing_unit.cpp b/cpp/arcticdb/processing/processing_unit.cpp index 2430022b6c..ee464a70f5 100644 --- a/cpp/arcticdb/processing/processing_unit.cpp +++ b/cpp/arcticdb/processing/processing_unit.cpp @@ -6,6 +6,7 @@ */ #include +#include namespace arcticdb { diff --git a/cpp/arcticdb/processing/processing_unit.hpp b/cpp/arcticdb/processing/processing_unit.hpp index eb7f6494d6..909bd04173 100644 --- a/cpp/arcticdb/processing/processing_unit.hpp +++ b/cpp/arcticdb/processing/processing_unit.hpp @@ -15,15 +15,13 @@ #include #include #include -#include #include #include #include #include -#include -#include namespace arcticdb { + struct ExpressionContext; enum class PipelineOptimisation : uint8_t { SPEED, MEMORY @@ -61,8 +59,8 @@ namespace arcticdb { std::optional&& row_range=std::nullopt, std::optional&& col_range=std::nullopt) { auto segment_in_memory = std::move(seg); - auto rows = row_range.value_or(RowRange(0, segment_in_memory.row_count())); - auto cols = col_range.value_or(ColRange(0, segment_in_memory.is_null() ? 0 : segment_in_memory.descriptor().field_count() - segment_in_memory.descriptor().index().field_count())); + auto rows = row_range.value_or(pipelines::RowRange(0, segment_in_memory.row_count())); + auto cols = col_range.value_or(pipelines::ColRange(0, segment_in_memory.is_null() ? 0 : segment_in_memory.descriptor().field_count() - segment_in_memory.descriptor().index().field_count())); segments_.emplace({std::make_shared(std::move(segment_in_memory))}); row_ranges_.emplace({std::make_shared(std::move(rows))}); col_ranges_.emplace({std::make_shared(std::move(cols))}); diff --git a/cpp/arcticdb/processing/test/benchmark_clause.cpp b/cpp/arcticdb/processing/test/benchmark_clause.cpp index 616ee3253e..f7177d570c 100644 --- a/cpp/arcticdb/processing/test/benchmark_clause.cpp +++ b/cpp/arcticdb/processing/test/benchmark_clause.cpp @@ -10,11 +10,10 @@ #include #include -#include #include -#include -#include - +#include +#include +#include using namespace arcticdb; // run like: --benchmark_time_unit=ms --benchmark_filter=.* --benchmark_min_time=5x diff --git a/cpp/arcticdb/processing/test/test_clause.cpp b/cpp/arcticdb/processing/test/test_clause.cpp index 62399fa3ae..e3673afb42 100644 --- a/cpp/arcticdb/processing/test/test_clause.cpp +++ b/cpp/arcticdb/processing/test/test_clause.cpp @@ -6,11 +6,11 @@ */ #include -#include #include #include -#include #include +#include +#include template void segment_scalar_assert_all_values_equal(const arcticdb::ProcessingUnit& proc_unit, const arcticdb::ColumnName& name, const std::unordered_set& expected, size_t expected_row_count) { diff --git a/cpp/arcticdb/processing/test/test_component_manager.cpp b/cpp/arcticdb/processing/test/test_component_manager.cpp index 181e060435..bf27d3f000 100644 --- a/cpp/arcticdb/processing/test/test_component_manager.cpp +++ b/cpp/arcticdb/processing/test/test_component_manager.cpp @@ -8,6 +8,9 @@ #include #include +#include +#include +#include using namespace arcticdb; using namespace arcticdb::pipelines; @@ -15,15 +18,15 @@ using namespace arcticdb::pipelines; TEST(ComponentManager, Simple) { ComponentManager component_manager; auto segment_0 = std::make_shared(); - auto row_range_0 = std::make_shared(0, 10); - auto col_range_0 = std::make_shared(10, 20); - auto key_0 = std::make_shared(AtomKeyBuilder().version_id(1).build("symbol_0", KeyType::TABLE_DATA)); + auto row_range_0 = std::make_shared(0, 10); + auto col_range_0 = std::make_shared(10, 20); + auto key_0 = std::make_shared(arcticdb::entity::AtomKeyBuilder().version_id(1).build("symbol_0", arcticdb::entity::KeyType::TABLE_DATA)); uint64_t expected_get_calls_0{1}; auto segment_1 = std::make_shared(); - auto row_range_1 = std::make_shared(20, 30); - auto col_range_1 = std::make_shared(30, 40); - auto key_1 = std::make_shared(AtomKeyBuilder().version_id(2).build("symbol_1", KeyType::TABLE_DATA)); + auto row_range_1 = std::make_shared(20, 30); + auto col_range_1 = std::make_shared(30, 40); + auto key_1 = std::make_shared(arcticdb::entity::AtomKeyBuilder().version_id(2).build("symbol_1", arcticdb::entity::KeyType::TABLE_DATA)); uint64_t expected_get_calls_1{2}; auto id_0 = component_manager.add(segment_0, std::nullopt, expected_get_calls_0); diff --git a/cpp/arcticdb/processing/test/test_has_valid_type_promotion.cpp b/cpp/arcticdb/processing/test/test_has_valid_type_promotion.cpp index 4c0689a328..551a429817 100644 --- a/cpp/arcticdb/processing/test/test_has_valid_type_promotion.cpp +++ b/cpp/arcticdb/processing/test/test_has_valid_type_promotion.cpp @@ -6,8 +6,8 @@ */ #include -#include #include +#include TEST(HasValidTypePromotion, DifferentDimensions) { using namespace arcticdb; diff --git a/cpp/arcticdb/python/normalization_checks.cpp b/cpp/arcticdb/python/normalization_checks.cpp index 17584bc941..90c549319b 100644 --- a/cpp/arcticdb/python/normalization_checks.cpp +++ b/cpp/arcticdb/python/normalization_checks.cpp @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include diff --git a/cpp/arcticdb/python/python_handlers.cpp b/cpp/arcticdb/python/python_handlers.cpp index 4e9aca7f33..5351f653bc 100644 --- a/cpp/arcticdb/python/python_handlers.cpp +++ b/cpp/arcticdb/python/python_handlers.cpp @@ -9,6 +9,7 @@ #include #include #include +#include namespace arcticdb { diff --git a/cpp/arcticdb/python/python_handlers.hpp b/cpp/arcticdb/python/python_handlers.hpp index 186c87cbd9..df0d054603 100644 --- a/cpp/arcticdb/python/python_handlers.hpp +++ b/cpp/arcticdb/python/python_handlers.hpp @@ -5,12 +5,14 @@ * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0. */ #pragma once - -#include -#include +#include +#include +#include namespace arcticdb { struct ColumnMapping; + struct BufferHolder; + enum class EncodingVersion : uint16_t; struct EmptyHandler { /// @see arcticdb::ITypeHandler diff --git a/cpp/arcticdb/python/python_module.cpp b/cpp/arcticdb/python/python_module.cpp index cb8be77a8d..1f817c2b69 100644 --- a/cpp/arcticdb/python/python_module.cpp +++ b/cpp/arcticdb/python/python_module.cpp @@ -20,7 +20,6 @@ #include #include #include -#include #include #include #include diff --git a/cpp/arcticdb/python/python_to_tensor_frame.cpp b/cpp/arcticdb/python/python_to_tensor_frame.cpp index fce30d3020..9ee82e82a5 100644 --- a/cpp/arcticdb/python/python_to_tensor_frame.cpp +++ b/cpp/arcticdb/python/python_to_tensor_frame.cpp @@ -7,12 +7,11 @@ #include -#include +#include #include #include #include #include -#include #include namespace arcticdb::convert { diff --git a/cpp/arcticdb/python/python_to_tensor_frame.hpp b/cpp/arcticdb/python/python_to_tensor_frame.hpp index 4ff566262a..61e54645e9 100644 --- a/cpp/arcticdb/python/python_to_tensor_frame.hpp +++ b/cpp/arcticdb/python/python_to_tensor_frame.hpp @@ -7,12 +7,14 @@ #pragma once -#include -#include #include #include #include +namespace arcticdb { + class ScopedGILLock; +} + namespace arcticdb::convert { namespace py = pybind11; diff --git a/cpp/arcticdb/storage/azure/azure_mock_client.cpp b/cpp/arcticdb/storage/azure/azure_mock_client.cpp index efafb98bd9..64f5a972b7 100644 --- a/cpp/arcticdb/storage/azure/azure_mock_client.cpp +++ b/cpp/arcticdb/storage/azure/azure_mock_client.cpp @@ -5,14 +5,10 @@ * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0. */ -#include -#include -#include -#include #include #include -#include +#include namespace arcticdb::storage::azure { diff --git a/cpp/arcticdb/storage/azure/azure_mock_client.hpp b/cpp/arcticdb/storage/azure/azure_mock_client.hpp index 147324f61e..9a20aa58fc 100644 --- a/cpp/arcticdb/storage/azure/azure_mock_client.hpp +++ b/cpp/arcticdb/storage/azure/azure_mock_client.hpp @@ -7,14 +7,12 @@ #pragma once -#include -#include -#include -#include -#include #include -#include + +namespace arcticdb::storage { + enum class StorageOperation; +} namespace arcticdb::storage::azure { diff --git a/cpp/arcticdb/storage/azure/azure_real_client.cpp b/cpp/arcticdb/storage/azure/azure_real_client.cpp index 60a8101d78..0f400f07ca 100644 --- a/cpp/arcticdb/storage/azure/azure_real_client.cpp +++ b/cpp/arcticdb/storage/azure/azure_real_client.cpp @@ -7,10 +7,7 @@ #include -#include -#include -#include #include #include #include diff --git a/cpp/arcticdb/storage/azure/azure_real_client.hpp b/cpp/arcticdb/storage/azure/azure_real_client.hpp index 9fc8116de0..3c2c7ce854 100644 --- a/cpp/arcticdb/storage/azure/azure_real_client.hpp +++ b/cpp/arcticdb/storage/azure/azure_real_client.hpp @@ -7,11 +7,7 @@ #pragma once -#include -#include -#include -#include #include namespace arcticdb::storage::azure { diff --git a/cpp/arcticdb/storage/azure/azure_storage.cpp b/cpp/arcticdb/storage/azure/azure_storage.cpp index 498cbe351d..e7c3e53313 100644 --- a/cpp/arcticdb/storage/azure/azure_storage.cpp +++ b/cpp/arcticdb/storage/azure/azure_storage.cpp @@ -7,29 +7,18 @@ #include -#include -#include #include -#include -#include -#include -#include #include -#include #include #include #include -#include #include #include -#include -#include -#include #include #include diff --git a/cpp/arcticdb/storage/azure/azure_storage.hpp b/cpp/arcticdb/storage/azure/azure_storage.hpp index 7dc04f28ed..2f08d0f117 100644 --- a/cpp/arcticdb/storage/azure/azure_storage.hpp +++ b/cpp/arcticdb/storage/azure/azure_storage.hpp @@ -8,19 +8,10 @@ #pragma once #include -#include -#include #include -#include #include -#include -#include -#include -#include -#include -#include +#include #include -#include namespace arcticdb::storage::azure { diff --git a/cpp/arcticdb/storage/config_resolvers.cpp b/cpp/arcticdb/storage/config_resolvers.cpp index 0003bd4827..23dba03d06 100644 --- a/cpp/arcticdb/storage/config_resolvers.cpp +++ b/cpp/arcticdb/storage/config_resolvers.cpp @@ -6,16 +6,8 @@ */ #include -#include -#include -#include -#include -#include -#include - -#include + #include -#include namespace arcticdb::storage::details { diff --git a/cpp/arcticdb/storage/config_resolvers.hpp b/cpp/arcticdb/storage/config_resolvers.hpp index 50e5d9fb7a..7e8f5bb462 100644 --- a/cpp/arcticdb/storage/config_resolvers.hpp +++ b/cpp/arcticdb/storage/config_resolvers.hpp @@ -7,13 +7,10 @@ #pragma once -#include -#include -#include #include -#include +#include +#include -#include namespace arcticdb::storage { class ConfigResolver { diff --git a/cpp/arcticdb/storage/failure_simulation.hpp b/cpp/arcticdb/storage/failure_simulation.hpp index 012ae7d88a..d9f94e082a 100644 --- a/cpp/arcticdb/storage/failure_simulation.hpp +++ b/cpp/arcticdb/storage/failure_simulation.hpp @@ -11,9 +11,8 @@ #include #include #include -#include #include - +#include namespace arcticdb { #ifdef _WIN32 diff --git a/cpp/arcticdb/storage/file/file_store.hpp b/cpp/arcticdb/storage/file/file_store.hpp index d34650577a..f6499058d0 100644 --- a/cpp/arcticdb/storage/file/file_store.hpp +++ b/cpp/arcticdb/storage/file/file_store.hpp @@ -4,7 +4,6 @@ * * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0. */ - #include #include #include @@ -23,12 +22,15 @@ #include #include #include +#include +#include +#include +#include namespace arcticdb { - size_t max_data_size( - const std::vector>& items, + const std::vector>& items, const arcticdb::proto::encoding::VariantCodec& codec_opts, EncodingVersion encoding_version) { auto max_file_size = 0UL; @@ -62,7 +64,7 @@ void write_dataframe_to_file_internal( auto slices = slice(*frame, slicing); ARCTICDB_SUBSAMPLE_DEFAULT(SliceAndWrite) - auto slice_and_rowcount = get_slice_and_rowcount(slices); + auto slice_and_rowcount = arcticdb::pipelines::get_slice_and_rowcount(slices); const size_t write_window = ConfigsMap::instance()->get_int("VersionStore.BatchWriteWindow", static_cast(2 * async::TaskScheduler::instance()->io_thread_count())); auto key_seg_futs = folly::collect(folly::window(std::move(slice_and_rowcount), @@ -93,6 +95,7 @@ void write_dataframe_to_file_internal( return store->async_write(key_seg, dedup_map); }, batch_size)).via(&async::io_executor()) .thenValue([&frame, stream_id, store] (auto&& slice_and_keys) { + using namespace arcticdb::pipelines; return index::write_index(frame, std::forward(slice_and_keys), IndexPartialKey{stream_id, VersionId{0}}, store); }); // TODO include key size and key offset in max size calculation @@ -107,7 +110,7 @@ void write_dataframe_to_file_internal( version_store::ReadVersionOutput read_dataframe_from_file_internal( const StreamId& stream_id, const std::string& path, - ReadQuery& read_query, + pipelines::ReadQuery& read_query, const ReadOptions& read_options, const arcticdb::proto::encoding::VariantCodec &codec_opts) { auto config = storage::file::pack_config(path, codec_opts); diff --git a/cpp/arcticdb/storage/file/mapped_file_storage.cpp b/cpp/arcticdb/storage/file/mapped_file_storage.cpp index b15672605e..7a482355ee 100644 --- a/cpp/arcticdb/storage/file/mapped_file_storage.cpp +++ b/cpp/arcticdb/storage/file/mapped_file_storage.cpp @@ -7,16 +7,12 @@ #include #include -#include -#include #include -#include #include #include #include #include #include - namespace arcticdb::storage::file { MappedFileStorage::MappedFileStorage(const LibraryPath &lib, OpenMode mode, Config conf) : diff --git a/cpp/arcticdb/storage/file/mapped_file_storage.hpp b/cpp/arcticdb/storage/file/mapped_file_storage.hpp index aedc4359e5..70bfc1e2bb 100644 --- a/cpp/arcticdb/storage/file/mapped_file_storage.hpp +++ b/cpp/arcticdb/storage/file/mapped_file_storage.hpp @@ -8,9 +8,8 @@ #pragma once #include -#include #include -#include +#include #include #include diff --git a/cpp/arcticdb/storage/library_manager.cpp b/cpp/arcticdb/storage/library_manager.cpp index 098f48c300..7a0f1de831 100644 --- a/cpp/arcticdb/storage/library_manager.cpp +++ b/cpp/arcticdb/storage/library_manager.cpp @@ -4,17 +4,13 @@ #include #include #include -#include #include #include #include -#include #include #include #include -#include -#include namespace arcticdb::storage { diff --git a/cpp/arcticdb/storage/library_manager.hpp b/cpp/arcticdb/storage/library_manager.hpp index ddb32edaa8..39595b1d61 100644 --- a/cpp/arcticdb/storage/library_manager.hpp +++ b/cpp/arcticdb/storage/library_manager.hpp @@ -8,23 +8,29 @@ #pragma once #include #include -#include #include #include -#include +namespace pybind11 { + class object; +} + +namespace arcticdb { + class Store; +} namespace arcticdb::storage { + class Library; class LibraryManager { public: explicit LibraryManager(const std::shared_ptr& library); ARCTICDB_NO_MOVE_OR_COPY(LibraryManager) - void write_library_config(const py::object& lib_cfg, const LibraryPath& path, const StorageOverride& storage_override, + void write_library_config(const pybind11::object& lib_cfg, const LibraryPath& path, const StorageOverride& storage_override, bool validate) const; - [[nodiscard]] py::object get_library_config(const LibraryPath& path, const StorageOverride& storage_override = StorageOverride{}) const; + [[nodiscard]] pybind11::object get_library_config(const LibraryPath& path, const StorageOverride& storage_override = StorageOverride{}) const; [[nodiscard]] bool is_library_config_ok(const LibraryPath& path, bool throw_on_failure) const; diff --git a/cpp/arcticdb/storage/library_path.hpp b/cpp/arcticdb/storage/library_path.hpp index fcef9d314e..607af9a7b7 100644 --- a/cpp/arcticdb/storage/library_path.hpp +++ b/cpp/arcticdb/storage/library_path.hpp @@ -29,8 +29,11 @@ class DefaultStringViewable : public std::shared_ptr { std::make_shared(args...)), hash_(arcticdb::hash(std::string_view{*this})) {} - DefaultStringViewable(const DefaultStringViewable &that) : - std::shared_ptr::shared_ptr(that), hash_(that.hash_) {} + DefaultStringViewable(const DefaultStringViewable&) = default; + DefaultStringViewable& operator=(const DefaultStringViewable&) = default; + + DefaultStringViewable(DefaultStringViewable&&) = default; + DefaultStringViewable& operator=(DefaultStringViewable&&) = default; operator std::string_view() const { return *this->get(); diff --git a/cpp/arcticdb/storage/lmdb/lmdb_mock_client.cpp b/cpp/arcticdb/storage/lmdb/lmdb_mock_client.cpp index 879089797c..ebb4373fed 100644 --- a/cpp/arcticdb/storage/lmdb/lmdb_mock_client.cpp +++ b/cpp/arcticdb/storage/lmdb/lmdb_mock_client.cpp @@ -8,9 +8,9 @@ #include #include -#include #include #include +#include namespace arcticdb::storage::lmdb { diff --git a/cpp/arcticdb/storage/lmdb/lmdb_mock_client.hpp b/cpp/arcticdb/storage/lmdb/lmdb_mock_client.hpp index 6e81bf6730..a20829169c 100644 --- a/cpp/arcticdb/storage/lmdb/lmdb_mock_client.hpp +++ b/cpp/arcticdb/storage/lmdb/lmdb_mock_client.hpp @@ -10,11 +10,11 @@ #include #include -#include #include -#include -#include +namespace arcticdb::storage { + enum class StorageOperation; +} namespace arcticdb::storage::lmdb { diff --git a/cpp/arcticdb/storage/lmdb/lmdb_real_client.cpp b/cpp/arcticdb/storage/lmdb/lmdb_real_client.cpp index 4eb35fcd92..1f86f8ad3e 100644 --- a/cpp/arcticdb/storage/lmdb/lmdb_real_client.cpp +++ b/cpp/arcticdb/storage/lmdb/lmdb_real_client.cpp @@ -6,7 +6,6 @@ */ #include -#include #include #include #include diff --git a/cpp/arcticdb/storage/lmdb/lmdb_real_client.hpp b/cpp/arcticdb/storage/lmdb/lmdb_real_client.hpp index 279ed5f006..f643bcd6c2 100644 --- a/cpp/arcticdb/storage/lmdb/lmdb_real_client.hpp +++ b/cpp/arcticdb/storage/lmdb/lmdb_real_client.hpp @@ -10,9 +10,7 @@ #include #include -#include #include -#include namespace arcticdb::storage::lmdb { diff --git a/cpp/arcticdb/storage/lmdb/lmdb_storage.cpp b/cpp/arcticdb/storage/lmdb/lmdb_storage.cpp index fa1f3c568d..186ee6b375 100644 --- a/cpp/arcticdb/storage/lmdb/lmdb_storage.cpp +++ b/cpp/arcticdb/storage/lmdb/lmdb_storage.cpp @@ -12,19 +12,16 @@ #include #include -#include #include #include #include #include #include -#include #include #include #include -#include +#include -#include #include namespace arcticdb::storage::lmdb { diff --git a/cpp/arcticdb/storage/lmdb/lmdb_storage.hpp b/cpp/arcticdb/storage/lmdb/lmdb_storage.hpp index 5629738084..cf0273b1bf 100644 --- a/cpp/arcticdb/storage/lmdb/lmdb_storage.hpp +++ b/cpp/arcticdb/storage/lmdb/lmdb_storage.hpp @@ -8,19 +8,27 @@ #pragma once #include -#include #include - -#include -#include -#include - +#include #include +// LMDB++ is using `std::is_pod` in `lmdb++.h`, which is deprecated as of C++20. +// See: https://github.com/drycpp/lmdbxx/blob/0b43ca87d8cfabba392dfe884eb1edb83874de02/lmdb%2B%2B.h#L1068 +// See: https://en.cppreference.com/w/cpp/types/is_pod +// This suppresses the warning. +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#ifdef ARCTICDB_USING_CONDA +#include +#else +#include +#endif + namespace fs = std::filesystem; namespace arcticdb::storage::lmdb { +class LmdbClientWrapper; class LmdbStorage final : public Storage { public: diff --git a/cpp/arcticdb/storage/memory/memory_storage.cpp b/cpp/arcticdb/storage/memory/memory_storage.cpp index bf11171930..228c99571a 100644 --- a/cpp/arcticdb/storage/memory/memory_storage.cpp +++ b/cpp/arcticdb/storage/memory/memory_storage.cpp @@ -9,7 +9,6 @@ #include -#include #include #include #include diff --git a/cpp/arcticdb/storage/memory/memory_storage.hpp b/cpp/arcticdb/storage/memory/memory_storage.hpp index a4b288bd62..07f70ed863 100644 --- a/cpp/arcticdb/storage/memory/memory_storage.hpp +++ b/cpp/arcticdb/storage/memory/memory_storage.hpp @@ -8,12 +8,9 @@ #pragma once #include -#include #include -#include -#include +#include #include -#include namespace arcticdb::storage::memory { diff --git a/cpp/arcticdb/storage/mongo/mongo_client.cpp b/cpp/arcticdb/storage/mongo/mongo_client.cpp index 608dd04b8a..74f6eaace6 100644 --- a/cpp/arcticdb/storage/mongo/mongo_client.cpp +++ b/cpp/arcticdb/storage/mongo/mongo_client.cpp @@ -17,12 +17,12 @@ #include #include #include -#include #include #include #include -#include #include +#include +#include namespace arcticdb::storage::mongo { diff --git a/cpp/arcticdb/storage/mongo/mongo_client.hpp b/cpp/arcticdb/storage/mongo/mongo_client.hpp index 1eb0078bce..4f682e639a 100644 --- a/cpp/arcticdb/storage/mongo/mongo_client.hpp +++ b/cpp/arcticdb/storage/mongo/mongo_client.hpp @@ -6,9 +6,7 @@ */ #pragma once -#include -#include #include #include diff --git a/cpp/arcticdb/storage/mongo/mongo_instance.hpp b/cpp/arcticdb/storage/mongo/mongo_instance.hpp index d8b7e4863b..cf32bf70d3 100644 --- a/cpp/arcticdb/storage/mongo/mongo_instance.hpp +++ b/cpp/arcticdb/storage/mongo/mongo_instance.hpp @@ -10,7 +10,6 @@ #include #include #include -#include namespace arcticdb::storage::mongo { diff --git a/cpp/arcticdb/storage/mongo/mongo_mock_client.cpp b/cpp/arcticdb/storage/mongo/mongo_mock_client.cpp index 605991d21b..07e0286ba9 100644 --- a/cpp/arcticdb/storage/mongo/mongo_mock_client.cpp +++ b/cpp/arcticdb/storage/mongo/mongo_mock_client.cpp @@ -5,15 +5,13 @@ * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0. */ -#include #include #include -#include #include #include #include #include - +#include namespace arcticdb::storage::mongo { diff --git a/cpp/arcticdb/storage/mongo/mongo_mock_client.hpp b/cpp/arcticdb/storage/mongo/mongo_mock_client.hpp index ec4ea72228..0839f163f3 100644 --- a/cpp/arcticdb/storage/mongo/mongo_mock_client.hpp +++ b/cpp/arcticdb/storage/mongo/mongo_mock_client.hpp @@ -7,11 +7,13 @@ #pragma once -#include #include -#include #include +namespace arcticdb::storage { + enum class StorageOperation; +} + namespace arcticdb::storage::mongo { struct MongoDocumentKey { diff --git a/cpp/arcticdb/storage/mongo/mongo_storage.cpp b/cpp/arcticdb/storage/mongo/mongo_storage.cpp index 1ca7f808ee..d5ecee27f2 100644 --- a/cpp/arcticdb/storage/mongo/mongo_storage.cpp +++ b/cpp/arcticdb/storage/mongo/mongo_storage.cpp @@ -8,20 +8,16 @@ #include -#include #include -#include #include #include -#include #include #include #include #include #include #include -#include namespace arcticdb::storage::mongo { diff --git a/cpp/arcticdb/storage/mongo/mongo_storage.hpp b/cpp/arcticdb/storage/mongo/mongo_storage.hpp index ced21689d8..89bb45af29 100644 --- a/cpp/arcticdb/storage/mongo/mongo_storage.hpp +++ b/cpp/arcticdb/storage/mongo/mongo_storage.hpp @@ -8,16 +8,14 @@ #pragma once #include -#include -#include #include -#include -#include +#include namespace arcticdb::storage::mongo { class MongoInstance; class MongoClient; +class MongoClientWrapper; class MongoStorage final : public Storage { public: diff --git a/cpp/arcticdb/storage/python_bindings.cpp b/cpp/arcticdb/storage/python_bindings.cpp index c573852554..fd66cf092a 100644 --- a/cpp/arcticdb/storage/python_bindings.cpp +++ b/cpp/arcticdb/storage/python_bindings.cpp @@ -8,10 +8,8 @@ #include #include #include -#include #include -#include #include #include @@ -19,6 +17,7 @@ #include #include #include +#include namespace py = pybind11; diff --git a/cpp/arcticdb/storage/python_bindings.hpp b/cpp/arcticdb/storage/python_bindings.hpp index 5bab4f25d5..5e51e402bb 100644 --- a/cpp/arcticdb/storage/python_bindings.hpp +++ b/cpp/arcticdb/storage/python_bindings.hpp @@ -8,11 +8,7 @@ #pragma once #include -#include -#include -#include -#include namespace arcticdb::storage::apy { diff --git a/cpp/arcticdb/storage/rocksdb/rocksdb_storage.hpp b/cpp/arcticdb/storage/rocksdb/rocksdb_storage.hpp index b0c76460ef..cc22055a2b 100644 --- a/cpp/arcticdb/storage/rocksdb/rocksdb_storage.hpp +++ b/cpp/arcticdb/storage/rocksdb/rocksdb_storage.hpp @@ -8,11 +8,8 @@ #pragma once #include -#include #include -#include -#include -#include +#include #include diff --git a/cpp/arcticdb/storage/s3/nfs_backed_storage.cpp b/cpp/arcticdb/storage/s3/nfs_backed_storage.cpp index b28a2fb8a8..a602b5404e 100644 --- a/cpp/arcticdb/storage/s3/nfs_backed_storage.cpp +++ b/cpp/arcticdb/storage/s3/nfs_backed_storage.cpp @@ -9,8 +9,10 @@ #include #include +#include #include #include +#include namespace arcticdb::storage::nfs_backed { diff --git a/cpp/arcticdb/storage/s3/nfs_backed_storage.hpp b/cpp/arcticdb/storage/s3/nfs_backed_storage.hpp index 10c1cdeb8b..f2224d41c1 100644 --- a/cpp/arcticdb/storage/s3/nfs_backed_storage.hpp +++ b/cpp/arcticdb/storage/s3/nfs_backed_storage.hpp @@ -8,19 +8,13 @@ #pragma once #include -#include -#include -#include -#include -#include -#include #include -#include -#include -#include #include -#include + +namespace arcticdb::storage::s3 { + class S3ApiInstance; +} namespace arcticdb::storage::nfs_backed { diff --git a/cpp/arcticdb/storage/s3/s3_api.cpp b/cpp/arcticdb/storage/s3/s3_api.cpp index 675c7c9b59..b551317cef 100644 --- a/cpp/arcticdb/storage/s3/s3_api.cpp +++ b/cpp/arcticdb/storage/s3/s3_api.cpp @@ -6,7 +6,6 @@ */ #include -#include #include #include #include diff --git a/cpp/arcticdb/storage/s3/s3_mock_client.cpp b/cpp/arcticdb/storage/s3/s3_mock_client.cpp index e0bff24f18..c4936c4e66 100644 --- a/cpp/arcticdb/storage/s3/s3_mock_client.cpp +++ b/cpp/arcticdb/storage/s3/s3_mock_client.cpp @@ -8,10 +8,7 @@ #include #include -#include -#include -#include #include diff --git a/cpp/arcticdb/storage/s3/s3_mock_client.hpp b/cpp/arcticdb/storage/s3/s3_mock_client.hpp index 04c6812218..aeee790396 100644 --- a/cpp/arcticdb/storage/s3/s3_mock_client.hpp +++ b/cpp/arcticdb/storage/s3/s3_mock_client.hpp @@ -7,22 +7,10 @@ #pragma once -#include #include -#include - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include + + namespace arcticdb::storage::s3 { diff --git a/cpp/arcticdb/storage/s3/s3_real_client.cpp b/cpp/arcticdb/storage/s3/s3_real_client.cpp index 37e18788e0..b96da089a9 100644 --- a/cpp/arcticdb/storage/s3/s3_real_client.cpp +++ b/cpp/arcticdb/storage/s3/s3_real_client.cpp @@ -11,15 +11,11 @@ #include #include -#include #include -#include -#include #include #include -#include #include #include #include diff --git a/cpp/arcticdb/storage/s3/s3_real_client.hpp b/cpp/arcticdb/storage/s3/s3_real_client.hpp index 57bfe87b65..6f023594ee 100644 --- a/cpp/arcticdb/storage/s3/s3_real_client.hpp +++ b/cpp/arcticdb/storage/s3/s3_real_client.hpp @@ -11,17 +11,7 @@ #include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include + namespace arcticdb::storage::s3{ diff --git a/cpp/arcticdb/storage/s3/s3_storage.cpp b/cpp/arcticdb/storage/s3/s3_storage.cpp index c82fe35627..86485fbf6f 100644 --- a/cpp/arcticdb/storage/s3/s3_storage.cpp +++ b/cpp/arcticdb/storage/s3/s3_storage.cpp @@ -9,35 +9,14 @@ #include -#include -#include -#include -#include #include -#include -#include -#include -#include #include #include -#include #include -#include -#include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include + #include #include diff --git a/cpp/arcticdb/storage/s3/s3_storage.hpp b/cpp/arcticdb/storage/s3/s3_storage.hpp index 1c8f870710..bb70a9c031 100644 --- a/cpp/arcticdb/storage/s3/s3_storage.hpp +++ b/cpp/arcticdb/storage/s3/s3_storage.hpp @@ -8,24 +8,19 @@ #pragma once #include -#include -#include -#include -#include #include -#include #include -#include #include -#include #include +#include +#include #include #include #include #include namespace arcticdb::storage::s3 { - +class S3ApiInstance; const std::string USE_AWS_CRED_PROVIDERS_TOKEN = "_RBAC_"; class S3Storage final : public Storage { diff --git a/cpp/arcticdb/storage/s3/s3_storage_tool.cpp b/cpp/arcticdb/storage/s3/s3_storage_tool.cpp index c87f7f95c7..7732cee1f2 100644 --- a/cpp/arcticdb/storage/s3/s3_storage_tool.cpp +++ b/cpp/arcticdb/storage/s3/s3_storage_tool.cpp @@ -6,12 +6,13 @@ */ #include - -#include +#include +#include #include #include #include #include +#include namespace arcticdb::storage::s3 { diff --git a/cpp/arcticdb/storage/s3/s3_storage_tool.hpp b/cpp/arcticdb/storage/s3/s3_storage_tool.hpp index ee21d384af..6572385770 100644 --- a/cpp/arcticdb/storage/s3/s3_storage_tool.hpp +++ b/cpp/arcticdb/storage/s3/s3_storage_tool.hpp @@ -7,10 +7,11 @@ #pragma once -#include #include +#include namespace arcticdb::storage::s3 { +class S3ApiInstance; class S3StorageTool { public: diff --git a/cpp/arcticdb/storage/storage.hpp b/cpp/arcticdb/storage/storage.hpp index 9825d4415c..40db18be25 100644 --- a/cpp/arcticdb/storage/storage.hpp +++ b/cpp/arcticdb/storage/storage.hpp @@ -9,7 +9,6 @@ #include #include #include -#include namespace arcticdb::storage { diff --git a/cpp/arcticdb/storage/storage_factory.hpp b/cpp/arcticdb/storage/storage_factory.hpp index 36478a1775..a9d1c8dad0 100644 --- a/cpp/arcticdb/storage/storage_factory.hpp +++ b/cpp/arcticdb/storage/storage_factory.hpp @@ -7,10 +7,8 @@ #pragma once -#include #include #include -#include namespace arcticdb { diff --git a/cpp/arcticdb/storage/test/in_memory_store.hpp b/cpp/arcticdb/storage/test/in_memory_store.hpp index 695c2fced6..1a34b8fdc7 100644 --- a/cpp/arcticdb/storage/test/in_memory_store.hpp +++ b/cpp/arcticdb/storage/test/in_memory_store.hpp @@ -14,6 +14,7 @@ #include #include #include +#include #include diff --git a/cpp/arcticdb/storage/test/test_azure_storage.cpp b/cpp/arcticdb/storage/test/test_azure_storage.cpp index 92e56ee2a6..3201edb179 100644 --- a/cpp/arcticdb/storage/test/test_azure_storage.cpp +++ b/cpp/arcticdb/storage/test/test_azure_storage.cpp @@ -6,12 +6,10 @@ */ #include -#include -#include #include #include -#include +#include #include using namespace arcticdb; diff --git a/cpp/arcticdb/storage/test/test_embedded.cpp b/cpp/arcticdb/storage/test/test_embedded.cpp index 60ae84f09c..8bf0a514d7 100644 --- a/cpp/arcticdb/storage/test/test_embedded.cpp +++ b/cpp/arcticdb/storage/test/test_embedded.cpp @@ -22,9 +22,6 @@ #include #include #include -#include -#include -#include namespace ac = arcticdb; namespace as = arcticdb::storage; diff --git a/cpp/arcticdb/storage/test/test_memory_storage.cpp b/cpp/arcticdb/storage/test/test_memory_storage.cpp index 730a596832..02935a86a5 100644 --- a/cpp/arcticdb/storage/test/test_memory_storage.cpp +++ b/cpp/arcticdb/storage/test/test_memory_storage.cpp @@ -6,7 +6,6 @@ */ #include -#include #include #include diff --git a/cpp/arcticdb/storage/test/test_mongo_storage.cpp b/cpp/arcticdb/storage/test/test_mongo_storage.cpp index 87f560d180..41ff2d78a1 100644 --- a/cpp/arcticdb/storage/test/test_mongo_storage.cpp +++ b/cpp/arcticdb/storage/test/test_mongo_storage.cpp @@ -7,12 +7,8 @@ #include -#include #include -#include -#include -#include const std::string test_server("mongodb://localhost:27017"); diff --git a/cpp/arcticdb/storage/test/test_s3_storage.cpp b/cpp/arcticdb/storage/test/test_s3_storage.cpp index 4365f4c748..83b5a7815a 100644 --- a/cpp/arcticdb/storage/test/test_s3_storage.cpp +++ b/cpp/arcticdb/storage/test/test_s3_storage.cpp @@ -11,12 +11,9 @@ #include #include #include -#include #include -#include #include -#include struct EnvFunctionShim : ::testing::Test { std::unordered_set env_vars_to_unset{}; diff --git a/cpp/arcticdb/storage/test/test_storage_exceptions.cpp b/cpp/arcticdb/storage/test/test_storage_exceptions.cpp index a7d422ec68..035f84cca0 100644 --- a/cpp/arcticdb/storage/test/test_storage_exceptions.cpp +++ b/cpp/arcticdb/storage/test/test_storage_exceptions.cpp @@ -7,7 +7,6 @@ #include -#include #include #include #include diff --git a/cpp/arcticdb/stream/aggregator.hpp b/cpp/arcticdb/stream/aggregator.hpp index 90d2f9753a..32a6c9d057 100644 --- a/cpp/arcticdb/stream/aggregator.hpp +++ b/cpp/arcticdb/stream/aggregator.hpp @@ -12,7 +12,6 @@ #include #include -#include namespace arcticdb::stream { diff --git a/cpp/arcticdb/stream/append_map.cpp b/cpp/arcticdb/stream/append_map.cpp index 0ed792e691..0a2d56fd98 100644 --- a/cpp/arcticdb/stream/append_map.cpp +++ b/cpp/arcticdb/stream/append_map.cpp @@ -6,11 +6,9 @@ */ #include -#include #include #include #include -#include #include #include #include @@ -250,7 +248,7 @@ std::vector get_incomplete( auto entries = get_incomplete_append_slices_for_stream_id(store, stream_id, via_iteration, load_data); util::variant_match(range, - [](const RowRange &) { + [](const pipelines::RowRange &) { util::raise_rte("Only timestamp based ranges supported for filtering."); }, [&entries](const IndexRange &index_range) { @@ -367,7 +365,7 @@ AppendMapEntry entry_from_key(const std::shared_ptr& store, const if(seg) seg->attach_descriptor(desc); - auto frame_slice = FrameSlice{desc, ColRange{index_field_count, field_count}, RowRange{0, entry.total_rows_}}; + auto frame_slice = FrameSlice{desc, pipelines::ColRange{index_field_count, field_count}, pipelines::RowRange{0, entry.total_rows_}}; entry.slice_and_key_ = SliceAndKey{std::move(frame_slice), key, std::move(seg)}; return entry; } diff --git a/cpp/arcticdb/stream/append_map.hpp b/cpp/arcticdb/stream/append_map.hpp index 39eb4fe9e0..c4f39f2101 100644 --- a/cpp/arcticdb/stream/append_map.hpp +++ b/cpp/arcticdb/stream/append_map.hpp @@ -10,9 +10,6 @@ #include #include #include -#include -#include -#include namespace arcticdb::pipelines { struct PythonOutputFrame; @@ -21,7 +18,9 @@ using FilterRange = std::variant; } namespace arcticdb { - +namespace stream { + struct StreamSource; +} std::pair, size_t> read_head( const std::shared_ptr& store, StreamId stream_id); diff --git a/cpp/arcticdb/stream/piloted_clock.hpp b/cpp/arcticdb/stream/piloted_clock.hpp index f2b79189ac..e14a311ad0 100644 --- a/cpp/arcticdb/stream/piloted_clock.hpp +++ b/cpp/arcticdb/stream/piloted_clock.hpp @@ -1,5 +1,4 @@ #include -#include namespace arcticdb { diff --git a/cpp/arcticdb/stream/python_bindings.cpp b/cpp/arcticdb/stream/python_bindings.cpp index 02cd52cfd1..3b8c518b18 100644 --- a/cpp/arcticdb/stream/python_bindings.cpp +++ b/cpp/arcticdb/stream/python_bindings.cpp @@ -7,17 +7,11 @@ #include -#include #include -#include #include #include -#include #include -#include -#include -#include namespace py = pybind11; diff --git a/cpp/arcticdb/stream/test/stream_test_common.cpp b/cpp/arcticdb/stream/test/stream_test_common.cpp index 3a02c8a639..57e38155ea 100644 --- a/cpp/arcticdb/stream/test/stream_test_common.cpp +++ b/cpp/arcticdb/stream/test/stream_test_common.cpp @@ -5,10 +5,10 @@ * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0. */ -#include - +#include +#include namespace arcticdb { -std::atomic PilotedClock::time_{0}; +std::atomic PilotedClock::time_{0}; } //namespace arcticdb \ No newline at end of file diff --git a/cpp/arcticdb/stream/test/stream_test_common.hpp b/cpp/arcticdb/stream/test/stream_test_common.hpp index 491a8d789e..5bd5239b46 100644 --- a/cpp/arcticdb/stream/test/stream_test_common.hpp +++ b/cpp/arcticdb/stream/test/stream_test_common.hpp @@ -8,25 +8,13 @@ #pragma once #include -#include #include -#include -#include -#include - -#include -#include -#include -#include -#include + #include #include -#include #include #include #include -#include - #include namespace fg = folly::gen; @@ -347,19 +335,4 @@ inline auto test_store(const std::string &lib_name) { return version_store; } -struct TestStore : ::testing::Test { -protected: - virtual std::string get_name() = 0; - - void SetUp() override { - test_store_ = test_store(get_name()); - } - - void TearDown() override { - test_store_->clear(); - } - - std::shared_ptr test_store_; -}; - } //namespace arcticdb diff --git a/cpp/arcticdb/stream/test/test_aggregator.cpp b/cpp/arcticdb/stream/test/test_aggregator.cpp index ead376d72f..8c63c65306 100644 --- a/cpp/arcticdb/stream/test/test_aggregator.cpp +++ b/cpp/arcticdb/stream/test/test_aggregator.cpp @@ -5,12 +5,10 @@ * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0. */ -#include #include #include -#include using namespace arcticdb; namespace as = arcticdb::stream; diff --git a/cpp/arcticdb/stream/test/test_append_map.cpp b/cpp/arcticdb/stream/test/test_append_map.cpp index fa7deef7d6..293a57b7a1 100644 --- a/cpp/arcticdb/stream/test/test_append_map.cpp +++ b/cpp/arcticdb/stream/test/test_append_map.cpp @@ -13,6 +13,7 @@ #include #include #include +#include TEST(Append, Simple) { using namespace arcticdb; diff --git a/cpp/arcticdb/stream/test/test_row_builder.cpp b/cpp/arcticdb/stream/test/test_row_builder.cpp index b7c2a79298..1a2626d3a3 100644 --- a/cpp/arcticdb/stream/test/test_row_builder.cpp +++ b/cpp/arcticdb/stream/test/test_row_builder.cpp @@ -7,7 +7,6 @@ #include -#include #include namespace as = arcticdb::stream; diff --git a/cpp/arcticdb/stream/test/test_segment_aggregator.cpp b/cpp/arcticdb/stream/test/test_segment_aggregator.cpp index eed31d9ac7..32816332b1 100644 --- a/cpp/arcticdb/stream/test/test_segment_aggregator.cpp +++ b/cpp/arcticdb/stream/test/test_segment_aggregator.cpp @@ -7,9 +7,9 @@ #include #include -#include #include #include +#include TEST(SegmentAggregator, Basic) { using namespace arcticdb; @@ -36,7 +36,7 @@ TEST(SegmentAggregator, Basic) { segments.emplace_back(std::move(wrapper.segment())); } - SegmentSinkWrapper seg_wrapper(symbol, TimeseriesIndex::default_index(), fields_from_range(std::vector{ + SegmentSinkWrapper seg_wrapper(symbol, stream::TimeseriesIndex::default_index(), fields_from_range(std::vector{ scalar_field(DataType::UINT64, "numbers"), scalar_field(DataType::ASCII_DYNAMIC64, "strings") })); diff --git a/cpp/arcticdb/stream/test/test_store_common.hpp b/cpp/arcticdb/stream/test/test_store_common.hpp new file mode 100644 index 0000000000..b54b0a0090 --- /dev/null +++ b/cpp/arcticdb/stream/test/test_store_common.hpp @@ -0,0 +1,26 @@ +/* Copyright 2024 Man Group Operations Limited + * + * Use of this software is governed by the Business Source License 1.1 included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software + * will be governed by the Apache License, version 2.0. + */ +#include +#include + +namespace arcticdb { + struct TestStore : ::testing::Test { + protected: + virtual std::string get_name() = 0; + + void SetUp() override { + test_store_ = test_store(get_name()); + } + + void TearDown() override { + test_store_->clear(); + } + + std::shared_ptr test_store_; + }; +} diff --git a/cpp/arcticdb/stream/test/test_types.cpp b/cpp/arcticdb/stream/test/test_types.cpp index fd3310d115..79ba0a7b47 100644 --- a/cpp/arcticdb/stream/test/test_types.cpp +++ b/cpp/arcticdb/stream/test/test_types.cpp @@ -10,7 +10,6 @@ #include -#include #include #include diff --git a/cpp/arcticdb/toolbox/library_tool.cpp b/cpp/arcticdb/toolbox/library_tool.cpp index 0ff5573a7b..fe92744f4d 100644 --- a/cpp/arcticdb/toolbox/library_tool.cpp +++ b/cpp/arcticdb/toolbox/library_tool.cpp @@ -9,14 +9,12 @@ #include #include -#include #include #include #include #include #include #include -#include #include #include diff --git a/cpp/arcticdb/toolbox/library_tool.hpp b/cpp/arcticdb/toolbox/library_tool.hpp index abc4867b4c..dad916c638 100644 --- a/cpp/arcticdb/toolbox/library_tool.hpp +++ b/cpp/arcticdb/toolbox/library_tool.hpp @@ -7,12 +7,9 @@ #pragma once -#include -#include #include #include -#include #include #include diff --git a/cpp/arcticdb/toolbox/python_bindings.cpp b/cpp/arcticdb/toolbox/python_bindings.cpp index c089d4cb72..a7107869d4 100644 --- a/cpp/arcticdb/toolbox/python_bindings.cpp +++ b/cpp/arcticdb/toolbox/python_bindings.cpp @@ -5,10 +5,8 @@ * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0. */ -#include #include -#include #include #include #include diff --git a/cpp/arcticdb/util/allocator.cpp b/cpp/arcticdb/util/allocator.cpp index 735389afec..a46b00fd52 100644 --- a/cpp/arcticdb/util/allocator.cpp +++ b/cpp/arcticdb/util/allocator.cpp @@ -6,7 +6,7 @@ */ #include - +#include #include #include #include @@ -15,7 +15,6 @@ #include #include -#include namespace arcticdb { diff --git a/cpp/arcticdb/util/buffer.hpp b/cpp/arcticdb/util/buffer.hpp index 905856c22c..22cf8a9181 100644 --- a/cpp/arcticdb/util/buffer.hpp +++ b/cpp/arcticdb/util/buffer.hpp @@ -7,8 +7,8 @@ #pragma once -#include -#include +#include +#include #include #include diff --git a/cpp/arcticdb/util/buffer_holder.hpp b/cpp/arcticdb/util/buffer_holder.hpp index 603500eeea..4684bbcf4e 100644 --- a/cpp/arcticdb/util/buffer_holder.hpp +++ b/cpp/arcticdb/util/buffer_holder.hpp @@ -7,7 +7,7 @@ #pragma once -#include +#include #include namespace arcticdb { diff --git a/cpp/arcticdb/util/buffer_pool.cpp b/cpp/arcticdb/util/buffer_pool.cpp index 3038717487..a22ca67944 100644 --- a/cpp/arcticdb/util/buffer_pool.cpp +++ b/cpp/arcticdb/util/buffer_pool.cpp @@ -6,7 +6,7 @@ */ #include -#include +#include namespace arcticdb { diff --git a/cpp/arcticdb/util/buffer_pool.hpp b/cpp/arcticdb/util/buffer_pool.hpp index 8f8efaab0b..97d3984f58 100644 --- a/cpp/arcticdb/util/buffer_pool.hpp +++ b/cpp/arcticdb/util/buffer_pool.hpp @@ -7,17 +7,19 @@ #pragma once -#include #ifdef ARCTICDB_USING_CONDA #include #else #include #endif - +#include +#include namespace arcticdb { +struct Buffer; + struct lock_policy { using mutex_type = std::mutex; using lock_type = std::lock_guard; diff --git a/cpp/arcticdb/util/decimal.cpp b/cpp/arcticdb/util/decimal.cpp index 337e59f8ae..fd3da5824a 100644 --- a/cpp/arcticdb/util/decimal.cpp +++ b/cpp/arcticdb/util/decimal.cpp @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include diff --git a/cpp/arcticdb/util/error_code.hpp b/cpp/arcticdb/util/error_code.hpp index 921f649ce6..e67e437608 100644 --- a/cpp/arcticdb/util/error_code.hpp +++ b/cpp/arcticdb/util/error_code.hpp @@ -6,11 +6,7 @@ */ #pragma once - #include - -#include - #include #include #include diff --git a/cpp/arcticdb/util/format_date.hpp b/cpp/arcticdb/util/format_date.hpp index d02f171118..7617e60c33 100644 --- a/cpp/arcticdb/util/format_date.hpp +++ b/cpp/arcticdb/util/format_date.hpp @@ -11,13 +11,14 @@ #include #include #include +#include namespace arcticdb::util { using Clock = std::chrono::system_clock; using TimePoint = std::chrono::time_point; -inline std::string format_timestamp(entity::timestamp ts) { +inline std::string format_timestamp(arcticdb::entity::timestamp ts) { std::stringstream ss; #if defined(_WIN32) or defined(__APPLE__) auto time_point = Clock::time_point(std::chrono::duration_cast(std::chrono::nanoseconds(ts))); diff --git a/cpp/arcticdb/util/memory_mapped_file.hpp b/cpp/arcticdb/util/memory_mapped_file.hpp index b3904a61fc..c5a15edfea 100644 --- a/cpp/arcticdb/util/memory_mapped_file.hpp +++ b/cpp/arcticdb/util/memory_mapped_file.hpp @@ -3,9 +3,8 @@ #include #ifdef WIN32 - +#include #include "Windows.h" -#include "winerror.h" namespace arcticdb { @@ -18,6 +17,7 @@ namespace arcticdb { public: MemoryMappedFile() = default; + ARCTICDB_NO_MOVE_OR_COPY(MemoryMappedFile) size_t get_file_size(const std::string& file_path) { LARGE_INTEGER size; @@ -122,7 +122,6 @@ namespace arcticdb { }; }//namespace arcticdb - #else #include #include diff --git a/cpp/arcticdb/util/memory_tracing.hpp b/cpp/arcticdb/util/memory_tracing.hpp index ac4431fbd5..01e2f37169 100644 --- a/cpp/arcticdb/util/memory_tracing.hpp +++ b/cpp/arcticdb/util/memory_tracing.hpp @@ -10,7 +10,7 @@ #include #include #include -#include +#include #ifdef WIN32 #include #else diff --git a/cpp/arcticdb/util/name_validation.cpp b/cpp/arcticdb/util/name_validation.cpp index d6765c50c3..2b1fe0c383 100644 --- a/cpp/arcticdb/util/name_validation.cpp +++ b/cpp/arcticdb/util/name_validation.cpp @@ -7,10 +7,9 @@ #include -#include #include #include -#include +#include #include namespace arcticdb { @@ -23,7 +22,7 @@ constexpr auto MAX_SIZE = 255; void verify_name( const std::string& name_type_for_error, - const StringId& name, + const entity::StringId& name, bool check_symbol_out_of_range = true, const std::set& unsupported_chars = UNSUPPORTED_S3_CHARS, std::optional unsupported_prefix = std::nullopt, @@ -74,7 +73,7 @@ void verify_name( } } -void verify_symbol_key(const StreamId& symbol_key) { +void verify_symbol_key(const entity::StreamId& symbol_key) { if (ConfigsMap::instance()->get_int("VersionStore.NoStrictSymbolCheck")) { ARCTICDB_DEBUG(log::version(), "Key with stream id {} will not be strictly checked because VersionStore.NoStrictSymbolCheck variable is set to 1.", @@ -84,13 +83,13 @@ void verify_symbol_key(const StreamId& symbol_key) { util::variant_match( symbol_key, - [](const NumericId &num_symbol_key) { + [](const entity::NumericId &num_symbol_key) { (void) num_symbol_key; // Suppresses -Wunused-parameter ARCTICDB_DEBUG(log::version(), "Nothing to verify in stream id {} as it contains a NumericId.", num_symbol_key); return; }, - [](const StringId &str_symbol_key) { + [](const entity::StringId &str_symbol_key) { verify_name("symbol key", str_symbol_key); } ); @@ -99,7 +98,7 @@ void verify_symbol_key(const StreamId& symbol_key) { // Library names starting with "/" fail if storage is LMDB and library parts starting with "/" will fail in Mongo constexpr auto UNSUPPORTED_LMDB_MONGO_PREFIX = '/'; -void verify_library_path(const StringId& library_path, char delim) { +void verify_library_path(const entity::StringId& library_path, char delim) { verify_name("library name", library_path, false, {}, {}, delim); } @@ -119,7 +118,7 @@ void verify_library_path_part(const std::string& library_part, char delim) { } } -void verify_library_path_on_write(const Store* store, const StringId& library_path) { +void verify_library_path_on_write(const Store* store, const entity::StringId& library_path) { verify_name("library name", library_path, true, UNSUPPORTED_S3_CHARS); user_input::check( store->is_path_valid(library_path), diff --git a/cpp/arcticdb/util/name_validation.hpp b/cpp/arcticdb/util/name_validation.hpp index ad0587e433..bc37fe0fcf 100644 --- a/cpp/arcticdb/util/name_validation.hpp +++ b/cpp/arcticdb/util/name_validation.hpp @@ -7,27 +7,25 @@ #pragma once -#include #include -#include -#include -#include namespace arcticdb { +class Store; + // Verifies whether a symbol_key is valid and raises UserInputException exceptions on invalid symbol names. // Should be used only when writing new symbols to allow for backwards compatibility with old symbols. -void verify_symbol_key(const StreamId &symbol_key); +void verify_symbol_key(const entity::StreamId &symbol_key); // Does strict checks on library names and raises UserInputException if it encounters an error. // Should be checked only when writing new libraries to allow for backwards compatibility // with old invalid libraries. -void verify_library_path_on_write(const Store* store, const StringId& library_path); +void verify_library_path_on_write(const Store* store, const entity::StringId& library_path); // These two do relaxed checks which should always be run on each library operation (including // already existing libraries). These raise friendly error messages instead of segfaulting or // raising an obscure internal error. -void verify_library_path(const StringId& library_path, char delim); +void verify_library_path(const entity::StringId& library_path, char delim); void verify_library_path_part(const std::string& library_part, char delim); diff --git a/cpp/arcticdb/util/test/gtest_main.cpp b/cpp/arcticdb/util/test/gtest_main.cpp index cb6151b775..2b5480781b 100644 --- a/cpp/arcticdb/util/test/gtest_main.cpp +++ b/cpp/arcticdb/util/test/gtest_main.cpp @@ -7,7 +7,7 @@ #include #include -#include // Must not directly include Python.h on Windows +#include // Must not directly include Python.h on Windows int main(int argc, char **argv) { ::testing::InitGoogleTest(&argc, argv); diff --git a/cpp/arcticdb/util/test/rapidcheck_generators.cpp b/cpp/arcticdb/util/test/rapidcheck_generators.cpp index 2b120cc9bd..69a1b39319 100644 --- a/cpp/arcticdb/util/test/rapidcheck_generators.cpp +++ b/cpp/arcticdb/util/test/rapidcheck_generators.cpp @@ -6,6 +6,7 @@ */ #include +#include #include #include diff --git a/cpp/arcticdb/util/test/rapidcheck_generators.hpp b/cpp/arcticdb/util/test/rapidcheck_generators.hpp index f7ad4afdaa..19fd2b428e 100644 --- a/cpp/arcticdb/util/test/rapidcheck_generators.hpp +++ b/cpp/arcticdb/util/test/rapidcheck_generators.hpp @@ -8,11 +8,16 @@ #pragma once #include -#include - +#include +#include +#include +#include +#include +#include #include -#include +#include +namespace ac = arcticdb; inline rc::Gen gen_numeric_datatype() { return rc::gen::element( arcticdb::entity::DataType::INT8, @@ -153,17 +158,17 @@ folly::Future write_frame_data(const TestDataFrame inline folly::Future write_test_frame(ac::StreamId stream_id, const TestDataFrame &data_frame, - std::shared_ptr store) { + std::shared_ptr store) { auto schema = schema_from_test_frame(data_frame, std::move(stream_id)); auto start_end = test_frame_range(data_frame); auto gen_id = arcticdb::VersionId(0); - ac::StreamWriter writer{ + ac::stream::StreamWriter writer{ std::move(schema), std::move(store), gen_id, start_end, - ac::RowCountSegmentPolicy{4} + ac::stream::RowCountSegmentPolicy{4} }; return write_frame_data(data_frame, writer); @@ -217,9 +222,9 @@ bool check_read_frame(const TestDataFrame &data_frame, ReaderType &reader, std:: inline bool check_test_frame(const TestDataFrame &data_frame, const arcticdb::entity::AtomKey &key, - std::shared_ptr store, + std::shared_ptr store, std::vector &errors) { - ac::StreamReader()>, arcticdb::SegmentInMemory::Row> stream_reader{ + ac::stream::StreamReader()>, arcticdb::SegmentInMemory::Row> stream_reader{ [&]() { return std::vector{key}; }, std::move(store) }; diff --git a/cpp/arcticdb/util/test/rapidcheck_string_pool.cpp b/cpp/arcticdb/util/test/rapidcheck_string_pool.cpp index dd1418e3f6..c0b4387da4 100644 --- a/cpp/arcticdb/util/test/rapidcheck_string_pool.cpp +++ b/cpp/arcticdb/util/test/rapidcheck_string_pool.cpp @@ -6,9 +6,9 @@ */ #include -#include #include #include +#include #include #include diff --git a/cpp/arcticdb/util/test/test_bitmagic.cpp b/cpp/arcticdb/util/test/test_bitmagic.cpp index 725b2d4c16..19163d947e 100644 --- a/cpp/arcticdb/util/test/test_bitmagic.cpp +++ b/cpp/arcticdb/util/test/test_bitmagic.cpp @@ -7,12 +7,11 @@ #include -#include -#include #include #include +#include +#include -#include TEST(BitMagic, Basic) { using namespace arcticdb; diff --git a/cpp/arcticdb/util/test/test_buffer_pool.cpp b/cpp/arcticdb/util/test/test_buffer_pool.cpp index ccae59f0d7..eb7522039b 100644 --- a/cpp/arcticdb/util/test/test_buffer_pool.cpp +++ b/cpp/arcticdb/util/test/test_buffer_pool.cpp @@ -7,6 +7,7 @@ #include #include +#include namespace arcticdb { TEST(BufferPool, Basic) { diff --git a/cpp/arcticdb/util/test/test_cursor.cpp b/cpp/arcticdb/util/test/test_cursor.cpp index a99f9947a6..151830e921 100644 --- a/cpp/arcticdb/util/test/test_cursor.cpp +++ b/cpp/arcticdb/util/test/test_cursor.cpp @@ -10,7 +10,6 @@ #include #include #include -#include #include #include diff --git a/cpp/arcticdb/util/test/test_format_date.cpp b/cpp/arcticdb/util/test/test_format_date.cpp index c74bdef6ee..daf0688f20 100644 --- a/cpp/arcticdb/util/test/test_format_date.cpp +++ b/cpp/arcticdb/util/test/test_format_date.cpp @@ -6,7 +6,6 @@ */ #include -#include #include TEST(FormatDate, ZeroTs) { diff --git a/cpp/arcticdb/util/test/test_hash.cpp b/cpp/arcticdb/util/test/test_hash.cpp index db3b24aa1b..6e7dbc84c9 100644 --- a/cpp/arcticdb/util/test/test_hash.cpp +++ b/cpp/arcticdb/util/test/test_hash.cpp @@ -7,7 +7,6 @@ #include #include -#include using namespace arcticdb; diff --git a/cpp/arcticdb/util/test/test_ranges_from_future.cpp b/cpp/arcticdb/util/test/test_ranges_from_future.cpp index eba54d80c1..2da11f60f9 100644 --- a/cpp/arcticdb/util/test/test_ranges_from_future.cpp +++ b/cpp/arcticdb/util/test/test_ranges_from_future.cpp @@ -7,7 +7,6 @@ #include #include - #include #include diff --git a/cpp/arcticdb/util/test/test_tracing_allocator.cpp b/cpp/arcticdb/util/test/test_tracing_allocator.cpp index dd45f4cd13..6fae853de8 100644 --- a/cpp/arcticdb/util/test/test_tracing_allocator.cpp +++ b/cpp/arcticdb/util/test/test_tracing_allocator.cpp @@ -7,7 +7,6 @@ #include #include -#include #include TEST(Allocator, Tracing) { diff --git a/cpp/arcticdb/util/type_handler.hpp b/cpp/arcticdb/util/type_handler.hpp index 87fcb6296f..d12f3cb3fa 100644 --- a/cpp/arcticdb/util/type_handler.hpp +++ b/cpp/arcticdb/util/type_handler.hpp @@ -9,7 +9,6 @@ #include #include -#include #include @@ -19,6 +18,7 @@ namespace arcticdb { struct BufferHolder; +struct ColumnMapping; struct ITypeHandler { template diff --git a/cpp/arcticdb/version/local_versioned_engine.cpp b/cpp/arcticdb/version/local_versioned_engine.cpp index 0fd631a72a..0b56349220 100644 --- a/cpp/arcticdb/version/local_versioned_engine.cpp +++ b/cpp/arcticdb/version/local_versioned_engine.cpp @@ -15,12 +15,15 @@ #include #include #include -#include #include #include #include #include #include +#include +#include +#include + namespace arcticdb::version_store { diff --git a/cpp/arcticdb/version/local_versioned_engine.hpp b/cpp/arcticdb/version/local_versioned_engine.hpp index 0c9a1e6bc3..d080a4a54b 100644 --- a/cpp/arcticdb/version/local_versioned_engine.hpp +++ b/cpp/arcticdb/version/local_versioned_engine.hpp @@ -8,21 +8,15 @@ #pragma once #include -#include -#include #include #include #include #include #include -#include -#include -#include #include #include #include -#include namespace arcticdb::version_store { /** diff --git a/cpp/arcticdb/version/python_bindings.cpp b/cpp/arcticdb/version/python_bindings.cpp index a75911105a..409bbcf59e 100644 --- a/cpp/arcticdb/version/python_bindings.cpp +++ b/cpp/arcticdb/version/python_bindings.cpp @@ -7,22 +7,20 @@ #include #include -#include #include #include #include #include -#include #include #include #include -#include #include #include #include #include #include #include +#include #include namespace arcticdb::version_store { diff --git a/cpp/arcticdb/version/python_bindings.hpp b/cpp/arcticdb/version/python_bindings.hpp index c3c9e86f18..b255fbbed4 100644 --- a/cpp/arcticdb/version/python_bindings.hpp +++ b/cpp/arcticdb/version/python_bindings.hpp @@ -8,10 +8,13 @@ #pragma once #include -#include namespace py = pybind11; +namespace arcticdb { + struct ArcticException; +} + namespace arcticdb::version_store { void register_bindings(py::module &m, py::exception& base_exception); diff --git a/cpp/arcticdb/version/schema_checks.hpp b/cpp/arcticdb/version/schema_checks.hpp index 28a7e3ed1d..2bef2e26b9 100644 --- a/cpp/arcticdb/version/schema_checks.hpp +++ b/cpp/arcticdb/version/schema_checks.hpp @@ -51,7 +51,7 @@ inline void check_normalization_index_match( const IndexDescriptor::Type old_idx_kind = old_descriptor.index().type(); const IndexDescriptor::Type new_idx_kind = frame.desc.index().type(); if (operation == UPDATE) { - const bool new_is_timeseries = std::holds_alternative(frame.index); + const bool new_is_timeseries = std::holds_alternative(frame.index); util::check_rte( (old_idx_kind == IndexDescriptor::TIMESTAMP || old_idx_kind == IndexDescriptor::EMPTY) && new_is_timeseries, "Update will not work as expected with a non-timeseries index" diff --git a/cpp/arcticdb/version/snapshot.cpp b/cpp/arcticdb/version/snapshot.cpp index 027d7b6114..9964351ccb 100644 --- a/cpp/arcticdb/version/snapshot.cpp +++ b/cpp/arcticdb/version/snapshot.cpp @@ -8,6 +8,8 @@ #include #include #include +#include +#include #include diff --git a/cpp/arcticdb/version/snapshot.hpp b/cpp/arcticdb/version/snapshot.hpp index 0c9c0f050b..5b3d3511e3 100644 --- a/cpp/arcticdb/version/snapshot.hpp +++ b/cpp/arcticdb/version/snapshot.hpp @@ -10,14 +10,7 @@ #include #include #include -#include #include -#include -#include -#include -#include -#include -#include namespace arcticdb { diff --git a/cpp/arcticdb/version/test/rapidcheck_version_map.cpp b/cpp/arcticdb/version/test/rapidcheck_version_map.cpp index 2bc9886bf5..6409fe612b 100644 --- a/cpp/arcticdb/version/test/rapidcheck_version_map.cpp +++ b/cpp/arcticdb/version/test/rapidcheck_version_map.cpp @@ -5,18 +5,12 @@ * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0. */ -#include -#include +#include #include -#include #include -#include -#include - -#include -#include #include + #include #include diff --git a/cpp/arcticdb/version/test/test_merge.cpp b/cpp/arcticdb/version/test/test_merge.cpp index e2be9abe93..fff9c662e0 100644 --- a/cpp/arcticdb/version/test/test_merge.cpp +++ b/cpp/arcticdb/version/test/test_merge.cpp @@ -12,6 +12,8 @@ #include #include #include +#include + struct MergeReadsTestStore : arcticdb::TestStore { protected: diff --git a/cpp/arcticdb/version/test/test_sparse.cpp b/cpp/arcticdb/version/test/test_sparse.cpp index 8789267713..0f8c7c83e6 100644 --- a/cpp/arcticdb/version/test/test_sparse.cpp +++ b/cpp/arcticdb/version/test/test_sparse.cpp @@ -14,6 +14,8 @@ #include #include #include +#include + struct SparseTestStore : arcticdb::TestStore { protected: diff --git a/cpp/arcticdb/version/test/test_symbol_list.cpp b/cpp/arcticdb/version/test/test_symbol_list.cpp index 5c6a0b515e..e5d24e4257 100644 --- a/cpp/arcticdb/version/test/test_symbol_list.cpp +++ b/cpp/arcticdb/version/test/test_symbol_list.cpp @@ -6,7 +6,6 @@ */ #include -#include // Included in gtest 1.10 #include #include @@ -14,8 +13,7 @@ #include #include #include - -#include +#include #include #include diff --git a/cpp/arcticdb/version/test/test_version_map.cpp b/cpp/arcticdb/version/test/test_version_map.cpp index 38d9a5c95b..d44c9709c5 100644 --- a/cpp/arcticdb/version/test/test_version_map.cpp +++ b/cpp/arcticdb/version/test/test_version_map.cpp @@ -6,16 +6,14 @@ */ #include -#include - +#include #include #include #include #include -#include -#include -#include #include +#include + namespace arcticdb { diff --git a/cpp/arcticdb/version/test/test_version_map_batch.cpp b/cpp/arcticdb/version/test/test_version_map_batch.cpp index 761aa1c01c..80ed0799c4 100644 --- a/cpp/arcticdb/version/test/test_version_map_batch.cpp +++ b/cpp/arcticdb/version/test/test_version_map_batch.cpp @@ -3,6 +3,9 @@ #include #include #include +#include +#include + using namespace arcticdb; using namespace arcticdb::pipelines; diff --git a/cpp/arcticdb/version/test/test_version_store.cpp b/cpp/arcticdb/version/test/test_version_store.cpp index 6910ce2154..aba051eb1b 100644 --- a/cpp/arcticdb/version/test/test_version_store.cpp +++ b/cpp/arcticdb/version/test/test_version_store.cpp @@ -8,16 +8,16 @@ #include #include -#include #include -#include #include #include #include -#include #include +#include +#include +#include + -#include #include #include @@ -48,7 +48,7 @@ auto write_version_frame( auto wrapper = get_test_simple_frame(stream_id, rows, start_val); auto& frame = wrapper.frame_; auto store = pvs._test_get_store(); - auto var_key = write_frame(std::move(pk), frame, slicing, store, de_dup_map).get(); + auto var_key = arcticdb::pipelines::write_frame(std::move(pk), frame, slicing, store, de_dup_map).get(); auto key = to_atom(var_key); // Moves if (update_version_map) { pvs._test_get_version_map()->write_version(store, key, previous_key); diff --git a/cpp/arcticdb/version/version_core-inl.hpp b/cpp/arcticdb/version/version_core-inl.hpp index 501a88263e..011884c9a3 100644 --- a/cpp/arcticdb/version/version_core-inl.hpp +++ b/cpp/arcticdb/version/version_core-inl.hpp @@ -13,6 +13,15 @@ #include #include #include +#include +#include +#include +#include +#include +#include +#include +#include + namespace arcticdb { @@ -33,7 +42,7 @@ void merge_frames_for_keys_impl( struct StreamMergeWrapper { StreamMergeWrapper( KeySupplier &&key_supplier, - std::shared_ptr store, + std::shared_ptr store, const IndexRange &index_range, StreamId id) : stream_reader_(std::move(key_supplier), std::move(store), storage::ReadKeyOpts{}, index_range), @@ -77,8 +86,8 @@ void merge_frames_for_keys_impl( input_streams.emplace(std::make_unique(std::move(key_func), store, index_range, index_key.id())); } - using AggregatorType = Aggregator; - AggregatorType agg{DynamicSchema{index.create_stream_descriptor(target_id, {}), index}, std::move(func), std::move(segmentation_policy)}; + using AggregatorType = stream::Aggregator; + AggregatorType agg{stream::DynamicSchema{index.create_stream_descriptor(target_id, {}), index}, std::move(func), std::move(segmentation_policy)}; do_merge(input_streams, agg, true); } diff --git a/cpp/arcticdb/version/version_core.cpp b/cpp/arcticdb/version/version_core.cpp index f27a53b830..4056082ca6 100644 --- a/cpp/arcticdb/version/version_core.cpp +++ b/cpp/arcticdb/version/version_core.cpp @@ -22,18 +22,17 @@ #include #include #include -#include #include #include #include -#include #include #include -#include #include #include #include #include +#include +#include namespace arcticdb::version_store { @@ -105,7 +104,7 @@ folly::Future async_write_dataframe_impl( if (validate_index && !index_is_not_timeseries_or_is_sorted_ascending(*frame)) { sorting::raise("When calling write with validate_index enabled, input data must be sorted"); } - return write_frame(std::move(partial_key), frame, slicing_arg, store, de_dup_map, sparsify_floats); + return arcticdb::pipelines::write_frame(std::move(partial_key), frame, slicing_arg, store, de_dup_map, sparsify_floats); } namespace { @@ -381,7 +380,7 @@ VersionedItem update_impl( orig_filter_range = idx_range; return intersecting_segments(affected_keys, idx_range, idx_range, update_info.next_version_id_, store); }, - [](const RowRange&)-> std::pair, std::vector> { + [](const pipelines::RowRange&)-> std::pair, std::vector> { util::raise_rte("Unexpected row_range in update query"); } ); @@ -498,10 +497,10 @@ Composite process_clauses( std::make_shared(std::move(segment_and_slice.segment_in_memory_)), entity_ids[idx], segment_proc_unit_counts[entity_ids[idx]]); component_manager->add( - std::make_shared(std::move(segment_and_slice.ranges_and_key_.row_range_)), + std::make_shared(std::move(segment_and_slice.ranges_and_key_.row_range_)), entity_ids[idx]); component_manager->add( - std::make_shared(std::move(segment_and_slice.ranges_and_key_.col_range_)), + std::make_shared(std::move(segment_and_slice.ranges_and_key_.col_range_)), entity_ids[idx]); component_manager->add( std::make_shared(std::move(segment_and_slice.ranges_and_key_.key_)), @@ -876,7 +875,7 @@ void check_incompletes_index_ranges_dont_overlap(const std::shared_ptris_pickled() && std::holds_alternative(read_query.row_filter)), "Cannot use head/tail/row_range with pickled data, use plain read instead"); + util::check_rte(!(pipeline_context->is_pickled() && std::holds_alternative(read_query.row_filter)), "Cannot use head/tail/row_range with pickled data, use plain read instead"); mark_index_slices(pipeline_context, opt_false(read_options.dynamic_schema_), pipeline_context->bucketize_dynamic_); frame = read_direct(store, pipeline_context, buffers, read_options); } diff --git a/cpp/arcticdb/version/version_core.hpp b/cpp/arcticdb/version/version_core.hpp index 9247c61136..8ccd23b2b1 100644 --- a/cpp/arcticdb/version/version_core.hpp +++ b/cpp/arcticdb/version/version_core.hpp @@ -10,24 +10,21 @@ #include #include #include -#include -#include -#include -#include -#include -#include #include -#include -#include #include -#include -#include + +namespace arcticdb{ + struct WriteOptions; + struct ReadOptions; + class DeDupMap; +} namespace arcticdb::version_store { using namespace arcticdb::entity; using namespace arcticdb::pipelines; +struct UpdateInfo; VersionedItem write_dataframe_impl( const std::shared_ptr& store, diff --git a/cpp/arcticdb/version/version_map_batch_methods.cpp b/cpp/arcticdb/version/version_map_batch_methods.cpp index 6e18165c40..2f61726c61 100644 --- a/cpp/arcticdb/version/version_map_batch_methods.cpp +++ b/cpp/arcticdb/version/version_map_batch_methods.cpp @@ -6,6 +6,7 @@ */ #include +#include namespace arcticdb { diff --git a/cpp/arcticdb/version/version_map_batch_methods.hpp b/cpp/arcticdb/version/version_map_batch_methods.hpp index e6b77b78b2..7f4474ffda 100644 --- a/cpp/arcticdb/version/version_map_batch_methods.hpp +++ b/cpp/arcticdb/version/version_map_batch_methods.hpp @@ -7,16 +7,13 @@ #pragma once -#include #include #include #include -#include -#include #include - #include + namespace arcticdb { struct SymbolStatus { diff --git a/cpp/arcticdb/version/version_store_api.cpp b/cpp/arcticdb/version/version_store_api.cpp index ff4d4d0ad4..ef2a2aba32 100644 --- a/cpp/arcticdb/version/version_store_api.cpp +++ b/cpp/arcticdb/version/version_store_api.cpp @@ -8,10 +8,7 @@ #include #include -#include #include -#include -#include #include #include #include @@ -19,19 +16,18 @@ #include #include #include -#include #include #include #include -#include #include #include +#include #include #include #include #include +#include -#include namespace arcticdb::version_store { @@ -52,7 +48,7 @@ VersionedItem PythonVersionStore::write_dataframe_specific_version( ARCTICDB_SAMPLE(WriteDataFrame, 0) ARCTICDB_DEBUG(log::version(), "write_dataframe_specific_version stream_id: {} , version_id: {}", stream_id, version_id); - if (auto version_key = ::arcticdb::get_specific_version(store(), version_map(), stream_id, version_id, VersionQuery{}); version_key) { + if (auto version_key = arcticdb::get_specific_version(store(), version_map(), stream_id, version_id, VersionQuery{}); version_key) { log::version().warn("Symbol stream_id: {} already exists with version_id: {}", stream_id, version_id); return {std::move(*version_key)}; } diff --git a/cpp/arcticdb/version/version_store_api.hpp b/cpp/arcticdb/version/version_store_api.hpp index 326d5d6124..4a690260e4 100644 --- a/cpp/arcticdb/version/version_store_api.hpp +++ b/cpp/arcticdb/version/version_store_api.hpp @@ -10,31 +10,11 @@ #include #include #include -#include -#include -#include -#include -#include #include -#include -#include -#include #include -#include #include -#include -#include -#include -#include -#include -#include -#include #include #include -#include - -#include -#include namespace arcticdb::version_store { diff --git a/cpp/arcticdb/version/version_utils.hpp b/cpp/arcticdb/version/version_utils.hpp index e21229280a..21fc031a43 100644 --- a/cpp/arcticdb/version/version_utils.hpp +++ b/cpp/arcticdb/version/version_utils.hpp @@ -9,7 +9,6 @@ #include #include -#include #include #include #include