diff --git a/src/duckdb/extension/parquet/decoder/delta_length_byte_array_decoder.cpp b/src/duckdb/extension/parquet/decoder/delta_length_byte_array_decoder.cpp index 9a0c1eac5..a2fd7abd9 100644 --- a/src/duckdb/extension/parquet/decoder/delta_length_byte_array_decoder.cpp +++ b/src/duckdb/extension/parquet/decoder/delta_length_byte_array_decoder.cpp @@ -34,13 +34,21 @@ void DeltaLengthByteArrayDecoder::InitializePage() { void DeltaLengthByteArrayDecoder::Read(shared_ptr &block_ref, uint8_t *defines, idx_t read_count, Vector &result, idx_t result_offset) { if (defines) { - ReadInternal(block_ref, defines, read_count, result, result_offset); + if (reader.Type().IsJSONType()) { + ReadInternal(block_ref, defines, read_count, result, result_offset); + } else { + ReadInternal(block_ref, defines, read_count, result, result_offset); + } } else { - ReadInternal(block_ref, defines, read_count, result, result_offset); + if (reader.Type().IsJSONType()) { + ReadInternal(block_ref, defines, read_count, result, result_offset); + } else { + ReadInternal(block_ref, defines, read_count, result, result_offset); + } } } -template +template void DeltaLengthByteArrayDecoder::ReadInternal(shared_ptr &block_ref, uint8_t *const defines, const idx_t read_count, Vector &result, const idx_t result_offset) { auto &block = *block_ref; @@ -58,6 +66,8 @@ void DeltaLengthByteArrayDecoder::ReadInternal(shared_ptr &blo } } + const auto &string_column_reader = reader.Cast(); + const auto start_ptr = block.ptr; for (idx_t row_idx = 0; row_idx < read_count; row_idx++) { const auto result_idx = result_offset + row_idx; @@ -75,11 +85,15 @@ void DeltaLengthByteArrayDecoder::ReadInternal(shared_ptr &blo } const auto &str_len = length_data[length_idx++]; result_data[result_idx] = string_t(char_ptr_cast(block.ptr), str_len); + if (VALIDATE_INDIVIDUAL_STRINGS) { + string_column_reader.VerifyString(char_ptr_cast(block.ptr), str_len); + } block.unsafe_inc(str_len); } - // Verify that the strings we read are valid UTF-8 - reader.Cast().VerifyString(char_ptr_cast(start_ptr), block.ptr - start_ptr); + if (!VALIDATE_INDIVIDUAL_STRINGS) { + string_column_reader.VerifyString(char_ptr_cast(start_ptr), NumericCast(block.ptr - start_ptr)); + } StringColumnReader::ReferenceBlock(result, block_ref); } diff --git a/src/duckdb/extension/parquet/include/decoder/delta_length_byte_array_decoder.hpp b/src/duckdb/extension/parquet/include/decoder/delta_length_byte_array_decoder.hpp index f8141e26e..9f304da25 100644 --- a/src/duckdb/extension/parquet/include/decoder/delta_length_byte_array_decoder.hpp +++ b/src/duckdb/extension/parquet/include/decoder/delta_length_byte_array_decoder.hpp @@ -27,7 +27,7 @@ class DeltaLengthByteArrayDecoder { void Skip(uint8_t *defines, idx_t skip_count); private: - template + template void ReadInternal(shared_ptr &block, uint8_t *defines, idx_t read_count, Vector &result, idx_t result_offset); template diff --git a/src/duckdb/extension/parquet/include/reader/string_column_reader.hpp b/src/duckdb/extension/parquet/include/reader/string_column_reader.hpp index bfc0692af..d0d18b80c 100644 --- a/src/duckdb/extension/parquet/include/reader/string_column_reader.hpp +++ b/src/duckdb/extension/parquet/include/reader/string_column_reader.hpp @@ -14,6 +14,7 @@ namespace duckdb { class StringColumnReader : public ColumnReader { +public: enum class StringColumnType : uint8_t { VARCHAR, JSON, OTHER }; static StringColumnType GetStringColumnType(const LogicalType &type) { @@ -36,7 +37,7 @@ class StringColumnReader : public ColumnReader { public: static void VerifyString(const char *str_data, uint32_t str_len, const bool isVarchar); - void VerifyString(const char *str_data, uint32_t str_len); + void VerifyString(const char *str_data, uint32_t str_len) const; static void ReferenceBlock(Vector &result, shared_ptr &block); diff --git a/src/duckdb/extension/parquet/reader/string_column_reader.cpp b/src/duckdb/extension/parquet/reader/string_column_reader.cpp index 867dbb4d8..019abd71a 100644 --- a/src/duckdb/extension/parquet/reader/string_column_reader.cpp +++ b/src/duckdb/extension/parquet/reader/string_column_reader.cpp @@ -31,7 +31,7 @@ void StringColumnReader::VerifyString(const char *str_data, uint32_t str_len, co } } -void StringColumnReader::VerifyString(const char *str_data, uint32_t str_len) { +void StringColumnReader::VerifyString(const char *str_data, uint32_t str_len) const { switch (string_column_type) { case StringColumnType::VARCHAR: VerifyString(str_data, str_len, true); diff --git a/src/duckdb/src/common/adbc/adbc.cpp b/src/duckdb/src/common/adbc/adbc.cpp index 0ed31eb39..2b6535055 100644 --- a/src/duckdb/src/common/adbc/adbc.cpp +++ b/src/duckdb/src/common/adbc/adbc.cpp @@ -537,7 +537,8 @@ static int get_schema(struct ArrowArrayStream *stream, struct ArrowSchema *out) auto count = duckdb_column_count(&result_wrapper->result); std::vector types(count); - std::vector owned_names(count); + std::vector owned_names; + owned_names.reserve(count); duckdb::vector names(count); for (idx_t i = 0; i < count; i++) { types[i] = duckdb_column_logical_type(&result_wrapper->result, i); @@ -793,7 +794,8 @@ AdbcStatusCode StatementGetParameterSchema(struct AdbcStatement *statement, stru count = 1; } std::vector types(count); - std::vector owned_names(count); + std::vector owned_names; + owned_names.reserve(count); duckdb::vector names(count); for (idx_t i = 0; i < count; i++) { diff --git a/src/duckdb/src/common/types/row/block_iterator.cpp b/src/duckdb/src/common/types/row/block_iterator.cpp deleted file mode 100644 index bebba60e1..000000000 --- a/src/duckdb/src/common/types/row/block_iterator.cpp +++ /dev/null @@ -1,34 +0,0 @@ -#include "duckdb/common/types/row/block_iterator.hpp" - -namespace duckdb { - -BlockIteratorStateType GetBlockIteratorStateType(const bool &external) { - return external ? BlockIteratorStateType::EXTERNAL : BlockIteratorStateType::IN_MEMORY; -} - -InMemoryBlockIteratorState::InMemoryBlockIteratorState(const TupleDataCollection &key_data) - : block_ptrs(ConvertBlockPointers(key_data.GetRowBlockPointers())), fast_mod(key_data.TuplesPerBlock()), - tuple_count(key_data.Count()) { -} - -unsafe_vector InMemoryBlockIteratorState::ConvertBlockPointers(const vector &block_ptrs) { - unsafe_vector converted_block_ptrs; - converted_block_ptrs.reserve(block_ptrs.size()); - for (const auto &block_ptr : block_ptrs) { - converted_block_ptrs.emplace_back(block_ptr); - } - return converted_block_ptrs; -} - -ExternalBlockIteratorState::ExternalBlockIteratorState(TupleDataCollection &key_data_p, - optional_ptr payload_data_p) - : tuple_count(key_data_p.Count()), current_chunk_idx(DConstants::INVALID_INDEX), key_data(key_data_p), - key_ptrs(FlatVector::GetData(key_scan_state.chunk_state.row_locations)), payload_data(payload_data_p), - keep_pinned(false), pin_payload(false) { - key_data.InitializeScan(key_scan_state); - if (payload_data) { - payload_data->InitializeScan(payload_scan_state); - } -} - -} // namespace duckdb diff --git a/src/duckdb/src/function/table/version/pragma_version.cpp b/src/duckdb/src/function/table/version/pragma_version.cpp index 45d770a67..d60fe187a 100644 --- a/src/duckdb/src/function/table/version/pragma_version.cpp +++ b/src/duckdb/src/function/table/version/pragma_version.cpp @@ -1,5 +1,5 @@ #ifndef DUCKDB_PATCH_VERSION -#define DUCKDB_PATCH_VERSION "2-dev146" +#define DUCKDB_PATCH_VERSION "2-dev185" #endif #ifndef DUCKDB_MINOR_VERSION #define DUCKDB_MINOR_VERSION 4 @@ -8,10 +8,10 @@ #define DUCKDB_MAJOR_VERSION 1 #endif #ifndef DUCKDB_VERSION -#define DUCKDB_VERSION "v1.4.2-dev146" +#define DUCKDB_VERSION "v1.4.2-dev185" #endif #ifndef DUCKDB_SOURCE_ID -#define DUCKDB_SOURCE_ID "b4d86dddcf" +#define DUCKDB_SOURCE_ID "307f9b41ff" #endif #include "duckdb/function/table/system_functions.hpp" #include "duckdb/main/database.hpp" diff --git a/src/duckdb/src/include/duckdb/common/types/row/block_iterator.hpp b/src/duckdb/src/include/duckdb/common/types/row/block_iterator.hpp index c29b094a8..e18c01e53 100644 --- a/src/duckdb/src/include/duckdb/common/types/row/block_iterator.hpp +++ b/src/duckdb/src/include/duckdb/common/types/row/block_iterator.hpp @@ -23,64 +23,98 @@ enum class BlockIteratorStateType : int8_t { EXTERNAL, }; -BlockIteratorStateType GetBlockIteratorStateType(const bool &external); +static BlockIteratorStateType GetBlockIteratorStateType(const bool &external) { + return external ? BlockIteratorStateType::EXTERNAL : BlockIteratorStateType::IN_MEMORY; +} -//! State for iterating over blocks of an in-memory TupleDataCollection -//! Multiple iterators can share the same state, everything is const -class InMemoryBlockIteratorState { -public: - explicit InMemoryBlockIteratorState(const TupleDataCollection &key_data); - -public: - template - T &GetValueAtIndex(const idx_t &block_idx, const idx_t &tuple_idx) const { - D_ASSERT(GetIndex(block_idx, tuple_idx) < tuple_count); - return reinterpret_cast(block_ptrs[block_idx])[tuple_idx]; +template +class BlockIteratorStateBase { +protected: + friend BLOCK_ITERATOR_STATE; + explicit BlockIteratorStateBase(const idx_t tuple_count_p) : tuple_count(tuple_count_p) { } - template - T &GetValueAtIndex(const idx_t &n) const { - const auto quotient = fast_mod.Div(n); - return GetValueAtIndex(quotient, fast_mod.Mod(n, quotient)); +public: + idx_t GetDivisor() const { + const auto &state = static_cast(*this); + return state.GetDivisor(); } - void RandomAccess(idx_t &block_idx, idx_t &tuple_idx, const idx_t &index) const { - block_idx = fast_mod.Div(index); - tuple_idx = fast_mod.Mod(index, block_idx); + void RandomAccess(idx_t &block_or_chunk_idx, idx_t &tuple_idx, const idx_t &index) const { + const auto &state = static_cast(*this); + state.RandomAccessInternal(block_or_chunk_idx, tuple_idx, index); } - void Add(idx_t &block_idx, idx_t &tuple_idx, const idx_t &value) const { + void Add(idx_t &block_or_chunk_idx, idx_t &tuple_idx, const idx_t &value) const { tuple_idx += value; - if (tuple_idx >= fast_mod.GetDivisor()) { - const auto div = fast_mod.Div(tuple_idx); - tuple_idx -= div * fast_mod.GetDivisor(); - block_idx += div; + if (tuple_idx >= GetDivisor()) { + RandomAccess(block_or_chunk_idx, tuple_idx, GetIndex(block_or_chunk_idx, tuple_idx)); } } - void Subtract(idx_t &block_idx, idx_t &tuple_idx, const idx_t &value) const { + void Subtract(idx_t &block_or_chunk_idx, idx_t &tuple_idx, const idx_t &value) const { tuple_idx -= value; - if (tuple_idx >= fast_mod.GetDivisor()) { - const auto div = fast_mod.Div(-tuple_idx); - tuple_idx += (div + 1) * fast_mod.GetDivisor(); - block_idx -= div + 1; + if (tuple_idx >= GetDivisor()) { + RandomAccess(block_or_chunk_idx, tuple_idx, GetIndex(block_or_chunk_idx, tuple_idx)); } } - void Increment(idx_t &block_idx, idx_t &tuple_idx) const { - const auto passed_boundary = ++tuple_idx == fast_mod.GetDivisor(); - block_idx += passed_boundary; - tuple_idx *= !passed_boundary; + void Increment(idx_t &block_or_chunk_idx, idx_t &tuple_idx) const { + const auto crossed_boundary = ++tuple_idx == GetDivisor(); + block_or_chunk_idx += crossed_boundary; + tuple_idx *= !crossed_boundary; } - void Decrement(idx_t &block_idx, idx_t &tuple_idx) const { + void Decrement(idx_t &block_or_chunk_idx, idx_t &tuple_idx) const { const auto crossed_boundary = tuple_idx-- == 0; - block_idx -= crossed_boundary; - tuple_idx += crossed_boundary * fast_mod.GetDivisor(); + block_or_chunk_idx -= crossed_boundary; + tuple_idx += crossed_boundary * GetDivisor(); + } + + idx_t GetIndex(const idx_t &block_or_chunk_idx, const idx_t &tuple_idx) const { + return block_or_chunk_idx * GetDivisor() + tuple_idx; + } + +protected: + const idx_t tuple_count; +}; + +template +class BlockIteratorState; + +//! State for iterating over blocks of an in-memory TupleDataCollection +//! Multiple iterators can share the same state, everything is const +template <> +class BlockIteratorState + : public BlockIteratorStateBase> { +public: + explicit BlockIteratorState(const TupleDataCollection &key_data) + : BlockIteratorStateBase(key_data.Count()), block_ptrs(ConvertBlockPointers(key_data.GetRowBlockPointers())), + fast_mod(key_data.TuplesPerBlock()) { + } + +public: + idx_t GetDivisor() const { + return fast_mod.GetDivisor(); + } + + void RandomAccessInternal(idx_t &block_idx, idx_t &tuple_idx, const idx_t &index) const { + block_idx = fast_mod.Div(index); + tuple_idx = fast_mod.Mod(index, block_idx); + } + + template + T &GetValueAtIndex(const idx_t &block_idx, const idx_t &tuple_idx) const { + D_ASSERT(GetIndex(block_idx, tuple_idx) < tuple_count); + return reinterpret_cast(block_ptrs[block_idx])[tuple_idx]; } - idx_t GetIndex(const idx_t &block_idx, const idx_t &tuple_idx) const { - return block_idx * fast_mod.GetDivisor() + tuple_idx; + template + T &GetValueAtIndex(const idx_t &index) const { + idx_t block_idx; + idx_t tuple_idx; + RandomAccess(block_idx, tuple_idx, index); + return GetValueAtIndex(block_idx, tuple_idx); } void SetKeepPinned(const bool &) { @@ -92,72 +126,63 @@ class InMemoryBlockIteratorState { } private: - static unsafe_vector ConvertBlockPointers(const vector &block_ptrs); + static unsafe_vector ConvertBlockPointers(const vector &block_ptrs) { + unsafe_vector converted_block_ptrs; + converted_block_ptrs.reserve(block_ptrs.size()); + for (const auto &block_ptr : block_ptrs) { + converted_block_ptrs.emplace_back(block_ptr); + } + return converted_block_ptrs; + } private: const unsafe_vector block_ptrs; const FastMod fast_mod; - const idx_t tuple_count; }; +using InMemoryBlockIteratorState = BlockIteratorState; + //! State for iterating over blocks of an external (larger-than-memory) TupleDataCollection //! This state cannot be shared by multiple iterators, it is stateful -class ExternalBlockIteratorState { +template <> +class BlockIteratorState + : public BlockIteratorStateBase> { public: - explicit ExternalBlockIteratorState(TupleDataCollection &key_data, optional_ptr payload_data); - -public: - template - T &GetValueAtIndex(const idx_t &chunk_idx, const idx_t &tuple_idx) { - if (chunk_idx != current_chunk_idx) { - InitializeChunk(chunk_idx); + explicit BlockIteratorState(TupleDataCollection &key_data_p, optional_ptr payload_data_p) + : BlockIteratorStateBase(key_data_p.Count()), current_chunk_idx(DConstants::INVALID_INDEX), + key_data(key_data_p), key_ptrs(FlatVector::GetData(key_scan_state.chunk_state.row_locations)), + payload_data(payload_data_p), keep_pinned(false), pin_payload(false) { + key_data.InitializeScan(key_scan_state); + if (payload_data) { + payload_data->InitializeScan(payload_scan_state); } - return *reinterpret_cast(key_ptrs)[tuple_idx]; } - template - T &GetValueAtIndex(const idx_t &n) { - D_ASSERT(n < tuple_count); - return GetValueAtIndex(n / STANDARD_VECTOR_SIZE, n % STANDARD_VECTOR_SIZE); +public: + static constexpr idx_t GetDivisor() { + return STANDARD_VECTOR_SIZE; } - static void RandomAccess(idx_t &chunk_idx, idx_t &tuple_idx, const idx_t &index) { + static void RandomAccessInternal(idx_t &chunk_idx, idx_t &tuple_idx, const idx_t &index) { chunk_idx = index / STANDARD_VECTOR_SIZE; tuple_idx = index % STANDARD_VECTOR_SIZE; } - static void Add(idx_t &chunk_idx, idx_t &tuple_idx, const idx_t &value) { - tuple_idx += value; - if (tuple_idx >= STANDARD_VECTOR_SIZE) { - const auto div = tuple_idx / STANDARD_VECTOR_SIZE; - tuple_idx -= div * STANDARD_VECTOR_SIZE; - chunk_idx += div; - } - } - - static void Subtract(idx_t &chunk_idx, idx_t &tuple_idx, const idx_t &value) { - tuple_idx -= value; - if (tuple_idx >= STANDARD_VECTOR_SIZE) { - const auto div = -tuple_idx / STANDARD_VECTOR_SIZE; - tuple_idx += (div + 1) * STANDARD_VECTOR_SIZE; - chunk_idx -= div + 1; + template + T &GetValueAtIndex(const idx_t &chunk_idx, const idx_t &tuple_idx) { + D_ASSERT(GetIndex(chunk_idx, tuple_idx) < tuple_count); + if (chunk_idx != current_chunk_idx) { + InitializeChunk(chunk_idx); } + return *reinterpret_cast(key_ptrs)[tuple_idx]; } - static void Increment(idx_t &chunk_idx, idx_t &tuple_idx) { - const auto passed_boundary = ++tuple_idx == STANDARD_VECTOR_SIZE; - chunk_idx += passed_boundary; - tuple_idx *= !passed_boundary; - } - - static void Decrement(idx_t &chunk_idx, idx_t &tuple_idx) { - const auto crossed_boundary = tuple_idx-- == 0; - chunk_idx -= crossed_boundary; - tuple_idx += crossed_boundary * static_cast(STANDARD_VECTOR_SIZE); - } - - static idx_t GetIndex(const idx_t &chunk_idx, const idx_t &tuple_idx) { - return chunk_idx * STANDARD_VECTOR_SIZE + tuple_idx; + template + T &GetValueAtIndex(const idx_t &index) { + idx_t chunk_idx; + idx_t tuple_idx; + RandomAccess(chunk_idx, tuple_idx, index); + return GetValueAtIndex(chunk_idx, tuple_idx); } void SetKeepPinned(const bool &enable) { @@ -201,7 +226,6 @@ class ExternalBlockIteratorState { } private: - const idx_t tuple_count; idx_t current_chunk_idx; TupleDataCollection &key_data; @@ -216,13 +240,7 @@ class ExternalBlockIteratorState { vector pins; }; -//! Utility so we can get the state using the type -template -using BlockIteratorState = typename std::conditional< - T == BlockIteratorStateType::IN_MEMORY, InMemoryBlockIteratorState, - typename std::conditional::type>::type; +using ExternalBlockIteratorState = BlockIteratorState; //! Iterator for data spread out over multiple blocks template @@ -305,16 +323,16 @@ class block_iterator_t { // NOLINT: match stl case return *this; } block_iterator_t operator+(const difference_type &n) const { - idx_t new_block_idx = block_or_chunk_idx; + idx_t new_block_or_chunk_idx = block_or_chunk_idx; idx_t new_tuple_idx = tuple_idx; - state->Add(new_block_idx, new_tuple_idx, n); - return block_iterator_t(*state, new_block_idx, new_tuple_idx); + state->Add(new_block_or_chunk_idx, new_tuple_idx, n); + return block_iterator_t(*state, new_block_or_chunk_idx, new_tuple_idx); } block_iterator_t operator-(const difference_type &n) const { - idx_t new_block_idx = block_or_chunk_idx; + idx_t new_block_or_chunk_idx = block_or_chunk_idx; idx_t new_tuple_idx = tuple_idx; - state->Subtract(new_block_idx, new_tuple_idx, n); - return block_iterator_t(*state, new_block_idx, new_tuple_idx); + state->Subtract(new_block_or_chunk_idx, new_tuple_idx, n); + return block_iterator_t(*state, new_block_or_chunk_idx, new_tuple_idx); } reference operator[](const difference_type &n) const { diff --git a/src/duckdb/src/include/duckdb/logging/log_manager.hpp b/src/duckdb/src/include/duckdb/logging/log_manager.hpp index 6ee88aeda..54f623a55 100644 --- a/src/duckdb/src/include/duckdb/logging/log_manager.hpp +++ b/src/duckdb/src/include/duckdb/logging/log_manager.hpp @@ -21,7 +21,7 @@ class LogType; // - Creates Loggers with cached configuration // - Main sink for logs (either by logging directly into this, or by syncing a pre-cached set of log entries) // - Holds the log storage -class LogManager : public enable_shared_from_this { +class LogManager { friend class ThreadSafeLogger; friend class ThreadLocalLogger; friend class MutableLogger; diff --git a/src/duckdb/src/include/duckdb/main/database.hpp b/src/duckdb/src/include/duckdb/main/database.hpp index 11936d5f7..860912130 100644 --- a/src/duckdb/src/include/duckdb/main/database.hpp +++ b/src/duckdb/src/include/duckdb/main/database.hpp @@ -90,7 +90,7 @@ class DatabaseInstance : public enable_shared_from_this { unique_ptr extension_manager; ValidChecker db_validity; unique_ptr db_file_system; - shared_ptr log_manager; + unique_ptr log_manager; unique_ptr external_file_cache; duckdb_ext_api_v1 (*create_api_v1)(); diff --git a/src/duckdb/src/main/database.cpp b/src/duckdb/src/main/database.cpp index 419d3304b..d7bb093eb 100644 --- a/src/duckdb/src/main/database.cpp +++ b/src/duckdb/src/main/database.cpp @@ -285,7 +285,7 @@ void DatabaseInstance::Initialize(const char *database_path, DBConfig *user_conf buffer_manager = make_uniq(*this, config.options.temporary_directory); } - log_manager = make_shared_ptr(*this, LogConfig()); + log_manager = make_uniq(*this, LogConfig()); log_manager->Initialize(); external_file_cache = make_uniq(*this, config.options.enable_external_file_cache); diff --git a/src/duckdb/src/planner/binder/tableref/bind_table_function.cpp b/src/duckdb/src/planner/binder/tableref/bind_table_function.cpp index 91bd5775d..f06e9e1ad 100644 --- a/src/duckdb/src/planner/binder/tableref/bind_table_function.cpp +++ b/src/duckdb/src/planner/binder/tableref/bind_table_function.cpp @@ -147,7 +147,7 @@ bool Binder::BindTableFunctionParameters(TableFunctionCatalogEntry &table_functi MoveCorrelatedExpressions(*subquery->binder); seen_subquery = true; arguments.emplace_back(LogicalTypeId::TABLE); - parameters.emplace_back(Value()); + parameters.emplace_back(); continue; } diff --git a/src/duckdb/src/storage/compression/zstd.cpp b/src/duckdb/src/storage/compression/zstd.cpp index 1fee17520..1ccb489a6 100644 --- a/src/duckdb/src/storage/compression/zstd.cpp +++ b/src/duckdb/src/storage/compression/zstd.cpp @@ -142,6 +142,11 @@ struct ZSTDAnalyzeState : public AnalyzeState { unique_ptr ZSTDStorage::StringInitAnalyze(ColumnData &col_data, PhysicalType type) { // check if the storage version we are writing to supports sztd auto &storage = col_data.GetStorageManager(); + auto &block_manager = col_data.GetBlockManager(); + if (block_manager.InMemory()) { + //! Can't use ZSTD in in-memory environment + return nullptr; + } if (storage.GetStorageVersion() < 4) { // compatibility mode with old versions - disable zstd return nullptr; diff --git a/src/duckdb/src/storage/storage_info.cpp b/src/duckdb/src/storage/storage_info.cpp index b4a7422e9..fb1d78d59 100644 --- a/src/duckdb/src/storage/storage_info.cpp +++ b/src/duckdb/src/storage/storage_info.cpp @@ -83,6 +83,8 @@ static const StorageVersionInfo storage_version_info[] = { {"v1.3.1", 66}, {"v1.3.2", 66}, {"v1.4.0", 67}, + {"v1.4.1", 67}, + {"v1.4.2", 67}, {nullptr, 0} }; // END OF STORAGE VERSION INFO @@ -108,6 +110,8 @@ static const SerializationVersionInfo serialization_version_info[] = { {"v1.3.1", 5}, {"v1.3.2", 5}, {"v1.4.0", 6}, + {"v1.4.1", 6}, + {"v1.4.2", 6}, {"latest", 6}, {nullptr, 0} }; diff --git a/src/duckdb/third_party/parquet/parquet_types.cpp b/src/duckdb/third_party/parquet/parquet_types.cpp index 91e221207..a508a69f2 100644 --- a/src/duckdb/third_party/parquet/parquet_types.cpp +++ b/src/duckdb/third_party/parquet/parquet_types.cpp @@ -184,7 +184,14 @@ int _kConvertedTypeValues[] = { * the provided duration. This duration of time is independent of any * particular timezone or date. */ - ConvertedType::INTERVAL + ConvertedType::INTERVAL, + /** + * Non-standard NULL value + * + * This was written by old writers - it is kept here for compatibility purposes. + * See https://github.com/duckdb/duckdb/pull/11774 + */ + ConvertedType::PARQUET_NULL }; const char* _kConvertedTypeNames[] = { /** @@ -308,9 +315,16 @@ const char* _kConvertedTypeNames[] = { * the provided duration. This duration of time is independent of any * particular timezone or date. */ - "INTERVAL" + "INTERVAL", + /** + * Non-standard NULL value + * + * This was written by old writers - it is kept here for compatibility purposes. + * See https://github.com/duckdb/duckdb/pull/11774 + */ + "PARQUET_NULL" }; -const std::map _ConvertedType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(22, _kConvertedTypeValues, _kConvertedTypeNames), ::apache::thrift::TEnumIterator(-1, nullptr, nullptr)); +const std::map _ConvertedType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(23, _kConvertedTypeValues, _kConvertedTypeNames), ::apache::thrift::TEnumIterator(-1, nullptr, nullptr)); std::ostream& operator<<(std::ostream& out, const ConvertedType::type& val) { std::map::const_iterator it = _ConvertedType_VALUES_TO_NAMES.find(val); diff --git a/src/duckdb/third_party/parquet/parquet_types.h b/src/duckdb/third_party/parquet/parquet_types.h index 4bfee7446..762d3533a 100644 --- a/src/duckdb/third_party/parquet/parquet_types.h +++ b/src/duckdb/third_party/parquet/parquet_types.h @@ -178,7 +178,14 @@ struct ConvertedType { * the provided duration. This duration of time is independent of any * particular timezone or date. */ - INTERVAL = 21 + INTERVAL = 21, + /** + * Non-standard NULL value + * + * This was written by old writers - it is kept here for compatibility purposes. + * See https://github.com/duckdb/duckdb/pull/11774 + */ + PARQUET_NULL = 24 }; }; diff --git a/src/duckdb/ub_src_common_types_row.cpp b/src/duckdb/ub_src_common_types_row.cpp index 3d4ff32c2..24bfa5fd4 100644 --- a/src/duckdb/ub_src_common_types_row.cpp +++ b/src/duckdb/ub_src_common_types_row.cpp @@ -1,5 +1,3 @@ -#include "src/common/types/row/block_iterator.cpp" - #include "src/common/types/row/partitioned_tuple_data.cpp" #include "src/common/types/row/row_data_collection.cpp"