From 558e185f8d98fa734f7aa9087adeb4f0db906da4 Mon Sep 17 00:00:00 2001 From: Hu Shenggang Date: Thu, 28 May 2026 11:27:02 +0800 Subject: [PATCH] [feature](be) Add deferred read plan for pruned complex columns ### What problem does this PR solve? Issue Number: None Related PR: #59263 Problem Summary: Nested column pruning can read only predicate subpaths before filtering. When a remaining conjunct also needs the same complex column as a common expression, pruned non-predicate subpaths still need to be materialized after row filtering. This change introduces an explicit deferred read phase for pruned complex columns. Predicate reads keep parent metadata and predicate subpaths, deferred reads materialize the remaining nested targets by selected rowids, and the finalization step removes temporary default placeholders before returning the column. The implementation keeps the phase and nested read plan on the iterator, propagates predicate/deferred intent through array/map/struct children, and exposes the prune switch through TQueryOptions. ### Release note Improve nested column pruning read behavior for complex columns used by both predicates and remaining expressions. ### Check List (For Author) - Test: Unit Test - DORIS_HOME=/mnt/disk7/hushenggang/doris-abs ninja -C be/ut_build_ASAN src/storage/CMakeFiles/Storage.dir/segment/column_reader.cpp.o src/storage/CMakeFiles/Storage.dir/segment/segment_iterator.cpp.o test/CMakeFiles/doris_be_test.dir/storage/segment/column_reader_test.cpp.o - build-support/clang-format.sh - git diff --check - Behavior changed: Yes. Pruned complex common-expression columns can defer non-predicate nested materialization until after filtering. - Does this need documentation: No --- be/src/exec/operator/olap_scan_operator.cpp | 2 + be/src/exec/operator/olap_scan_operator.h | 1 + be/src/exec/scan/olap_scanner.cpp | 1 + be/src/runtime/runtime_state.h | 5 + be/src/storage/olap_common.h | 1 + be/src/storage/segment/column_reader.cpp | 553 ++++++--- be/src/storage/segment/column_reader.h | 133 +- be/src/storage/segment/segment_iterator.cpp | 107 +- be/src/storage/segment/segment_iterator.h | 6 +- .../storage/segment/column_reader_test.cpp | 1084 +++++++++++++++++ .../org/apache/doris/qe/SessionVariable.java | 2 + gensrc/thrift/PaloInternalService.thrift | 3 + .../complex_types/test_pruned_columns.out | 395 +++++- .../complex_types/test_pruned_columns.groovy | 189 ++- 14 files changed, 2242 insertions(+), 240 deletions(-) diff --git a/be/src/exec/operator/olap_scan_operator.cpp b/be/src/exec/operator/olap_scan_operator.cpp index 4ce564241aabcc..ebeff39c4e1cff 100644 --- a/be/src/exec/operator/olap_scan_operator.cpp +++ b/be/src/exec/operator/olap_scan_operator.cpp @@ -217,6 +217,8 @@ Status OlapScanLocalState::_init_profile() { _lazy_read_seek_timer = ADD_TIMER(_segment_profile, "LazyReadSeekTime"); _lazy_read_seek_counter = ADD_COUNTER(_segment_profile, "LazyReadSeekCount", TUnit::UNIT); + _deferred_nested_read_timer = ADD_TIMER(_segment_profile, "DeferredNestedReadTime"); + _output_col_timer = ADD_TIMER(_segment_profile, "OutputColumnTime"); _stats_filtered_counter = ADD_COUNTER(_segment_profile, "RowsStatsFiltered", TUnit::UNIT); diff --git a/be/src/exec/operator/olap_scan_operator.h b/be/src/exec/operator/olap_scan_operator.h index 04df02fec80fed..031da708378432 100644 --- a/be/src/exec/operator/olap_scan_operator.h +++ b/be/src/exec/operator/olap_scan_operator.h @@ -208,6 +208,7 @@ class OlapScanLocalState final : public ScanLocalState { RuntimeProfile::Counter* _lazy_read_timer = nullptr; RuntimeProfile::Counter* _lazy_read_seek_timer = nullptr; RuntimeProfile::Counter* _lazy_read_seek_counter = nullptr; + RuntimeProfile::Counter* _deferred_nested_read_timer = nullptr; // total pages read // used by segment v2 diff --git a/be/src/exec/scan/olap_scanner.cpp b/be/src/exec/scan/olap_scanner.cpp index 8656576dbeb4cd..8ebdcb41354631 100644 --- a/be/src/exec/scan/olap_scanner.cpp +++ b/be/src/exec/scan/olap_scanner.cpp @@ -770,6 +770,7 @@ void OlapScanner::_collect_profile_before_close() { COUNTER_UPDATE(local_state->_predicate_column_read_seek_counter, stats.predicate_column_read_seek_num); COUNTER_UPDATE(local_state->_lazy_read_timer, stats.lazy_read_ns); + COUNTER_UPDATE(local_state->_deferred_nested_read_timer, stats.deferred_nested_read_ns); COUNTER_UPDATE(local_state->_lazy_read_seek_timer, stats.block_lazy_read_seek_ns); COUNTER_UPDATE(local_state->_lazy_read_seek_counter, stats.block_lazy_read_seek_num); COUNTER_UPDATE(local_state->_output_col_timer, stats.output_col_ns); diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h index 619b842eef7de7..64fd4405219061 100644 --- a/be/src/runtime/runtime_state.h +++ b/be/src/runtime/runtime_state.h @@ -615,6 +615,11 @@ class RuntimeState { _query_options.enable_aggregate_function_null_v2; } + bool enable_prune_nested_column() const { + return _query_options.__isset.enable_prune_nested_column && + _query_options.enable_prune_nested_column; + } + bool is_read_csv_empty_line_as_null() const { return _query_options.__isset.read_csv_empty_line_as_null && _query_options.read_csv_empty_line_as_null; diff --git a/be/src/storage/olap_common.h b/be/src/storage/olap_common.h index 1fb680ee747cc7..d61f192d8108af 100644 --- a/be/src/storage/olap_common.h +++ b/be/src/storage/olap_common.h @@ -327,6 +327,7 @@ struct OlapReaderStatistics { int64_t lazy_read_ns = 0; int64_t block_lazy_read_seek_num = 0; int64_t block_lazy_read_seek_ns = 0; + int64_t deferred_nested_read_ns = 0; int64_t raw_rows_read = 0; diff --git a/be/src/storage/segment/column_reader.cpp b/be/src/storage/segment/column_reader.cpp index 7fc5d5e14663a6..fdc0babc7ed72c 100644 --- a/be/src/storage/segment/column_reader.cpp +++ b/be/src/storage/segment/column_reader.cpp @@ -89,6 +89,14 @@ inline bool read_as_string(PrimitiveType type) { type == PrimitiveType::TYPE_BITMAP || type == PrimitiveType::TYPE_FIXED_LENGTH_OBJECT; } +TColumnAccessPath make_full_access_path_for_child(const std::string& column_name) { + TColumnAccessPath access_path; + TDataAccessPath data_access_path; + data_access_path.__set_path({column_name}); + access_path.__set_data_access_path(data_access_path); + return access_path; +} + Status ColumnReader::create_array(const ColumnReaderOptions& opts, const ColumnMetaPB& meta, const io::FileReaderSPtr& file_reader, std::shared_ptr* reader) { @@ -855,6 +863,25 @@ Status ColumnReader::new_struct_iterator(ColumnIteratorUPtr* iterator, return Status::OK(); } +void ColumnIterator::_append_deferred_defaults(MutableColumnPtr& dst, size_t count) { + if (_reading_flag != ReadingFlag::SKIP_READING && _read_phase == ReadPhase::PREDICATE) { + _nested_read_plan.has_deferred_defaults = true; + } + + if (_read_phase != ReadPhase::DEFERRED) { + dst->insert_many_defaults(count); + } +} + +void ColumnIterator::_drop_deferred_defaults(MutableColumnPtr& dst) { + if (_read_phase == ReadPhase::DEFERRED) { + if (_nested_read_plan.has_deferred_defaults) { + dst->clear(); + _nested_read_plan.has_deferred_defaults = false; + } + } +} + Result ColumnIterator::_get_sub_access_paths( const TColumnAccessPaths& access_paths) { TColumnAccessPaths sub_access_paths = access_paths; @@ -882,6 +909,37 @@ Result ColumnIterator::_get_sub_access_paths( return sub_access_paths; } +Result ColumnIterator::_normalize_access_paths( + const TColumnAccessPaths& access_paths, const bool is_predicate) { + TColumnAccessPaths sub_access_paths = access_paths; + for (auto it = sub_access_paths.begin(); it != sub_access_paths.end();) { + TColumnAccessPath& name_path = *it; + if (name_path.data_access_path.path.empty()) { + return ResultError( + Status::InternalError("Invalid access path for struct column: path is empty")); + } + + if (!StringCaseEqual()(name_path.data_access_path.path[0], _column_name)) { + return ResultError(Status::InternalError( + R"(Invalid access path for column: expected name "{}", got "{}")", _column_name, + name_path.data_access_path.path[0])); + } + + name_path.data_access_path.path.erase(name_path.data_access_path.path.begin()); + if (!name_path.data_access_path.path.empty()) { + ++it; + } else { + if (is_predicate) { + _reading_flag = ReadingFlag::READING_FOR_PREDICATE; + } else { + set_need_to_read(); + } + it = sub_access_paths.erase(it); + } + } + return sub_access_paths; +} + ///====================== MapFileColumnIterator ============================//// MapFileColumnIterator::MapFileColumnIterator(std::shared_ptr reader, ColumnIteratorUPtr null_iterator, @@ -912,7 +970,7 @@ Status MapFileColumnIterator::init(const ColumnIteratorOptions& opts) { } Status MapFileColumnIterator::seek_to_ordinal(ordinal_t ord) { - if (_reading_flag == ReadingFlag::SKIP_READING) { + if (!need_to_read()) { DLOG(INFO) << "Map column iterator column " << _column_name << " skip reading."; return Status::OK(); } @@ -954,23 +1012,37 @@ Status MapFileColumnIterator::init_prefetcher(const SegmentPrefetchParams& param void MapFileColumnIterator::collect_prefetchers( std::map>& prefetchers, PrefetcherInitMethod init_method) { - _offsets_iterator->collect_prefetchers(prefetchers, init_method); + if (!need_to_read()) { + return; + } + if (!read_null_map_only()) { + _offsets_iterator->collect_prefetchers(prefetchers, init_method); + } if (_map_reader->is_nullable()) { _null_iterator->collect_prefetchers(prefetchers, init_method); } + if (read_offset_only() || read_null_map_only()) { + return; + } // the actual data pages to read of key/value column depends on the read result of offset column, // so we can't init prefetch blocks according to rowids, just prefetch all data blocks here. - _key_iterator->collect_prefetchers(prefetchers, PrefetcherInitMethod::ALL_DATA_BLOCKS); - _val_iterator->collect_prefetchers(prefetchers, PrefetcherInitMethod::ALL_DATA_BLOCKS); + if (_key_iterator->need_to_read()) { + _key_iterator->collect_prefetchers(prefetchers, PrefetcherInitMethod::ALL_DATA_BLOCKS); + } + if (_val_iterator->need_to_read()) { + _val_iterator->collect_prefetchers(prefetchers, PrefetcherInitMethod::ALL_DATA_BLOCKS); + } } Status MapFileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bool* has_null) { - if (_reading_flag == ReadingFlag::SKIP_READING) { + if (!need_to_read()) { DLOG(INFO) << "Map column iterator column " << _column_name << " skip reading."; - dst->insert_many_defaults(*n); + _append_deferred_defaults(dst, *n); return Status::OK(); } + _drop_deferred_defaults(dst); + if (read_null_map_only()) { // NULL_MAP_ONLY mode: read null map, fill nested ColumnMap with empty defaults DORIS_CHECK(dst->is_nullable()); @@ -997,8 +1069,22 @@ Status MapFileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bool* auto& column_map = assert_cast( dst->is_nullable() ? static_cast(*dst).get_nested_column() : *dst); - auto column_offsets_ptr = IColumn::mutate(std::move(column_map.get_offsets_ptr())); - Defer defer_offsets {[&] { column_map.get_offsets_ptr() = std::move(column_offsets_ptr); }}; + const bool read_meta_columns = need_to_read_meta_columns(); + MutableColumnPtr column_offsets_ptr; + if (read_meta_columns) { + column_offsets_ptr = IColumn::mutate(std::move(column_map.get_offsets_ptr())); + } else { + const auto base_offset = + column_map.get_offsets().empty() ? 0 : column_map.get_offsets().back(); + column_offsets_ptr = ColumnMap::COffsets::create(); + assert_cast(*column_offsets_ptr) + .insert_value(base_offset); + } + Defer defer_offsets {[&] { + if (read_meta_columns) { + column_map.get_offsets_ptr() = std::move(column_offsets_ptr); + } + }}; bool offsets_has_null = false; ssize_t start = column_offsets_ptr->size(); RETURN_IF_ERROR(_offsets_iterator->next_batch(n, column_offsets_ptr, &offsets_has_null)); @@ -1030,7 +1116,7 @@ Status MapFileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bool* } } - if (dst->is_nullable()) { + if (dst->is_nullable() && read_meta_columns) { size_t num_read = *n; auto null_map_ptr = static_cast(*dst).get_null_map_column_ptr(); // in not-null to null linked-schemachange mode, @@ -1052,12 +1138,14 @@ Status MapFileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bool* Status MapFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t count, MutableColumnPtr& dst) { - if (_reading_flag == ReadingFlag::SKIP_READING) { - DLOG(INFO) << "File column iterator column " << _column_name << " skip reading."; - dst->insert_many_defaults(count); + if (!need_to_read()) { + DLOG(INFO) << "Map column iterator column " << _column_name << " skip reading."; + _append_deferred_defaults(dst, count); return Status::OK(); } + _drop_deferred_defaults(dst); + if (read_null_map_only()) { // NULL_MAP_ONLY mode: read null map by rowids, fill nested ColumnMap with empty defaults DORIS_CHECK(dst->is_nullable()); @@ -1081,39 +1169,74 @@ Status MapFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t if (count == 0) { return Status::OK(); } + // resolve ColumnMap and nullable wrapper auto& column_map = assert_cast( dst->is_nullable() ? static_cast(*dst).get_nested_column() : *dst); - auto offsets_ptr = IColumn::mutate(std::move(column_map.get_offsets_ptr())); - Defer defer_offsets {[&] { column_map.get_offsets_ptr() = std::move(offsets_ptr); }}; + const bool read_meta_columns = need_to_read_meta_columns(); + MutableColumnPtr offsets_ptr; + if (read_meta_columns) { + offsets_ptr = IColumn::mutate(std::move(column_map.get_offsets_ptr())); + } else { + const auto base_offset = + column_map.get_offsets().empty() ? 0 : column_map.get_offsets().back(); + offsets_ptr = ColumnMap::COffsets::create(); + assert_cast(*offsets_ptr) + .insert_value(base_offset); + } + Defer defer_offsets {[&] { + if (read_meta_columns) { + column_map.get_offsets_ptr() = std::move(offsets_ptr); + } + }}; auto& offsets = static_cast(*offsets_ptr); size_t base = offsets.get_data().empty() ? 0 : offsets.get_data().back(); // 1. bulk read null-map if nullable std::vector null_mask; // 0: not null, 1: null - if (_map_reader->is_nullable()) { - // For nullable map columns, the destination column must also be nullable. + if (read_meta_columns) { + if (_map_reader->is_nullable()) { + // For nullable map columns, the destination column must also be nullable. + if (UNLIKELY(!dst->is_nullable())) { + return Status::InternalError( + "unexpected non-nullable destination column for nullable map reader"); + } + MutableColumnPtr null_map_ptr = + static_cast(*dst).get_null_map_column_ptr(); + size_t null_before = null_map_ptr->size(); + RETURN_IF_ERROR(_null_iterator->read_by_rowids(rowids, count, null_map_ptr)); + // extract a light-weight view to decide element reads + auto& null_map_col = assert_cast(*null_map_ptr); + null_mask.reserve(count); + for (size_t i = 0; i < count; ++i) { + null_mask.push_back(null_map_col.get_element(null_before + i)); + } + } else if (dst->is_nullable()) { + // in not-null to null linked-schemachange mode, + // actually we do not change dat data include meta in footer, + // so may dst from changed meta which is nullable but old data is not nullable, + // if so, we should set null_map to all null by default + MutableColumnPtr null_map_ptr = + static_cast(*dst).get_null_map_column_ptr(); + auto& null_map = assert_cast(*null_map_ptr); + null_map.insert_many_vals(0, count); + } + } else if (_map_reader->is_nullable()) { + // In deferred phase the parent null map has already been materialized during predicate read. + // Read a temporary null map here only to compute child element ranges correctly. if (UNLIKELY(!dst->is_nullable())) { return Status::InternalError( "unexpected non-nullable destination column for nullable map reader"); } - auto null_map_ptr = static_cast(*dst).get_null_map_column_ptr(); - size_t null_before = null_map_ptr->size(); - auto* null_map_col = null_map_ptr.get(); - MutableColumnPtr null_map_column = std::move(null_map_ptr); - RETURN_IF_ERROR(_null_iterator->read_by_rowids(rowids, count, null_map_column)); - // extract a light-weight view to decide element reads + + MutableColumnPtr null_map_ptr = ColumnVector::create(); + RETURN_IF_ERROR(_null_iterator->read_by_rowids(rowids, count, null_map_ptr)); + + auto& null_map_col = assert_cast(*null_map_ptr); null_mask.reserve(count); for (size_t i = 0; i < count; ++i) { - null_mask.push_back(null_map_col->get_element(null_before + i)); + null_mask.push_back(null_map_col.get_element(i)); } - } else if (dst->is_nullable()) { - // in not-null to null linked-schemachange mode, - // actually we do not change dat data include meta in footer, - // so may dst from changed meta which is nullable but old data is not nullable, - // if so, we should set null_map to all null by default - auto null_map_ptr = static_cast(*dst).get_null_map_column_ptr(); - null_map_ptr->insert_many_vals(0, count); } // 2. bulk read start ordinals for requested rows @@ -1156,16 +1279,19 @@ Status MapFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t auto& next_starts_data = assert_cast(*next_starts_col).get_data(); std::vector sizes(count, 0); size_t acc = base; - const auto original_size = offsets.get_data().back(); - offsets.get_data().reserve(offsets.get_data().size() + count); + if (read_meta_columns) { + offsets.get_data().reserve(offsets.get_data().size() + count); + } for (size_t i = 0; i < count; ++i) { - size_t sz = static_cast(next_starts_data[i] - starts_data[i]); + auto sz = static_cast(next_starts_data[i] - starts_data[i]); if (_map_reader->is_nullable() && !null_mask.empty() && null_mask[i]) { sz = 0; // null rows do not consume elements } sizes[i] = sz; acc += sz; - offsets.get_data().push_back(acc); + if (read_meta_columns) { + offsets.get_data().push_back(acc); + } } // 6. read key/value elements for non-empty sizes @@ -1188,18 +1314,14 @@ Status MapFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t bool dummy_has_null = false; if (this_run != 0) { - if (_key_iterator->reading_flag() != ReadingFlag::SKIP_READING) { - RETURN_IF_ERROR(_key_iterator->seek_to_ordinal(start_idx)); - RETURN_IF_ERROR(_key_iterator->next_batch(&n, keys_ptr, &dummy_has_null)); - DCHECK(n == this_run); - } - - if (_val_iterator->reading_flag() != ReadingFlag::SKIP_READING) { - n = this_run; - RETURN_IF_ERROR(_val_iterator->seek_to_ordinal(start_idx)); - RETURN_IF_ERROR(_val_iterator->next_batch(&n, vals_ptr, &dummy_has_null)); - DCHECK(n == this_run); - } + RETURN_IF_ERROR(_key_iterator->seek_to_ordinal(start_idx)); + RETURN_IF_ERROR(_key_iterator->next_batch(&n, keys_ptr, &dummy_has_null)); + DCHECK(n == this_run); + + n = this_run; + RETURN_IF_ERROR(_val_iterator->seek_to_ordinal(start_idx)); + RETURN_IF_ERROR(_val_iterator->next_batch(&n, vals_ptr, &dummy_has_null)); + DCHECK(n == this_run); } start_idx = start; this_run = sz; @@ -1212,29 +1334,17 @@ Status MapFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t } size_t n = this_run; - const size_t total_count = offsets.get_data().back() - original_size; bool dummy_has_null = false; - if (_key_iterator->reading_flag() != ReadingFlag::SKIP_READING) { - if (this_run != 0) { - RETURN_IF_ERROR(_key_iterator->seek_to_ordinal(start_idx)); - RETURN_IF_ERROR(_key_iterator->next_batch(&n, keys_ptr, &dummy_has_null)); - DCHECK(n == this_run); - } - } else { - keys_ptr->insert_many_defaults(total_count); - } + if (this_run != 0) { + RETURN_IF_ERROR(_key_iterator->seek_to_ordinal(start_idx)); + RETURN_IF_ERROR(_key_iterator->next_batch(&n, keys_ptr, &dummy_has_null)); + DCHECK(n == this_run); - if (_val_iterator->reading_flag() != ReadingFlag::SKIP_READING) { - if (this_run != 0) { - n = this_run; - RETURN_IF_ERROR(_val_iterator->seek_to_ordinal(start_idx)); - RETURN_IF_ERROR(_val_iterator->next_batch(&n, vals_ptr, &dummy_has_null)); - DCHECK(n == this_run); - } - } else { - vals_ptr->insert_many_defaults(total_count); + n = this_run; + RETURN_IF_ERROR(_val_iterator->seek_to_ordinal(start_idx)); + RETURN_IF_ERROR(_val_iterator->next_batch(&n, vals_ptr, &dummy_has_null)); + DCHECK(n == this_run); } - return Status::OK(); } @@ -1261,10 +1371,20 @@ Status MapFileColumnIterator::set_access_paths(const TColumnAccessPaths& all_acc << " to READING_FOR_PREDICATE"; } - auto sub_all_access_paths = DORIS_TRY(_get_sub_access_paths(all_access_paths)); - auto sub_predicate_access_paths = DORIS_TRY(_get_sub_access_paths(predicate_access_paths)); + _nested_read_plan.pruned = true; - if (sub_all_access_paths.empty()) { + auto sub_all_access_paths = DORIS_TRY(_normalize_access_paths(all_access_paths, false)); + auto sub_predicate_access_paths = + DORIS_TRY(_normalize_access_paths(predicate_access_paths, true)); + + if (sub_predicate_access_paths.empty() && _reading_flag == ReadingFlag::READING_FOR_PREDICATE) { + // if no sub-column in predicate_access_paths, but current column is READING_FOR_PREDICATE, + // then we should set key/value iterator to READING_FOR_PREDICATE too. + _key_iterator->set_reading_flag_recursively(ReadingFlag::READING_FOR_PREDICATE); + _val_iterator->set_reading_flag_recursively(ReadingFlag::READING_FOR_PREDICATE); + } + + if (sub_all_access_paths.empty() && sub_predicate_access_paths.empty()) { return Status::OK(); } @@ -1274,14 +1394,14 @@ Status MapFileColumnIterator::set_access_paths(const TColumnAccessPaths& all_acc _key_iterator->set_reading_flag(ReadingFlag::SKIP_READING); _val_iterator->set_reading_flag(ReadingFlag::SKIP_READING); DLOG(INFO) << "Map column iterator set column " << _column_name - << " to OFFSET_ONLY reading mode, key/value columns set to SKIP_READING"; + << " to OFFSET_ONLY read phase, key/value columns set to SKIP_READING"; return Status::OK(); } if (read_null_map_only()) { _key_iterator->set_reading_flag(ReadingFlag::SKIP_READING); _val_iterator->set_reading_flag(ReadingFlag::SKIP_READING); DLOG(INFO) << "Map column iterator set column " << _column_name - << " to NULL_MAP_ONLY reading mode, key/value columns set to SKIP_READING"; + << " to NULL_MAP_ONLY read phase, key/value columns set to SKIP_READING"; return Status::OK(); } @@ -1290,27 +1410,34 @@ Status MapFileColumnIterator::set_access_paths(const TColumnAccessPaths& all_acc TColumnAccessPaths key_predicate_access_paths; TColumnAccessPaths val_predicate_access_paths; - for (auto paths : sub_all_access_paths) { - if (paths.data_access_path.path[0] == ACCESS_ALL) { - // ACCESS_ALL means element_at(map, key) style access: the key column must be - // fully read so that the runtime can match the requested key, while any sub-path - // qualifiers (e.g. OFFSET) apply only to the value column. - // For key: create a path with just the column name (= full data access). - TColumnAccessPath key_path; - key_path.__set_type(paths.type); - TDataAccessPath key_data_path; - key_data_path.__set_path({_key_iterator->column_name()}); - key_path.__set_data_access_path(key_data_path); - key_all_access_paths.emplace_back(std::move(key_path)); - // For value: pass the full sub-path so qualifiers like OFFSET propagate. - paths.data_access_path.path[0] = _val_iterator->column_name(); - val_all_access_paths.emplace_back(paths); - } else if (paths.data_access_path.path[0] == ACCESS_MAP_KEYS) { - paths.data_access_path.path[0] = _key_iterator->column_name(); - key_all_access_paths.emplace_back(paths); - } else if (paths.data_access_path.path[0] == ACCESS_MAP_VALUES) { - paths.data_access_path.path[0] = _val_iterator->column_name(); - val_all_access_paths.emplace_back(paths); + if (sub_all_access_paths.empty()) { + key_all_access_paths.emplace_back( + make_full_access_path_for_child(_key_iterator->column_name())); + val_all_access_paths.emplace_back( + make_full_access_path_for_child(_val_iterator->column_name())); + } else { + for (auto paths : sub_all_access_paths) { + if (paths.data_access_path.path[0] == ACCESS_ALL) { + // ACCESS_ALL means element_at(map, key) style access: the key column must be + // fully read so that the runtime can match the requested key, while any sub-path + // qualifiers (e.g. OFFSET) apply only to the value column. + // For key: create a path with just the column name (= full data access). + TColumnAccessPath key_path; + key_path.__set_type(paths.type); + TDataAccessPath key_data_path; + key_data_path.__set_path({_key_iterator->column_name()}); + key_path.__set_data_access_path(key_data_path); + key_all_access_paths.emplace_back(std::move(key_path)); + // For value: pass the full sub-path so qualifiers like OFFSET propagate. + paths.data_access_path.path[0] = _val_iterator->column_name(); + val_all_access_paths.emplace_back(paths); + } else if (paths.data_access_path.path[0] == ACCESS_MAP_KEYS) { + paths.data_access_path.path[0] = _key_iterator->column_name(); + key_all_access_paths.emplace_back(paths); + } else if (paths.data_access_path.path[0] == ACCESS_MAP_VALUES) { + paths.data_access_path.path[0] = _val_iterator->column_name(); + val_all_access_paths.emplace_back(paths); + } } } const auto need_read_keys = !key_all_access_paths.empty(); @@ -1356,6 +1483,35 @@ Status MapFileColumnIterator::set_access_paths(const TColumnAccessPaths& all_acc return Status::OK(); } +void MapFileColumnIterator::activate_read_phase(ReadPhase mode) { + ColumnIterator::activate_read_phase(mode); + _key_iterator->activate_read_phase(mode); + _val_iterator->activate_read_phase(mode); +} + +void MapFileColumnIterator::finish_deferred_read(MutableColumnPtr& dst) { + _drop_deferred_defaults(dst); + auto& map_column = assert_cast( + dst->is_nullable() ? static_cast(*dst).get_nested_column() : *dst); + auto keys_ptr = IColumn::mutate(std::move(map_column.get_keys_ptr())); + auto vals_ptr = IColumn::mutate(std::move(map_column.get_values_ptr())); + _key_iterator->finish_deferred_read(keys_ptr); + _val_iterator->finish_deferred_read(vals_ptr); + map_column.get_keys_ptr() = std::move(keys_ptr); + map_column.get_values_ptr() = std::move(vals_ptr); +} + +void MapFileColumnIterator::set_reading_flag_recursively(ReadingFlag flag) { + set_reading_flag(flag); + _key_iterator->set_reading_flag_recursively(flag); + _val_iterator->set_reading_flag_recursively(flag); +} + +bool MapFileColumnIterator::has_deferred_read_target() const { + return _reading_flag == ReadingFlag::NEED_TO_READ || + _key_iterator->has_deferred_read_target() || _val_iterator->has_deferred_read_target(); +} + //////////////////////////////////////////////////////////////////////////////// StructFileColumnIterator::StructFileColumnIterator( @@ -1383,12 +1539,14 @@ Status StructFileColumnIterator::init(const ColumnIteratorOptions& opts) { } Status StructFileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bool* has_null) { - if (_reading_flag == ReadingFlag::SKIP_READING) { + if (!need_to_read()) { DLOG(INFO) << "Struct column iterator column " << _column_name << " skip reading."; - dst->insert_many_defaults(*n); + _append_deferred_defaults(dst, *n); return Status::OK(); } + _drop_deferred_defaults(dst); + if (read_null_map_only()) { // NULL_MAP_ONLY mode: read null map, fill nested ColumnStruct with empty defaults DORIS_CHECK(dst->is_nullable()); @@ -1426,7 +1584,7 @@ Status StructFileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bo DCHECK(num_read == *n); } - if (dst->is_nullable()) { + if (dst->is_nullable() && need_to_read_meta_columns()) { size_t num_read = *n; auto null_map_ptr = static_cast(*dst).get_null_map_column_ptr(); // in not-null to null linked-schemachange mode, @@ -1448,7 +1606,7 @@ Status StructFileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bo } Status StructFileColumnIterator::seek_to_ordinal(ordinal_t ord) { - if (_reading_flag == ReadingFlag::SKIP_READING) { + if (!need_to_read()) { DLOG(INFO) << "Struct column iterator column " << _column_name << " skip reading."; return Status::OK(); } @@ -1464,7 +1622,8 @@ Status StructFileColumnIterator::seek_to_ordinal(ordinal_t ord) { for (auto& column_iterator : _sub_column_iterators) { RETURN_IF_ERROR(column_iterator->seek_to_ordinal(ord)); } - if (_struct_reader->is_nullable()) { + + if (_struct_reader->is_nullable() && need_to_read_meta_columns()) { RETURN_IF_ERROR(_null_iterator->seek_to_ordinal(ord)); } return Status::OK(); @@ -1483,22 +1642,32 @@ Status StructFileColumnIterator::init_prefetcher(const SegmentPrefetchParams& pa void StructFileColumnIterator::collect_prefetchers( std::map>& prefetchers, PrefetcherInitMethod init_method) { - for (auto& column_iterator : _sub_column_iterators) { - column_iterator->collect_prefetchers(prefetchers, init_method); + if (!need_to_read()) { + return; } if (_struct_reader->is_nullable()) { _null_iterator->collect_prefetchers(prefetchers, init_method); } + if (read_null_map_only()) { + return; + } + for (auto& column_iterator : _sub_column_iterators) { + if (column_iterator->need_to_read()) { + column_iterator->collect_prefetchers(prefetchers, init_method); + } + } } Status StructFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t count, MutableColumnPtr& dst) { - if (_reading_flag == ReadingFlag::SKIP_READING) { + if (!need_to_read()) { DLOG(INFO) << "Struct column iterator column " << _column_name << " skip reading."; - dst->insert_many_defaults(count); + _append_deferred_defaults(dst, count); return Status::OK(); } + _drop_deferred_defaults(dst); + if (count == 0) { return Status::OK(); } @@ -1562,8 +1731,12 @@ Status StructFileColumnIterator::set_access_paths( DLOG(INFO) << "Struct column iterator set sub-column " << _column_name << " to READING_FOR_PREDICATE"; } - auto sub_all_access_paths = DORIS_TRY(_get_sub_access_paths(all_access_paths)); - auto sub_predicate_access_paths = DORIS_TRY(_get_sub_access_paths(predicate_access_paths)); + + _nested_read_plan.pruned = true; + + auto sub_all_access_paths = DORIS_TRY(_normalize_access_paths(all_access_paths, false)); + auto sub_predicate_access_paths = + DORIS_TRY(_normalize_access_paths(predicate_access_paths, true)); // Check for NULL_MAP_ONLY mode: only read null map, skip all sub-columns _check_and_set_meta_read_mode(sub_all_access_paths); @@ -1572,7 +1745,7 @@ Status StructFileColumnIterator::set_access_paths( sub_iterator->set_reading_flag(ReadingFlag::SKIP_READING); } DLOG(INFO) << "Struct column iterator set column " << _column_name - << " to NULL_MAP_ONLY reading mode, all sub-columns set to SKIP_READING"; + << " to NULL_MAP_ONLY read phase, all sub-columns set to SKIP_READING"; return Status::OK(); } @@ -1609,6 +1782,10 @@ Status StructFileColumnIterator::set_access_paths( sub_predicate_access_paths_of_this.emplace_back(paths); } } + } else if (_reading_flag == ReadingFlag::READING_FOR_PREDICATE) { + // if no sub-column in predicate_access_paths, but current column is READING_FOR_PREDICATE, + // then we should set sub iterator to READING_FOR_PREDICATE too. + sub_iterator->set_reading_flag_recursively(ReadingFlag::READING_FOR_PREDICATE); } RETURN_IF_ERROR(sub_iterator->set_access_paths(sub_all_access_paths_of_this, @@ -1617,6 +1794,43 @@ Status StructFileColumnIterator::set_access_paths( return Status::OK(); } +void StructFileColumnIterator::activate_read_phase(ReadPhase mode) { + ColumnIterator::activate_read_phase(mode); + for (auto& sub_iterator : _sub_column_iterators) { + sub_iterator->activate_read_phase(mode); + } +} + +void StructFileColumnIterator::finish_deferred_read(MutableColumnPtr& dst) { + _drop_deferred_defaults(dst); + auto& column_struct = assert_cast( + dst->is_nullable() ? static_cast(*dst).get_nested_column() : *dst); + + for (size_t i = 0; i < _sub_column_iterators.size(); ++i) { + auto& sub_column = column_struct.get_column_ptr(i); + MutableColumnPtr mutable_sub_column = IColumn::mutate(std::move(sub_column)); + _sub_column_iterators[i]->finish_deferred_read(mutable_sub_column); + sub_column = std::move(mutable_sub_column); + } +} + +void StructFileColumnIterator::set_reading_flag_recursively(ReadingFlag flag) { + set_reading_flag(flag); + for (const auto& sub_column_iterator : _sub_column_iterators) { + sub_column_iterator->set_reading_flag_recursively(flag); + } +} + +bool StructFileColumnIterator::has_deferred_read_target() const { + if (_reading_flag == ReadingFlag::NEED_TO_READ) { + return true; + } + return std::any_of(_sub_column_iterators.begin(), _sub_column_iterators.end(), + [](const auto& sub_column_iterator) { + return sub_column_iterator->has_deferred_read_target(); + }); +} + //////////////////////////////////////////////////////////////////////////////// Status OffsetFileColumnIterator::init(const ColumnIteratorOptions& opts) { RETURN_IF_ERROR(_offset_iterator->init(opts)); @@ -1727,7 +1941,7 @@ Status ArrayFileColumnIterator::_seek_by_offsets(ordinal_t ord) { } Status ArrayFileColumnIterator::seek_to_ordinal(ordinal_t ord) { - if (_reading_flag == ReadingFlag::SKIP_READING) { + if (!need_to_read()) { DLOG(INFO) << "Array column iterator column " << _column_name << " skip reading."; return Status::OK(); } @@ -1748,12 +1962,16 @@ Status ArrayFileColumnIterator::seek_to_ordinal(ordinal_t ord) { } Status ArrayFileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bool* has_null) { - if (_reading_flag == ReadingFlag::SKIP_READING) { - DLOG(INFO) << "Array column iterator column " << _column_name << " skip reading."; - dst->insert_many_defaults(*n); + if (!need_to_read()) { + DLOG(INFO) << "Array column iterator column " << _column_name << " skip reading, read phase" + << static_cast(_read_phase) + << ", reading flag: " << static_cast(_reading_flag); + _append_deferred_defaults(dst, *n); return Status::OK(); } + _drop_deferred_defaults(dst); + if (read_null_map_only()) { // NULL_MAP_ONLY mode: read null map, fill nested ColumnArray with empty defaults DORIS_CHECK(dst->is_nullable()); @@ -1782,8 +2000,22 @@ Status ArrayFileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, boo dst->is_nullable() ? static_cast(*dst).get_nested_column() : *dst); bool offsets_has_null = false; - auto column_offsets_ptr = IColumn::mutate(std::move(column_array.get_offsets_ptr())); - Defer defer_offsets {[&] { column_array.get_offsets_ptr() = std::move(column_offsets_ptr); }}; + const bool read_meta_columns = need_to_read_meta_columns(); + MutableColumnPtr column_offsets_ptr; + if (read_meta_columns) { + column_offsets_ptr = IColumn::mutate(std::move(column_array.get_offsets_ptr())); + } else { + const auto base_offset = + column_array.get_offsets().empty() ? 0 : column_array.get_offsets().back(); + column_offsets_ptr = ColumnArray::ColumnOffsets::create(); + assert_cast(*column_offsets_ptr) + .insert_value(base_offset); + } + Defer defer_offsets {[&] { + if (read_meta_columns) { + column_array.get_offsets_ptr() = std::move(column_offsets_ptr); + } + }}; ssize_t start = column_offsets_ptr->size(); RETURN_IF_ERROR(_offset_iterator->next_batch(n, column_offsets_ptr, &offsets_has_null)); if (*n == 0) { @@ -1808,7 +2040,7 @@ Status ArrayFileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, boo } } - if (dst->is_nullable()) { + if (dst->is_nullable() && read_meta_columns) { auto null_map_ptr = static_cast(*dst).get_null_map_column_ptr(); size_t num_read = *n; // in not-null to null linked-schemachange mode, @@ -1841,23 +2073,35 @@ Status ArrayFileColumnIterator::init_prefetcher(const SegmentPrefetchParams& par void ArrayFileColumnIterator::collect_prefetchers( std::map>& prefetchers, PrefetcherInitMethod init_method) { - _offset_iterator->collect_prefetchers(prefetchers, init_method); - // the actual data pages to read of item column depends on the read result of offset column, - // so we can't init prefetch blocks according to rowids, just prefetch all data blocks here. - _item_iterator->collect_prefetchers(prefetchers, PrefetcherInitMethod::ALL_DATA_BLOCKS); + if (!need_to_read()) { + return; + } + if (!read_null_map_only()) { + _offset_iterator->collect_prefetchers(prefetchers, init_method); + } if (_array_reader->is_nullable()) { _null_iterator->collect_prefetchers(prefetchers, init_method); } + if (read_offset_only() || read_null_map_only()) { + return; + } + // the actual data pages to read of item column depends on the read result of offset column, + // so we can't init prefetch blocks according to rowids, just prefetch all data blocks here. + if (_item_iterator->need_to_read()) { + _item_iterator->collect_prefetchers(prefetchers, PrefetcherInitMethod::ALL_DATA_BLOCKS); + } } Status ArrayFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t count, MutableColumnPtr& dst) { - if (_reading_flag == ReadingFlag::SKIP_READING) { + if (!need_to_read()) { DLOG(INFO) << "Array column iterator column " << _column_name << " skip reading."; - dst->insert_many_defaults(count); + _append_deferred_defaults(dst, count); return Status::OK(); } + _drop_deferred_defaults(dst); + for (size_t i = 0; i < count; ++i) { // TODO(cambyszju): now read array one by one, need optimize later RETURN_IF_ERROR(seek_to_ordinal(rowids[i])); @@ -1876,6 +2120,29 @@ void ArrayFileColumnIterator::remove_pruned_sub_iterators() { _item_iterator->remove_pruned_sub_iterators(); } +void ArrayFileColumnIterator::activate_read_phase(ReadPhase mode) { + ColumnIterator::activate_read_phase(mode); + _item_iterator->activate_read_phase(mode); +} + +void ArrayFileColumnIterator::finish_deferred_read(MutableColumnPtr& dst) { + _drop_deferred_defaults(dst); + auto& column_array = assert_cast( + dst->is_nullable() ? static_cast(*dst).get_nested_column() : *dst); + auto item_column_ptr = IColumn::mutate(std::move(column_array.get_data_ptr())); + _item_iterator->finish_deferred_read(item_column_ptr); + column_array.get_data_ptr() = std::move(item_column_ptr); +} + +void ArrayFileColumnIterator::set_reading_flag_recursively(ReadingFlag flag) { + set_reading_flag(flag); + _item_iterator->set_reading_flag_recursively(flag); +} + +bool ArrayFileColumnIterator::has_deferred_read_target() const { + return _reading_flag == ReadingFlag::NEED_TO_READ || _item_iterator->has_deferred_read_target(); +} + Status ArrayFileColumnIterator::set_access_paths(const TColumnAccessPaths& all_access_paths, const TColumnAccessPaths& predicate_access_paths) { if (all_access_paths.empty()) { @@ -1888,21 +2155,24 @@ Status ArrayFileColumnIterator::set_access_paths(const TColumnAccessPaths& all_a << " to READING_FOR_PREDICATE"; } - auto sub_all_access_paths = DORIS_TRY(_get_sub_access_paths(all_access_paths)); - auto sub_predicate_access_paths = DORIS_TRY(_get_sub_access_paths(predicate_access_paths)); + _nested_read_plan.pruned = true; + + auto sub_all_access_paths = DORIS_TRY(_normalize_access_paths(all_access_paths, false)); + auto sub_predicate_access_paths = + DORIS_TRY(_normalize_access_paths(predicate_access_paths, true)); // Check for meta-only modes (OFFSET_ONLY or NULL_MAP_ONLY) _check_and_set_meta_read_mode(sub_all_access_paths); if (read_offset_only()) { _item_iterator->set_reading_flag(ReadingFlag::SKIP_READING); DLOG(INFO) << "Array column iterator set column " << _column_name - << " to OFFSET_ONLY reading mode, item column set to SKIP_READING"; + << " to OFFSET_ONLY read phase, item column set to SKIP_READING"; return Status::OK(); } if (read_null_map_only()) { _item_iterator->set_reading_flag(ReadingFlag::SKIP_READING); DLOG(INFO) << "Array column iterator set column " << _column_name - << " to NULL_MAP_ONLY reading mode, item column set to SKIP_READING"; + << " to NULL_MAP_ONLY read phase, item column set to SKIP_READING"; return Status::OK(); } @@ -1915,6 +2185,9 @@ Status ArrayFileColumnIterator::set_access_paths(const TColumnAccessPaths& all_a path.data_access_path.path[0] = _item_iterator->column_name(); } } + } else if (!no_predicate_sub_column) { + sub_all_access_paths.emplace_back( + make_full_access_path_for_child(_item_iterator->column_name())); } if (!no_predicate_sub_column) { @@ -1923,6 +2196,10 @@ Status ArrayFileColumnIterator::set_access_paths(const TColumnAccessPaths& all_a path.data_access_path.path[0] = _item_iterator->column_name(); } } + } else if (_reading_flag == ReadingFlag::READING_FOR_PREDICATE) { + // if no sub-column in predicate_access_paths, but current column is READING_FOR_PREDICATE, + // then we should set item_iterator to READING_FOR_PREDICATE too. + _item_iterator->set_reading_flag_recursively(ReadingFlag::READING_FOR_PREDICATE); } if (!no_sub_column_to_skip || !no_predicate_sub_column) { @@ -1989,10 +2266,10 @@ Status StringFileColumnIterator::set_access_paths( } if (read_offset_only()) { DLOG(INFO) << "String column iterator set column " << _column_name - << " to OFFSET_ONLY reading mode"; + << " to OFFSET_ONLY read phase"; } else if (read_null_map_only()) { DLOG(INFO) << "String column iterator set column " << _column_name - << " to NULL_MAP_ONLY reading mode"; + << " to NULL_MAP_ONLY read phase"; } return Status::OK(); @@ -2063,7 +2340,7 @@ void FileColumnIterator::_trigger_prefetch_if_eligible(ordinal_t ord) { } Status FileColumnIterator::seek_to_ordinal(ordinal_t ord) { - if (_reading_flag == ReadingFlag::SKIP_READING) { + if (!need_to_read()) { DLOG(INFO) << "File column iterator column " << _column_name << " skip reading."; return Status::OK(); } @@ -2124,6 +2401,14 @@ Status FileColumnIterator::next_batch_of_zone_map(size_t* n, MutableColumnPtr& d } Status FileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bool* has_null) { + if (!need_to_read()) { + DLOG(INFO) << "File column iterator column " << _column_name << " skip reading."; + _append_deferred_defaults(dst, *n); + return Status::OK(); + } + + _drop_deferred_defaults(dst); + if (read_null_map_only()) { DLOG(INFO) << "File column iterator column " << _column_name << " in NULL_MAP_ONLY mode, reading only null map."; @@ -2172,12 +2457,6 @@ Status FileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bool* ha return Status::OK(); } - if (_reading_flag == ReadingFlag::SKIP_READING) { - DLOG(INFO) << "File column iterator column " << _column_name << " skip reading."; - dst->insert_many_defaults(*n); - return Status::OK(); - } - size_t curr_size = dst->byte_size(); dst->reserve(*n); size_t remaining = *n; @@ -2237,6 +2516,14 @@ Status FileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bool* ha Status FileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t count, MutableColumnPtr& dst) { + if (!need_to_read()) { + DLOG(INFO) << "File column iterator column " << _column_name << " skip reading."; + _append_deferred_defaults(dst, count); + return Status::OK(); + } + + _drop_deferred_defaults(dst); + if (read_null_map_only()) { DLOG(INFO) << "File column iterator column " << _column_name << " in NULL_MAP_ONLY mode, reading only null map by rowids."; @@ -2300,12 +2587,6 @@ Status FileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t co return Status::OK(); } - if (_reading_flag == ReadingFlag::SKIP_READING) { - DLOG(INFO) << "File column iterator column " << _column_name << " skip reading."; - dst->insert_many_defaults(count); - return Status::OK(); - } - size_t remaining = count; size_t total_read_count = 0; size_t nrows_to_read = 0; diff --git a/be/src/storage/segment/column_reader.h b/be/src/storage/segment/column_reader.h index dcee88dcbdf578..69f0cd8258a9d0 100644 --- a/be/src/storage/segment/column_reader.h +++ b/be/src/storage/segment/column_reader.h @@ -23,7 +23,8 @@ #include // for size_t #include // for uint32_t -#include // for unique_ptr +#include +#include // for unique_ptr #include #include #include @@ -424,18 +425,81 @@ class ColumnIterator { bool read_offset_only() const { return _read_mode == ReadMode::OFFSET_ONLY; } bool read_null_map_only() const { return _read_mode == ReadMode::NULL_MAP_ONLY; } + enum class ReadPhase : int { FULL, PREDICATE, DEFERRED }; + + virtual void activate_read_phase(ReadPhase mode) { + _read_phase = mode; + if (mode == ReadPhase::PREDICATE) { + _nested_read_plan.has_deferred_defaults = false; + } + } + + virtual bool need_to_read() const { + switch (_read_phase) { + case ReadPhase::FULL: + return _reading_flag != ReadingFlag::SKIP_READING; + case ReadPhase::PREDICATE: + return _reading_flag == ReadingFlag::READING_FOR_PREDICATE; + case ReadPhase::DEFERRED: + return _reading_flag == ReadingFlag::NEED_TO_READ; + default: + return false; + } + } + + // Whether need to read meta columns, such as null map column, offset column. + bool need_to_read_meta_columns() const { + if (_reading_flag == ReadingFlag::SKIP_READING) { + return false; + } + switch (_read_phase) { + case ReadPhase::FULL: + case ReadPhase::PREDICATE: + return true; + case ReadPhase::DEFERRED: + return _reading_flag != ReadingFlag::READING_FOR_PREDICATE; + } + return false; + } + + virtual void finish_deferred_read(MutableColumnPtr& dst) { _drop_deferred_defaults(dst); } + + virtual void set_reading_flag_recursively(ReadingFlag flag) { set_reading_flag(flag); } + + // Whether this iterator or any nested iterator has data that must be materialized + // in deferred phase. Predicate-only and meta-only branches are read before filtering and + // must not be re-read in the lazy phase. + virtual bool has_deferred_read_target() const { + return _reading_flag == ReadingFlag::NEED_TO_READ; + } + + bool is_pruned() const { return _nested_read_plan.pruned; } + protected: + void _append_deferred_defaults(MutableColumnPtr& dst, size_t count); + + void _drop_deferred_defaults(MutableColumnPtr& dst); + // Checks sub access paths for OFFSET or NULL meta-only modes and // updates _read_mode accordingly. Use the accessor helpers // read_offset_only() / read_null_map_only() to query the current mode. void _check_and_set_meta_read_mode(const TColumnAccessPaths& sub_all_access_paths); Result _get_sub_access_paths(const TColumnAccessPaths& access_paths); + Result _normalize_access_paths(const TColumnAccessPaths& access_paths, + const bool is_predicate); ColumnIteratorOptions _opts; ReadingFlag _reading_flag {ReadingFlag::NORMAL_READING}; ReadMode _read_mode = ReadMode::DEFAULT; + ReadPhase _read_phase {ReadPhase::FULL}; std::string _column_name; + + struct NestedReadPlan { + bool pruned {false}; + bool has_deferred_defaults {false}; + }; + NestedReadPlan _nested_read_plan; }; // This iterator is used to read column data from file @@ -633,6 +697,29 @@ class MapFileColumnIterator final : public ColumnIterator { void remove_pruned_sub_iterators() override; + void activate_read_phase(ReadPhase mode) override; + + bool need_to_read() const override { + switch (_read_phase) { + case ReadPhase::FULL: + return _reading_flag != ReadingFlag::SKIP_READING; + case ReadPhase::PREDICATE: + return _reading_flag == ReadingFlag::READING_FOR_PREDICATE; + case ReadPhase::DEFERRED: + // In deferred phase, read this map only when at least one key/value branch still + // has non-predicate data to materialize. + return has_deferred_read_target(); + default: + return false; + } + } + + void finish_deferred_read(MutableColumnPtr& dst) override; + + void set_reading_flag_recursively(ReadingFlag flag) override; + + bool has_deferred_read_target() const override; + private: std::shared_ptr _map_reader = nullptr; ColumnIteratorUPtr _null_iterator; @@ -682,6 +769,27 @@ class StructFileColumnIterator final : public ColumnIterator { std::map>& prefetchers, PrefetcherInitMethod init_method) override; + void activate_read_phase(ReadPhase mode) override; + + bool need_to_read() const override { + switch (_read_phase) { + case ReadPhase::FULL: + return _reading_flag != ReadingFlag::SKIP_READING; + case ReadPhase::PREDICATE: + return _reading_flag == ReadingFlag::READING_FOR_PREDICATE; + case ReadPhase::DEFERRED: + // In deferred phase, read this struct only when at least one nested branch still + // has non-predicate data to materialize. + return has_deferred_read_target(); + default: + return false; + } + } + + void finish_deferred_read(MutableColumnPtr& dst) override; + void set_reading_flag_recursively(ReadingFlag flag) override; + bool has_deferred_read_target() const override; + private: std::shared_ptr _struct_reader = nullptr; ColumnIteratorUPtr _null_iterator; @@ -729,6 +837,29 @@ class ArrayFileColumnIterator final : public ColumnIterator { std::map>& prefetchers, PrefetcherInitMethod init_method) override; + void activate_read_phase(ReadPhase mode) override; + + bool need_to_read() const override { + switch (_read_phase) { + case ReadPhase::FULL: + return _reading_flag != ReadingFlag::SKIP_READING; + case ReadPhase::PREDICATE: + return _reading_flag == ReadingFlag::READING_FOR_PREDICATE; + case ReadPhase::DEFERRED: + // In deferred phase, read this array only when its item branch still has + // non-predicate data to materialize. + return has_deferred_read_target(); + default: + return false; + } + } + + void finish_deferred_read(MutableColumnPtr& dst) override; + + void set_reading_flag_recursively(ReadingFlag flag) override; + + bool has_deferred_read_target() const override; + private: std::shared_ptr _array_reader = nullptr; std::unique_ptr _offset_iterator; diff --git a/be/src/storage/segment/segment_iterator.cpp b/be/src/storage/segment/segment_iterator.cpp index 0d979ac9fa1df1..d2adb05b52d014 100644 --- a/be/src/storage/segment/segment_iterator.cpp +++ b/be/src/storage/segment/segment_iterator.cpp @@ -117,6 +117,19 @@ using namespace ErrorCode; namespace segment_v2 { namespace { +class ScopedColumnReadPhase { +public: + ScopedColumnReadPhase(ColumnIterator* iterator, ColumnIterator::ReadPhase phase) + : _iterator(iterator) { + _iterator->activate_read_phase(phase); + } + + ~ScopedColumnReadPhase() { _iterator->activate_read_phase(ColumnIterator::ReadPhase::FULL); } + +private: + ColumnIterator* _iterator; +}; + Status tablet_column_id_by_slot(const TabletSchemaSPtr& tablet_schema, const SlotDescriptor* slot, ColumnId* cid) { int32_t field_index = -1; @@ -530,6 +543,10 @@ Status SegmentIterator::_init_impl(const StorageReadOptions& opts) { _score_runtime = _opts.score_runtime; _ann_topn_runtime = _opts.ann_topn_runtime; + _enable_prune_nested_column = _opts.io_ctx.reader_type == ReaderType::READER_QUERY && + _opts.runtime_state && + _opts.runtime_state->enable_prune_nested_column(); + if (opts.output_columns != nullptr) { _output_columns = *(opts.output_columns); } @@ -773,8 +790,14 @@ void SegmentIterator::_init_segment_prefetchers() { ? PrefetcherInitMethod::FROM_ROWIDS : PrefetcherInitMethod::ALL_DATA_BLOCKS; std::map> prefetchers; - for (const auto& column_iter : _column_iterators) { + for (size_t idx = 0; idx < _column_iterators.size(); ++idx) { + auto cid = cast_set(idx); + auto* column_iter = _column_iterators[cid].get(); if (column_iter != nullptr) { + ScopedColumnReadPhase phase_scope(column_iter, + _deferred_nested_columns.contains(cid) + ? ColumnIterator::ReadPhase::PREDICATE + : ColumnIterator::ReadPhase::FULL); column_iter->collect_prefetchers(prefetchers, init_method); } } @@ -2095,6 +2118,25 @@ Status SegmentIterator::_vec_init_lazy_materialization() { if (_is_common_expr_column[cid] || _is_pred_column[cid]) { auto loc = _schema_block_id_map[cid]; _columns_to_filter.push_back(loc); + + const auto field_type = _schema->column(cid)->type(); + if (_is_common_expr_column[cid] && _enable_prune_nested_column && + (field_type == FieldType::OLAP_FIELD_TYPE_STRUCT || + field_type == FieldType::OLAP_FIELD_TYPE_ARRAY || + field_type == FieldType::OLAP_FIELD_TYPE_MAP)) { + DCHECK(_column_iterators[cid]); + if (_column_iterators[cid]->is_pruned() && + _column_iterators[cid]->reading_flag() == + ColumnIterator::ReadingFlag::READING_FOR_PREDICATE && + _column_iterators[cid]->has_deferred_read_target()) { + // Only split lazy recovery for pruned complex common expr columns + // whose predicate access path is explicit. When a remaining conjunct + // references the whole complex column without predicate sub-paths, + // the predicate phase must read the pruned column normally because the + // expression may need nested values, not only parent metadata. + _deferred_nested_columns.emplace(cid); + } + } } } @@ -2455,16 +2497,22 @@ Status SegmentIterator::_read_columns_by_index(uint32_t nrows_read_limit, uint16 }) } + auto* column_iter = _column_iterators[cid].get(); + ScopedColumnReadPhase phase_scope(column_iter, + _deferred_nested_columns.contains(cid) + ? ColumnIterator::ReadPhase::PREDICATE + : ColumnIterator::ReadPhase::FULL); + if (is_continuous) { size_t rows_read = nrows_read; _opts.stats->predicate_column_read_seek_num += 1; if (_opts.runtime_state && _opts.runtime_state->enable_profile()) { SCOPED_RAW_TIMER(&_opts.stats->predicate_column_read_seek_ns); - RETURN_IF_ERROR(_column_iterators[cid]->seek_to_ordinal(_block_rowids[0])); + RETURN_IF_ERROR(column_iter->seek_to_ordinal(_block_rowids[0])); } else { - RETURN_IF_ERROR(_column_iterators[cid]->seek_to_ordinal(_block_rowids[0])); + RETURN_IF_ERROR(column_iter->seek_to_ordinal(_block_rowids[0])); } - RETURN_IF_ERROR(_column_iterators[cid]->next_batch(&rows_read, column)); + RETURN_IF_ERROR(column_iter->next_batch(&rows_read, column)); if (rows_read != nrows_read) { return Status::Error("nrows({}) != rows_read({})", nrows_read, rows_read); @@ -2484,20 +2532,18 @@ Status SegmentIterator::_read_columns_by_index(uint32_t nrows_read_limit, uint16 _opts.stats->predicate_column_read_seek_num += 1; if (_opts.runtime_state && _opts.runtime_state->enable_profile()) { SCOPED_RAW_TIMER(&_opts.stats->predicate_column_read_seek_ns); - RETURN_IF_ERROR( - _column_iterators[cid]->seek_to_ordinal(_block_rowids[processed])); + RETURN_IF_ERROR(column_iter->seek_to_ordinal(_block_rowids[processed])); } else { - RETURN_IF_ERROR( - _column_iterators[cid]->seek_to_ordinal(_block_rowids[processed])); + RETURN_IF_ERROR(column_iter->seek_to_ordinal(_block_rowids[processed])); } - RETURN_IF_ERROR(_column_iterators[cid]->next_batch(&rows_read, column)); + RETURN_IF_ERROR(column_iter->next_batch(&rows_read, column)); if (rows_read != current_batch_size) { return Status::Error( "batch nrows({}) != rows_read({})", current_batch_size, rows_read); } } else { - RETURN_IF_ERROR(_column_iterators[cid]->read_by_rowids( - &_block_rowids[processed], current_batch_size, column)); + RETURN_IF_ERROR(column_iter->read_by_rowids(&_block_rowids[processed], + current_batch_size, column)); } processed += current_batch_size; } @@ -2797,7 +2843,8 @@ Status SegmentIterator::_read_columns_by_rowids(std::vector& read_colu std::vector& rowid_vector, uint16_t* sel_rowid_idx, size_t select_size, MutableColumns* mutable_columns, - bool init_condition_cache) { + bool init_condition_cache, + bool read_for_predicate) { SCOPED_RAW_TIMER(&_opts.stats->lazy_read_ns); std::vector rowids(select_size); @@ -2841,8 +2888,15 @@ Status SegmentIterator::_read_columns_by_rowids(std::vector& read_colu "SegmentIterator meet invalid column, return columns size {}, cid {}", _current_return_columns.size(), cid); } - RETURN_IF_ERROR(_column_iterators[cid]->read_by_rowids(rowids.data(), select_size, - _current_return_columns[cid])); + + auto* column_iter = _column_iterators[cid].get(); + ScopedColumnReadPhase phase_scope( + column_iter, read_for_predicate && _deferred_nested_columns.contains(cid) + ? ColumnIterator::ReadPhase::PREDICATE + : ColumnIterator::ReadPhase::FULL); + + RETURN_IF_ERROR(column_iter->read_by_rowids(rowids.data(), select_size, + _current_return_columns[cid])); } return Status::OK(); @@ -3062,7 +3116,7 @@ Status SegmentIterator::_next_batch_internal(Block* block) { SCOPED_RAW_TIMER(&_opts.stats->non_predicate_read_ns); RETURN_IF_ERROR(_read_columns_by_rowids( _common_expr_column_ids, _block_rowids, _sel_rowid_idx.data(), - _selected_size, &_current_return_columns)); + _selected_size, &_current_return_columns, false, true)); _replace_version_col_if_needed(_common_expr_column_ids, _selected_size); _update_lsn_col_if_needed(_common_expr_column_ids, _selected_size); _update_tso_col_if_needed(_common_expr_column_ids, _selected_size); @@ -3097,7 +3151,7 @@ Status SegmentIterator::_next_batch_internal(Block* block) { RETURN_IF_ERROR(_read_columns_by_rowids( _non_predicate_columns, _block_rowids, _sel_rowid_idx.data(), _selected_size, &_current_return_columns, - _opts.condition_cache_digest && !_find_condition_cache)); + _opts.condition_cache_digest && !_find_condition_cache, false)); _replace_version_col_if_needed(_non_predicate_columns, _selected_size); _update_lsn_col_if_needed(_non_predicate_columns, _selected_size); _update_tso_col_if_needed(_non_predicate_columns, _selected_size); @@ -3111,6 +3165,27 @@ Status SegmentIterator::_next_batch_internal(Block* block) { } } } + + if (!_deferred_nested_columns.empty()) { + SCOPED_RAW_TIMER(&_opts.stats->deferred_nested_read_ns); + DorisVector rowids(_selected_size); + for (size_t i = 0; i < _selected_size; ++i) { + rowids[i] = _block_rowids[_sel_rowid_idx[i]]; + } + + for (auto cid : _deferred_nested_columns) { + auto loc = _schema_block_id_map[cid]; + auto column = IColumn::mutate(std::move(block->get_by_position(loc).column)); + auto* column_iter = _column_iterators[cid].get(); + ScopedColumnReadPhase phase_scope(column_iter, ColumnIterator::ReadPhase::DEFERRED); + if (_selected_size > 0) { + RETURN_IF_ERROR( + column_iter->read_by_rowids(rowids.data(), _selected_size, column)); + } + column_iter->finish_deferred_read(column); + block->get_by_position(loc).column = std::move(column); + } + } } // step5: output columns diff --git a/be/src/storage/segment/segment_iterator.h b/be/src/storage/segment/segment_iterator.h index a54aae7db89ec3..62a31ea39e8884 100644 --- a/be/src/storage/segment/segment_iterator.h +++ b/be/src/storage/segment/segment_iterator.h @@ -233,7 +233,8 @@ class SegmentIterator : public RowwiseIterator { std::vector& rowid_vector, uint16_t* sel_rowid_idx, size_t select_size, MutableColumns* mutable_columns, - bool init_condition_cache = false); + bool init_condition_cache = false, + bool read_for_predicate = false); Status copy_column_data_by_selector(IColumn* input_col_ptr, MutableColumnPtr& output_col, uint16_t* sel_rowid_idx, uint16_t select_size, @@ -378,6 +379,9 @@ class SegmentIterator : public RowwiseIterator { bool _is_need_short_eval = false; bool _is_need_expr_eval = false; + std::set _deferred_nested_columns; + bool _enable_prune_nested_column = false; + // fields for vectorization execution std::vector _vec_pred_column_ids; // keep columnId of columns for vectorized predicate evaluation diff --git a/be/test/storage/segment/column_reader_test.cpp b/be/test/storage/segment/column_reader_test.cpp index 54bf94a2061ae0..e8b9c25fad5ee8 100644 --- a/be/test/storage/segment/column_reader_test.cpp +++ b/be/test/storage/segment/column_reader_test.cpp @@ -38,6 +38,40 @@ #include "util/json/path_in_data.h" namespace doris::segment_v2 { +namespace { +class TestColumnIterator final : public ColumnIterator { +public: + Status seek_to_ordinal(ordinal_t /* ord */) override { return Status::OK(); } + + Status next_batch(size_t* n, MutableColumnPtr& dst, bool* has_null) override { + dst->insert_many_defaults(*n); + if (has_null != nullptr) { + *has_null = false; + } + return Status::OK(); + } + + Status read_by_rowids(const rowid_t* /* rowids */, const size_t count, + MutableColumnPtr& dst) override { + dst->insert_many_defaults(count); + return Status::OK(); + } + + ordinal_t get_current_ordinal() const override { return 0; } + + void force_set_reading_flag(ReadingFlag flag) { _reading_flag = flag; } + + Result process_sub_access_paths(const TColumnAccessPaths& access_paths, + bool is_predicate) { + return _normalize_access_paths(access_paths, is_predicate); + } + + void append_deferred_defaults(MutableColumnPtr& dst, size_t count) { + _append_deferred_defaults(dst, count); + } +}; +} // namespace + class ColumnReaderTest : public ::testing::Test { protected: void SetUp() override {} @@ -117,6 +151,1056 @@ TEST_F(ColumnReaderTest, StructAccessPaths) { ColumnIterator::ReadingFlag::NEED_TO_READ); } +TEST_F(ColumnReaderTest, ReadPhaseMatrix) { + TestColumnIterator iterator; + + iterator.force_set_reading_flag(ColumnIterator::ReadingFlag::SKIP_READING); + iterator.activate_read_phase(ColumnIterator::ReadPhase::FULL); + EXPECT_FALSE(iterator.need_to_read()); + EXPECT_FALSE(iterator.need_to_read_meta_columns()); + + iterator.force_set_reading_flag(ColumnIterator::ReadingFlag::NEED_TO_READ); + EXPECT_TRUE(iterator.need_to_read()); + EXPECT_TRUE(iterator.need_to_read_meta_columns()); + + iterator.force_set_reading_flag(ColumnIterator::ReadingFlag::NORMAL_READING); + iterator.activate_read_phase(ColumnIterator::ReadPhase::PREDICATE); + EXPECT_FALSE(iterator.need_to_read()); + EXPECT_TRUE(iterator.need_to_read_meta_columns()); + + iterator.force_set_reading_flag(ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); + EXPECT_TRUE(iterator.need_to_read()); + EXPECT_TRUE(iterator.need_to_read_meta_columns()); + + iterator.force_set_reading_flag(ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); + iterator.activate_read_phase(ColumnIterator::ReadPhase::DEFERRED); + EXPECT_FALSE(iterator.need_to_read()); + EXPECT_FALSE(iterator.need_to_read_meta_columns()); + + iterator.force_set_reading_flag(ColumnIterator::ReadingFlag::NEED_TO_READ); + EXPECT_TRUE(iterator.need_to_read()); + EXPECT_TRUE(iterator.need_to_read_meta_columns()); +} + +TEST_F(ColumnReaderTest, ReadingFlagPriorityAndNeedToRead) { + TestColumnIterator iterator; + + iterator.set_reading_flag(ColumnIterator::ReadingFlag::SKIP_READING); + EXPECT_EQ(iterator.reading_flag(), ColumnIterator::ReadingFlag::SKIP_READING); + + iterator.set_need_to_read(); + EXPECT_EQ(iterator.reading_flag(), ColumnIterator::ReadingFlag::NEED_TO_READ); + + iterator.set_reading_flag(ColumnIterator::ReadingFlag::SKIP_READING); + EXPECT_EQ(iterator.reading_flag(), ColumnIterator::ReadingFlag::NEED_TO_READ); + + iterator.set_reading_flag(ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); + EXPECT_EQ(iterator.reading_flag(), ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); + + iterator.set_reading_flag(ColumnIterator::ReadingFlag::NEED_TO_READ); + EXPECT_EQ(iterator.reading_flag(), ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); +} + +TEST_F(ColumnReaderTest, DeferredDefaultsLifecycle) { + TestColumnIterator iterator; + iterator.force_set_reading_flag(ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); + iterator.activate_read_phase(ColumnIterator::ReadPhase::PREDICATE); + + MutableColumnPtr dst = ColumnInt32::create(); + iterator.append_deferred_defaults(dst, 3); + + EXPECT_EQ(3, dst->size()); + EXPECT_TRUE(iterator._nested_read_plan.has_deferred_defaults); + + iterator.activate_read_phase(ColumnIterator::ReadPhase::DEFERRED); + iterator.finish_deferred_read(dst); + EXPECT_EQ(0, dst->size()); + EXPECT_FALSE(iterator._nested_read_plan.has_deferred_defaults); + + MutableColumnPtr lazy_dst = ColumnInt32::create(); + iterator.force_set_reading_flag(ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); + iterator.activate_read_phase(ColumnIterator::ReadPhase::DEFERRED); + iterator.append_deferred_defaults(lazy_dst, 4); + EXPECT_EQ(0, lazy_dst->size()); +} + +TEST_F(ColumnReaderTest, DeferredDefaultsRecoveryAfterColumnReplacement) { + TestColumnIterator iterator; + iterator.force_set_reading_flag(ColumnIterator::ReadingFlag::NEED_TO_READ); + iterator.activate_read_phase(ColumnIterator::ReadPhase::PREDICATE); + + MutableColumnPtr dst = ColumnInt32::create(); + iterator.append_deferred_defaults(dst, 3); + EXPECT_TRUE(iterator._nested_read_plan.has_deferred_defaults); + + IColumn::Filter filter; + filter.resize(3); + filter[0] = 1; + filter[1] = 0; + filter[2] = 1; + dst = IColumn::mutate(dst->filter(filter, 2)); + EXPECT_EQ(2, dst->size()); + + iterator.activate_read_phase(ColumnIterator::ReadPhase::DEFERRED); + iterator.finish_deferred_read(dst); + EXPECT_EQ(0, dst->size()); + EXPECT_FALSE(iterator._nested_read_plan.has_deferred_defaults); + + dst->insert_many_defaults(2); + iterator.activate_read_phase(ColumnIterator::ReadPhase::PREDICATE); + iterator.activate_read_phase(ColumnIterator::ReadPhase::DEFERRED); + iterator.finish_deferred_read(dst); + EXPECT_EQ(2, dst->size()); +} + +TEST_F(ColumnReaderTest, SetReadingFlagRecursivelyPropagates) { + auto null_iter = std::make_unique(std::make_shared()); + std::vector struct_sub_iters; + + auto sub_col = std::make_unique(std::make_shared()); + sub_col->set_column_name("sub_col"); + struct_sub_iters.emplace_back(std::move(sub_col)); + + auto array_item = std::make_unique(std::make_shared()); + array_item->set_column_name("item"); + auto array_offsets = std::make_unique( + std::make_unique(std::make_shared())); + auto array_null = std::make_unique(std::make_shared()); + auto array_iter = std::make_unique( + std::make_shared(), std::move(array_offsets), std::move(array_item), + std::move(array_null)); + array_iter->set_column_name("arr"); + struct_sub_iters.emplace_back(std::move(array_iter)); + + StructFileColumnIterator struct_iter(std::make_shared(), std::move(null_iter), + std::move(struct_sub_iters)); + struct_iter.set_reading_flag_recursively(ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); + + EXPECT_EQ(struct_iter.reading_flag(), ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); + EXPECT_EQ(struct_iter._sub_column_iterators[0]->_reading_flag, + ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); + + auto* nested_array = + static_cast(struct_iter._sub_column_iterators[1].get()); + EXPECT_EQ(nested_array->_reading_flag, ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); + EXPECT_EQ(nested_array->_item_iterator->_reading_flag, + ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); + + auto map_null_iter = std::make_unique(std::make_shared()); + auto map_offsets_iter = std::make_unique( + std::make_unique(std::make_shared())); + auto map_key_iter = std::make_unique(std::make_shared()); + auto map_val_iter = std::make_unique(std::make_shared()); + MapFileColumnIterator map_iter(std::make_shared(), std::move(map_null_iter), + std::move(map_offsets_iter), std::move(map_key_iter), + std::move(map_val_iter)); + map_iter.set_reading_flag_recursively(ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); + EXPECT_EQ(map_iter._key_iterator->_reading_flag, + ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); + EXPECT_EQ(map_iter._val_iterator->_reading_flag, + ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); +} + +TEST_F(ColumnReaderTest, RemovePrunedSubIterators) { + auto struct_null_iter = std::make_unique(std::make_shared()); + std::vector struct_sub_iters; + auto sub_keep = std::make_unique(std::make_shared()); + sub_keep->set_column_name("keep"); + auto sub_prune = std::make_unique(std::make_shared()); + sub_prune->set_column_name("prune"); + sub_prune->set_reading_flag(ColumnIterator::ReadingFlag::SKIP_READING); + struct_sub_iters.emplace_back(std::move(sub_keep)); + struct_sub_iters.emplace_back(std::move(sub_prune)); + + auto array_item_null = std::make_unique(std::make_shared()); + std::vector item_struct_sub_iters; + auto item_keep = std::make_unique(std::make_shared()); + item_keep->set_column_name("keep"); + auto item_prune = std::make_unique(std::make_shared()); + item_prune->set_column_name("prune"); + item_prune->set_reading_flag(ColumnIterator::ReadingFlag::SKIP_READING); + item_struct_sub_iters.emplace_back(std::move(item_keep)); + item_struct_sub_iters.emplace_back(std::move(item_prune)); + auto item_struct = std::make_unique(std::make_shared(), + std::move(array_item_null), + std::move(item_struct_sub_iters)); + + auto array_offsets = std::make_unique( + std::make_unique(std::make_shared())); + auto array_null = std::make_unique(std::make_shared()); + auto array_iter = std::make_unique( + std::make_shared(), std::move(array_offsets), std::move(item_struct), + std::move(array_null)); + struct_sub_iters.emplace_back(std::move(array_iter)); + + StructFileColumnIterator struct_iter(std::make_shared(), + std::move(struct_null_iter), std::move(struct_sub_iters)); + ASSERT_EQ(3, struct_iter._sub_column_iterators.size()); + struct_iter.remove_pruned_sub_iterators(); + ASSERT_EQ(2, struct_iter._sub_column_iterators.size()); + + auto* nested_array = + static_cast(struct_iter._sub_column_iterators[1].get()); + auto* nested_struct = + static_cast(nested_array->_item_iterator.get()); + ASSERT_EQ(1, nested_struct->_sub_column_iterators.size()); + EXPECT_EQ(nested_struct->_sub_column_iterators[0]->column_name(), "keep"); +} + +TEST_F(ColumnReaderTest, FinishDeferredReadOnNestedStruct) { + auto sub_iter = std::make_unique(); + auto* sub_iter_ptr = sub_iter.get(); + auto null_iter = std::make_unique(std::make_shared()); + std::vector sub_iters; + sub_iters.emplace_back(std::move(sub_iter)); + + StructFileColumnIterator struct_iter(std::make_shared(), std::move(null_iter), + std::move(sub_iters)); + struct_iter.activate_read_phase(ColumnIterator::ReadPhase::PREDICATE); + sub_iter_ptr->activate_read_phase(ColumnIterator::ReadPhase::PREDICATE); + + MutableColumnPtr nested_column = ColumnInt32::create(); + MutableColumnPtr nested_mut = IColumn::mutate(std::move(nested_column)); + sub_iter_ptr->append_deferred_defaults(nested_mut, 5); + EXPECT_EQ(5, nested_mut->size()); + + Columns struct_columns; + struct_columns.emplace_back(std::move(nested_mut)); + auto struct_column = ColumnStruct::create(struct_columns); + MutableColumnPtr struct_mut = std::move(struct_column); + struct_iter.activate_read_phase(ColumnIterator::ReadPhase::DEFERRED); + sub_iter_ptr->activate_read_phase(ColumnIterator::ReadPhase::DEFERRED); + struct_iter.finish_deferred_read(struct_mut); + + auto& column_struct = assert_cast(*struct_mut); + auto nested_after = column_struct.get_column_ptr(0); + EXPECT_EQ(0, nested_after->size()); +} + +TEST_F(ColumnReaderTest, NormalizeAccessPathsSetsPredicateFlag) { + TestColumnIterator iterator; + iterator.set_column_name("self"); + + TColumnAccessPaths access_paths; + access_paths.emplace_back(); + access_paths[0].data_access_path.path = {"self"}; + + iterator.force_set_reading_flag(ColumnIterator::ReadingFlag::NORMAL_READING); + auto sub_paths = TEST_TRY(iterator.process_sub_access_paths(access_paths, false)); + EXPECT_TRUE(sub_paths.empty()); + EXPECT_EQ(iterator._reading_flag, ColumnIterator::ReadingFlag::NEED_TO_READ); + + iterator.force_set_reading_flag(ColumnIterator::ReadingFlag::NORMAL_READING); + sub_paths = TEST_TRY(iterator.process_sub_access_paths(access_paths, true)); + EXPECT_TRUE(sub_paths.empty()); + EXPECT_EQ(iterator._reading_flag, ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); +} + +TEST_F(ColumnReaderTest, NestedIteratorsPropagateReadPhase) { + auto struct_null_iterator = + std::make_unique(std::make_shared()); + std::vector struct_sub_iters; + struct_sub_iters.emplace_back( + std::make_unique(std::make_shared())); + struct_sub_iters.emplace_back( + std::make_unique(std::make_shared())); + auto struct_iterator = std::make_unique( + std::make_shared(), std::move(struct_null_iterator), + std::move(struct_sub_iters)); + + struct_iterator->activate_read_phase(ColumnIterator::ReadPhase::DEFERRED); + EXPECT_EQ(struct_iterator->_sub_column_iterators[0]->_read_phase, + ColumnIterator::ReadPhase::DEFERRED); + EXPECT_EQ(struct_iterator->_sub_column_iterators[1]->_read_phase, + ColumnIterator::ReadPhase::DEFERRED); + + auto array_item_iterator = + std::make_unique(std::make_shared()); + auto array_offsets_iter = std::make_unique( + std::make_unique(std::make_shared())); + auto array_null_iter = std::make_unique(std::make_shared()); + ArrayFileColumnIterator array_iterator( + std::make_shared(), std::move(array_offsets_iter), + std::move(array_item_iterator), std::move(array_null_iter)); + array_iterator.activate_read_phase(ColumnIterator::ReadPhase::PREDICATE); + EXPECT_EQ(array_iterator._item_iterator->_read_phase, ColumnIterator::ReadPhase::PREDICATE); + + auto map_null_iter = std::make_unique(std::make_shared()); + auto map_offsets_iter = std::make_unique( + std::make_unique(std::make_shared())); + auto map_key_iter = std::make_unique(std::make_shared()); + auto map_val_iter = std::make_unique(std::make_shared()); + MapFileColumnIterator map_iterator(std::make_shared(), std::move(map_null_iter), + std::move(map_offsets_iter), std::move(map_key_iter), + std::move(map_val_iter)); + map_iterator.activate_read_phase(ColumnIterator::ReadPhase::DEFERRED); + EXPECT_EQ(map_iterator._key_iterator->_read_phase, ColumnIterator::ReadPhase::DEFERRED); + EXPECT_EQ(map_iterator._val_iterator->_read_phase, ColumnIterator::ReadPhase::DEFERRED); +} + +TEST_F(ColumnReaderTest, AccessPathsPropagatePredicateToChildren) { + auto struct_null_iterator = + std::make_unique(std::make_shared()); + std::vector struct_sub_iters; + struct_sub_iters.emplace_back( + std::make_unique(std::make_shared())); + struct_sub_iters.emplace_back( + std::make_unique(std::make_shared())); + auto struct_iterator = std::make_unique( + std::make_shared(), std::move(struct_null_iterator), + std::move(struct_sub_iters)); + struct_iterator->set_column_name("s"); + + TColumnAccessPaths all_access_paths; + all_access_paths.emplace_back(); + all_access_paths[0].data_access_path.path = {"s"}; + TColumnAccessPaths predicate_access_paths = all_access_paths; + + auto st = struct_iterator->set_access_paths(all_access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << "failed to set struct access paths: " << st.to_string(); + EXPECT_TRUE(struct_iterator->is_pruned()); + EXPECT_EQ(struct_iterator->_reading_flag, ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); + EXPECT_EQ(struct_iterator->_sub_column_iterators[0]->_reading_flag, + ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); + EXPECT_EQ(struct_iterator->_sub_column_iterators[1]->_reading_flag, + ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); + + auto array_item_iterator = + std::make_unique(std::make_shared()); + auto array_offsets_iter = std::make_unique( + std::make_unique(std::make_shared())); + auto array_null_iter = std::make_unique(std::make_shared()); + ArrayFileColumnIterator array_iterator( + std::make_shared(), std::move(array_offsets_iter), + std::move(array_item_iterator), std::move(array_null_iter)); + array_iterator.set_column_name("a"); + TColumnAccessPaths array_access_paths; + array_access_paths.emplace_back(); + array_access_paths[0].data_access_path.path = {"a"}; + TColumnAccessPaths array_predicate_paths = array_access_paths; + st = array_iterator.set_access_paths(array_access_paths, array_predicate_paths); + ASSERT_TRUE(st.ok()) << "failed to set array access paths: " << st.to_string(); + EXPECT_TRUE(array_iterator.is_pruned()); + EXPECT_EQ(array_iterator._reading_flag, ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); + EXPECT_EQ(array_iterator._item_iterator->_reading_flag, + ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); + + auto map_null_iter = std::make_unique(std::make_shared()); + auto map_offsets_iter = std::make_unique( + std::make_unique(std::make_shared())); + auto map_key_iter = std::make_unique(std::make_shared()); + auto map_val_iter = std::make_unique(std::make_shared()); + MapFileColumnIterator map_iterator(std::make_shared(), std::move(map_null_iter), + std::move(map_offsets_iter), std::move(map_key_iter), + std::move(map_val_iter)); + map_iterator.set_column_name("m"); + TColumnAccessPaths map_access_paths; + map_access_paths.emplace_back(); + map_access_paths[0].data_access_path.path = {"m"}; + TColumnAccessPaths map_predicate_paths = map_access_paths; + st = map_iterator.set_access_paths(map_access_paths, map_predicate_paths); + ASSERT_TRUE(st.ok()) << "failed to set map access paths: " << st.to_string(); + EXPECT_TRUE(map_iterator.is_pruned()); + EXPECT_EQ(map_iterator._reading_flag, ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); + EXPECT_EQ(map_iterator._key_iterator->_reading_flag, + ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); + EXPECT_EQ(map_iterator._val_iterator->_reading_flag, + ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); +} + +TEST_F(ColumnReaderTest, NestedStructArrayMapStructAccessPaths) { + auto make_value_struct = []() { + auto null_iter = std::make_unique(std::make_shared()); + std::vector sub_iters; + auto sub_a = std::make_unique(std::make_shared()); + sub_a->set_column_name("a"); + auto sub_b = std::make_unique(std::make_shared()); + sub_b->set_column_name("b"); + sub_iters.emplace_back(std::move(sub_a)); + sub_iters.emplace_back(std::move(sub_b)); + + auto value_struct = std::make_unique( + std::make_shared(), std::move(null_iter), std::move(sub_iters)); + value_struct->set_column_name("value"); + return value_struct; + }; + + auto map_null_iter = std::make_unique(std::make_shared()); + auto map_offsets_iter = std::make_unique( + std::make_unique(std::make_shared())); + auto map_key_iter = std::make_unique(std::make_shared()); + map_key_iter->set_column_name("key"); + auto map_val_iter = make_value_struct(); + auto map_iterator = std::make_unique( + std::make_shared(), std::move(map_null_iter), std::move(map_offsets_iter), + std::move(map_key_iter), std::move(map_val_iter)); + map_iterator->set_column_name("item"); + + auto array_null_iter = std::make_unique(std::make_shared()); + auto array_offsets_iter = std::make_unique( + std::make_unique(std::make_shared())); + auto array_iterator = std::make_unique( + std::make_shared(), std::move(array_offsets_iter), + std::move(map_iterator), std::move(array_null_iter)); + array_iterator->set_column_name("col2"); + + auto struct_null_iter = std::make_unique(std::make_shared()); + std::vector struct_sub_iters; + auto sub_col1 = std::make_unique(std::make_shared()); + sub_col1->set_column_name("col1"); + struct_sub_iters.emplace_back(std::move(sub_col1)); + struct_sub_iters.emplace_back(std::move(array_iterator)); + auto top_struct = std::make_unique(std::make_shared(), + std::move(struct_null_iter), + std::move(struct_sub_iters)); + top_struct->set_column_name("root"); + + TColumnAccessPaths access_paths; + access_paths.emplace_back(); + access_paths[0].data_access_path.path = {"root", "col2", "*", "VALUES", "a"}; + TColumnAccessPaths predicate_access_paths = access_paths; + + auto st = top_struct->set_access_paths(access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << "failed to set nested access paths: " << st.to_string(); + + EXPECT_TRUE(top_struct->is_pruned()); + EXPECT_EQ(top_struct->_reading_flag, ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); + EXPECT_EQ(top_struct->_sub_column_iterators[0]->_reading_flag, + ColumnIterator::ReadingFlag::SKIP_READING); + EXPECT_EQ(top_struct->_sub_column_iterators[1]->_reading_flag, + ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); + + auto* array_iter = + static_cast(top_struct->_sub_column_iterators[1].get()); + EXPECT_TRUE(array_iter->is_pruned()); + auto* map_iter = static_cast(array_iter->_item_iterator.get()); + EXPECT_TRUE(map_iter->is_pruned()); + EXPECT_EQ(map_iter->_key_iterator->_reading_flag, ColumnIterator::ReadingFlag::SKIP_READING); + EXPECT_EQ(map_iter->_val_iterator->_reading_flag, + ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); + + auto* value_struct = static_cast(map_iter->_val_iterator.get()); + EXPECT_TRUE(value_struct->is_pruned()); + EXPECT_EQ(value_struct->_sub_column_iterators[0]->_reading_flag, + ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); + EXPECT_EQ(value_struct->_sub_column_iterators[1]->_reading_flag, + ColumnIterator::ReadingFlag::SKIP_READING); +} + +TEST_F(ColumnReaderTest, NestedStructArrayMapStructAccessPathsVariants) { + auto build_nested_iterator = []() { + auto make_value_struct = []() { + auto null_iter = std::make_unique(std::make_shared()); + std::vector sub_iters; + auto sub_a = std::make_unique(std::make_shared()); + sub_a->set_column_name("a"); + auto sub_b = std::make_unique(std::make_shared()); + sub_b->set_column_name("b"); + sub_iters.emplace_back(std::move(sub_a)); + sub_iters.emplace_back(std::move(sub_b)); + + auto value_struct = std::make_unique( + std::make_shared(), std::move(null_iter), std::move(sub_iters)); + value_struct->set_column_name("value"); + return value_struct; + }; + + auto map_null_iter = std::make_unique(std::make_shared()); + auto map_offsets_iter = std::make_unique( + std::make_unique(std::make_shared())); + auto map_key_iter = std::make_unique(std::make_shared()); + map_key_iter->set_column_name("key"); + auto map_val_iter = make_value_struct(); + auto map_iterator = std::make_unique( + std::make_shared(), std::move(map_null_iter), + std::move(map_offsets_iter), std::move(map_key_iter), std::move(map_val_iter)); + map_iterator->set_column_name("item"); + + auto array_null_iter = + std::make_unique(std::make_shared()); + auto array_offsets_iter = std::make_unique( + std::make_unique(std::make_shared())); + auto array_iterator = std::make_unique( + std::make_shared(), std::move(array_offsets_iter), + std::move(map_iterator), std::move(array_null_iter)); + array_iterator->set_column_name("col2"); + + auto struct_null_iter = + std::make_unique(std::make_shared()); + std::vector struct_sub_iters; + auto sub_col1 = std::make_unique(std::make_shared()); + sub_col1->set_column_name("col1"); + struct_sub_iters.emplace_back(std::move(sub_col1)); + struct_sub_iters.emplace_back(std::move(array_iterator)); + auto top_struct = std::make_unique( + std::make_shared(), std::move(struct_null_iter), + std::move(struct_sub_iters)); + top_struct->set_column_name("root"); + return top_struct; + }; + + { + auto top_struct = build_nested_iterator(); + TColumnAccessPaths all_access_paths; + all_access_paths.emplace_back(); + all_access_paths[0].data_access_path.path = {"root", "col1"}; + TColumnAccessPaths predicate_access_paths; + + auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string(); + + EXPECT_TRUE(top_struct->is_pruned()); + EXPECT_EQ(top_struct->_reading_flag, ColumnIterator::ReadingFlag::NEED_TO_READ); + EXPECT_EQ(top_struct->_sub_column_iterators[0]->_reading_flag, + ColumnIterator::ReadingFlag::NEED_TO_READ); + EXPECT_EQ(top_struct->_sub_column_iterators[1]->_reading_flag, + ColumnIterator::ReadingFlag::SKIP_READING); + } + + { + auto top_struct = build_nested_iterator(); + TColumnAccessPaths all_access_paths; + all_access_paths.emplace_back(); + all_access_paths[0].data_access_path.path = {"root", "col2", "*", "KEYS"}; + TColumnAccessPaths predicate_access_paths; + predicate_access_paths.emplace_back(); + predicate_access_paths[0].data_access_path.path = {"root", "col2", "*", "VALUES", "b"}; + + auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); + EXPECT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string(); + } + + { + auto top_struct = build_nested_iterator(); + TColumnAccessPaths all_access_paths; + all_access_paths.emplace_back(); + all_access_paths[0].data_access_path.path = {"root", "col2"}; + TColumnAccessPaths predicate_access_paths = all_access_paths; + + auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string(); + + EXPECT_EQ(top_struct->_reading_flag, ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); + EXPECT_EQ(top_struct->_sub_column_iterators[1]->_reading_flag, + ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); + + auto* array_iter = + static_cast(top_struct->_sub_column_iterators[1].get()); + auto* map_iter = static_cast(array_iter->_item_iterator.get()); + EXPECT_EQ(map_iter->_key_iterator->_reading_flag, + ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); + EXPECT_EQ(map_iter->_val_iterator->_reading_flag, + ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); + + auto* value_struct = static_cast(map_iter->_val_iterator.get()); + EXPECT_EQ(value_struct->_sub_column_iterators[0]->_reading_flag, + ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); + EXPECT_EQ(value_struct->_sub_column_iterators[1]->_reading_flag, + ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); + } + + { + auto top_struct = build_nested_iterator(); + TColumnAccessPaths all_access_paths; + all_access_paths.emplace_back(); + all_access_paths[0].data_access_path.path = {"root", "col2"}; + TColumnAccessPaths predicate_access_paths; + predicate_access_paths.emplace_back(); + predicate_access_paths[0].data_access_path.path = {"root", "col2", "*", "VALUES", "a"}; + + auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string(); + + auto* array_iter = + static_cast(top_struct->_sub_column_iterators[1].get()); + auto* map_iter = static_cast(array_iter->_item_iterator.get()); + EXPECT_EQ(map_iter->_key_iterator->_reading_flag, + ColumnIterator::ReadingFlag::NEED_TO_READ); + + auto* value_struct = static_cast(map_iter->_val_iterator.get()); + EXPECT_EQ(value_struct->_sub_column_iterators[0]->_reading_flag, + ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); + EXPECT_EQ(value_struct->_sub_column_iterators[1]->_reading_flag, + ColumnIterator::ReadingFlag::NEED_TO_READ); + } + + { + auto top_struct = build_nested_iterator(); + TColumnAccessPaths all_access_paths; + TColumnAccessPaths predicate_access_paths; + predicate_access_paths.emplace_back(); + predicate_access_paths[0].data_access_path.path = {"root", "col2", "*", "VALUES", "a"}; + + auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); + EXPECT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string(); + } + { + auto top_struct = build_nested_iterator(); + TColumnAccessPaths all_access_paths; + all_access_paths.emplace_back(); + all_access_paths[0].data_access_path.path = {"root", "col2", "*", "KEYS"}; + TColumnAccessPaths predicate_access_paths; + + auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string(); + + auto* array_iter = + static_cast(top_struct->_sub_column_iterators[1].get()); + auto* map_iter = static_cast(array_iter->_item_iterator.get()); + EXPECT_EQ(map_iter->_key_iterator->_reading_flag, + ColumnIterator::ReadingFlag::NEED_TO_READ); + EXPECT_EQ(map_iter->_val_iterator->_reading_flag, + ColumnIterator::ReadingFlag::SKIP_READING); + } + + { + auto top_struct = build_nested_iterator(); + TColumnAccessPaths all_access_paths; + all_access_paths.emplace_back(); + all_access_paths[0].data_access_path.path = {"root", "col2", "*", "VALUES"}; + TColumnAccessPaths predicate_access_paths; + + auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string(); + + auto* array_iter = + static_cast(top_struct->_sub_column_iterators[1].get()); + auto* map_iter = static_cast(array_iter->_item_iterator.get()); + EXPECT_EQ(map_iter->_key_iterator->_reading_flag, + ColumnIterator::ReadingFlag::SKIP_READING); + EXPECT_EQ(map_iter->_val_iterator->_reading_flag, + ColumnIterator::ReadingFlag::NEED_TO_READ); + + auto* value_struct = static_cast(map_iter->_val_iterator.get()); + EXPECT_EQ(value_struct->_sub_column_iterators[0]->_reading_flag, + ColumnIterator::ReadingFlag::NEED_TO_READ); + EXPECT_EQ(value_struct->_sub_column_iterators[1]->_reading_flag, + ColumnIterator::ReadingFlag::NEED_TO_READ); + } + + { + auto top_struct = build_nested_iterator(); + TColumnAccessPaths all_access_paths; + all_access_paths.emplace_back(); + all_access_paths[0].data_access_path.path = {"root", "col2", "*"}; + TColumnAccessPaths predicate_access_paths; + + auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string(); + + auto* array_iter = + static_cast(top_struct->_sub_column_iterators[1].get()); + auto* map_iter = static_cast(array_iter->_item_iterator.get()); + EXPECT_EQ(map_iter->_key_iterator->_reading_flag, + ColumnIterator::ReadingFlag::NEED_TO_READ); + EXPECT_EQ(map_iter->_val_iterator->_reading_flag, + ColumnIterator::ReadingFlag::NEED_TO_READ); + + auto* value_struct = static_cast(map_iter->_val_iterator.get()); + EXPECT_EQ(value_struct->_sub_column_iterators[0]->_reading_flag, + ColumnIterator::ReadingFlag::NEED_TO_READ); + EXPECT_EQ(value_struct->_sub_column_iterators[1]->_reading_flag, + ColumnIterator::ReadingFlag::NEED_TO_READ); + } + + { + auto top_struct = build_nested_iterator(); + TColumnAccessPaths all_access_paths; + all_access_paths.emplace_back(); + all_access_paths[0].data_access_path.path = {"root", "col2", "*", "VALUES", "a"}; + TColumnAccessPaths predicate_access_paths = all_access_paths; + + auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string(); + + EXPECT_EQ(top_struct->_reading_flag, ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); + EXPECT_EQ(top_struct->_sub_column_iterators[1]->_reading_flag, + ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); + + auto* array_iter = + static_cast(top_struct->_sub_column_iterators[1].get()); + auto* map_iter = static_cast(array_iter->_item_iterator.get()); + EXPECT_EQ(map_iter->_key_iterator->_reading_flag, + ColumnIterator::ReadingFlag::SKIP_READING); + EXPECT_EQ(map_iter->_val_iterator->_reading_flag, + ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); + + auto* value_struct = static_cast(map_iter->_val_iterator.get()); + EXPECT_EQ(value_struct->_sub_column_iterators[0]->_reading_flag, + ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); + EXPECT_EQ(value_struct->_sub_column_iterators[1]->_reading_flag, + ColumnIterator::ReadingFlag::SKIP_READING); + } + + { + auto top_struct = build_nested_iterator(); + TColumnAccessPaths all_access_paths; + all_access_paths.emplace_back(); + all_access_paths[0].data_access_path.path = {"root", "col2", "*"}; + TColumnAccessPaths predicate_access_paths; + predicate_access_paths.emplace_back(); + predicate_access_paths[0].data_access_path.path = {"root", "col2", "*", "VALUES"}; + + auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string(); + + auto* array_iter = + static_cast(top_struct->_sub_column_iterators[1].get()); + auto* map_iter = static_cast(array_iter->_item_iterator.get()); + EXPECT_EQ(map_iter->_key_iterator->_reading_flag, + ColumnIterator::ReadingFlag::NEED_TO_READ); + EXPECT_EQ(map_iter->_val_iterator->_reading_flag, + ColumnIterator::ReadingFlag::NEED_TO_READ); + } + + { + auto top_struct = build_nested_iterator(); + TColumnAccessPaths all_access_paths; + all_access_paths.emplace_back(); + all_access_paths[0].data_access_path.path = {}; + TColumnAccessPaths predicate_access_paths; + + auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); + EXPECT_FALSE(st.ok()); + } + + { + auto top_struct = build_nested_iterator(); + TColumnAccessPaths all_access_paths; + all_access_paths.emplace_back(); + all_access_paths[0].data_access_path.path = {"wrong_root", "col2"}; + TColumnAccessPaths predicate_access_paths; + + auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); + EXPECT_FALSE(st.ok()); + } + + { + auto top_struct = build_nested_iterator(); + TColumnAccessPaths all_access_paths; + all_access_paths.emplace_back(); + all_access_paths[0].data_access_path.path = {"root", "col2", "wrong_item"}; + TColumnAccessPaths predicate_access_paths; + + auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); + EXPECT_FALSE(st.ok()); + } +} + +TEST_F(ColumnReaderTest, DeepNestedAccessPathsFiveLevels) { + auto make_item_struct = []() { + auto null_iter = std::make_unique(std::make_shared()); + std::vector sub_iters; + auto sub_p = std::make_unique(std::make_shared()); + sub_p->set_column_name("p"); + auto sub_q = std::make_unique(std::make_shared()); + sub_q->set_column_name("q"); + sub_iters.emplace_back(std::move(sub_p)); + sub_iters.emplace_back(std::move(sub_q)); + + auto item_struct = std::make_unique( + std::make_shared(), std::move(null_iter), std::move(sub_iters)); + item_struct->set_column_name("item"); + return item_struct; + }; + + auto make_value_struct = [make_item_struct]() { + auto null_iter = std::make_unique(std::make_shared()); + std::vector sub_iters; + auto array_offsets = std::make_unique( + std::make_unique(std::make_shared())); + auto array_null = std::make_unique(std::make_shared()); + auto array_iter = std::make_unique( + std::make_shared(), std::move(array_offsets), make_item_struct(), + std::move(array_null)); + array_iter->set_column_name("arr"); + sub_iters.emplace_back(std::move(array_iter)); + + auto sub_z = std::make_unique(std::make_shared()); + sub_z->set_column_name("z"); + sub_iters.emplace_back(std::move(sub_z)); + + auto value_struct = std::make_unique( + std::make_shared(), std::move(null_iter), std::move(sub_iters)); + value_struct->set_column_name("value"); + return value_struct; + }; + + auto map_null_iter = std::make_unique(std::make_shared()); + auto map_offsets_iter = std::make_unique( + std::make_unique(std::make_shared())); + auto map_key_iter = std::make_unique(std::make_shared()); + map_key_iter->set_column_name("key"); + auto map_val_iter = make_value_struct(); + auto map_iter = std::make_unique( + std::make_shared(), std::move(map_null_iter), std::move(map_offsets_iter), + std::move(map_key_iter), std::move(map_val_iter)); + map_iter->set_column_name("m"); + + auto struct_null_iter = std::make_unique(std::make_shared()); + std::vector struct_sub_iters; + auto sub_x = std::make_unique(std::make_shared()); + sub_x->set_column_name("x"); + struct_sub_iters.emplace_back(std::move(sub_x)); + struct_sub_iters.emplace_back(std::move(map_iter)); + auto top_struct = std::make_unique(std::make_shared(), + std::move(struct_null_iter), + std::move(struct_sub_iters)); + top_struct->set_column_name("root"); + + TColumnAccessPaths all_access_paths; + all_access_paths.emplace_back(); + all_access_paths[0].data_access_path.path = {"root", "m", "VALUES", "arr", "*"}; + TColumnAccessPaths predicate_access_paths; + predicate_access_paths.emplace_back(); + predicate_access_paths[0].data_access_path.path = {"root", "m", "VALUES", "arr", "*", "q"}; + + auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << "failed to set deep access paths: " << st.to_string(); + + auto* map_ptr = static_cast(top_struct->_sub_column_iterators[1].get()); + EXPECT_EQ(map_ptr->_key_iterator->_reading_flag, ColumnIterator::ReadingFlag::SKIP_READING); + EXPECT_EQ(map_ptr->_val_iterator->_reading_flag, + ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); + + auto* value_struct = static_cast(map_ptr->_val_iterator.get()); + auto* array_iter = + static_cast(value_struct->_sub_column_iterators[0].get()); + auto* item_struct = static_cast(array_iter->_item_iterator.get()); + EXPECT_EQ(item_struct->_sub_column_iterators[0]->_reading_flag, + ColumnIterator::ReadingFlag::NEED_TO_READ); + EXPECT_EQ(item_struct->_sub_column_iterators[1]->_reading_flag, + ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); +} + +TEST_F(ColumnReaderTest, NestedNeedToReadInDeferredPredicatePhase) { + auto struct_null_iterator = + std::make_unique(std::make_shared()); + std::vector struct_sub_iters; + struct_sub_iters.emplace_back( + std::make_unique(std::make_shared())); + struct_sub_iters.emplace_back( + std::make_unique(std::make_shared())); + StructFileColumnIterator struct_iterator(std::make_shared(), + std::move(struct_null_iterator), + std::move(struct_sub_iters)); + struct_iterator.activate_read_phase(ColumnIterator::ReadPhase::DEFERRED); + struct_iterator.set_reading_flag(ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); + EXPECT_FALSE(struct_iterator.has_deferred_read_target()); + EXPECT_FALSE(struct_iterator.need_to_read()); + struct_iterator._sub_column_iterators[0]->set_need_to_read(); + EXPECT_TRUE(struct_iterator.has_deferred_read_target()); + EXPECT_TRUE(struct_iterator.need_to_read()); + + auto array_item_iterator = + std::make_unique(std::make_shared()); + auto array_offsets_iter = std::make_unique( + std::make_unique(std::make_shared())); + auto array_null_iter = std::make_unique(std::make_shared()); + ArrayFileColumnIterator array_iterator( + std::make_shared(), std::move(array_offsets_iter), + std::move(array_item_iterator), std::move(array_null_iter)); + array_iterator.activate_read_phase(ColumnIterator::ReadPhase::DEFERRED); + array_iterator.set_reading_flag(ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); + EXPECT_FALSE(array_iterator.has_deferred_read_target()); + EXPECT_FALSE(array_iterator.need_to_read()); + array_iterator._item_iterator->set_need_to_read(); + EXPECT_TRUE(array_iterator.has_deferred_read_target()); + EXPECT_TRUE(array_iterator.need_to_read()); + + auto map_null_iter = std::make_unique(std::make_shared()); + auto map_offsets_iter = std::make_unique( + std::make_unique(std::make_shared())); + auto map_key_iter = std::make_unique(std::make_shared()); + auto map_val_iter = std::make_unique(std::make_shared()); + MapFileColumnIterator map_iterator(std::make_shared(), std::move(map_null_iter), + std::move(map_offsets_iter), std::move(map_key_iter), + std::move(map_val_iter)); + map_iterator.activate_read_phase(ColumnIterator::ReadPhase::DEFERRED); + map_iterator.set_reading_flag(ColumnIterator::ReadingFlag::READING_FOR_PREDICATE); + EXPECT_FALSE(map_iterator.has_deferred_read_target()); + EXPECT_FALSE(map_iterator.need_to_read()); + map_iterator._val_iterator->set_need_to_read(); + EXPECT_TRUE(map_iterator.has_deferred_read_target()); + EXPECT_TRUE(map_iterator.need_to_read()); +} + +TEST_F(ColumnReaderTest, NestedReadPhaseNeedToReadMatrix) { + auto build_nested_iterator = []() { + auto make_value_struct = []() { + auto null_iter = std::make_unique(std::make_shared()); + std::vector sub_iters; + auto sub_a = std::make_unique(std::make_shared()); + sub_a->set_column_name("a"); + auto sub_b = std::make_unique(std::make_shared()); + sub_b->set_column_name("b"); + sub_iters.emplace_back(std::move(sub_a)); + sub_iters.emplace_back(std::move(sub_b)); + + auto value_struct = std::make_unique( + std::make_shared(), std::move(null_iter), std::move(sub_iters)); + value_struct->set_column_name("value"); + return value_struct; + }; + + auto map_null_iter = std::make_unique(std::make_shared()); + auto map_offsets_iter = std::make_unique( + std::make_unique(std::make_shared())); + auto map_key_iter = std::make_unique(std::make_shared()); + map_key_iter->set_column_name("key"); + auto map_val_iter = make_value_struct(); + auto map_iterator = std::make_unique( + std::make_shared(), std::move(map_null_iter), + std::move(map_offsets_iter), std::move(map_key_iter), std::move(map_val_iter)); + map_iterator->set_column_name("item"); + + auto array_null_iter = + std::make_unique(std::make_shared()); + auto array_offsets_iter = std::make_unique( + std::make_unique(std::make_shared())); + auto array_iterator = std::make_unique( + std::make_shared(), std::move(array_offsets_iter), + std::move(map_iterator), std::move(array_null_iter)); + array_iterator->set_column_name("col2"); + + auto struct_null_iter = + std::make_unique(std::make_shared()); + std::vector struct_sub_iters; + auto sub_col1 = std::make_unique(std::make_shared()); + sub_col1->set_column_name("col1"); + struct_sub_iters.emplace_back(std::move(sub_col1)); + struct_sub_iters.emplace_back(std::move(array_iterator)); + auto top_struct = std::make_unique( + std::make_shared(), std::move(struct_null_iter), + std::move(struct_sub_iters)); + top_struct->set_column_name("root"); + return top_struct; + }; + + auto assert_need_to_read = [](StructFileColumnIterator* top_struct) { + auto* array_iter = + static_cast(top_struct->_sub_column_iterators[1].get()); + auto* map_iter = static_cast(array_iter->_item_iterator.get()); + auto* value_struct = static_cast(map_iter->_val_iterator.get()); + auto expect_scalar = [](ColumnIterator::ReadingFlag flag, ColumnIterator::ReadPhase mode) { + switch (mode) { + case ColumnIterator::ReadPhase::FULL: + return flag != ColumnIterator::ReadingFlag::SKIP_READING; + case ColumnIterator::ReadPhase::PREDICATE: + return flag == ColumnIterator::ReadingFlag::READING_FOR_PREDICATE; + case ColumnIterator::ReadPhase::DEFERRED: + return flag == ColumnIterator::ReadingFlag::NEED_TO_READ; + default: + return false; + } + }; + auto expect_nested = [](ColumnIterator::ReadingFlag flag, ColumnIterator::ReadPhase mode) { + switch (mode) { + case ColumnIterator::ReadPhase::FULL: + return flag != ColumnIterator::ReadingFlag::SKIP_READING; + case ColumnIterator::ReadPhase::PREDICATE: + return flag == ColumnIterator::ReadingFlag::READING_FOR_PREDICATE; + default: + return false; + } + }; + + top_struct->activate_read_phase(ColumnIterator::ReadPhase::FULL); + EXPECT_EQ(expect_nested(top_struct->reading_flag(), ColumnIterator::ReadPhase::FULL), + top_struct->need_to_read()); + EXPECT_EQ(expect_nested(array_iter->reading_flag(), ColumnIterator::ReadPhase::FULL), + array_iter->need_to_read()); + EXPECT_EQ(expect_nested(map_iter->reading_flag(), ColumnIterator::ReadPhase::FULL), + map_iter->need_to_read()); + EXPECT_EQ(expect_nested(value_struct->reading_flag(), ColumnIterator::ReadPhase::FULL), + value_struct->need_to_read()); + EXPECT_EQ(expect_scalar(map_iter->_key_iterator->reading_flag(), + ColumnIterator::ReadPhase::FULL), + map_iter->_key_iterator->need_to_read()); + EXPECT_EQ(expect_nested(map_iter->_val_iterator->reading_flag(), + ColumnIterator::ReadPhase::FULL), + map_iter->_val_iterator->need_to_read()); + + top_struct->activate_read_phase(ColumnIterator::ReadPhase::PREDICATE); + EXPECT_EQ(expect_nested(top_struct->reading_flag(), ColumnIterator::ReadPhase::PREDICATE), + top_struct->need_to_read()); + EXPECT_EQ(expect_nested(array_iter->reading_flag(), ColumnIterator::ReadPhase::PREDICATE), + array_iter->need_to_read()); + EXPECT_EQ(expect_nested(map_iter->reading_flag(), ColumnIterator::ReadPhase::PREDICATE), + map_iter->need_to_read()); + EXPECT_EQ(expect_nested(value_struct->reading_flag(), ColumnIterator::ReadPhase::PREDICATE), + value_struct->need_to_read()); + EXPECT_EQ(expect_scalar(map_iter->_key_iterator->reading_flag(), + ColumnIterator::ReadPhase::PREDICATE), + map_iter->_key_iterator->need_to_read()); + EXPECT_EQ(expect_nested(map_iter->_val_iterator->reading_flag(), + ColumnIterator::ReadPhase::PREDICATE), + map_iter->_val_iterator->need_to_read()); + + top_struct->activate_read_phase(ColumnIterator::ReadPhase::DEFERRED); + EXPECT_EQ(top_struct->has_deferred_read_target(), top_struct->need_to_read()); + EXPECT_EQ(array_iter->has_deferred_read_target(), array_iter->need_to_read()); + EXPECT_EQ(map_iter->has_deferred_read_target(), map_iter->need_to_read()); + EXPECT_EQ(value_struct->has_deferred_read_target(), value_struct->need_to_read()); + EXPECT_EQ(expect_scalar(map_iter->_key_iterator->reading_flag(), + ColumnIterator::ReadPhase::DEFERRED), + map_iter->_key_iterator->need_to_read()); + EXPECT_EQ(map_iter->_val_iterator->has_deferred_read_target(), + map_iter->_val_iterator->need_to_read()); + }; + + { + auto top_struct = build_nested_iterator(); + TColumnAccessPaths all_access_paths; + all_access_paths.emplace_back(); + all_access_paths[0].data_access_path.path = {"root", "col2", "*", "VALUES", "a"}; + TColumnAccessPaths predicate_access_paths = all_access_paths; + + auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string(); + assert_need_to_read(top_struct.get()); + } + + { + auto top_struct = build_nested_iterator(); + TColumnAccessPaths all_access_paths; + all_access_paths.emplace_back(); + all_access_paths[0].data_access_path.path = {"root", "col2", "*", "KEYS"}; + TColumnAccessPaths predicate_access_paths; + + auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string(); + assert_need_to_read(top_struct.get()); + } + + { + auto top_struct = build_nested_iterator(); + TColumnAccessPaths all_access_paths; + all_access_paths.emplace_back(); + all_access_paths[0].data_access_path.path = {"root", "col2", "*"}; + TColumnAccessPaths predicate_access_paths; + predicate_access_paths.emplace_back(); + predicate_access_paths[0].data_access_path.path = {"root", "col2", "*", "VALUES"}; + + auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string(); + assert_need_to_read(top_struct.get()); + } + + { + auto top_struct = build_nested_iterator(); + TColumnAccessPaths all_access_paths; + all_access_paths.emplace_back(); + all_access_paths[0].data_access_path.path = {"root", "col2", "*", "VALUES"}; + TColumnAccessPaths predicate_access_paths; + predicate_access_paths.emplace_back(); + predicate_access_paths[0].data_access_path.path = {"root", "col2", "*", "KEYS"}; + + auto st = top_struct->set_access_paths(all_access_paths, predicate_access_paths); + EXPECT_TRUE(st.ok()); + } +} + TEST_F(ColumnReaderTest, MultiAccessPaths) { auto create_struct_iterator = []() { auto null_reader = std::make_shared(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 54095c9e08a08a..52128c19ef7cf1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -5603,6 +5603,8 @@ public TQueryOptions toThrift() { tResult.setReadCsvEmptyLineAsNull(readCsvEmptyLineAsNull); tResult.setSerdeDialect(getSerdeDialect()); + tResult.setEnablePruneNestedColumn(enablePruneNestedColumns); + tResult.setEnableMatchWithoutInvertedIndex(enableMatchWithoutInvertedIndex); tResult.setEnableFallbackOnMissingInvertedIndex(enableFallbackOnMissingInvertedIndex); tResult.setEnableInvertedIndexSearcherCache(enableInvertedIndexSearcherCache); diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift index 211f38dccb5896..0360aea7902d1e 100644 --- a/gensrc/thrift/PaloInternalService.thrift +++ b/gensrc/thrift/PaloInternalService.thrift @@ -493,6 +493,9 @@ struct TQueryOptions { 219: optional bool enable_segment_limit_pushdown = true 220: optional bool enable_ann_index_result_cache = true + + 221: optional bool enable_prune_nested_column = false; + // For cloud, to control if the content would be written into file cache // In write path, to control if the content would be written into file cache. // In read path, read from file cache or remote storage when execute query. diff --git a/regression-test/data/datatype_p0/complex_types/test_pruned_columns.out b/regression-test/data/datatype_p0/complex_types/test_pruned_columns.out index b3312aa670c066..054c14dc57348d 100644 --- a/regression-test/data/datatype_p0/complex_types/test_pruned_columns.out +++ b/regression-test/data/datatype_p0/complex_types/test_pruned_columns.out @@ -1,86 +1,367 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !sql -- -1 {"city":"beijing", "data":[{1:{"a":10, "b":20}, 2:{"a":30, "b":40}}], "value":1} -2 {"city":"shanghai", "data":[{2:{"a":50, "b":40}, 1:{"a":70, "b":80}}], "value":2} -3 {"city":"guangzhou", "data":[{1:{"a":90, "b":60}, 2:{"a":110, "b":40}}], "value":3} -4 {"city":"shenzhen", "data":[{2:{"a":130, "b":20}, 1:{"a":150, "b":40}}], "value":4} -5 {"city":"hangzhou", "data":[{1:{"a":170, "b":80}, 2:{"a":190, "b":40}}], "value":5} -6 {"city":"nanjing", "data":[{2:{"a":210, "b":60}, 1:{"a":230, "b":40}}], "value":6} -7 {"city":"tianjin", "data":[{1:{"a":250, "b":20}, 2:{"a":270, "b":40}}], "value":7} -8 {"city":"chongqing", "data":[{2:{"a":290, "b":80}, 1:{"a":310, "b":40}}], "value":8} -9 {"city":"wuhan", "data":[{1:{"a":330, "b":60}, 2:{"a":350, "b":40}}], "value":9} -10 {"city":"xian", "data":[{2:{"a":370, "b":20}, 1:{"a":390, "b":40}}], "value":10} -11 {"city":"changsha", "data":[{1:{"a":410, "b":80}, 2:{"a":430, "b":40}}], "value":11} -12 {"city":"qingdao", "data":[{2:{"a":450, "b":60}, 1:{"a":470, "b":40}}], "value":12} -13 {"city":"dalian", "data":[{1:{"a":490, "b":20}, 2:{"a":510, "b":40}}], "value":13} +\N 300 +beijing 300 +chengdu 300 +guangzhou 300 +hangzhou 300 +nanjing 300 +shanghai 300 +shenzhen 300 +wuhan 300 +xian 300 -- !sql1 -- -1 [10] +1 [10, 5] + +-- !sql1_1 -- + +-- !sql1_2 -- -- !sql2 -- -1 beijing -2 shanghai +0 beijing +1 shanghai +2 shenzhen 3 guangzhou -4 shenzhen -5 hangzhou -6 nanjing -7 tianjin -8 chongqing -9 wuhan -10 xian -11 changsha -12 qingdao -13 dalian +4 hangzhou +5 chengdu +6 wuhan +7 xian +8 nanjing +9 \N +10 beijing +11 shanghai +12 shenzhen +13 guangzhou +14 hangzhou +15 chengdu +16 wuhan +17 xian +18 nanjing +19 \N + +-- !sql2_1 -- +100 beijing +101 shanghai +102 shenzhen +103 guangzhou +104 hangzhou +105 chengdu +106 wuhan +107 xian +108 nanjing +109 \N +110 beijing +111 shanghai +112 shenzhen +113 guangzhou +114 hangzhou +115 chengdu +116 wuhan +117 xian +118 nanjing +119 \N + +-- !sql2_2 -- +2999 \N +2998 nanjing +2997 xian +2996 wuhan +2995 chengdu +2994 hangzhou +2993 guangzhou +2992 shenzhen +2991 shanghai +2990 beijing +2989 \N +2988 nanjing +2987 xian +2986 wuhan +2985 chengdu +2984 hangzhou +2983 guangzhou +2982 shenzhen +2981 shanghai +2980 beijing -- !sql3 -- -1 [{1:{"a":10, "b":20}, 2:{"a":30, "b":40}}] -2 [{2:{"a":50, "b":40}, 1:{"a":70, "b":80}}] -3 [{1:{"a":90, "b":60}, 2:{"a":110, "b":40}}] -4 [{2:{"a":130, "b":20}, 1:{"a":150, "b":40}}] -5 [{1:{"a":170, "b":80}, 2:{"a":190, "b":40}}] -6 [{2:{"a":210, "b":60}, 1:{"a":230, "b":40}}] -7 [{1:{"a":250, "b":20}, 2:{"a":270, "b":40}}] -8 [{2:{"a":290, "b":80}, 1:{"a":310, "b":40}}] -9 [{1:{"a":330, "b":60}, 2:{"a":350, "b":40}}] -10 [{2:{"a":370, "b":20}, 1:{"a":390, "b":40}}] -11 [{1:{"a":410, "b":80}, 2:{"a":430, "b":40}}] -12 [{2:{"a":450, "b":60}, 1:{"a":470, "b":40}}] -13 [{1:{"a":490, "b":20}, 2:{"a":510, "b":40}}] +0 [{1:{"a":0, "b":0}, 2:{"a":20, "b":10}}, {1:{"a":0, "b":0}, 2:{"a":0, "b":0}}] +1 [{1:{"a":10, "b":11}, 2:{"a":30, "b":20}}, {2:{"a":5, "b":2.5}, 3:{"a":3, "b":1.5}}] +2 [{1:{"a":20, "b":22}, 2:{"a":40, "b":30}}, {3:{"a":10, "b":5}, 4:{"a":6, "b":3}}] +3 [{1:{"a":30, "b":33}, 2:{"a":50, "b":40}}, {1:{"a":15, "b":7.5}, 5:{"a":9, "b":4.5}}] +4 [{1:{"a":40, "b":44}, 2:{"a":60, "b":50}}, {2:{"a":20, "b":10}, 6:{"a":12, "b":6}}] +5 [{1:{"a":50, "b":50}, 2:{"a":70, "b":60}}, {3:{"a":25, "b":12.5}, 2:{"a":15, "b":7.5}}] +6 [{1:{"a":60, "b":61}, 2:{"a":80, "b":70}}, {1:{"a":30, "b":15}, 3:{"a":18, "b":9}}] +7 [{1:{"a":70, "b":72}, 2:{"a":90, "b":80}}, {2:{"a":35, "b":17.5}, 4:{"a":21, "b":10.5}}] +8 [{1:{"a":80, "b":83}, 2:{"a":100, "b":90}}, {3:{"a":40, "b":20}, 5:{"a":24, "b":12}}] +9 [{1:{"a":90, "b":94}, 2:{"a":110, "b":100}}, {1:{"a":45, "b":22.5}, 6:{"a":27, "b":13.5}}] +10 [{1:{"a":100, "b":100}, 2:{"a":120, "b":10}}, {2:{"a":30, "b":15}}] +11 [{1:{"a":110, "b":111}, 2:{"a":130, "b":20}}, {3:{"a":33, "b":16.5}}] +12 [{1:{"a":120, "b":122}, 2:{"a":140, "b":30}}, {1:{"a":60, "b":30}, 4:{"a":36, "b":18}}] +13 [{1:{"a":130, "b":133}, 2:{"a":150, "b":40}}, {2:{"a":65, "b":32.5}, 5:{"a":39, "b":19.5}}] +14 [{1:{"a":140, "b":144}, 2:{"a":160, "b":50}}, {3:{"a":70, "b":35}, 6:{"a":42, "b":21}}] +15 [{1:{"a":150, "b":150}, 2:{"a":170, "b":60}}, {1:{"a":75, "b":37.5}, 2:{"a":45, "b":22.5}}] +16 [{1:{"a":160, "b":161}, 2:{"a":180, "b":70}}, {2:{"a":80, "b":40}, 3:{"a":48, "b":24}}] +17 [{1:{"a":170, "b":172}, 2:{"a":190, "b":80}}, {3:{"a":85, "b":42.5}, 4:{"a":51, "b":25.5}}] +18 [{1:{"a":180, "b":183}, 2:{"a":200, "b":90}}, {1:{"a":90, "b":45}, 5:{"a":54, "b":27}}] +19 [{1:{"a":190, "b":194}, 2:{"a":210, "b":100}}, {2:{"a":95, "b":47.5}, 6:{"a":57, "b":28.5}}] + +-- !sql3_1 -- +200 [{1:{"a":2000, "b":2000}, 2:{"a":2020, "b":10}}, {3:{"a":1000, "b":500}, 2:{"a":600, "b":300}}] +201 [{1:{"a":2010, "b":2011}, 2:{"a":2030, "b":20}}, {1:{"a":1005, "b":502.5}, 3:{"a":603, "b":301.5}}] +202 [{1:{"a":2020, "b":2022}, 2:{"a":2040, "b":30}}, {2:{"a":1010, "b":505}, 4:{"a":606, "b":303}}] +203 [{1:{"a":2030, "b":2033}, 2:{"a":2050, "b":40}}, {3:{"a":1015, "b":507.5}, 5:{"a":609, "b":304.5}}] +204 [{1:{"a":2040, "b":2044}, 2:{"a":2060, "b":50}}, {1:{"a":1020, "b":510}, 6:{"a":612, "b":306}}] +205 [{1:{"a":2050, "b":2050}, 2:{"a":2070, "b":60}}, {2:{"a":615, "b":307.5}}] +206 [{1:{"a":2060, "b":2061}, 2:{"a":2080, "b":70}}, {3:{"a":618, "b":309}}] +207 [{1:{"a":2070, "b":2072}, 2:{"a":2090, "b":80}}, {1:{"a":1035, "b":517.5}, 4:{"a":621, "b":310.5}}] +208 [{1:{"a":2080, "b":2083}, 2:{"a":2100, "b":90}}, {2:{"a":1040, "b":520}, 5:{"a":624, "b":312}}] +209 [{1:{"a":2090, "b":2094}, 2:{"a":2110, "b":100}}, {3:{"a":1045, "b":522.5}, 6:{"a":627, "b":313.5}}] +210 [{1:{"a":2100, "b":2100}, 2:{"a":2120, "b":10}}, {1:{"a":1050, "b":525}, 2:{"a":630, "b":315}}] +211 [{1:{"a":2110, "b":2111}, 2:{"a":2130, "b":20}}, {2:{"a":1055, "b":527.5}, 3:{"a":633, "b":316.5}}] +212 [{1:{"a":2120, "b":2122}, 2:{"a":2140, "b":30}}, {3:{"a":1060, "b":530}, 4:{"a":636, "b":318}}] +213 [{1:{"a":2130, "b":2133}, 2:{"a":2150, "b":40}}, {1:{"a":1065, "b":532.5}, 5:{"a":639, "b":319.5}}] +214 [{1:{"a":2140, "b":2144}, 2:{"a":2160, "b":50}}, {2:{"a":1070, "b":535}, 6:{"a":642, "b":321}}] +215 [{1:{"a":2150, "b":2150}, 2:{"a":2170, "b":60}}, {3:{"a":1075, "b":537.5}, 2:{"a":645, "b":322.5}}] +216 [{1:{"a":2160, "b":2161}, 2:{"a":2180, "b":70}}, {1:{"a":1080, "b":540}, 3:{"a":648, "b":324}}] +217 [{1:{"a":2170, "b":2172}, 2:{"a":2190, "b":80}}, {2:{"a":1085, "b":542.5}, 4:{"a":651, "b":325.5}}] +218 [{1:{"a":2180, "b":2183}, 2:{"a":2200, "b":90}}, {3:{"a":1090, "b":545}, 5:{"a":654, "b":327}}] +219 [{1:{"a":2190, "b":2194}, 2:{"a":2210, "b":100}}, {1:{"a":1095, "b":547.5}, 6:{"a":657, "b":328.5}}] + +-- !sql3_2 -- +2999 [{1:{"a":29990, "b":29994}, 2:{"a":30010, "b":100}}, {3:{"a":14995, "b":7497.5}, 6:{"a":8997, "b":4498.5}}] +2998 [{1:{"a":29980, "b":29983}, 2:{"a":30000, "b":90}}, {2:{"a":14990, "b":7495}, 5:{"a":8994, "b":4497}}] +2997 [{1:{"a":29970, "b":29972}, 2:{"a":29990, "b":80}}, {1:{"a":14985, "b":7492.5}, 4:{"a":8991, "b":4495.5}}] +2996 [{1:{"a":29960, "b":29961}, 2:{"a":29980, "b":70}}, {3:{"a":8988, "b":4494}}] +2995 [{1:{"a":29950, "b":29950}, 2:{"a":29970, "b":60}}, {2:{"a":8985, "b":4492.5}}] +2994 [{1:{"a":29940, "b":29944}, 2:{"a":29960, "b":50}}, {1:{"a":14970, "b":7485}, 6:{"a":8982, "b":4491}}] +2993 [{1:{"a":29930, "b":29933}, 2:{"a":29950, "b":40}}, {3:{"a":14965, "b":7482.5}, 5:{"a":8979, "b":4489.5}}] +2992 [{1:{"a":29920, "b":29922}, 2:{"a":29940, "b":30}}, {2:{"a":14960, "b":7480}, 4:{"a":8976, "b":4488}}] +2991 [{1:{"a":29910, "b":29911}, 2:{"a":29930, "b":20}}, {1:{"a":14955, "b":7477.5}, 3:{"a":8973, "b":4486.5}}] +2990 [{1:{"a":29900, "b":29900}, 2:{"a":29920, "b":10}}, {3:{"a":14950, "b":7475}, 2:{"a":8970, "b":4485}}] +2989 [{1:{"a":29890, "b":29894}, 2:{"a":29910, "b":100}}, {2:{"a":14945, "b":7472.5}, 6:{"a":8967, "b":4483.5}}] +2988 [{1:{"a":29880, "b":29883}, 2:{"a":29900, "b":90}}, {1:{"a":14940, "b":7470}, 5:{"a":8964, "b":4482}}] +2987 [{1:{"a":29870, "b":29872}, 2:{"a":29890, "b":80}}, {3:{"a":14935, "b":7467.5}, 4:{"a":8961, "b":4480.5}}] +2986 [{1:{"a":29860, "b":29861}, 2:{"a":29880, "b":70}}, {2:{"a":14930, "b":7465}, 3:{"a":8958, "b":4479}}] +2985 [{1:{"a":29850, "b":29850}, 2:{"a":29870, "b":60}}, {1:{"a":14925, "b":7462.5}, 2:{"a":8955, "b":4477.5}}] +2984 [{1:{"a":29840, "b":29844}, 2:{"a":29860, "b":50}}, {3:{"a":14920, "b":7460}, 6:{"a":8952, "b":4476}}] +2983 [{1:{"a":29830, "b":29833}, 2:{"a":29850, "b":40}}, {2:{"a":14915, "b":7457.5}, 5:{"a":8949, "b":4474.5}}] +2982 [{1:{"a":29820, "b":29822}, 2:{"a":29840, "b":30}}, {1:{"a":14910, "b":7455}, 4:{"a":8946, "b":4473}}] +2981 [{1:{"a":29810, "b":29811}, 2:{"a":29830, "b":20}}, {3:{"a":8943, "b":4471.5}}] +2980 [{1:{"a":29800, "b":29800}, 2:{"a":29820, "b":10}}, {2:{"a":8940, "b":4470}}] -- !sql4 -- -1 [{1:{"a":10, "b":20}, 2:{"a":30, "b":40}}] -2 [{2:{"a":50, "b":40}, 1:{"a":70, "b":80}}] -3 [{1:{"a":90, "b":60}, 2:{"a":110, "b":40}}] -5 [{1:{"a":170, "b":80}, 2:{"a":190, "b":40}}] -7 [{1:{"a":250, "b":20}, 2:{"a":270, "b":40}}] -9 [{1:{"a":330, "b":60}, 2:{"a":350, "b":40}}] -11 [{1:{"a":410, "b":80}, 2:{"a":430, "b":40}}] -13 [{1:{"a":490, "b":20}, 2:{"a":510, "b":40}}] +3 [{1:{"a":30, "b":33}, 2:{"a":50, "b":40}}, {1:{"a":15, "b":7.5}, 5:{"a":9, "b":4.5}}] +13 [{1:{"a":130, "b":133}, 2:{"a":150, "b":40}}, {2:{"a":65, "b":32.5}, 5:{"a":39, "b":19.5}}] +23 [{1:{"a":230, "b":233}, 2:{"a":250, "b":40}}, {3:{"a":115, "b":57.5}, 5:{"a":69, "b":34.5}}] +33 [{1:{"a":330, "b":333}, 2:{"a":350, "b":40}}, {1:{"a":165, "b":82.5}, 5:{"a":99, "b":49.5}}] +43 [{1:{"a":430, "b":433}, 2:{"a":450, "b":40}}, {2:{"a":215, "b":107.5}, 5:{"a":129, "b":64.5}}] +53 [{1:{"a":530, "b":533}, 2:{"a":550, "b":40}}, {3:{"a":265, "b":132.5}, 5:{"a":159, "b":79.5}}] +63 [{1:{"a":630, "b":633}, 2:{"a":650, "b":40}}, {1:{"a":315, "b":157.5}, 5:{"a":189, "b":94.5}}] +73 [{1:{"a":730, "b":733}, 2:{"a":750, "b":40}}, {2:{"a":365, "b":182.5}, 5:{"a":219, "b":109.5}}] +83 [{1:{"a":830, "b":833}, 2:{"a":850, "b":40}}, {3:{"a":415, "b":207.5}, 5:{"a":249, "b":124.5}}] +93 [{1:{"a":930, "b":933}, 2:{"a":950, "b":40}}, {1:{"a":465, "b":232.5}, 5:{"a":279, "b":139.5}}] +103 [{1:{"a":1030, "b":1033}, 2:{"a":1050, "b":40}}, {2:{"a":515, "b":257.5}, 5:{"a":309, "b":154.5}}] +113 [{1:{"a":1130, "b":1133}, 2:{"a":1150, "b":40}}, {3:{"a":565, "b":282.5}, 5:{"a":339, "b":169.5}}] +123 [{1:{"a":1230, "b":1233}, 2:{"a":1250, "b":40}}, {1:{"a":615, "b":307.5}, 5:{"a":369, "b":184.5}}] +133 [{1:{"a":1330, "b":1333}, 2:{"a":1350, "b":40}}, {2:{"a":665, "b":332.5}, 5:{"a":399, "b":199.5}}] +143 [{1:{"a":1430, "b":1433}, 2:{"a":1450, "b":40}}, {3:{"a":715, "b":357.5}, 5:{"a":429, "b":214.5}}] +153 [{1:{"a":1530, "b":1533}, 2:{"a":1550, "b":40}}, {1:{"a":765, "b":382.5}, 5:{"a":459, "b":229.5}}] +163 [{1:{"a":1630, "b":1633}, 2:{"a":1650, "b":40}}, {2:{"a":815, "b":407.5}, 5:{"a":489, "b":244.5}}] +173 [{1:{"a":1730, "b":1733}, 2:{"a":1750, "b":40}}, {3:{"a":865, "b":432.5}, 5:{"a":519, "b":259.5}}] +183 [{1:{"a":1830, "b":1833}, 2:{"a":1850, "b":40}}, {1:{"a":915, "b":457.5}, 5:{"a":549, "b":274.5}}] +193 [{1:{"a":1930, "b":1933}, 2:{"a":1950, "b":40}}, {2:{"a":965, "b":482.5}, 5:{"a":579, "b":289.5}}] + +-- !sql4_1 -- +1003 [{1:{"a":10030, "b":10033}, 2:{"a":10050, "b":40}}, {2:{"a":5015, "b":2507.5}, 5:{"a":3009, "b":1504.5}}] +1013 [{1:{"a":10130, "b":10133}, 2:{"a":10150, "b":40}}, {3:{"a":5065, "b":2532.5}, 5:{"a":3039, "b":1519.5}}] +1023 [{1:{"a":10230, "b":10233}, 2:{"a":10250, "b":40}}, {1:{"a":5115, "b":2557.5}, 5:{"a":3069, "b":1534.5}}] +1033 [{1:{"a":10330, "b":10333}, 2:{"a":10350, "b":40}}, {2:{"a":5165, "b":2582.5}, 5:{"a":3099, "b":1549.5}}] +1043 [{1:{"a":10430, "b":10433}, 2:{"a":10450, "b":40}}, {3:{"a":5215, "b":2607.5}, 5:{"a":3129, "b":1564.5}}] +1053 [{1:{"a":10530, "b":10533}, 2:{"a":10550, "b":40}}, {1:{"a":5265, "b":2632.5}, 5:{"a":3159, "b":1579.5}}] +1063 [{1:{"a":10630, "b":10633}, 2:{"a":10650, "b":40}}, {2:{"a":5315, "b":2657.5}, 5:{"a":3189, "b":1594.5}}] +1073 [{1:{"a":10730, "b":10733}, 2:{"a":10750, "b":40}}, {3:{"a":5365, "b":2682.5}, 5:{"a":3219, "b":1609.5}}] +1083 [{1:{"a":10830, "b":10833}, 2:{"a":10850, "b":40}}, {1:{"a":5415, "b":2707.5}, 5:{"a":3249, "b":1624.5}}] +1093 [{1:{"a":10930, "b":10933}, 2:{"a":10950, "b":40}}, {2:{"a":5465, "b":2732.5}, 5:{"a":3279, "b":1639.5}}] +1103 [{1:{"a":11030, "b":11033}, 2:{"a":11050, "b":40}}, {3:{"a":5515, "b":2757.5}, 5:{"a":3309, "b":1654.5}}] +1113 [{1:{"a":11130, "b":11133}, 2:{"a":11150, "b":40}}, {1:{"a":5565, "b":2782.5}, 5:{"a":3339, "b":1669.5}}] +1123 [{1:{"a":11230, "b":11233}, 2:{"a":11250, "b":40}}, {2:{"a":5615, "b":2807.5}, 5:{"a":3369, "b":1684.5}}] +1133 [{1:{"a":11330, "b":11333}, 2:{"a":11350, "b":40}}, {3:{"a":5665, "b":2832.5}, 5:{"a":3399, "b":1699.5}}] +1143 [{1:{"a":11430, "b":11433}, 2:{"a":11450, "b":40}}, {1:{"a":5715, "b":2857.5}, 5:{"a":3429, "b":1714.5}}] +1153 [{1:{"a":11530, "b":11533}, 2:{"a":11550, "b":40}}, {2:{"a":5765, "b":2882.5}, 5:{"a":3459, "b":1729.5}}] +1163 [{1:{"a":11630, "b":11633}, 2:{"a":11650, "b":40}}, {3:{"a":5815, "b":2907.5}, 5:{"a":3489, "b":1744.5}}] +1173 [{1:{"a":11730, "b":11733}, 2:{"a":11750, "b":40}}, {1:{"a":5865, "b":2932.5}, 5:{"a":3519, "b":1759.5}}] +1183 [{1:{"a":11830, "b":11833}, 2:{"a":11850, "b":40}}, {2:{"a":5915, "b":2957.5}, 5:{"a":3549, "b":1774.5}}] +1193 [{1:{"a":11930, "b":11933}, 2:{"a":11950, "b":40}}, {3:{"a":5965, "b":2982.5}, 5:{"a":3579, "b":1789.5}}] + +-- !sql4_2 -- +2993 [{1:{"a":29930, "b":29933}, 2:{"a":29950, "b":40}}, {3:{"a":14965, "b":7482.5}, 5:{"a":8979, "b":4489.5}}] +2983 [{1:{"a":29830, "b":29833}, 2:{"a":29850, "b":40}}, {2:{"a":14915, "b":7457.5}, 5:{"a":8949, "b":4474.5}}] +2973 [{1:{"a":29730, "b":29733}, 2:{"a":29750, "b":40}}, {1:{"a":14865, "b":7432.5}, 5:{"a":8919, "b":4459.5}}] +2963 [{1:{"a":29630, "b":29633}, 2:{"a":29650, "b":40}}, {3:{"a":14815, "b":7407.5}, 5:{"a":8889, "b":4444.5}}] +2953 [{1:{"a":29530, "b":29533}, 2:{"a":29550, "b":40}}, {2:{"a":14765, "b":7382.5}, 5:{"a":8859, "b":4429.5}}] +2943 [{1:{"a":29430, "b":29433}, 2:{"a":29450, "b":40}}, {1:{"a":14715, "b":7357.5}, 5:{"a":8829, "b":4414.5}}] +2933 [{1:{"a":29330, "b":29333}, 2:{"a":29350, "b":40}}, {3:{"a":14665, "b":7332.5}, 5:{"a":8799, "b":4399.5}}] +2923 [{1:{"a":29230, "b":29233}, 2:{"a":29250, "b":40}}, {2:{"a":14615, "b":7307.5}, 5:{"a":8769, "b":4384.5}}] +2913 [{1:{"a":29130, "b":29133}, 2:{"a":29150, "b":40}}, {1:{"a":14565, "b":7282.5}, 5:{"a":8739, "b":4369.5}}] +2903 [{1:{"a":29030, "b":29033}, 2:{"a":29050, "b":40}}, {3:{"a":14515, "b":7257.5}, 5:{"a":8709, "b":4354.5}}] +2893 [{1:{"a":28930, "b":28933}, 2:{"a":28950, "b":40}}, {2:{"a":14465, "b":7232.5}, 5:{"a":8679, "b":4339.5}}] +2883 [{1:{"a":28830, "b":28833}, 2:{"a":28850, "b":40}}, {1:{"a":14415, "b":7207.5}, 5:{"a":8649, "b":4324.5}}] +2873 [{1:{"a":28730, "b":28733}, 2:{"a":28750, "b":40}}, {3:{"a":14365, "b":7182.5}, 5:{"a":8619, "b":4309.5}}] +2863 [{1:{"a":28630, "b":28633}, 2:{"a":28650, "b":40}}, {2:{"a":14315, "b":7157.5}, 5:{"a":8589, "b":4294.5}}] +2853 [{1:{"a":28530, "b":28533}, 2:{"a":28550, "b":40}}, {1:{"a":14265, "b":7132.5}, 5:{"a":8559, "b":4279.5}}] +2843 [{1:{"a":28430, "b":28433}, 2:{"a":28450, "b":40}}, {3:{"a":14215, "b":7107.5}, 5:{"a":8529, "b":4264.5}}] +2833 [{1:{"a":28330, "b":28333}, 2:{"a":28350, "b":40}}, {2:{"a":14165, "b":7082.5}, 5:{"a":8499, "b":4249.5}}] +2823 [{1:{"a":28230, "b":28233}, 2:{"a":28250, "b":40}}, {1:{"a":14115, "b":7057.5}, 5:{"a":8469, "b":4234.5}}] +2813 [{1:{"a":28130, "b":28133}, 2:{"a":28150, "b":40}}, {3:{"a":14065, "b":7032.5}, 5:{"a":8439, "b":4219.5}}] +2803 [{1:{"a":28030, "b":28033}, 2:{"a":28050, "b":40}}, {2:{"a":14015, "b":7007.5}, 5:{"a":8409, "b":4204.5}}] -- !sql5 -- -1 beijing -2 shanghai 3 guangzhou -5 hangzhou -7 tianjin -9 wuhan -11 changsha -13 dalian +13 guangzhou +23 guangzhou +33 guangzhou +43 guangzhou +53 guangzhou +63 guangzhou +73 guangzhou +83 guangzhou +93 guangzhou +103 guangzhou +113 guangzhou +123 guangzhou +133 guangzhou +143 guangzhou +153 guangzhou +163 guangzhou +173 guangzhou +183 guangzhou +193 guangzhou -- !sql5_1 -- -61 +1003 guangzhou +1013 guangzhou +1023 guangzhou +1033 guangzhou +1043 guangzhou +1053 guangzhou +1063 guangzhou +1073 guangzhou +1083 guangzhou +1093 guangzhou +1103 guangzhou +1113 guangzhou +1123 guangzhou +1133 guangzhou +1143 guangzhou +1153 guangzhou +1163 guangzhou +1173 guangzhou +1183 guangzhou +1193 guangzhou -- !sql5_2 -- +2993 guangzhou +2983 guangzhou +2973 guangzhou +2963 guangzhou +2953 guangzhou +2943 guangzhou +2933 guangzhou +2923 guangzhou +2913 guangzhou +2903 guangzhou +2893 guangzhou +2883 guangzhou +2873 guangzhou +2863 guangzhou +2853 guangzhou +2843 guangzhou +2833 guangzhou +2823 guangzhou +2813 guangzhou +2803 guangzhou + +-- !sql5_3 -- +61 + +-- !sql5_4 -- 61 -- !sql6 -- -2 +5 12.5 +15 \N +25 \N +35 87.5 +45 \N +55 \N +65 162.5 +75 \N +85 \N +95 237.5 +105 \N +115 \N +125 312.5 +135 \N +145 \N +155 387.5 +165 \N +175 \N +185 462.5 +195 \N + +-- !sql6_1 -- +1005 \N +1015 \N +1025 2562.5 +1035 \N +1045 \N +1055 2637.5 +1065 \N +1075 \N +1085 2712.5 +1095 \N +1105 \N +1115 2787.5 +1125 \N +1135 \N +1145 2862.5 +1155 \N +1165 \N +1175 2937.5 +1185 \N +1195 \N + +-- !sql6_2 -- +2995 \N +2985 \N +2975 7437.5 +2965 \N +2955 \N +2945 7362.5 +2935 \N +2925 \N +2915 7287.5 +2905 \N +2895 \N +2885 7212.5 +2875 \N +2865 \N +2855 7137.5 +2845 \N +2835 \N +2825 7062.5 +2815 \N +2805 \N -- !sql7 -- +2 + +-- !sql8 -- 0.41 0.99 --- !sql8 -- +-- !sql9 -- \N added_z diff --git a/regression-test/suites/datatype_p0/complex_types/test_pruned_columns.groovy b/regression-test/suites/datatype_p0/complex_types/test_pruned_columns.groovy index a99b7b6da5998b..062dbab3c94b5a 100644 --- a/regression-test/suites/datatype_p0/complex_types/test_pruned_columns.groovy +++ b/regression-test/suites/datatype_p0/complex_types/test_pruned_columns.groovy @@ -16,6 +16,7 @@ // under the License. suite("test_pruned_columns") { + sql "set batch_size = 32;" sql """DROP TABLE IF EXISTS `tbl_test_pruned_columns`""" sql """ CREATE TABLE `tbl_test_pruned_columns` ( @@ -23,58 +24,161 @@ suite("test_pruned_columns") { `s` struct>>, value:int> NULL ) ENGINE=OLAP DUPLICATE KEY(`id`) - DISTRIBUTED BY RANDOM BUCKETS AUTO + DISTRIBUTED BY RANDOM BUCKETS 2 PROPERTIES ( "replication_allocation" = "tag.location.default: 1" ); """ sql """ - insert into `tbl_test_pruned_columns` values - (1, named_struct('city', 'beijing', 'data', array(map(1, named_struct('a', 10, 'b', 20.0), 2, named_struct('a', 30, 'b', 40))), 'value', 1)), - (2, named_struct('city', 'shanghai', 'data', array(map(2, named_struct('a', 50, 'b', 40.0), 1, named_struct('a', 70, 'b', 80))), 'value', 2)), - (3, named_struct('city', 'guangzhou', 'data', array(map(1, named_struct('a', 90, 'b', 60.0), 2, named_struct('a', 110, 'b', 40))), 'value', 3)), - (4, named_struct('city', 'shenzhen', 'data', array(map(2, named_struct('a', 130, 'b', 20.0), 1, named_struct('a', 150, 'b', 40))), 'value', 4)), - (5, named_struct('city', 'hangzhou', 'data', array(map(1, named_struct('a', 170, 'b', 80.0), 2, named_struct('a', 190, 'b', 40))), 'value', 5)), - (6, named_struct('city', 'nanjing', 'data', array(map(2, named_struct('a', 210, 'b', 60.0), 1, named_struct('a', 230, 'b', 40))), 'value', 6)), - (7, named_struct('city', 'tianjin', 'data', array(map(1, named_struct('a', 250, 'b', 20.0), 2, named_struct('a', 270, 'b', 40))), 'value', 7)), - (8, named_struct('city', 'chongqing', 'data', array(map(2, named_struct('a', 290, 'b', 80.0), 1, named_struct('a', 310, 'b', 40))), 'value', 8)), - (9, named_struct('city', 'wuhan', 'data', array(map(1, named_struct('a', 330, 'b', 60.0), 2, named_struct('a', 350, 'b', 40))), 'value', 9)), - (10, named_struct('city', 'xian', 'data', array(map(2, named_struct('a', 370, 'b', 20.0), 1, named_struct('a', 390, 'b', 40))), 'value', 10)), - (11, named_struct('city', 'changsha', 'data', array(map(1, named_struct('a', 410, 'b', 80.0), 2, named_struct('a', 430, 'b', 40))), 'value', 11)), - (12, named_struct('city', 'qingdao', 'data', array(map(2, named_struct('a', 450, 'b', 60.0), 1, named_struct('a', 470, 'b', 40))), 'value', 12)), - (13, named_struct('city', 'dalian', 'data', array(map(1, named_struct('a', 490, 'b', 20.0), 2, named_struct('a', 510, 'b', 40))), 'value', 13)); + insert into `tbl_test_pruned_columns` + select + number as id, + named_struct( + 'city', + case (number % 10) + when 0 then 'beijing' + when 1 then 'shanghai' + when 2 then 'shenzhen' + when 3 then 'guangzhou' + when 4 then 'hangzhou' + when 5 then 'chengdu' + when 6 then 'wuhan' + when 7 then 'xian' + when 8 then 'nanjing' + else null + end, + 'data', + array( + map( + 1, named_struct('a', number * 10, 'b', (number * 10 + number % 5) * 1.0), + 2, named_struct('a', number * 10 + 20, 'b', (number % 10 + 1) * 10.0) + ), + map( + (number % 3 + 1), named_struct('a', number * 5, 'b', number * 2.5), + (number % 5 + 2), named_struct('a', number * 3, 'b', number * 1.5) + ) + ), + 'value', + number + ) as s + from numbers("number" = "3000"); """ qt_sql """ - select * from `tbl_test_pruned_columns` order by 1; + select struct_element(s, 'city'), count() from `tbl_test_pruned_columns` group by struct_element(s, 'city') order by 1, 2; """ qt_sql1 """ - select b.id, array_map(x -> struct_element(map_values(x)[1], 'a'), struct_element(s, 'data')) from `tbl_test_pruned_columns` t join (select 1 id) b on t.id = b.id order by 1; + select + b.id + , array_map(x -> struct_element(map_values(x)[1], 'a') + , struct_element(s, 'data')) + from `tbl_test_pruned_columns` t join (select 1 id) b on t.id = b.id + order by 1, 2 limit 0, 20; + """ + + qt_sql1_1 """ + select + b.id + , array_map(x -> struct_element(map_values(x)[1], 'a') + , struct_element(s, 'data')) + from `tbl_test_pruned_columns` t join (select 1 id) b on t.id = b.id + order by 1, 2 limit 100, 20; + """ + + qt_sql1_2 """ + select + b.id + , array_map(x -> struct_element(map_values(x)[1], 'a') + , struct_element(s, 'data')) + from `tbl_test_pruned_columns` t join (select 1 id) b on t.id = b.id + order by 1 desc, 2 limit 100, 20; """ qt_sql2 """ - select id, struct_element(s, 'city') from `tbl_test_pruned_columns` order by 1; + select id, struct_element(s, 'city') from `tbl_test_pruned_columns` order by 1 limit 0, 20; + """ + + qt_sql2_1 """ + select id, struct_element(s, 'city') from `tbl_test_pruned_columns` order by 1 limit 100, 20; + """ + + qt_sql2_2 """ + select id, struct_element(s, 'city') from `tbl_test_pruned_columns` order by 1 desc limit 0, 20; """ qt_sql3 """ - select id, struct_element(s, 'data') from `tbl_test_pruned_columns` order by 1; + select id, struct_element(s, 'data') from `tbl_test_pruned_columns` order by 1 limit 0, 20; + """ + + qt_sql3_1 """ + select id, struct_element(s, 'data') from `tbl_test_pruned_columns` order by 1 limit 200, 20; + """ + + qt_sql3_2 """ + select id, struct_element(s, 'data') from `tbl_test_pruned_columns` order by 1 desc limit 0, 20; """ qt_sql4 """ - select id, struct_element(s, 'data') from `tbl_test_pruned_columns` where struct_element(struct_element(s, 'data')[1][2], 'b') = 40 order by 1; + select + id + , struct_element(s, 'data') + from `tbl_test_pruned_columns` + where struct_element(struct_element(s, 'data')[1][2], 'b') = 40 + order by 1 limit 0, 20; + """ + + qt_sql4_1 """ + select + id + , struct_element(s, 'data') + from `tbl_test_pruned_columns` + where struct_element(struct_element(s, 'data')[1][2], 'b') = 40 + order by 1 limit 100, 20; + """ + + qt_sql4_2 """ + select + id + , struct_element(s, 'data') + from `tbl_test_pruned_columns` + where struct_element(struct_element(s, 'data')[1][2], 'b') = 40 + order by 1 desc limit 0, 20; """ qt_sql5 """ - select id, struct_element(s, 'city') from `tbl_test_pruned_columns` where struct_element(struct_element(s, 'data')[1][2], 'b') = 40 order by 1; + select + id + , struct_element(s, 'city') + from `tbl_test_pruned_columns` + where struct_element(struct_element(s, 'data')[1][2], 'b') = 40 + order by 1, 2 limit 0, 20; """ qt_sql5_1 """ - select /*+ set enable_prune_nested_column = 1; */ sum(s.value) from `tbl_test_pruned_columns` where id in(1,2,3,4,8,9,10,11,13); + select + id + , struct_element(s, 'city') + from `tbl_test_pruned_columns` + where struct_element(struct_element(s, 'data')[1][2], 'b') = 40 + order by 1, 2 limit 100, 20; """ qt_sql5_2 """ + select + id + , struct_element(s, 'city') + from `tbl_test_pruned_columns` + where struct_element(struct_element(s, 'data')[1][2], 'b') = 40 + order by 1 desc, 2 limit 0, 20; + """ + + qt_sql5_3 """ + select /*+ set enable_prune_nested_column = 1; */ sum(s.value) from `tbl_test_pruned_columns` where id in(1,2,3,4,8,9,10,11,13); + """ + + qt_sql5_4 """ select /*+ set enable_prune_nested_column = 0; */ sum(s.value) from `tbl_test_pruned_columns` where id in(1,2,3,4,8,9,10,11,13); """ @@ -98,10 +202,37 @@ suite("test_pruned_columns") { """ qt_sql6 """ - select count(struct_element(dynamic_attributes['theme_preference'], 'confidence_score')) from `tbl_test_pruned_columns_map`; + select + id + , struct_element(struct_element(s, 'data')[2][3], 'b') + from `tbl_test_pruned_columns` + where struct_element(s, 'city') = 'chengdu' + order by 1, 2 limit 0, 20; + """ + + qt_sql6_1 """ + select + id + , struct_element(struct_element(s, 'data')[2][3], 'b') + from `tbl_test_pruned_columns` + where struct_element(s, 'city') = 'chengdu' + order by 1, 2 limit 100, 20; + """ + + qt_sql6_2 """ + select + id + , struct_element(struct_element(s, 'data')[2][3], 'b') + from `tbl_test_pruned_columns` + where struct_element(s, 'city') = 'chengdu' + order by 1 desc, 2 limit 0, 20; """ qt_sql7 """ + select count(struct_element(dynamic_attributes['theme_preference'], 'confidence_score')) from `tbl_test_pruned_columns_map`; + """ + + qt_sql8 """ select struct_element(dynamic_attributes['theme_preference'], 'confidence_score') from `tbl_test_pruned_columns_map` order by id; """ @@ -113,12 +244,12 @@ suite("test_pruned_columns") { `s_info` STRUCT, `arr_s` ARRAY>, `map_s` MAP> - ) - UNIQUE KEY(`id`) - DISTRIBUTED BY HASH(`id`) BUCKETS 4 + ) + UNIQUE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 4 PROPERTIES ( "replication_num" = "1", - "light_schema_change" = "true" + "light_schema_change" = "true" ); """ sql """ @@ -134,7 +265,7 @@ suite("test_pruned_columns") { INSERT INTO nested_sc_tbl VALUES (3, struct(30.5, 'v3', 888), array(struct(500, 600, 'added_z'), struct(501, 601, 'added_z_2')), map('k3', struct(3, 3.3))); """ - qt_sql8 """ + qt_sql9 """ select struct_element(element_at(arr_s, 1), 'z') as inner_z FROM nested_sc_tbl ORDER BY id; """ -} \ No newline at end of file +}