From 715b28c6500cedc662ceff4a62d570b6b26b8089 Mon Sep 17 00:00:00 2001 From: yoruet <1559650411@qq.com> Date: Tue, 15 Oct 2024 18:43:30 +0800 Subject: [PATCH 1/6] add time counter and bytes counter for compaction --- be/src/olap/merger.cpp | 2 +- .../olap/rowset/segment_v2/column_reader.cpp | 7 +++- be/src/olap/rowset/segment_v2/column_reader.h | 20 +++++++++- be/src/olap/rowset/segment_v2/page_io.cpp | 1 + .../rowset/segment_v2/segment_iterator.cpp | 5 +++ .../olap/rowset/segment_v2/segment_iterator.h | 39 ++++++++++++++++++- be/src/vec/olap/vertical_block_reader.cpp | 12 ++++++ be/src/vec/olap/vertical_block_reader.h | 5 ++- 8 files changed, 85 insertions(+), 6 deletions(-) diff --git a/be/src/olap/merger.cpp b/be/src/olap/merger.cpp index ab034123ac883c..f8cb1c1cc0165e 100644 --- a/be/src/olap/merger.cpp +++ b/be/src/olap/merger.cpp @@ -243,7 +243,7 @@ Status Merger::vertical_compact_one_group( CompactionSampleInfo* sample_info) { // build tablet reader VLOG_NOTICE << "vertical compact one group, max_rows_per_segment=" << max_rows_per_segment; - vectorized::VerticalBlockReader reader(row_source_buf); + vectorized::VerticalBlockReader reader(row_source_buf, true); TabletReader::ReaderParams reader_params; reader_params.is_key_column_group = is_key; reader_params.key_group_cluster_key_idxes = key_group_cluster_key_idxes; diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp index 69242352f78d01..d578e7fde95f21 100644 --- a/be/src/olap/rowset/segment_v2/column_reader.cpp +++ b/be/src/olap/rowset/segment_v2/column_reader.cpp @@ -369,7 +369,10 @@ Status ColumnReader::read_page(const ColumnIteratorOptions& iter_opts, const Pag }; // index page should not pre decode if (iter_opts.type == INDEX_PAGE) opts.pre_decode = false; - return PageIO::read_and_decompress_page(opts, handle, page_body, footer); + Status s = PageIO::read_and_decompress_page(opts, handle, page_body, footer); + _compaction_io_time_ns += opts.stats->io_ns; + _compaction_io_bytes += opts.stats->bytes_read; + return s; } Status ColumnReader::get_row_ranges_by_zone_map( @@ -1352,6 +1355,8 @@ Status FileColumnIterator::_read_data_page(const OrdinalPageIndexIterator& iter) _opts.type = DATA_PAGE; RETURN_IF_ERROR( _reader->read_page(_opts, iter.page(), &handle, &page_body, &footer, _compress_codec)); + _compaction_io_time_ns += doris::segment_v2::ColumnReader::get_compaction_io_time_ns(); + _compaction_io_bytes += doris::segment_v2::ColumnReader::get_compaction_io_bytes(); // parse data page RETURN_IF_ERROR(ParsedPage::create(std::move(handle), page_body, footer.data_page_footer(), _reader->encoding_info(), iter.page(), iter.page_index(), diff --git a/be/src/olap/rowset/segment_v2/column_reader.h b/be/src/olap/rowset/segment_v2/column_reader.h index b013c712e66733..1994b4f857f631 100644 --- a/be/src/olap/rowset/segment_v2/column_reader.h +++ b/be/src/olap/rowset/segment_v2/column_reader.h @@ -206,6 +206,14 @@ class ColumnReader : public MetadataAdder { FieldType get_meta_type() { return _meta_type; } + static int64_t get_compaction_io_time_ns() { + return _compaction_io_time_ns; + } + + static size_t get_compaction_io_bytes() { + return _compaction_io_bytes; + } + private: ColumnReader(const ColumnReaderOptions& opts, const ColumnMetaPB& meta, uint64_t num_rows, io::FileReaderSPtr file_reader); @@ -254,6 +262,9 @@ class ColumnReader : public MetadataAdder { bool _use_index_page_cache; int _be_exec_version = -1; + static int64_t _compaction_io_time_ns; + static size_t _compaction_io_bytes; + PagePointer _meta_dict_page; CompressionTypePB _meta_compression; @@ -342,6 +353,9 @@ class ColumnIterator { virtual bool is_all_dict_encoding() const { return false; } + virtual int64_t get_compaction_io_time_ns() const { return 0; } + virtual size_t get_compaction_io_bytes() const { return 0; } + protected: ColumnIteratorOptions _opts; }; @@ -389,12 +403,16 @@ class FileColumnIterator final : public ColumnIterator { bool is_all_dict_encoding() const override { return _is_all_dict_encoding; } + int64_t get_compaction_io_time_ns() const override { return _compaction_io_time_ns; } + size_t get_compaction_io_bytes() const override { return _compaction_io_bytes; } + private: void _seek_to_pos_in_page(ParsedPage* page, ordinal_t offset_in_page) const; Status _load_next_page(bool* eos); Status _read_data_page(const OrdinalPageIndexIterator& iter); Status _read_dict_data(); - + int64_t _compaction_io_time_ns = 0; + size_t _compaction_io_bytes = 0; ColumnReader* _reader = nullptr; // iterator owned compress codec, should NOT be shared by threads, initialized in init() diff --git a/be/src/olap/rowset/segment_v2/page_io.cpp b/be/src/olap/rowset/segment_v2/page_io.cpp index 07d5656ee8a44b..796b176e05e81d 100644 --- a/be/src/olap/rowset/segment_v2/page_io.cpp +++ b/be/src/olap/rowset/segment_v2/page_io.cpp @@ -154,6 +154,7 @@ Status PageIO::read_and_decompress_page_(const PageReadOptions& opts, PageHandle &opts.io_ctx)); DCHECK_EQ(bytes_read, page_size); opts.stats->compressed_bytes_read += page_size; + opts.stats->bytes_read += bytes_read; } if (opts.verify_checksum) { diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp index a921674a1e506d..22546274fc1a4f 100644 --- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp +++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp @@ -1895,6 +1895,11 @@ Status SegmentIterator::_read_columns_by_rowids(std::vector& read_colu RETURN_IF_ERROR(_column_iterators[cid]->read_by_rowids(rowids.data(), select_size, _current_return_columns[cid])); + if (_is_compaction) { + _compaction_io_time_ns += _column_iterators[cid]->get_compaction_io_time_ns(); + _compaction_io_bytes += _column_iterators[cid]->get_compaction_io_bytes(); + LOG(INFO) << "compaction io time: " << _compaction_io_time_ns << " ns, compaction io bytes: " << _compaction_io_bytes << " bytes"; + } } return Status::OK(); diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.h b/be/src/olap/rowset/segment_v2/segment_iterator.h index c2e2139e8ad411..f986fcbc412008 100644 --- a/be/src/olap/rowset/segment_v2/segment_iterator.h +++ b/be/src/olap/rowset/segment_v2/segment_iterator.h @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -161,9 +162,45 @@ class SegmentIterator : public RowwiseIterator { [](const auto& iterator) { return iterator != nullptr; }); } + int64_t get_compaction_io_time_ns() const { + return _compaction_io_time_ns; + } + + size_t get_compaction_io_bytes() const { + return _compaction_io_bytes; + } + + bool get_is_compaction() const { + return _is_compaction; + } + + void set_is_compaction(bool is_compaction) { + _is_compaction = is_compaction; + } + + bool get_is_cache_io() const { + return _is_cache_io; + } + + void set_is_cache_io(bool is_cache_io) { + _is_cache_io = is_cache_io; + } + + bool get_is_s3_io() const { + return _is_s3_io; + } + + void set_is_s3_io(bool is_s3_io) { + _is_s3_io = is_s3_io; + } + private: + bool _is_compaction = false; + bool _is_cache_io = false; + bool _is_s3_io = false; + int64_t _compaction_io_time_ns = 0; + size_t _compaction_io_bytes = 0; Status _next_batch_internal(vectorized::Block* block); - template bool _update_profile(RuntimeProfile* profile, const Container& predicates, const std::string& title) { diff --git a/be/src/vec/olap/vertical_block_reader.cpp b/be/src/vec/olap/vertical_block_reader.cpp index 5367729f637e15..239a1ef2ebd8ea 100644 --- a/be/src/vec/olap/vertical_block_reader.cpp +++ b/be/src/vec/olap/vertical_block_reader.cpp @@ -32,6 +32,7 @@ #include "olap/rowset/rowset.h" #include "olap/rowset/rowset_reader.h" #include "olap/rowset/rowset_reader_context.h" +#include "olap/rowset/segment_v2/segment_iterator.h" #include "olap/tablet_schema.h" #include "vec/aggregate_functions/aggregate_function_reader.h" #include "vec/columns/column_nullable.h" @@ -84,6 +85,17 @@ Status VerticalBlockReader::_get_segment_iterators(const ReaderParams& read_para bool use_cache = !rs_split.rs_reader->rowset()->is_local(); RETURN_IF_ERROR(rs_split.rs_reader->get_segment_iterators(&_reader_context, segment_iters, use_cache)); + for (auto& iter : *segment_iters) { + auto* seg_iter = dynamic_cast(iter.get()); + if (seg_iter != nullptr) { + seg_iter->set_is_compaction(_is_compaction); + if (use_cache) { + seg_iter->set_is_cache_io(true); + } else { + seg_iter->set_is_s3_io(true); + } + } + } // if segments overlapping, all segment iterator should be inited in // heap merge iterator. If segments are none overlapping, only first segment of this // rowset will be inited and push to heap, other segment will be inited later when current diff --git a/be/src/vec/olap/vertical_block_reader.h b/be/src/vec/olap/vertical_block_reader.h index 2043db4b00a590..efea3b51c91612 100644 --- a/be/src/vec/olap/vertical_block_reader.h +++ b/be/src/vec/olap/vertical_block_reader.h @@ -47,8 +47,8 @@ class RowSourcesBuffer; class VerticalBlockReader final : public TabletReader { public: - VerticalBlockReader(RowSourcesBuffer* row_sources_buffer) - : _row_sources_buffer(row_sources_buffer) { + VerticalBlockReader(RowSourcesBuffer* row_sources_buffer, bool is_compaction = false) + : _row_sources_buffer(row_sources_buffer), _is_compaction(is_compaction) { _id = nextId++; } @@ -103,6 +103,7 @@ class VerticalBlockReader final : public TabletReader { Status (VerticalBlockReader::*_next_block_func)(Block* block, bool* eof) = nullptr; RowSourcesBuffer* _row_sources_buffer; + bool _is_compaction; ColumnPtr _delete_filter_column; // for agg mode From 95b22be46390b61940d23f29a81535ea3ebfbb55 Mon Sep 17 00:00:00 2001 From: yoruet <1559650411@qq.com> Date: Tue, 15 Oct 2024 22:08:34 +0800 Subject: [PATCH 2/6] remove const of column_reader and change the function to non-static --- be/src/olap/rowset/segment_v2/column_reader.cpp | 6 +++--- be/src/olap/rowset/segment_v2/column_reader.h | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp index d578e7fde95f21..891c958f94d59c 100644 --- a/be/src/olap/rowset/segment_v2/column_reader.cpp +++ b/be/src/olap/rowset/segment_v2/column_reader.cpp @@ -353,7 +353,7 @@ Status ColumnReader::new_inverted_index_iterator( Status ColumnReader::read_page(const ColumnIteratorOptions& iter_opts, const PagePointer& pp, PageHandle* handle, Slice* page_body, PageFooterPB* footer, - BlockCompressionCodec* codec) const { + BlockCompressionCodec* codec) { iter_opts.sanity_check(); PageReadOptions opts { .verify_checksum = _opts.verify_checksum, @@ -1355,8 +1355,8 @@ Status FileColumnIterator::_read_data_page(const OrdinalPageIndexIterator& iter) _opts.type = DATA_PAGE; RETURN_IF_ERROR( _reader->read_page(_opts, iter.page(), &handle, &page_body, &footer, _compress_codec)); - _compaction_io_time_ns += doris::segment_v2::ColumnReader::get_compaction_io_time_ns(); - _compaction_io_bytes += doris::segment_v2::ColumnReader::get_compaction_io_bytes(); + _compaction_io_time_ns += _reader->get_compaction_io_time_ns(); + _compaction_io_bytes += _reader->get_compaction_io_bytes(); // parse data page RETURN_IF_ERROR(ParsedPage::create(std::move(handle), page_body, footer.data_page_footer(), _reader->encoding_info(), iter.page(), iter.page_index(), diff --git a/be/src/olap/rowset/segment_v2/column_reader.h b/be/src/olap/rowset/segment_v2/column_reader.h index 1994b4f857f631..7111114de77917 100644 --- a/be/src/olap/rowset/segment_v2/column_reader.h +++ b/be/src/olap/rowset/segment_v2/column_reader.h @@ -206,11 +206,11 @@ class ColumnReader : public MetadataAdder { FieldType get_meta_type() { return _meta_type; } - static int64_t get_compaction_io_time_ns() { + int64_t get_compaction_io_time_ns() const { return _compaction_io_time_ns; } - static size_t get_compaction_io_bytes() { + size_t get_compaction_io_bytes() const { return _compaction_io_bytes; } @@ -262,8 +262,8 @@ class ColumnReader : public MetadataAdder { bool _use_index_page_cache; int _be_exec_version = -1; - static int64_t _compaction_io_time_ns; - static size_t _compaction_io_bytes; + int64_t _compaction_io_time_ns; + size_t _compaction_io_bytes; PagePointer _meta_dict_page; CompressionTypePB _meta_compression; From c1eaa7fe53483ccdd81af5b36f058dcfc3c99ef1 Mon Sep 17 00:00:00 2001 From: yoruet <1559650411@qq.com> Date: Tue, 15 Oct 2024 22:12:33 +0800 Subject: [PATCH 3/6] change column_reader.h to match the change of column_reader.cpp --- be/src/olap/rowset/segment_v2/column_reader.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/olap/rowset/segment_v2/column_reader.h b/be/src/olap/rowset/segment_v2/column_reader.h index 7111114de77917..2c9222cc21ca83 100644 --- a/be/src/olap/rowset/segment_v2/column_reader.h +++ b/be/src/olap/rowset/segment_v2/column_reader.h @@ -155,7 +155,7 @@ class ColumnReader : public MetadataAdder { // read a page from file into a page handle Status read_page(const ColumnIteratorOptions& iter_opts, const PagePointer& pp, PageHandle* handle, Slice* page_body, PageFooterPB* footer, - BlockCompressionCodec* codec) const; + BlockCompressionCodec* codec); bool is_nullable() const { return _meta_is_nullable; } From 4f73056fbc12d396e4582d6fad46b50111bf7bac Mon Sep 17 00:00:00 2001 From: yoruet <1559650411@qq.com> Date: Tue, 15 Oct 2024 22:50:47 +0800 Subject: [PATCH 4/6] divide bytes_read to cache and s3/local --- be/src/olap/olap_common.h | 3 ++ .../olap/rowset/segment_v2/column_reader.cpp | 4 ++- be/src/olap/rowset/segment_v2/column_reader.h | 28 +++++++++++---- be/src/olap/rowset/segment_v2/page_io.cpp | 9 +++-- .../rowset/segment_v2/segment_iterator.cpp | 21 ++++++++++-- .../olap/rowset/segment_v2/segment_iterator.h | 34 +++++++------------ 6 files changed, 67 insertions(+), 32 deletions(-) diff --git a/be/src/olap/olap_common.h b/be/src/olap/olap_common.h index c1d3038050fbd4..56aba87d824a1b 100644 --- a/be/src/olap/olap_common.h +++ b/be/src/olap/olap_common.h @@ -293,6 +293,9 @@ struct OlapReaderStatistics { // total read bytes in memory int64_t bytes_read = 0; + int64_t cache_bytes_read = 0; + int64_t local_bytes_read = 0; + int64_t s3_bytes_read = 0; int64_t block_fetch_ns = 0; // time of rowset reader's `next_batch()` call int64_t block_load_ns = 0; diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp index 891c958f94d59c..7fc5d54e28cd3f 100644 --- a/be/src/olap/rowset/segment_v2/column_reader.cpp +++ b/be/src/olap/rowset/segment_v2/column_reader.cpp @@ -371,7 +371,9 @@ Status ColumnReader::read_page(const ColumnIteratorOptions& iter_opts, const Pag if (iter_opts.type == INDEX_PAGE) opts.pre_decode = false; Status s = PageIO::read_and_decompress_page(opts, handle, page_body, footer); _compaction_io_time_ns += opts.stats->io_ns; - _compaction_io_bytes += opts.stats->bytes_read; + _compaction_cache_bytes += opts.stats->cache_bytes_read; + _compaction_local_bytes += opts.stats->local_bytes_read; + _compaction_s3_bytes += opts.stats->s3_bytes_read; return s; } diff --git a/be/src/olap/rowset/segment_v2/column_reader.h b/be/src/olap/rowset/segment_v2/column_reader.h index 2c9222cc21ca83..974f0599e2c9d9 100644 --- a/be/src/olap/rowset/segment_v2/column_reader.h +++ b/be/src/olap/rowset/segment_v2/column_reader.h @@ -210,8 +210,16 @@ class ColumnReader : public MetadataAdder { return _compaction_io_time_ns; } - size_t get_compaction_io_bytes() const { - return _compaction_io_bytes; + int64_t get_compaction_cache_bytes() const { + return _compaction_cache_bytes; + } + + int64_t get_compaction_local_bytes() const { + return _compaction_local_bytes; + } + + int64_t get_compaction_s3_bytes() const { + return _compaction_s3_bytes; } private: @@ -263,7 +271,9 @@ class ColumnReader : public MetadataAdder { int _be_exec_version = -1; int64_t _compaction_io_time_ns; - size_t _compaction_io_bytes; + int64_t _compaction_cache_bytes; + int64_t _compaction_local_bytes; + int64_t _compaction_s3_bytes; PagePointer _meta_dict_page; CompressionTypePB _meta_compression; @@ -354,7 +364,9 @@ class ColumnIterator { virtual bool is_all_dict_encoding() const { return false; } virtual int64_t get_compaction_io_time_ns() const { return 0; } - virtual size_t get_compaction_io_bytes() const { return 0; } + virtual int64_t get_compaction_cache_bytes() const { return 0; } + virtual int64_t get_compaction_local_bytes() const { return 0; } + virtual int64_t get_compaction_s3_bytes() const { return 0; } protected: ColumnIteratorOptions _opts; @@ -404,7 +416,9 @@ class FileColumnIterator final : public ColumnIterator { bool is_all_dict_encoding() const override { return _is_all_dict_encoding; } int64_t get_compaction_io_time_ns() const override { return _compaction_io_time_ns; } - size_t get_compaction_io_bytes() const override { return _compaction_io_bytes; } + int64_t get_compaction_cache_bytes() const override { return _compaction_cache_bytes; } + int64_t get_compaction_local_bytes() const override { return _compaction_local_bytes; } + int64_t get_compaction_s3_bytes() const override { return _compaction_s3_bytes; } private: void _seek_to_pos_in_page(ParsedPage* page, ordinal_t offset_in_page) const; @@ -412,7 +426,9 @@ class FileColumnIterator final : public ColumnIterator { Status _read_data_page(const OrdinalPageIndexIterator& iter); Status _read_dict_data(); int64_t _compaction_io_time_ns = 0; - size_t _compaction_io_bytes = 0; + int64_t _compaction_cache_bytes = 0; + int64_t _compaction_local_bytes = 0; + int64_t _compaction_s3_bytes = 0; ColumnReader* _reader = nullptr; // iterator owned compress codec, should NOT be shared by threads, initialized in init() diff --git a/be/src/olap/rowset/segment_v2/page_io.cpp b/be/src/olap/rowset/segment_v2/page_io.cpp index 796b176e05e81d..560a28c3301d9b 100644 --- a/be/src/olap/rowset/segment_v2/page_io.cpp +++ b/be/src/olap/rowset/segment_v2/page_io.cpp @@ -27,6 +27,7 @@ #include #include +#include "cloud/config.h" #include "common/logging.h" #include "gutil/strings/substitute.h" #include "io/fs/file_reader.h" @@ -133,6 +134,7 @@ Status PageIO::read_and_decompress_page_(const PageReadOptions& opts, PageHandle footer_size, opts.file_reader->path().native()); } *body = Slice(page_slice.data, page_slice.size - 4 - footer_size); + opts.stats->cache_bytes_read += page_slice.size; return Status::OK(); } @@ -154,9 +156,12 @@ Status PageIO::read_and_decompress_page_(const PageReadOptions& opts, PageHandle &opts.io_ctx)); DCHECK_EQ(bytes_read, page_size); opts.stats->compressed_bytes_read += page_size; - opts.stats->bytes_read += bytes_read; + if (config::is_cloud_mode()) { + opts.stats->s3_bytes_read += bytes_read; + } else { + opts.stats->local_bytes_read += bytes_read; + } } - if (opts.verify_checksum) { uint32_t expect = decode_fixed32_le((uint8_t*)page_slice.data + page_slice.size - 4); uint32_t actual = crc32c::Value(page_slice.data, page_slice.size - 4); diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp index 22546274fc1a4f..a3f6b2c32106f7 100644 --- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp +++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp @@ -31,6 +31,7 @@ #include #include +#include "cloud/config.h" #include "common/compiler_util.h" // IWYU pragma: keep #include "common/config.h" #include "common/consts.h" @@ -1897,8 +1898,24 @@ Status SegmentIterator::_read_columns_by_rowids(std::vector& read_colu _current_return_columns[cid])); if (_is_compaction) { _compaction_io_time_ns += _column_iterators[cid]->get_compaction_io_time_ns(); - _compaction_io_bytes += _column_iterators[cid]->get_compaction_io_bytes(); - LOG(INFO) << "compaction io time: " << _compaction_io_time_ns << " ns, compaction io bytes: " << _compaction_io_bytes << " bytes"; + _compaction_cache_bytes += _column_iterators[cid]->get_compaction_cache_bytes(); + _compaction_local_bytes += _column_iterators[cid]->get_compaction_local_bytes(); + _compaction_s3_bytes += _column_iterators[cid]->get_compaction_s3_bytes(); + if (config::is_cloud_mode()) { + int64_t total_bytes = _compaction_cache_bytes + _compaction_s3_bytes; + double cache_percent = total_bytes > 0 ? (_compaction_cache_bytes * 100.0 / total_bytes) : 0; + double s3_percent = 1 - cache_percent; + LOG(INFO) << "Compaction IO time: " << _compaction_io_time_ns << " ns, " + << "Cache bytes: " << _compaction_cache_bytes << " bytes (" << std::fixed << std::setprecision(2) << cache_percent << "%), " + << "S3 bytes: " << _compaction_s3_bytes << " bytes (" << std::fixed << std::setprecision(2) << s3_percent << "%)"; + } else { + int64_t total_bytes = _compaction_cache_bytes + _compaction_local_bytes; + double cache_percent = total_bytes > 0 ? (_compaction_cache_bytes * 100.0 / total_bytes) : 0; + double local_percent = 1 - cache_percent; + LOG(INFO) << "Compaction IO time: " << _compaction_io_time_ns << " ns, " + << "Cache bytes: " << _compaction_cache_bytes << " bytes (" << std::fixed << std::setprecision(2) << cache_percent << "%), " + << "Local bytes: " << _compaction_local_bytes << " bytes (" << std::fixed << std::setprecision(2) << local_percent << "%)"; + } } } diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.h b/be/src/olap/rowset/segment_v2/segment_iterator.h index f986fcbc412008..2d197f36cb0ca9 100644 --- a/be/src/olap/rowset/segment_v2/segment_iterator.h +++ b/be/src/olap/rowset/segment_v2/segment_iterator.h @@ -21,7 +21,6 @@ #include #include -#include #include #include #include @@ -166,40 +165,33 @@ class SegmentIterator : public RowwiseIterator { return _compaction_io_time_ns; } - size_t get_compaction_io_bytes() const { - return _compaction_io_bytes; + int64_t get_compaction_cache_bytes() const { + return _compaction_cache_bytes; } - bool get_is_compaction() const { - return _is_compaction; + int64_t get_compaction_local_bytes() const { + return _compaction_local_bytes; } - void set_is_compaction(bool is_compaction) { - _is_compaction = is_compaction; + int64_t get_compaction_s3_bytes() const { + return _compaction_s3_bytes; } - bool get_is_cache_io() const { - return _is_cache_io; + bool get_is_compaction() const { + return _is_compaction; } - void set_is_cache_io(bool is_cache_io) { - _is_cache_io = is_cache_io; - } - - bool get_is_s3_io() const { - return _is_s3_io; + void set_is_compaction(bool is_compaction) { + _is_compaction = is_compaction; } - void set_is_s3_io(bool is_s3_io) { - _is_s3_io = is_s3_io; - } private: bool _is_compaction = false; - bool _is_cache_io = false; - bool _is_s3_io = false; int64_t _compaction_io_time_ns = 0; - size_t _compaction_io_bytes = 0; + int64_t _compaction_cache_bytes = 0; + int64_t _compaction_local_bytes = 0; + int64_t _compaction_s3_bytes = 0; Status _next_batch_internal(vectorized::Block* block); template bool _update_profile(RuntimeProfile* profile, const Container& predicates, From 120d09a6f51f9e663bb5798fa4e013c3cf8f3fb4 Mon Sep 17 00:00:00 2001 From: yoruet <1559650411@qq.com> Date: Tue, 15 Oct 2024 23:31:57 +0800 Subject: [PATCH 5/6] fix bug in column_reader.cpp --- be/src/olap/rowset/segment_v2/column_reader.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp index 7fc5d54e28cd3f..7a7c8d4e778194 100644 --- a/be/src/olap/rowset/segment_v2/column_reader.cpp +++ b/be/src/olap/rowset/segment_v2/column_reader.cpp @@ -1358,7 +1358,9 @@ Status FileColumnIterator::_read_data_page(const OrdinalPageIndexIterator& iter) RETURN_IF_ERROR( _reader->read_page(_opts, iter.page(), &handle, &page_body, &footer, _compress_codec)); _compaction_io_time_ns += _reader->get_compaction_io_time_ns(); - _compaction_io_bytes += _reader->get_compaction_io_bytes(); + _compaction_cache_bytes += _reader->get_compaction_cache_bytes(); + _compaction_s3_bytes += _reader->get_compaction_s3_bytes(); + _compaction_local_bytes += _reader->get_compaction_local_bytes(); // parse data page RETURN_IF_ERROR(ParsedPage::create(std::move(handle), page_body, footer.data_page_footer(), _reader->encoding_info(), iter.page(), iter.page_index(), From b49e5ab502a2f97663c0faef8df141ca5a4abea3 Mon Sep 17 00:00:00 2001 From: yoruet <1559650411@qq.com> Date: Wed, 16 Oct 2024 00:31:15 +0800 Subject: [PATCH 6/6] fix bug in vertical_block_reader.cpp --- be/src/vec/olap/vertical_block_reader.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/be/src/vec/olap/vertical_block_reader.cpp b/be/src/vec/olap/vertical_block_reader.cpp index 239a1ef2ebd8ea..e63b0eac84e125 100644 --- a/be/src/vec/olap/vertical_block_reader.cpp +++ b/be/src/vec/olap/vertical_block_reader.cpp @@ -89,11 +89,6 @@ Status VerticalBlockReader::_get_segment_iterators(const ReaderParams& read_para auto* seg_iter = dynamic_cast(iter.get()); if (seg_iter != nullptr) { seg_iter->set_is_compaction(_is_compaction); - if (use_cache) { - seg_iter->set_is_cache_io(true); - } else { - seg_iter->set_is_s3_io(true); - } } } // if segments overlapping, all segment iterator should be inited in