Skip to content

[Improvementation](profile) add some profile on vcollect_iterator #32794

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
BiteTheDDDDt merged 1 commit into from
Mar 26, 2024
Merged

[Improvementation](profile) add some profile on vcollect_iterator #32794

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions be/src/olap/olap_common.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -369,6 +369,9 @@ struct OlapReaderStatistics {

io::FileCacheStatistics file_cache_stats;
int64_t load_segments_timer = 0;

int64_t collect_iterator_merge_next_timer = 0;
int64_t collect_iterator_normal_next_timer = 0;
};

using ColumnId = uint32_t;
Expand Down
2 changes: 2 additions & 0 deletions be/src/pipeline/exec/olap_scan_operator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,8 @@ Status OlapScanLocalState::_init_profile() {
_block_conditions_filtered_timer = ADD_TIMER(_segment_profile, "BlockConditionsFilteredTime");
_block_conditions_filtered_bf_timer =
ADD_TIMER(_segment_profile, "BlockConditionsFilteredBloomFilterTime");
_collect_iterator_merge_next_timer = ADD_TIMER(_segment_profile, "CollectIteratorMergeTime");
_collect_iterator_normal_next_timer = ADD_TIMER(_segment_profile, "CollectIteratorNormalTime");
_block_conditions_filtered_zonemap_timer =
ADD_TIMER(_segment_profile, "BlockConditionsFilteredZonemapTime");
_block_conditions_filtered_zonemap_rp_timer =
Expand Down
2 changes: 2 additions & 0 deletions be/src/pipeline/exec/olap_scan_operator.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,8 @@ class OlapScanLocalState final : public ScanLocalState<OlapScanLocalState> {
RuntimeProfile::Counter* _block_init_seek_counter = nullptr;
RuntimeProfile::Counter* _block_conditions_filtered_timer = nullptr;
RuntimeProfile::Counter* _block_conditions_filtered_bf_timer = nullptr;
RuntimeProfile::Counter* _collect_iterator_merge_next_timer = nullptr;
RuntimeProfile::Counter* _collect_iterator_normal_next_timer = nullptr;
RuntimeProfile::Counter* _block_conditions_filtered_zonemap_timer = nullptr;
RuntimeProfile::Counter* _block_conditions_filtered_zonemap_rp_timer = nullptr;
RuntimeProfile::Counter* _block_conditions_filtered_dict_timer = nullptr;
Expand Down
2 changes: 2 additions & 0 deletions be/src/vec/exec/scan/new_olap_scan_node.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,8 @@ Status NewOlapScanNode::_init_profile() {
_block_conditions_filtered_timer = ADD_TIMER(_segment_profile, "BlockConditionsFilteredTime");
_block_conditions_filtered_bf_timer =
ADD_TIMER(_segment_profile, "BlockConditionsFilteredBloomFilterTime");
_collect_iterator_merge_next_timer = ADD_TIMER(_segment_profile, "CollectIteratorMergeTime");
_collect_iterator_normal_next_timer = ADD_TIMER(_segment_profile, "CollectIteratorNormalTime");
_block_conditions_filtered_zonemap_timer =
ADD_TIMER(_segment_profile, "BlockConditionsFilteredZonemapTime");
_block_conditions_filtered_zonemap_rp_timer =
Expand Down
2 changes: 2 additions & 0 deletions be/src/vec/exec/scan/new_olap_scan_node.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,8 @@ class NewOlapScanNode : public VScanNode {
RuntimeProfile::Counter* _block_init_seek_counter = nullptr;
RuntimeProfile::Counter* _block_conditions_filtered_timer = nullptr;
RuntimeProfile::Counter* _block_conditions_filtered_bf_timer = nullptr;
RuntimeProfile::Counter* _collect_iterator_merge_next_timer = nullptr;
RuntimeProfile::Counter* _collect_iterator_normal_next_timer = nullptr;
RuntimeProfile::Counter* _block_conditions_filtered_zonemap_timer = nullptr;
RuntimeProfile::Counter* _block_conditions_filtered_zonemap_rp_timer = nullptr;
RuntimeProfile::Counter* _block_conditions_filtered_dict_timer = nullptr;
Expand Down
5 changes: 5 additions & 0 deletions be/src/vec/exec/scan/new_olap_scanner.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,9 @@ Status NewOlapScanner::init() {

Status NewOlapScanner::open(RuntimeState* state) {
RETURN_IF_ERROR(VScanner::open(state));
auto* timer = _parent ? ((NewOlapScanNode*)_parent)->_reader_init_timer
: ((pipeline::OlapScanLocalState*)_local_state)->_reader_init_timer;
SCOPED_TIMER(timer);

auto res = _tablet_reader->init(_tablet_reader_params);
if (!res.ok()) {
Expand Down Expand Up @@ -585,6 +588,8 @@ void NewOlapScanner::_collect_profile_before_close() {
COUNTER_UPDATE(Parent->_block_conditions_filtered_timer, stats.block_conditions_filtered_ns); \
COUNTER_UPDATE(Parent->_block_conditions_filtered_bf_timer, \
stats.block_conditions_filtered_bf_ns); \
COUNTER_UPDATE(Parent->_collect_iterator_merge_next_timer, \
stats.collect_iterator_merge_next_timer); \
COUNTER_UPDATE(Parent->_block_conditions_filtered_zonemap_timer, \
stats.block_conditions_filtered_zonemap_ns); \
COUNTER_UPDATE(Parent->_block_conditions_filtered_zonemap_rp_timer, \
Expand Down
6 changes: 3 additions & 3 deletions be/src/vec/exec/scan/vscanner.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ Status VScanner::get_block(RuntimeState* state, Block* block, bool* eof) {
SCOPED_RAW_TIMER(&_per_scanner_timer);
int64_t rows_read_threshold = _num_rows_read + config::doris_scanner_row_num;
if (!block->mem_reuse()) {
for (const auto slot_desc : _output_tuple_desc->slots()) {
for (auto* const slot_desc : _output_tuple_desc->slots()) {
if (!slot_desc->need_materialize()) {
// should be ignore from reading
continue;
Expand All @@ -112,7 +112,7 @@ Status VScanner::get_block(RuntimeState* state, Block* block, bool* eof) {
// 1. Get input block from scanner
{
// get block time
auto timer = _parent ? _parent->_scan_timer : _local_state->_scan_timer;
auto* timer = _parent ? _parent->_scan_timer : _local_state->_scan_timer;
SCOPED_TIMER(timer);
RETURN_IF_ERROR(_get_block_impl(state, block, eof));
if (*eof) {
Expand All @@ -125,7 +125,7 @@ Status VScanner::get_block(RuntimeState* state, Block* block, bool* eof) {

// 2. Filter the output block finally.
{
auto timer = _parent ? _parent->_filter_timer : _local_state->_filter_timer;
auto* timer = _parent ? _parent->_filter_timer : _local_state->_filter_timer;
SCOPED_TIMER(timer);
RETURN_IF_ERROR(_filter_output_block(block));
}
Expand Down
8 changes: 6 additions & 2 deletions be/src/vec/olap/vcollect_iterator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@

#include <algorithm>
#include <iterator>
#include <memory>
#include <ostream>
#include <set>
#include <utility>
Expand All @@ -38,6 +39,7 @@
#include "runtime/query_context.h"
#include "runtime/runtime_predicate.h"
#include "runtime/runtime_state.h"
#include "util/runtime_profile.h"
#include "vec/columns/column.h"
#include "vec/core/column_with_type_and_name.h"
#include "vec/core/field.h"
Expand Down Expand Up @@ -467,7 +469,7 @@ Status VCollectIterator::Level0Iterator::init(bool get_data_by_ref) {
}

auto st = refresh_current_row();
if (_get_data_by_ref && _block_view.size()) {
if (_get_data_by_ref && !_block_view.empty()) {
_ref = _block_view[0];
} else {
_ref = {_block, 0, false};
Expand Down Expand Up @@ -664,7 +666,7 @@ Status VCollectIterator::Level1Iterator::init(bool get_data_by_ref) {
break;
}
}
_heap.reset(new MergeHeap {LevelIteratorComparator(sequence_loc, _is_reverse)});
_heap = std::make_unique<MergeHeap>(LevelIteratorComparator(sequence_loc, _is_reverse));
for (auto&& child : _children) {
DCHECK(child != nullptr);
//DCHECK(child->current_row().ok());
Expand Down Expand Up @@ -770,6 +772,7 @@ Status VCollectIterator::Level1Iterator::_normal_next(IteratorRowRef* ref) {
}

Status VCollectIterator::Level1Iterator::_merge_next(Block* block) {
SCOPED_RAW_TIMER(&_reader->_stats.collect_iterator_merge_next_timer);
int target_block_row = 0;
auto target_columns = block->mutate_columns();
size_t column_count = target_columns.size();
Expand Down Expand Up @@ -851,6 +854,7 @@ Status VCollectIterator::Level1Iterator::_merge_next(Block* block) {
}

Status VCollectIterator::Level1Iterator::_normal_next(Block* block) {
SCOPED_RAW_TIMER(&_reader->_stats.collect_iterator_normal_next_timer);
auto res = _cur_child->next(block);
if (LIKELY(res.ok())) {
return Status::OK();
Expand Down
2 changes: 1 addition & 1 deletion be/src/vec/olap/vcollect_iterator.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,7 @@ class VCollectIterator {

Status init(bool get_data_by_ref = false) override;

virtual void init_for_union(bool get_data_by_ref) override;
void init_for_union(bool get_data_by_ref) override;

/* For unique and agg, rows is aggregated in block_reader, which access
* first row so we need prepare the first row ref while duplicated
Expand Down