Revert "[Improvement](sink) optimization for parallel result sink (#3… by BiteTheDDDDt · Pull Request #36628 · apache/doris

BiteTheDDDDt · 2024-06-20T12:50:55Z

…6305)"

This reverts commit fdb5891.

Proposed changes

Issue Number: close #xxx

…ache#36305)" This reverts commit fdb5891.

BiteTheDDDDt · 2024-06-20T12:51:06Z

run performance 20

github-actions

clang-tidy made some suggestions

github-actions · 2024-06-20T13:00:24Z

be/src/runtime/buffer_control_block.h

+namespace google {
+namespace protobuf {


warning: nested namespaces can be concatenated [modernize-concat-nested-namespaces]

Suggested change

namespace google {

namespace protobuf {

namespace google::protobuf {

be/src/runtime/buffer_control_block.h:36:

- } - } // namespace google + } // namespace google

github-actions · 2024-06-20T13:00:24Z

be/src/vec/sink/vmysql_result_writer.cpp


 template <bool is_binary_format>
-Status VMysqlResultWriter<is_binary_format>::write(RuntimeState* state, Block& input_block) {
+Status VMysqlResultWriter<is_binary_format>::write(Block& input_block) {


warning: function 'write' exceeds recommended size/complexity thresholds [readability-function-size]

Status VMysqlResultWriter<is_binary_format>::write(Block& input_block) { ^

Additional context

be/src/vec/sink/vmysql_result_writer.cpp:104: 107 lines including whitespace and comments (threshold 80)

Status VMysqlResultWriter<is_binary_format>::write(Block& input_block) { ^

github-actions · 2024-06-20T13:00:24Z

be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp

 }

-Status VIcebergTableWriter::write(RuntimeState* state, vectorized::Block& block) {
+Status VIcebergTableWriter::write(vectorized::Block& block) {


warning: function 'write' has cognitive complexity of 85 (threshold 50) [readability-function-cognitive-complexity]

Status VIcebergTableWriter::write(vectorized::Block& block) { ^

Additional context

be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:109: +1, including nesting penalty of 0, nesting level increased to 1

if (block.rows() == 0) { ^

be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:113: +1, including nesting penalty of 0, nesting level increased to 1

RETURN_IF_ERROR(vectorized::VExprContext::get_output_block_after_execute_exprs( ^

be/src/common/status.h:626: expanded from macro 'RETURN_IF_ERROR'

do { \ ^

be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:113: +2, including nesting penalty of 1, nesting level increased to 2

RETURN_IF_ERROR(vectorized::VExprContext::get_output_block_after_execute_exprs( ^

be/src/common/status.h:628: expanded from macro 'RETURN_IF_ERROR'

if (UNLIKELY(!_status_.ok())) { \ ^

be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:120: +1, including nesting penalty of 0, nesting level increased to 1

if (_iceberg_partition_columns.empty()) { ^

be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:125: +2, including nesting penalty of 1, nesting level increased to 2

if (writer_iter == _partitions_to_writers.end()) { ^

be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:128: +3, including nesting penalty of 2, nesting level increased to 3

} catch (doris::Exception& e) { ^

be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:132: +3, including nesting penalty of 2, nesting level increased to 3

RETURN_IF_ERROR(writer->open(_state, _profile)); ^

be/src/common/status.h:626: expanded from macro 'RETURN_IF_ERROR'

do { \ ^

be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:132: +4, including nesting penalty of 3, nesting level increased to 4

RETURN_IF_ERROR(writer->open(_state, _profile)); ^

be/src/common/status.h:628: expanded from macro 'RETURN_IF_ERROR'

if (UNLIKELY(!_status_.ok())) { \ ^

be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:133: +1, nesting level increased to 2

} else { ^

be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:134: +3, including nesting penalty of 2, nesting level increased to 3

if (writer_iter->second->written_len() > config::iceberg_sink_max_file_size) { ^

be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:145: +4, including nesting penalty of 3, nesting level increased to 4

} catch (doris::Exception& e) { ^

be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:149: +4, including nesting penalty of 3, nesting level increased to 4

RETURN_IF_ERROR(writer->open(_state, _profile)); ^

be/src/common/status.h:626: expanded from macro 'RETURN_IF_ERROR'

do { \ ^

be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:149: +5, including nesting penalty of 4, nesting level increased to 5

RETURN_IF_ERROR(writer->open(_state, _profile)); ^

be/src/common/status.h:628: expanded from macro 'RETURN_IF_ERROR'

if (UNLIKELY(!_status_.ok())) { \ ^

be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:150: +1, nesting level increased to 3

} else { ^

be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:157: +2, including nesting penalty of 1, nesting level increased to 2

RETURN_IF_ERROR(writer->write(output_block)); ^

be/src/common/status.h:626: expanded from macro 'RETURN_IF_ERROR'

do { \ ^

be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:157: +3, including nesting penalty of 2, nesting level increased to 3

RETURN_IF_ERROR(writer->write(output_block)); ^

be/src/common/status.h:628: expanded from macro 'RETURN_IF_ERROR'

if (UNLIKELY(!_status_.ok())) { \ ^

be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:168: +1, including nesting penalty of 0, nesting level increased to 1

for (int i = 0; i < output_block.rows(); ++i) { ^

be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:172: +2, including nesting penalty of 1, nesting level increased to 2

} catch (doris::Exception& e) { ^

be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:179: +2, including nesting penalty of 1, nesting level increased to 2

} catch (doris::Exception& e) { ^

be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:183: nesting level increased to 2

[&](const std::string& partition_name, int position, ^

be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:189: +3, including nesting penalty of 2, nesting level increased to 3

RETURN_IF_ERROR(writer->open(_state, _profile)); ^

be/src/common/status.h:626: expanded from macro 'RETURN_IF_ERROR'

do { \ ^

be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:189: +4, including nesting penalty of 3, nesting level increased to 4

RETURN_IF_ERROR(writer->open(_state, _profile)); ^

be/src/common/status.h:628: expanded from macro 'RETURN_IF_ERROR'

if (UNLIKELY(!_status_.ok())) { \ ^

be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:195: +3, including nesting penalty of 2, nesting level increased to 3

} catch (doris::Exception& e) { ^

be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:202: +2, including nesting penalty of 1, nesting level increased to 2

if (writer_iter == _partitions_to_writers.end()) { ^

be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:204: +3, including nesting penalty of 2, nesting level increased to 3

if (_partitions_to_writers.size() + 1 > ^

be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:210: +3, including nesting penalty of 2, nesting level increased to 3

RETURN_IF_ERROR(create_and_open_writer(partition_name, i, nullptr, 0, writer)); ^

be/src/common/status.h:626: expanded from macro 'RETURN_IF_ERROR'

do { \ ^

be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:210: +4, including nesting penalty of 3, nesting level increased to 4

RETURN_IF_ERROR(create_and_open_writer(partition_name, i, nullptr, 0, writer)); ^

be/src/common/status.h:628: expanded from macro 'RETURN_IF_ERROR'

if (UNLIKELY(!_status_.ok())) { \ ^

be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:211: +1, nesting level increased to 2

} else { ^

be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:213: +3, including nesting penalty of 2, nesting level increased to 3

if (writer_iter->second->written_len() > config::iceberg_sink_max_file_size) { ^

be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:222: +4, including nesting penalty of 3, nesting level increased to 4

RETURN_IF_ERROR(create_and_open_writer(partition_name, i, &file_name, ^

be/src/common/status.h:626: expanded from macro 'RETURN_IF_ERROR'

do { \ ^

be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:222: +5, including nesting penalty of 4, nesting level increased to 5

RETURN_IF_ERROR(create_and_open_writer(partition_name, i, &file_name, ^

be/src/common/status.h:628: expanded from macro 'RETURN_IF_ERROR'

if (UNLIKELY(!_status_.ok())) { \ ^

be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:224: +1, nesting level increased to 3

} else { ^

be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:228: +3, including nesting penalty of 2, nesting level increased to 3

if (writer_pos_iter == writer_positions.end()) { ^

be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:232: +1, nesting level increased to 3

} else { ^

github-actions · 2024-06-20T13:00:24Z

be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp

 }

-Status VIcebergTableWriter::write(RuntimeState* state, vectorized::Block& block) {
+Status VIcebergTableWriter::write(vectorized::Block& block) {


warning: function 'write' exceeds recommended size/complexity thresholds [readability-function-size]

Status VIcebergTableWriter::write(vectorized::Block& block) { ^

Additional context

be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp:107: 139 lines including whitespace and comments (threshold 80)

Status VIcebergTableWriter::write(vectorized::Block& block) { ^

github-actions · 2024-06-20T13:00:24Z

be/src/vec/sink/writer/vhive_table_writer.cpp

 }

-Status VHiveTableWriter::write(RuntimeState* state, vectorized::Block& block) {
+Status VHiveTableWriter::write(vectorized::Block& block) {


warning: function 'write' has cognitive complexity of 83 (threshold 50) [readability-function-cognitive-complexity]

Status VHiveTableWriter::write(vectorized::Block& block) { ^

Additional context

be/src/vec/sink/writer/vhive_table_writer.cpp:86: +1, including nesting penalty of 0, nesting level increased to 1

if (block.rows() == 0) { ^

be/src/vec/sink/writer/vhive_table_writer.cpp:90: +1, including nesting penalty of 0, nesting level increased to 1

RETURN_IF_ERROR(vectorized::VExprContext::get_output_block_after_execute_exprs( ^

be/src/common/status.h:626: expanded from macro 'RETURN_IF_ERROR'

do { \ ^

be/src/vec/sink/writer/vhive_table_writer.cpp:90: +2, including nesting penalty of 1, nesting level increased to 2

RETURN_IF_ERROR(vectorized::VExprContext::get_output_block_after_execute_exprs( ^

be/src/common/status.h:628: expanded from macro 'RETURN_IF_ERROR'

if (UNLIKELY(!_status_.ok())) { \ ^

be/src/vec/sink/writer/vhive_table_writer.cpp:98: +1, including nesting penalty of 0, nesting level increased to 1

if (_partition_columns_input_index.empty()) { ^

be/src/vec/sink/writer/vhive_table_writer.cpp:103: +2, including nesting penalty of 1, nesting level increased to 2

if (writer_iter == _partitions_to_writers.end()) { ^

be/src/vec/sink/writer/vhive_table_writer.cpp:106: +3, including nesting penalty of 2, nesting level increased to 3

} catch (doris::Exception& e) { ^

be/src/vec/sink/writer/vhive_table_writer.cpp:110: +3, including nesting penalty of 2, nesting level increased to 3

RETURN_IF_ERROR(writer->open(_state, _profile)); ^

be/src/common/status.h:626: expanded from macro 'RETURN_IF_ERROR'

do { \ ^

be/src/vec/sink/writer/vhive_table_writer.cpp:110: +4, including nesting penalty of 3, nesting level increased to 4

RETURN_IF_ERROR(writer->open(_state, _profile)); ^

be/src/common/status.h:628: expanded from macro 'RETURN_IF_ERROR'

if (UNLIKELY(!_status_.ok())) { \ ^

be/src/vec/sink/writer/vhive_table_writer.cpp:111: +1, nesting level increased to 2

} else { ^

be/src/vec/sink/writer/vhive_table_writer.cpp:112: +3, including nesting penalty of 2, nesting level increased to 3

if (writer_iter->second->written_len() > config::hive_sink_max_file_size) { ^

be/src/vec/sink/writer/vhive_table_writer.cpp:123: +4, including nesting penalty of 3, nesting level increased to 4

} catch (doris::Exception& e) { ^

be/src/vec/sink/writer/vhive_table_writer.cpp:127: +4, including nesting penalty of 3, nesting level increased to 4

RETURN_IF_ERROR(writer->open(_state, _profile)); ^

be/src/common/status.h:626: expanded from macro 'RETURN_IF_ERROR'

do { \ ^

be/src/vec/sink/writer/vhive_table_writer.cpp:127: +5, including nesting penalty of 4, nesting level increased to 5

RETURN_IF_ERROR(writer->open(_state, _profile)); ^

be/src/common/status.h:628: expanded from macro 'RETURN_IF_ERROR'

if (UNLIKELY(!_status_.ok())) { \ ^

be/src/vec/sink/writer/vhive_table_writer.cpp:128: +1, nesting level increased to 3

} else { ^

be/src/vec/sink/writer/vhive_table_writer.cpp:135: +2, including nesting penalty of 1, nesting level increased to 2

RETURN_IF_ERROR(writer->write(output_block)); ^

be/src/common/status.h:626: expanded from macro 'RETURN_IF_ERROR'

do { \ ^

be/src/vec/sink/writer/vhive_table_writer.cpp:135: +3, including nesting penalty of 2, nesting level increased to 3

RETURN_IF_ERROR(writer->write(output_block)); ^

be/src/common/status.h:628: expanded from macro 'RETURN_IF_ERROR'

if (UNLIKELY(!_status_.ok())) { \ ^

be/src/vec/sink/writer/vhive_table_writer.cpp:141: +1, including nesting penalty of 0, nesting level increased to 1

for (int i = 0; i < output_block.rows(); ++i) { ^

be/src/vec/sink/writer/vhive_table_writer.cpp:145: +2, including nesting penalty of 1, nesting level increased to 2

} catch (doris::Exception& e) { ^

be/src/vec/sink/writer/vhive_table_writer.cpp:152: nesting level increased to 2

[&](const std::string& partition_name, int position, ^

be/src/vec/sink/writer/vhive_table_writer.cpp:158: +3, including nesting penalty of 2, nesting level increased to 3

RETURN_IF_ERROR(writer->open(_state, _profile)); ^

be/src/common/status.h:626: expanded from macro 'RETURN_IF_ERROR'

do { \ ^

be/src/vec/sink/writer/vhive_table_writer.cpp:158: +4, including nesting penalty of 3, nesting level increased to 4

RETURN_IF_ERROR(writer->open(_state, _profile)); ^

be/src/common/status.h:628: expanded from macro 'RETURN_IF_ERROR'

if (UNLIKELY(!_status_.ok())) { \ ^

be/src/vec/sink/writer/vhive_table_writer.cpp:164: +3, including nesting penalty of 2, nesting level increased to 3

} catch (doris::Exception& e) { ^

be/src/vec/sink/writer/vhive_table_writer.cpp:171: +2, including nesting penalty of 1, nesting level increased to 2

if (writer_iter == _partitions_to_writers.end()) { ^

be/src/vec/sink/writer/vhive_table_writer.cpp:173: +3, including nesting penalty of 2, nesting level increased to 3

if (_partitions_to_writers.size() + 1 > ^

be/src/vec/sink/writer/vhive_table_writer.cpp:179: +3, including nesting penalty of 2, nesting level increased to 3

RETURN_IF_ERROR(create_and_open_writer(partition_name, i, nullptr, 0, writer)); ^

be/src/common/status.h:626: expanded from macro 'RETURN_IF_ERROR'

do { \ ^

be/src/vec/sink/writer/vhive_table_writer.cpp:179: +4, including nesting penalty of 3, nesting level increased to 4

RETURN_IF_ERROR(create_and_open_writer(partition_name, i, nullptr, 0, writer)); ^

be/src/common/status.h:628: expanded from macro 'RETURN_IF_ERROR'

if (UNLIKELY(!_status_.ok())) { \ ^

be/src/vec/sink/writer/vhive_table_writer.cpp:180: +1, nesting level increased to 2

} else { ^

be/src/vec/sink/writer/vhive_table_writer.cpp:182: +3, including nesting penalty of 2, nesting level increased to 3

if (writer_iter->second->written_len() > config::hive_sink_max_file_size) { ^

be/src/vec/sink/writer/vhive_table_writer.cpp:191: +4, including nesting penalty of 3, nesting level increased to 4

RETURN_IF_ERROR(create_and_open_writer(partition_name, i, &file_name, ^

be/src/common/status.h:626: expanded from macro 'RETURN_IF_ERROR'

do { \ ^

be/src/vec/sink/writer/vhive_table_writer.cpp:191: +5, including nesting penalty of 4, nesting level increased to 5

RETURN_IF_ERROR(create_and_open_writer(partition_name, i, &file_name, ^

be/src/common/status.h:628: expanded from macro 'RETURN_IF_ERROR'

if (UNLIKELY(!_status_.ok())) { \ ^

be/src/vec/sink/writer/vhive_table_writer.cpp:193: +1, nesting level increased to 3

} else { ^

be/src/vec/sink/writer/vhive_table_writer.cpp:197: +3, including nesting penalty of 2, nesting level increased to 3

if (writer_pos_iter == writer_positions.end()) { ^

be/src/vec/sink/writer/vhive_table_writer.cpp:201: +1, nesting level increased to 3

} else { ^

github-actions · 2024-06-20T13:00:25Z

be/src/vec/sink/writer/vhive_table_writer.cpp

 }

-Status VHiveTableWriter::write(RuntimeState* state, vectorized::Block& block) {
+Status VHiveTableWriter::write(vectorized::Block& block) {


warning: function 'write' exceeds recommended size/complexity thresholds [readability-function-size]

Status VHiveTableWriter::write(vectorized::Block& block) { ^

Additional context

be/src/vec/sink/writer/vhive_table_writer.cpp:83: 132 lines including whitespace and comments (threshold 80)

Status VHiveTableWriter::write(vectorized::Block& block) { ^

github-actions · 2024-06-20T13:00:25Z

be/src/vec/sink/writer/vhive_table_writer.h

    VHiveTableWriter(const TDataSink& t_sink, const VExprContextSPtrs& output_exprs);

-    ~VHiveTableWriter() override = default;
+    ~VHiveTableWriter() = default;


warning: annotate this function with 'override' or (rarely) 'final' [modernize-use-override]

Suggested change

~VHiveTableWriter() = default;

~VHiveTableWriter() override = default;

BiteTheDDDDt · 2024-06-20T14:08:35Z

run buildall

github-actions · 2024-06-21T01:43:52Z

PR approved by at least one committer and no changes requested.

github-actions · 2024-06-21T01:43:53Z

PR approved by anyone and no changes requested.

#36628) …6305)" This reverts commit fdb5891.

## Proposed changes #36305 #36628 reverted coz some bug about local exchange, this pr do not change local exchange now

Revert "[Improvement](sink) optimization for parallel result sink (ap…

18cf613

…ache#36305)" This reverts commit fdb5891.

github-actions bot reviewed Jun 20, 2024

View reviewed changes

Gabriel39 approved these changes Jun 21, 2024

View reviewed changes

github-actions bot added the approved Indicates a PR has been approved by one committer. label Jun 21, 2024

github-actions bot added the reviewed label Jun 21, 2024

hello-stephen approved these changes Jun 21, 2024

View reviewed changes

Gabriel39 merged commit 46457bc into apache:master Jun 21, 2024

BiteTheDDDDt mentioned this pull request Jun 21, 2024

[Improvement](sink) optimization for parallel result sink #36667

Merged

dataroaring pushed a commit that referenced this pull request Jun 21, 2024

Revert "[Improvement](sink) optimization for parallel result sink (#3… (

4bdc67b

#36628) …6305)" This reverts commit fdb5891.

BiteTheDDDDt added a commit that referenced this pull request Jun 24, 2024

[Improvement](sink) optimization for parallel result sink (#36667)

502f431

## Proposed changes #36305 #36628 reverted coz some bug about local exchange, this pr do not change local exchange now

dataroaring pushed a commit that referenced this pull request Jun 26, 2024

[Improvement](sink) optimization for parallel result sink (#36667)

f93b915

## Proposed changes #36305 #36628 reverted coz some bug about local exchange, this pr do not change local exchange now

dataroaring added the dev/3.0.0-merged label Aug 5, 2024

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Revert "[Improvement](sink) optimization for parallel result sink (#3…#36628

Revert "[Improvement](sink) optimization for parallel result sink (#3…#36628
Gabriel39 merged 1 commit intoapache:masterfrom
BiteTheDDDDt:rv_0620

BiteTheDDDDt commented Jun 20, 2024

Uh oh!

BiteTheDDDDt commented Jun 20, 2024

Uh oh!

github-actions bot left a comment

Uh oh!

github-actions bot Jun 20, 2024

Uh oh!

github-actions bot Jun 20, 2024

Uh oh!

github-actions bot Jun 20, 2024

Uh oh!

github-actions bot Jun 20, 2024

Uh oh!

github-actions bot Jun 20, 2024

Uh oh!

github-actions bot Jun 20, 2024

Uh oh!

github-actions bot Jun 20, 2024

Uh oh!

BiteTheDDDDt commented Jun 20, 2024

Uh oh!

github-actions bot commented Jun 21, 2024

Uh oh!

github-actions bot commented Jun 21, 2024

Uh oh!

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

4 participants

	namespace google {
	namespace protobuf {
	namespace google::protobuf {

	~VHiveTableWriter() = default;
	~VHiveTableWriter() override = default;

Conversation

BiteTheDDDDt commented Jun 20, 2024

Proposed changes

Uh oh!

BiteTheDDDDt commented Jun 20, 2024

Uh oh!

github-actions bot left a comment

Choose a reason for hiding this comment

Uh oh!

github-actions bot Jun 20, 2024

Choose a reason for hiding this comment

Uh oh!

github-actions bot Jun 20, 2024

Choose a reason for hiding this comment

Uh oh!

github-actions bot Jun 20, 2024

Choose a reason for hiding this comment

Uh oh!

github-actions bot Jun 20, 2024

Choose a reason for hiding this comment

Uh oh!

github-actions bot Jun 20, 2024

Choose a reason for hiding this comment

Uh oh!

github-actions bot Jun 20, 2024

Choose a reason for hiding this comment

Uh oh!

github-actions bot Jun 20, 2024

Choose a reason for hiding this comment

Uh oh!

BiteTheDDDDt commented Jun 20, 2024

Uh oh!

github-actions bot commented Jun 21, 2024

Uh oh!

github-actions bot commented Jun 21, 2024

Uh oh!

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

4 participants