New join test #26730

BiteTheDDDDt · 2023-11-10T02:36:41Z

Proposed changes

Issue Number: close #xxx

Further comments

If this is a relatively large or complex change, kick off the discussion at dev@doris.apache.org by explaining why you chose the solution you did and what alternatives you considered, etc...

Co-authored-by: BiteTheDDDDt <pxl290@qq.com>

update

github-actions

clang-tidy made some suggestions

github-actions · 2023-11-10T02:48:01Z

be/src/exprs/bloom_filter_func.h


 struct DateTimeFindOp : public CommonFindOp<VecDateTimeValue> {
-    bool find_olap_engine(const BloomFilterAdaptor& bloom_filter, const void* data) const {
+    bool find_olap_engine(const BloomFilterAdaptor& bloom_filter, const void* data) const override {


warning: method 'find_olap_engine' can be made static [readability-convert-member-functions-to-static]

Suggested change

bool find_olap_engine(const BloomFilterAdaptor& bloom_filter, const void* data) const override {

static bool find_olap_engine(const BloomFilterAdaptor& bloom_filter, const void* data) override {

github-actions · 2023-11-10T02:48:01Z

be/src/exprs/bloom_filter_func.h


 struct DateFindOp : public CommonFindOp<VecDateTimeValue> {
-    bool find_olap_engine(const BloomFilterAdaptor& bloom_filter, const void* data) const {
+    bool find_olap_engine(const BloomFilterAdaptor& bloom_filter, const void* data) const override {


warning: method 'find_olap_engine' can be made static [readability-convert-member-functions-to-static]

Suggested change

bool find_olap_engine(const BloomFilterAdaptor& bloom_filter, const void* data) const override {

static bool find_olap_engine(const BloomFilterAdaptor& bloom_filter, const void* data) override {

github-actions · 2023-11-10T02:48:01Z

be/src/exprs/bloom_filter_func.h


 struct DecimalV2FindOp : public CommonFindOp<DecimalV2Value> {
-    bool find_olap_engine(const BloomFilterAdaptor& bloom_filter, const void* data) const {
+    bool find_olap_engine(const BloomFilterAdaptor& bloom_filter, const void* data) const override {


warning: method 'find_olap_engine' can be made static [readability-convert-member-functions-to-static]

Suggested change

bool find_olap_engine(const BloomFilterAdaptor& bloom_filter, const void* data) const override {

static bool find_olap_engine(const BloomFilterAdaptor& bloom_filter, const void* data) override {

github-actions · 2023-11-10T02:48:01Z

be/src/exprs/runtime_filter.cpp


-    void bitmap_filter_insert_batch(const vectorized::ColumnPtr column,
-                                    const std::vector<int>& rows) {
+    void bitmap_filter_insert_batch(const vectorized::ColumnPtr column, size_t start) {


warning: method 'bitmap_filter_insert_batch' can be made static [readability-convert-member-functions-to-static]

Suggested change

void bitmap_filter_insert_batch(const vectorized::ColumnPtr column, size_t start) {

static void bitmap_filter_insert_batch(const vectorized::ColumnPtr column, size_t start) {

github-actions · 2023-11-10T02:48:01Z

be/src/exprs/runtime_filter_slots.h

            : _build_expr_context(build_expr_ctxs), _runtime_filter_descs(runtime_filter_descs) {}

-    Status init(RuntimeState* state, int64_t hash_table_size, size_t build_bf_cardinality) {
+    Status init(RuntimeState* state, int64_t hash_table_size) {


warning: method 'init' can be made static [readability-convert-member-functions-to-static]

Suggested change

Status init(RuntimeState* state, int64_t hash_table_size) {

static Status init(RuntimeState* state, int64_t hash_table_size) {

github-actions · 2023-11-10T02:48:02Z

be/src/vec/columns/column_string.cpp

+    offsets.resize(offsets.size() + indices_end - indices_begin);
+    auto* dst_offsets_data = offsets.data();
+
+    for (auto x = indices_begin; x != indices_end; ++x) {


warning: 'auto x' can be declared as 'const auto *x' [readability-qualified-auto]

Suggested change

for (auto x = indices_begin; x != indices_end; ++x) {

for (const auto *x = indices_begin; x != indices_end; ++x) {

github-actions · 2023-11-10T02:48:02Z

be/src/vec/columns/column_string.cpp

+    auto* dst_data_ptr = chars.data();
+
+    size_t dst_chars_pos = old_char_size;
+    for (auto x = indices_begin; x != indices_end; ++x) {


warning: 'auto x' can be declared as 'const auto *x' [readability-qualified-auto]

Suggested change

for (auto x = indices_begin; x != indices_end; ++x) {

for (const auto *x = indices_begin; x != indices_end; ++x) {

github-actions · 2023-11-10T02:48:02Z

be/src/vec/columns/column_struct.cpp

    }
 }

+void ColumnStruct::insert_indices_from_join(const IColumn& src, const uint32_t* indices_begin,


warning: method 'insert_indices_from_join' can be made static [readability-convert-member-functions-to-static]

be/src/vec/columns/column_struct.h:127:

- void insert_indices_from_join(const IColumn& src, const uint32_t* indices_begin, + static void insert_indices_from_join(const IColumn& src, const uint32_t* indices_begin,

github-actions · 2023-11-10T02:48:02Z

be/src/vec/common/hash_table/hash_map.h


 #pragma once

+#include <gen_cpp/PlanNodes_types.h>


warning: 'gen_cpp/PlanNodes_types.h' file not found [clang-diagnostic-error]

#include <gen_cpp/PlanNodes_types.h> ^

github-actions · 2023-11-10T02:48:02Z

be/src/vec/common/hash_table/hash_map.h

+    using HashMapTable<Key, Cell, Hash, Grower, Allocator>::HashMapTable;
+
+    static uint32_t calc_bucket_size(size_t num_elem) {
+        size_t expect_bucket_size = num_elem + (num_elem - 1) / 7;


warning: 7 is a magic number; consider replacing it with a named constant [readability-magic-numbers]

size_t expect_bucket_size = num_elem + (num_elem - 1) / 7; ^

github-actions

clang-tidy made some suggestions

github-actions · 2023-11-18T15:18:48Z

be/src/vec/common/hash_table/hash_map.h

+    std::vector<uint8_t> visited;
+
+    uint32_t bucket_size = 1;
+    int max_batch_size = 4064;


warning: 4064 is a magic number; consider replacing it with a named constant [readability-magic-numbers]

int max_batch_size = 4064; ^

* update some fix on join * save code --------- Co-authored-by: BiteTheDDDDt <pxl290@qq.com>

HappenLee and others added 11 commits November 3, 2023 13:05

Join rewrite (#26140)

41b2073

Co-authored-by: BiteTheDDDDt <pxl290@qq.com>

update rf

8a077b1

update

fix bug in shared hash table (#26158)

76592a3

support tpch q21 (#26248)

fa9578d

fix rf and outer join performance (#26320)

1c85069

opt for rf

d99292d

fix tpch q13

6c9e593

fix profile and change timer profile (#26468)

20b3070

opt for some join (#26513)

707b7a4

opt for right anti (#26697)

40f81a2

fix runtime filter performance (#26727)

a7bc11f

github-actions bot reviewed Nov 10, 2023

View reviewed changes

some fix (#26761)

f82f9f1

github-actions bot reviewed Nov 18, 2023

View reviewed changes

HappenLee and others added 3 commits November 21, 2023 12:04

fix outer join and other join conjuncts (#27319)

5d5e3bc

* update some fix on join * save code --------- Co-authored-by: BiteTheDDDDt <pxl290@qq.com>

[Rf](bf) fix bug of in change bf (#27355)

c771965

[performance] opt the join and other join conjuncts (#27512)

41dd3bf

BiteTheDDDDt closed this Nov 29, 2023

morningman deleted the new_join branch December 18, 2023 04:45

	bool find_olap_engine(const BloomFilterAdaptor& bloom_filter, const void* data) const override {
	static bool find_olap_engine(const BloomFilterAdaptor& bloom_filter, const void* data) override {

	void bitmap_filter_insert_batch(const vectorized::ColumnPtr column, size_t start) {
	static void bitmap_filter_insert_batch(const vectorized::ColumnPtr column, size_t start) {

	Status init(RuntimeState* state, int64_t hash_table_size) {
	static Status init(RuntimeState* state, int64_t hash_table_size) {

	for (auto x = indices_begin; x != indices_end; ++x) {
	for (const auto *x = indices_begin; x != indices_end; ++x) {

New join test #26730

New join test #26730

Uh oh!

Conversation

BiteTheDDDDt commented Nov 10, 2023

Proposed changes

Further comments

Uh oh!

github-actions bot left a comment

Choose a reason for hiding this comment

Uh oh!

github-actions bot Nov 10, 2023

Choose a reason for hiding this comment

Uh oh!

github-actions bot Nov 10, 2023

Choose a reason for hiding this comment

Uh oh!

github-actions bot Nov 10, 2023

Choose a reason for hiding this comment

Uh oh!

github-actions bot Nov 10, 2023

Choose a reason for hiding this comment

Uh oh!

github-actions bot Nov 10, 2023

Choose a reason for hiding this comment

Uh oh!

github-actions bot Nov 10, 2023

Choose a reason for hiding this comment

Uh oh!

github-actions bot Nov 10, 2023

Choose a reason for hiding this comment

Uh oh!

github-actions bot Nov 10, 2023

Choose a reason for hiding this comment

Uh oh!

github-actions bot Nov 10, 2023

Choose a reason for hiding this comment

Uh oh!

github-actions bot Nov 10, 2023

Choose a reason for hiding this comment

Uh oh!

github-actions bot left a comment

Choose a reason for hiding this comment

Uh oh!

github-actions bot Nov 18, 2023

Choose a reason for hiding this comment

Uh oh!

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

3 participants