From aa06570ebcf503adb3e0b170f4e6f04a0da17088 Mon Sep 17 00:00:00 2001 From: liuhangyuan1 Date: Tue, 26 Mar 2019 18:39:31 +0800 Subject: [PATCH 01/11] Fix syntax error lable -> label --- docs/help/Contents/Data Manipulation/streaming.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/help/Contents/Data Manipulation/streaming.md b/docs/help/Contents/Data Manipulation/streaming.md index ed4eeda72f23ba..5872eaf85a744b 100644 --- a/docs/help/Contents/Data Manipulation/streaming.md +++ b/docs/help/Contents/Data Manipulation/streaming.md @@ -65,19 +65,19 @@ ## example 1. 将本地文件'testData'中的数据导入到数据库'testDb'中'testTbl'的表，使用Label用于去重 - curl --location-trusted -u root -H "lable:123" -T testData http://host:port/api/testDb/testTbl/_stream_load + curl --location-trusted -u root -H "label:123" -T testData http://host:port/api/testDb/testTbl/_stream_load 2. 将本地文件'testData'中的数据导入到数据库'testDb'中'testTbl'的表，使用Label用于去重, 并且只导入k1等于20180601的数据 - curl --location-trusted -u root -H "lable:123" -H "where: k1=20180601" -T testData http://host:port/api/testDb/testTbl/_stream_load + curl --location-trusted -u root -H "label:123" -H "where: k1=20180601" -T testData http://host:port/api/testDb/testTbl/_stream_load 3. 将本地文件'testData'中的数据导入到数据库'testDb'中'testTbl'的表, 允许20%的错误率（用户是defalut_cluster中的） - curl --location-trusted -u root -H "lable:123" -H "max_filter_ratio:0.2" -T testData http://host:port/api/testDb/testTbl/_stream_load + curl --location-trusted -u root -H "label:123" -H "max_filter_ratio:0.2" -T testData http://host:port/api/testDb/testTbl/_stream_load 4. 将本地文件'testData'中的数据导入到数据库'testDb'中'testTbl'的表, 允许20%的错误率，并且指定文件的列名（用户是defalut_cluster中的） - curl --location-trusted -u root -H "lable:123" -H "max_filter_ratio:0.2" -H "columns: k2, k1, v1" -T testData http://host:port/api/testDb/testTbl/_stream_load + curl --location-trusted -u root -H "label:123" -H "max_filter_ratio:0.2" -H "columns: k2, k1, v1" -T testData http://host:port/api/testDb/testTbl/_stream_load 5. 将本地文件'testData'中的数据导入到数据库'testDb'中'testTbl'的表中的p1, p2分区, 允许20%的错误率。 - curl --location-trusted -u root -H "lable:123" -H "max_filter_ratio:0.2" -H "partitions: p1, p2" -T testData http://host:port/api/testDb/testTbl/_stream_load + curl --location-trusted -u root -H "label:123" -H "max_filter_ratio:0.2" -H "partitions: p1, p2" -T testData http://host:port/api/testDb/testTbl/_stream_load 6. 使用streaming方式导入（用户是defalut_cluster中的） seq 1 10 | awk '{OFS="\t"}{print $1, $1 * 10}' | curl --location-trusted -u root -T - http://host:port/api/testDb/testTbl/_stream_load From 41f6b5eb52fc1f45c31090ca309c5db508109362 Mon Sep 17 00:00:00 2001 From: liuhangyuan1 Date: Thu, 4 Apr 2019 16:06:40 +0800 Subject: [PATCH 02/11] Fix docs 'CANCEL DECOMMISSION' --- docs/help/Contents/Administration/admin_stmt.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/help/Contents/Administration/admin_stmt.md b/docs/help/Contents/Administration/admin_stmt.md index c0e9c0fe3eb076..95e469c533839d 100644 --- a/docs/help/Contents/Administration/admin_stmt.md +++ b/docs/help/Contents/Administration/admin_stmt.md @@ -27,7 +27,7 @@ 2) heartbeat_port 为该节点的心跳端口 3) 增加和删除节点为同步操作。这两种操作不考虑节点上已有的数据，节点直接从元数据中删除，请谨慎使用。 4) 节点下线操作用于安全下线节点。该操作为异步操作。如果成功，节点最终会从元数据中删除。如果失败，则不会完成下线。 - 5) 可以手动取消节点下线操作。详见 CANCEL ALTER SYSTEM + 5) 可以手动取消节点下线操作。详见 CANCEL DECOMMISSION 6) Load error hub: 当前支持两种类型的 Hub：Mysql 和 Broker。需在 PROPERTIES 中指定 "type" = "mysql" 或 "type" = "broker"。 如果需要删除当前的 load error hub，可以将 type 设为 null。 @@ -92,20 +92,20 @@ ## keyword ALTER,SYSTEM,BACKEND,BROKER,FREE -# CANCEL ALTER SYSTEM +# CANCEL DECOMMISSION ## description 该语句用于撤销一个节点下线操作。（仅管理员使用！） 语法： - CANCEL ALTER SYSTEM DECOMMISSION BACKEND "host:heartbeat_port"[,"host:heartbeat_port"...]; + CANCEL DECOMMISSION BACKEND "host:heartbeat_port"[,"host:heartbeat_port"...]; ## example 1. 取消两个节点的下线操作： - CANCEL ALTER SYSTEM DECOMMISSION BACKEND "host1:port", "host2:port"; + CANCEL DECOMMISSION BACKEND "host1:port", "host2:port"; ## keyword - CANCEL,ALTER,SYSTEM,BACKEND + CANCEL,DECOMMISSION,BACKEND # CREATE CLUSTER ## description From 90e2c4fe5e8c1949e5a99d9659c7d1a8e9770f92 Mon Sep 17 00:00:00 2001 From: liuhangyuan1 Date: Thu, 11 Apr 2019 16:00:03 +0800 Subject: [PATCH 03/11] Fixed a bug in the Bloom filter where column names could not be found due to spaces --- .../main/java/org/apache/doris/common/util/PropertyAnalyzer.java | 1 + 1 file changed, 1 insertion(+) diff --git a/fe/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index ecb7b8f69ecd99..78c0d94ad69417 100644 --- a/fe/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -257,6 +257,7 @@ public static Set analyzeBloomFilterColumns(Map properti String[] bfColumnArr = bfColumnsStr.split(COMMA_SEPARATOR); Set bfColumnSet = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); for (String bfColumn : bfColumnArr) { + bfColumn = bfColumn.trim(); boolean found = false; for (Column column : columns) { if (column.getName().equalsIgnoreCase(bfColumn)) { From 7e43e8ea04f1a9b1418a7b8721cb9e2ece21b9f1 Mon Sep 17 00:00:00 2001 From: liuhangyuan1 Date: Mon, 15 Apr 2019 14:35:26 +0800 Subject: [PATCH 04/11] Fix bug in concat_ws --- be/src/exprs/string_functions.cpp | 23 ++++++++++++------- .../doris/rewrite/FoldConstantsRule.java | 1 + 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/be/src/exprs/string_functions.cpp b/be/src/exprs/string_functions.cpp index 7fcd5dc3a212b4..777b0279b31819 100644 --- a/be/src/exprs/string_functions.cpp +++ b/be/src/exprs/string_functions.cpp @@ -516,17 +516,17 @@ StringVal StringFunctions::concat_ws( return strs[0]; } - if (strs[0].is_null) { + if (strs[0].is_null && 0 == sep.len) { return StringVal::null(); } - int32_t total_size = strs[0].len; + int32_t total_size = strs[0].is_null ? 0 : strs[0].len; // Loop once to compute the final size and reserve space. for (int32_t i = 1; i < num_children; ++i) { - if (strs[i].is_null) { + if (strs[i].is_null && 0 == sep.len) { return StringVal::null(); } - total_size += sep.len + strs[i].len; + total_size += strs[i].is_null ? 0 : (sep.len + strs[i].len); } // TODO pengyubing @@ -535,11 +535,18 @@ StringVal StringFunctions::concat_ws( uint8_t* ptr = result.ptr; // Loop again to append the data. - memcpy(ptr, strs[0].ptr, strs[0].len); - ptr += strs[0].len; + if(!strs[0].is_null) { + memcpy(ptr, strs[0].ptr, strs[0].len); + ptr += strs[0].len; + } for (int32_t i = 1; i < num_children; ++i) { - memcpy(ptr, sep.ptr, sep.len); - ptr += sep.len; + if(strs[i].is_null) { + continue; + } + if(!(strs[0].is_null && 1 == i)) { + memcpy(ptr, sep.ptr, sep.len); + ptr += sep.len; + } memcpy(ptr, strs[i].ptr, strs[i].len); ptr += strs[i].len; } diff --git a/fe/src/main/java/org/apache/doris/rewrite/FoldConstantsRule.java b/fe/src/main/java/org/apache/doris/rewrite/FoldConstantsRule.java index 16b06e50b9dcc6..28e6a621f03b81 100644 --- a/fe/src/main/java/org/apache/doris/rewrite/FoldConstantsRule.java +++ b/fe/src/main/java/org/apache/doris/rewrite/FoldConstantsRule.java @@ -204,6 +204,7 @@ private synchronized void registerFunctions() { new ImmutableSet.Builder(); setBuilder.add("if"); setBuilder.add("hll_hash"); + setBuilder.add("concat_ws"); this.nonNullResultWithNullParamFunctions = setBuilder.build(); } From 80d6a8d53de141949020a1ec4295db3744e34f1e Mon Sep 17 00:00:00 2001 From: liuhangyuan1 Date: Mon, 15 Apr 2019 15:40:13 +0800 Subject: [PATCH 05/11] Fix bug in concat_ws --- be/src/exprs/string_functions.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/be/src/exprs/string_functions.cpp b/be/src/exprs/string_functions.cpp index 777b0279b31819..0837941d422651 100644 --- a/be/src/exprs/string_functions.cpp +++ b/be/src/exprs/string_functions.cpp @@ -516,14 +516,14 @@ StringVal StringFunctions::concat_ws( return strs[0]; } - if (strs[0].is_null && 0 == sep.len) { + if (strs[0].is_null && sep.is_null) { return StringVal::null(); } int32_t total_size = strs[0].is_null ? 0 : strs[0].len; // Loop once to compute the final size and reserve space. for (int32_t i = 1; i < num_children; ++i) { - if (strs[i].is_null && 0 == sep.len) { + if (strs[i].is_null && sep.is_null) { return StringVal::null(); } total_size += strs[i].is_null ? 0 : (sep.len + strs[i].len); @@ -535,15 +535,15 @@ StringVal StringFunctions::concat_ws( uint8_t* ptr = result.ptr; // Loop again to append the data. - if(!strs[0].is_null) { + if (!strs[0].is_null) { memcpy(ptr, strs[0].ptr, strs[0].len); ptr += strs[0].len; } for (int32_t i = 1; i < num_children; ++i) { - if(strs[i].is_null) { + if (strs[i].is_null) { continue; } - if(!(strs[0].is_null && 1 == i)) { + if (!(strs[0].is_null && 1 == i)) { memcpy(ptr, sep.ptr, sep.len); ptr += sep.len; } From 029bc1527ef321462a0e9c48fba3589e8ad093ff Mon Sep 17 00:00:00 2001 From: liuhangyuan1 Date: Mon, 15 Apr 2019 16:46:50 +0800 Subject: [PATCH 06/11] Fix bug in concat_ws --- be/src/exprs/string_functions.cpp | 63 ++++++++++++++++++++----------- 1 file changed, 41 insertions(+), 22 deletions(-) diff --git a/be/src/exprs/string_functions.cpp b/be/src/exprs/string_functions.cpp index 0837941d422651..082b32dd9b3751 100644 --- a/be/src/exprs/string_functions.cpp +++ b/be/src/exprs/string_functions.cpp @@ -500,7 +500,37 @@ StringVal StringFunctions::regexp_replace( StringVal StringFunctions::concat( FunctionContext* context, int num_children, const StringVal* strs) { - return concat_ws(context, StringVal(), num_children, strs); + DCHECK_GE(num_children, 1); + + // Pass through if there's only one argument + if (num_children == 1) { + return strs[0]; + } + + if (strs[0].is_null) { + return StringVal::null(); + } + + int32_t total_size = strs[0].len; + // Loop once to compute the final size and reserve space. + for (int32_t i = 1; i < num_children; ++i) { + if (strs[i].is_null) { + return StringVal::null(); + } + total_size += strs[i].is_null ? 0 : strs[i].len; + } + + // TODO pengyubing + // StringVal result = StringVal::create_temp_string_val(context, total_size); + StringVal result(context, total_size); + uint8_t* ptr = result.ptr; + + // Loop again to append the data. + for (int32_t i = 0; i < num_children; ++i) { + memcpy(ptr, strs[i].ptr, strs[i].len); + ptr += strs[i].len; + } + return result; } StringVal StringFunctions::concat_ws( @@ -516,16 +546,9 @@ StringVal StringFunctions::concat_ws( return strs[0]; } - if (strs[0].is_null && sep.is_null) { - return StringVal::null(); - } int32_t total_size = strs[0].is_null ? 0 : strs[0].len; - // Loop once to compute the final size and reserve space. for (int32_t i = 1; i < num_children; ++i) { - if (strs[i].is_null && sep.is_null) { - return StringVal::null(); - } total_size += strs[i].is_null ? 0 : (sep.len + strs[i].len); } @@ -533,22 +556,18 @@ StringVal StringFunctions::concat_ws( // StringVal result = StringVal::create_temp_string_val(context, total_size); StringVal result(context, total_size); uint8_t* ptr = result.ptr; - + bool not_first = false; // Loop again to append the data. - if (!strs[0].is_null) { - memcpy(ptr, strs[0].ptr, strs[0].len); - ptr += strs[0].len; - } - for (int32_t i = 1; i < num_children; ++i) { - if (strs[i].is_null) { - continue; - } - if (!(strs[0].is_null && 1 == i)) { - memcpy(ptr, sep.ptr, sep.len); - ptr += sep.len; + for (int32_t i = 0; i < num_children; ++i) { + if (!strs[i].is_null) { + if (not_first) { + memcpy(ptr, sep.ptr, sep.len); + ptr += sep.len; + } + memcpy(ptr, strs[i].ptr, strs[i].len); + ptr += strs[i].len; + not_first = true; } - memcpy(ptr, strs[i].ptr, strs[i].len); - ptr += strs[i].len; } return result; } From 86a18f788afc70920531cf3afc7227fcbde6ff79 Mon Sep 17 00:00:00 2001 From: liuhangyuan1 Date: Mon, 15 Apr 2019 16:56:02 +0800 Subject: [PATCH 07/11] Fix bug in concat_ws --- be/src/exprs/string_functions.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/be/src/exprs/string_functions.cpp b/be/src/exprs/string_functions.cpp index 082b32dd9b3751..7cebd5b60891d0 100644 --- a/be/src/exprs/string_functions.cpp +++ b/be/src/exprs/string_functions.cpp @@ -541,11 +541,6 @@ StringVal StringFunctions::concat_ws( return StringVal::null(); } - // Pass through if there's only one argument - if (num_children == 1) { - return strs[0]; - } - int32_t total_size = strs[0].is_null ? 0 : strs[0].len; // Loop once to compute the final size and reserve space. for (int32_t i = 1; i < num_children; ++i) { From 28f13a7a25ad11d4f68269a0cd55eb27ed4931ba Mon Sep 17 00:00:00 2001 From: liuhangyuan1 Date: Mon, 15 Apr 2019 17:38:59 +0800 Subject: [PATCH 08/11] Fix bug in concat_ws --- be/src/exprs/string_functions.cpp | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/be/src/exprs/string_functions.cpp b/be/src/exprs/string_functions.cpp index 7cebd5b60891d0..51f9e2fd01ccad 100644 --- a/be/src/exprs/string_functions.cpp +++ b/be/src/exprs/string_functions.cpp @@ -507,17 +507,13 @@ StringVal StringFunctions::concat( return strs[0]; } - if (strs[0].is_null) { - return StringVal::null(); - } - - int32_t total_size = strs[0].len; // Loop once to compute the final size and reserve space. - for (int32_t i = 1; i < num_children; ++i) { + int32_t total_size = 0; + for (int32_t i = 0; i < num_children; ++i) { if (strs[i].is_null) { return StringVal::null(); } - total_size += strs[i].is_null ? 0 : strs[i].len; + total_size += strs[i].len; } // TODO pengyubing @@ -541,9 +537,9 @@ StringVal StringFunctions::concat_ws( return StringVal::null(); } - int32_t total_size = strs[0].is_null ? 0 : strs[0].len; // Loop once to compute the final size and reserve space. - for (int32_t i = 1; i < num_children; ++i) { + int32_t total_size = 0; + for (int32_t i = 0; i < num_children; ++i) { total_size += strs[i].is_null ? 0 : (sep.len + strs[i].len); } @@ -554,15 +550,16 @@ StringVal StringFunctions::concat_ws( bool not_first = false; // Loop again to append the data. for (int32_t i = 0; i < num_children; ++i) { - if (!strs[i].is_null) { - if (not_first) { - memcpy(ptr, sep.ptr, sep.len); - ptr += sep.len; - } - memcpy(ptr, strs[i].ptr, strs[i].len); - ptr += strs[i].len; - not_first = true; + if (strs[i].is_null) { + continue; + } + if (not_first) { + memcpy(ptr, sep.ptr, sep.len); + ptr += sep.len; } + memcpy(ptr, strs[i].ptr, strs[i].len); + ptr += strs[i].len; + not_first = true; } return result; } From 4990c8294c94f2c85380b5236667d0708c998511 Mon Sep 17 00:00:00 2001 From: liuhangyuan1 Date: Mon, 15 Apr 2019 18:45:59 +0800 Subject: [PATCH 09/11] Fix bug in concat_ws --- be/src/exprs/string_functions.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/be/src/exprs/string_functions.cpp b/be/src/exprs/string_functions.cpp index 51f9e2fd01ccad..f80ae515bb7879 100644 --- a/be/src/exprs/string_functions.cpp +++ b/be/src/exprs/string_functions.cpp @@ -539,15 +539,23 @@ StringVal StringFunctions::concat_ws( // Loop once to compute the final size and reserve space. int32_t total_size = 0; + bool not_first = false; for (int32_t i = 0; i < num_children; ++i) { - total_size += strs[i].is_null ? 0 : (sep.len + strs[i].len); + if (strs[i].is_null) { + continue; + } + if (not_first) { + total_size += sep.len; + } + total_size += strs[i].len; + not_first = true; } // TODO pengyubing // StringVal result = StringVal::create_temp_string_val(context, total_size); StringVal result(context, total_size); uint8_t* ptr = result.ptr; - bool not_first = false; + not_first = false; // Loop again to append the data. for (int32_t i = 0; i < num_children; ++i) { if (strs[i].is_null) { From 161d2c3a5216decc265efe1ed21a96674f2904e9 Mon Sep 17 00:00:00 2001 From: liuhangyuan1 Date: Mon, 15 Apr 2019 19:18:14 +0800 Subject: [PATCH 10/11] Fix bug in concat_ws --- be/src/exprs/string_functions.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/be/src/exprs/string_functions.cpp b/be/src/exprs/string_functions.cpp index f80ae515bb7879..b0961d11f22690 100644 --- a/be/src/exprs/string_functions.cpp +++ b/be/src/exprs/string_functions.cpp @@ -516,8 +516,6 @@ StringVal StringFunctions::concat( total_size += strs[i].len; } - // TODO pengyubing - // StringVal result = StringVal::create_temp_string_val(context, total_size); StringVal result(context, total_size); uint8_t* ptr = result.ptr; @@ -551,8 +549,6 @@ StringVal StringFunctions::concat_ws( not_first = true; } - // TODO pengyubing - // StringVal result = StringVal::create_temp_string_val(context, total_size); StringVal result(context, total_size); uint8_t* ptr = result.ptr; not_first = false; From 299f52d6a8dfad9f042c90bae032e89456e0a8d8 Mon Sep 17 00:00:00 2001 From: liuhangyuan1 Date: Fri, 19 Apr 2019 20:05:58 +0800 Subject: [PATCH 11/11] Fix doc in alter bloom filter --- docs/help/Contents/Data Definition/ddl_stmt.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/help/Contents/Data Definition/ddl_stmt.md b/docs/help/Contents/Data Definition/ddl_stmt.md index b1daca6e51b338..eaf13fa65c32bc 100644 --- a/docs/help/Contents/Data Definition/ddl_stmt.md +++ b/docs/help/Contents/Data Definition/ddl_stmt.md @@ -580,8 +580,7 @@ ORDER BY (k3,k1,k2,v2,v1) FROM example_rollup_index; 11. 修改表的 bloom filter 列 - ALTER TABLE example_db.my_table - PROPERTIES ("bloom_filter_columns"="k1,k2,k3"); + ALTER TABLE example_db.my_table set ("bloom_filter_columns"="k1,k2,k3"); 也可以合并到上面的 schema change 操作中 ALTER TABLE example_db.my_table