From aa06570ebcf503adb3e0b170f4e6f04a0da17088 Mon Sep 17 00:00:00 2001 From: liuhangyuan1 Date: Tue, 26 Mar 2019 18:39:31 +0800 Subject: [PATCH 1/4] Fix syntax error lable -> label --- docs/help/Contents/Data Manipulation/streaming.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/help/Contents/Data Manipulation/streaming.md b/docs/help/Contents/Data Manipulation/streaming.md index ed4eeda72f23ba..5872eaf85a744b 100644 --- a/docs/help/Contents/Data Manipulation/streaming.md +++ b/docs/help/Contents/Data Manipulation/streaming.md @@ -65,19 +65,19 @@ ## example 1. 将本地文件'testData'中的数据导入到数据库'testDb'中'testTbl'的表，使用Label用于去重 - curl --location-trusted -u root -H "lable:123" -T testData http://host:port/api/testDb/testTbl/_stream_load + curl --location-trusted -u root -H "label:123" -T testData http://host:port/api/testDb/testTbl/_stream_load 2. 将本地文件'testData'中的数据导入到数据库'testDb'中'testTbl'的表，使用Label用于去重, 并且只导入k1等于20180601的数据 - curl --location-trusted -u root -H "lable:123" -H "where: k1=20180601" -T testData http://host:port/api/testDb/testTbl/_stream_load + curl --location-trusted -u root -H "label:123" -H "where: k1=20180601" -T testData http://host:port/api/testDb/testTbl/_stream_load 3. 将本地文件'testData'中的数据导入到数据库'testDb'中'testTbl'的表, 允许20%的错误率（用户是defalut_cluster中的） - curl --location-trusted -u root -H "lable:123" -H "max_filter_ratio:0.2" -T testData http://host:port/api/testDb/testTbl/_stream_load + curl --location-trusted -u root -H "label:123" -H "max_filter_ratio:0.2" -T testData http://host:port/api/testDb/testTbl/_stream_load 4. 将本地文件'testData'中的数据导入到数据库'testDb'中'testTbl'的表, 允许20%的错误率，并且指定文件的列名（用户是defalut_cluster中的） - curl --location-trusted -u root -H "lable:123" -H "max_filter_ratio:0.2" -H "columns: k2, k1, v1" -T testData http://host:port/api/testDb/testTbl/_stream_load + curl --location-trusted -u root -H "label:123" -H "max_filter_ratio:0.2" -H "columns: k2, k1, v1" -T testData http://host:port/api/testDb/testTbl/_stream_load 5. 将本地文件'testData'中的数据导入到数据库'testDb'中'testTbl'的表中的p1, p2分区, 允许20%的错误率。 - curl --location-trusted -u root -H "lable:123" -H "max_filter_ratio:0.2" -H "partitions: p1, p2" -T testData http://host:port/api/testDb/testTbl/_stream_load + curl --location-trusted -u root -H "label:123" -H "max_filter_ratio:0.2" -H "partitions: p1, p2" -T testData http://host:port/api/testDb/testTbl/_stream_load 6. 使用streaming方式导入（用户是defalut_cluster中的） seq 1 10 | awk '{OFS="\t"}{print $1, $1 * 10}' | curl --location-trusted -u root -T - http://host:port/api/testDb/testTbl/_stream_load From 41f6b5eb52fc1f45c31090ca309c5db508109362 Mon Sep 17 00:00:00 2001 From: liuhangyuan1 Date: Thu, 4 Apr 2019 16:06:40 +0800 Subject: [PATCH 2/4] Fix docs 'CANCEL DECOMMISSION' --- docs/help/Contents/Administration/admin_stmt.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/help/Contents/Administration/admin_stmt.md b/docs/help/Contents/Administration/admin_stmt.md index c0e9c0fe3eb076..95e469c533839d 100644 --- a/docs/help/Contents/Administration/admin_stmt.md +++ b/docs/help/Contents/Administration/admin_stmt.md @@ -27,7 +27,7 @@ 2) heartbeat_port 为该节点的心跳端口 3) 增加和删除节点为同步操作。这两种操作不考虑节点上已有的数据，节点直接从元数据中删除，请谨慎使用。 4) 节点下线操作用于安全下线节点。该操作为异步操作。如果成功，节点最终会从元数据中删除。如果失败，则不会完成下线。 - 5) 可以手动取消节点下线操作。详见 CANCEL ALTER SYSTEM + 5) 可以手动取消节点下线操作。详见 CANCEL DECOMMISSION 6) Load error hub: 当前支持两种类型的 Hub：Mysql 和 Broker。需在 PROPERTIES 中指定 "type" = "mysql" 或 "type" = "broker"。 如果需要删除当前的 load error hub，可以将 type 设为 null。 @@ -92,20 +92,20 @@ ## keyword ALTER,SYSTEM,BACKEND,BROKER,FREE -# CANCEL ALTER SYSTEM +# CANCEL DECOMMISSION ## description 该语句用于撤销一个节点下线操作。（仅管理员使用！） 语法： - CANCEL ALTER SYSTEM DECOMMISSION BACKEND "host:heartbeat_port"[,"host:heartbeat_port"...]; + CANCEL DECOMMISSION BACKEND "host:heartbeat_port"[,"host:heartbeat_port"...]; ## example 1. 取消两个节点的下线操作： - CANCEL ALTER SYSTEM DECOMMISSION BACKEND "host1:port", "host2:port"; + CANCEL DECOMMISSION BACKEND "host1:port", "host2:port"; ## keyword - CANCEL,ALTER,SYSTEM,BACKEND + CANCEL,DECOMMISSION,BACKEND # CREATE CLUSTER ## description From 90e2c4fe5e8c1949e5a99d9659c7d1a8e9770f92 Mon Sep 17 00:00:00 2001 From: liuhangyuan1 Date: Thu, 11 Apr 2019 16:00:03 +0800 Subject: [PATCH 3/4] Fixed a bug in the Bloom filter where column names could not be found due to spaces --- .../main/java/org/apache/doris/common/util/PropertyAnalyzer.java | 1 + 1 file changed, 1 insertion(+) diff --git a/fe/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index ecb7b8f69ecd99..78c0d94ad69417 100644 --- a/fe/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -257,6 +257,7 @@ public static Set analyzeBloomFilterColumns(Map properti String[] bfColumnArr = bfColumnsStr.split(COMMA_SEPARATOR); Set bfColumnSet = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); for (String bfColumn : bfColumnArr) { + bfColumn = bfColumn.trim(); boolean found = false; for (Column column : columns) { if (column.getName().equalsIgnoreCase(bfColumn)) { From 7e43e8ea04f1a9b1418a7b8721cb9e2ece21b9f1 Mon Sep 17 00:00:00 2001 From: liuhangyuan1 Date: Mon, 15 Apr 2019 14:35:26 +0800 Subject: [PATCH 4/4] Fix bug in concat_ws --- be/src/exprs/string_functions.cpp | 23 ++++++++++++------- .../doris/rewrite/FoldConstantsRule.java | 1 + 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/be/src/exprs/string_functions.cpp b/be/src/exprs/string_functions.cpp index 7fcd5dc3a212b4..777b0279b31819 100644 --- a/be/src/exprs/string_functions.cpp +++ b/be/src/exprs/string_functions.cpp @@ -516,17 +516,17 @@ StringVal StringFunctions::concat_ws( return strs[0]; } - if (strs[0].is_null) { + if (strs[0].is_null && 0 == sep.len) { return StringVal::null(); } - int32_t total_size = strs[0].len; + int32_t total_size = strs[0].is_null ? 0 : strs[0].len; // Loop once to compute the final size and reserve space. for (int32_t i = 1; i < num_children; ++i) { - if (strs[i].is_null) { + if (strs[i].is_null && 0 == sep.len) { return StringVal::null(); } - total_size += sep.len + strs[i].len; + total_size += strs[i].is_null ? 0 : (sep.len + strs[i].len); } // TODO pengyubing @@ -535,11 +535,18 @@ StringVal StringFunctions::concat_ws( uint8_t* ptr = result.ptr; // Loop again to append the data. - memcpy(ptr, strs[0].ptr, strs[0].len); - ptr += strs[0].len; + if(!strs[0].is_null) { + memcpy(ptr, strs[0].ptr, strs[0].len); + ptr += strs[0].len; + } for (int32_t i = 1; i < num_children; ++i) { - memcpy(ptr, sep.ptr, sep.len); - ptr += sep.len; + if(strs[i].is_null) { + continue; + } + if(!(strs[0].is_null && 1 == i)) { + memcpy(ptr, sep.ptr, sep.len); + ptr += sep.len; + } memcpy(ptr, strs[i].ptr, strs[i].len); ptr += strs[i].len; } diff --git a/fe/src/main/java/org/apache/doris/rewrite/FoldConstantsRule.java b/fe/src/main/java/org/apache/doris/rewrite/FoldConstantsRule.java index 16b06e50b9dcc6..28e6a621f03b81 100644 --- a/fe/src/main/java/org/apache/doris/rewrite/FoldConstantsRule.java +++ b/fe/src/main/java/org/apache/doris/rewrite/FoldConstantsRule.java @@ -204,6 +204,7 @@ private synchronized void registerFunctions() { new ImmutableSet.Builder(); setBuilder.add("if"); setBuilder.add("hll_hash"); + setBuilder.add("concat_ws"); this.nonNullResultWithNullParamFunctions = setBuilder.build(); }