From 54b5b2594e3272b0a64672751f0008eee84fb966 Mon Sep 17 00:00:00 2001 From: Xuan Ronaldo Date: Thu, 14 Dec 2023 16:14:54 +0800 Subject: [PATCH 1/9] update code and add a UT --- .../extractor/IoTDBDataRegionExtractor.java | 21 +++++++------------ .../IoTDBDataRegionExtractorTest.java | 20 ++++++++++++++++++ 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java index c1a1b4405795a..7c3bbd6b74987 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.pipe.extractor; import org.apache.iotdb.commons.consensus.DataRegionId; +import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.db.pipe.config.plugin.env.PipeTaskExtractorRuntimeEnvironment; import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent; import org.apache.iotdb.db.pipe.extractor.historical.PipeHistoricalDataRegionExtractor; @@ -48,20 +49,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; -import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_HISTORY_ENABLE_DEFAULT_VALUE; -import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_HISTORY_ENABLE_KEY; -import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_REALTIME_ENABLE_DEFAULT_VALUE; -import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_REALTIME_ENABLE_KEY; -import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_REALTIME_MODE_BATCH_MODE_VALUE; -import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_REALTIME_MODE_FILE_VALUE; -import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_REALTIME_MODE_FORCED_LOG_VALUE; -import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_REALTIME_MODE_HYBRID_VALUE; -import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_REALTIME_MODE_KEY; -import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_REALTIME_MODE_LOG_VALUE; -import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_REALTIME_MODE_STREAM_MODE_VALUE; -import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.SOURCE_HISTORY_ENABLE_KEY; -import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.SOURCE_REALTIME_ENABLE_KEY; -import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.SOURCE_REALTIME_MODE_KEY; +import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.*; public class IoTDBDataRegionExtractor implements PipeExtractor { @@ -84,6 +72,11 @@ public IoTDBDataRegionExtractor() { @Override public void validate(PipeParameterValidator validator) throws Exception { + // Check whether the pattern is legal + PathUtils.isLegalPath( + validator + .getParameters() + .getStringOrDefault(EXTRACTOR_PATTERN_KEY, EXTRACTOR_PATTERN_DEFAULT_VALUE)); // Validate extractor.history.enable and extractor.realtime.enable validator .validateAttributeValueRange( diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractorTest.java index 1914316fd2820..86d3668875d65 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractorTest.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.pipe.extractor; +import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant; import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator; import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; @@ -53,4 +54,23 @@ public void testIoTDBDataRegionExtractor() { Assert.fail(); } } + + @Test + public void testIoTDBDataRegionExtractorWithIllegalPattern() { + IoTDBDataRegionExtractor extractor = new IoTDBDataRegionExtractor(); + try { + extractor.validate( + new PipeParameterValidator( + new PipeParameters( + new HashMap() { + { + put( + PipeExtractorConstant.EXTRACTOR_PATTERN_KEY, + "root.a-b"); + } + }))); + } catch (Exception e) { + Assert.assertEquals(e.getClass(), IllegalPathException.class); + } + } } From de70d7f66597e8bcf176bab8551c2d8746da9069 Mon Sep 17 00:00:00 2001 From: Xuan Ronaldo Date: Thu, 14 Dec 2023 16:21:05 +0800 Subject: [PATCH 2/9] reformat code --- .../extractor/IoTDBDataRegionExtractor.java | 17 ++++++++++++++++- .../extractor/IoTDBDataRegionExtractorTest.java | 4 +--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java index 7c3bbd6b74987..dc68a23ed065e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java @@ -49,7 +49,22 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; -import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.*; +import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_HISTORY_ENABLE_DEFAULT_VALUE; +import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_HISTORY_ENABLE_KEY; +import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_PATTERN_DEFAULT_VALUE; +import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_PATTERN_KEY; +import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_REALTIME_ENABLE_DEFAULT_VALUE; +import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_REALTIME_ENABLE_KEY; +import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_REALTIME_MODE_BATCH_MODE_VALUE; +import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_REALTIME_MODE_FILE_VALUE; +import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_REALTIME_MODE_FORCED_LOG_VALUE; +import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_REALTIME_MODE_HYBRID_VALUE; +import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_REALTIME_MODE_KEY; +import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_REALTIME_MODE_LOG_VALUE; +import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_REALTIME_MODE_STREAM_MODE_VALUE; +import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.SOURCE_HISTORY_ENABLE_KEY; +import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.SOURCE_REALTIME_ENABLE_KEY; +import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.SOURCE_REALTIME_MODE_KEY; public class IoTDBDataRegionExtractor implements PipeExtractor { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractorTest.java index 86d3668875d65..8d83ee6d7853d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractorTest.java @@ -64,9 +64,7 @@ public void testIoTDBDataRegionExtractorWithIllegalPattern() { new PipeParameters( new HashMap() { { - put( - PipeExtractorConstant.EXTRACTOR_PATTERN_KEY, - "root.a-b"); + put(PipeExtractorConstant.EXTRACTOR_PATTERN_KEY, "root.a-b"); } }))); } catch (Exception e) { From 7226f581dff6545189206c72a902d5d3af566aea Mon Sep 17 00:00:00 2001 From: Steve Yurong Su Date: Thu, 14 Dec 2023 16:25:56 +0800 Subject: [PATCH 3/9] Update iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java --- .../apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java | 1 + 1 file changed, 1 insertion(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java index dc68a23ed065e..71a979dffcb9d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java @@ -92,6 +92,7 @@ public void validate(PipeParameterValidator validator) throws Exception { validator .getParameters() .getStringOrDefault(EXTRACTOR_PATTERN_KEY, EXTRACTOR_PATTERN_DEFAULT_VALUE)); + // Validate extractor.history.enable and extractor.realtime.enable validator .validateAttributeValueRange( From b12d457c29f21f4c120b620d692218524f99a27c Mon Sep 17 00:00:00 2001 From: Xuan Ronaldo Date: Thu, 14 Dec 2023 16:33:12 +0800 Subject: [PATCH 4/9] add SOURCE_PATTERN_KEY to param's list --- .../iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java index dc68a23ed065e..aa5b40b73484f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java @@ -63,6 +63,7 @@ import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_REALTIME_MODE_LOG_VALUE; import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_REALTIME_MODE_STREAM_MODE_VALUE; import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.SOURCE_HISTORY_ENABLE_KEY; +import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.SOURCE_PATTERN_KEY; import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.SOURCE_REALTIME_ENABLE_KEY; import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.SOURCE_REALTIME_MODE_KEY; @@ -91,7 +92,9 @@ public void validate(PipeParameterValidator validator) throws Exception { PathUtils.isLegalPath( validator .getParameters() - .getStringOrDefault(EXTRACTOR_PATTERN_KEY, EXTRACTOR_PATTERN_DEFAULT_VALUE)); + .getStringOrDefault( + Arrays.asList(EXTRACTOR_PATTERN_KEY, SOURCE_PATTERN_KEY), + EXTRACTOR_PATTERN_DEFAULT_VALUE)); // Validate extractor.history.enable and extractor.realtime.enable validator .validateAttributeValueRange( From 2e7cb87edd99acedd5beb0702596035dfda3c226 Mon Sep 17 00:00:00 2001 From: Xuan Ronaldo Date: Thu, 14 Dec 2023 16:34:50 +0800 Subject: [PATCH 5/9] add SOURCE_PATTERN_KEY to param's list --- .../iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java index dc68a23ed065e..227c75abe58fc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java @@ -63,6 +63,7 @@ import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_REALTIME_MODE_LOG_VALUE; import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_REALTIME_MODE_STREAM_MODE_VALUE; import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.SOURCE_HISTORY_ENABLE_KEY; +import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.SOURCE_PATTERN_KEY; import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.SOURCE_REALTIME_ENABLE_KEY; import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.SOURCE_REALTIME_MODE_KEY; @@ -91,7 +92,10 @@ public void validate(PipeParameterValidator validator) throws Exception { PathUtils.isLegalPath( validator .getParameters() - .getStringOrDefault(EXTRACTOR_PATTERN_KEY, EXTRACTOR_PATTERN_DEFAULT_VALUE)); + .getStringOrDefault( + Arrays.asList(EXTRACTOR_PATTERN_KEY, SOURCE_PATTERN_KEY), + EXTRACTOR_PATTERN_DEFAULT_VALUE)); + // Validate extractor.history.enable and extractor.realtime.enable validator .validateAttributeValueRange( From edf2132c5eb9ec0f788cfe15b343c209fc0e3bd9 Mon Sep 17 00:00:00 2001 From: Xuan Ronaldo Date: Sat, 16 Dec 2023 12:20:25 +0800 Subject: [PATCH 6/9] fix UTs --- .../extractor/IoTDBDataRegionExtractor.java | 21 ++++++++++++++++-- .../IoTDBDataRegionExtractorTest.java | 22 ++++++++++++++----- 2 files changed, 35 insertions(+), 8 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java index 227c75abe58fc..0e923c42ee2d3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.pipe.extractor; import org.apache.iotdb.commons.consensus.DataRegionId; +import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.db.pipe.config.plugin.env.PipeTaskExtractorRuntimeEnvironment; import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent; @@ -41,6 +42,7 @@ import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent; import org.apache.iotdb.pipe.api.exception.PipeException; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -89,12 +91,27 @@ public IoTDBDataRegionExtractor() { @Override public void validate(PipeParameterValidator validator) throws Exception { // Check whether the pattern is legal - PathUtils.isLegalPath( + String pattern = validator .getParameters() .getStringOrDefault( Arrays.asList(EXTRACTOR_PATTERN_KEY, SOURCE_PATTERN_KEY), - EXTRACTOR_PATTERN_DEFAULT_VALUE)); + EXTRACTOR_PATTERN_DEFAULT_VALUE); + try { + PathUtils.isLegalPath(pattern); + } catch (IllegalPathException e) { + try { + String[] pathNodes = StringUtils.split(pattern, "\\."); + PathUtils.splitPathToDetachedNodes( + String.join(".", Arrays.copyOfRange(pathNodes, 0, pathNodes.length - 1))); + String lastNode = pathNodes[pathNodes.length - 1]; + if (!"".equals(lastNode)) { + Double.parseDouble(lastNode); + } + } catch (NumberFormatException | IllegalPathException ignored) { + throw new IllegalArgumentException(); + } + } // Validate extractor.history.enable and extractor.realtime.enable validator diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractorTest.java index 8d83ee6d7853d..e5df8cf786c4f 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractorTest.java @@ -19,7 +19,6 @@ package org.apache.iotdb.db.pipe.extractor; -import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant; import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator; import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; @@ -56,19 +55,30 @@ public void testIoTDBDataRegionExtractor() { } @Test - public void testIoTDBDataRegionExtractorWithIllegalPattern() { - IoTDBDataRegionExtractor extractor = new IoTDBDataRegionExtractor(); - try { + public void testIoTDBDataRegionExtractorWithPattern() { + Assert.assertEquals( + testIoTDBDataRegionExtractorWithPattern("root.a-b").getClass(), + IllegalArgumentException.class); + Assert.assertEquals( + testIoTDBDataRegionExtractorWithPattern("root.1.a").getClass(), + IllegalArgumentException.class); + Assert.assertNull(testIoTDBDataRegionExtractorWithPattern("root.`a-b`")); + Assert.assertNull(testIoTDBDataRegionExtractorWithPattern("root.1")); + } + + public Exception testIoTDBDataRegionExtractorWithPattern(String pattern) { + try (IoTDBDataRegionExtractor extractor = new IoTDBDataRegionExtractor()) { extractor.validate( new PipeParameterValidator( new PipeParameters( new HashMap() { { - put(PipeExtractorConstant.EXTRACTOR_PATTERN_KEY, "root.a-b"); + put(PipeExtractorConstant.EXTRACTOR_PATTERN_KEY, pattern); } }))); } catch (Exception e) { - Assert.assertEquals(e.getClass(), IllegalPathException.class); + return e; } + return null; } } From ddcf45215e9a819fa737b30cf232c5b38359751f Mon Sep 17 00:00:00 2001 From: Xuan Ronaldo Date: Mon, 18 Dec 2023 10:36:24 +0800 Subject: [PATCH 7/9] update the method of validatePattern, and add some UTs for it. --- .../extractor/IoTDBDataRegionExtractor.java | 45 ++++++++++++------- .../IoTDBDataRegionExtractorTest.java | 19 ++++++++ 2 files changed, 47 insertions(+), 17 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java index 0e923c42ee2d3..ef7212f07af3f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java @@ -91,27 +91,12 @@ public IoTDBDataRegionExtractor() { @Override public void validate(PipeParameterValidator validator) throws Exception { // Check whether the pattern is legal - String pattern = + validatePattern( validator .getParameters() .getStringOrDefault( Arrays.asList(EXTRACTOR_PATTERN_KEY, SOURCE_PATTERN_KEY), - EXTRACTOR_PATTERN_DEFAULT_VALUE); - try { - PathUtils.isLegalPath(pattern); - } catch (IllegalPathException e) { - try { - String[] pathNodes = StringUtils.split(pattern, "\\."); - PathUtils.splitPathToDetachedNodes( - String.join(".", Arrays.copyOfRange(pathNodes, 0, pathNodes.length - 1))); - String lastNode = pathNodes[pathNodes.length - 1]; - if (!"".equals(lastNode)) { - Double.parseDouble(lastNode); - } - } catch (NumberFormatException | IllegalPathException ignored) { - throw new IllegalArgumentException(); - } - } + EXTRACTOR_PATTERN_DEFAULT_VALUE)); // Validate extractor.history.enable and extractor.realtime.enable validator @@ -163,6 +148,32 @@ public void validate(PipeParameterValidator validator) throws Exception { realtimeExtractor.validate(validator); } + private void validatePattern(String pattern) { + if (!pattern.startsWith("root")) { + throw new IllegalArgumentException( + "The argument `extractor.pattern` or `source.pattern` is an illegal path."); + } + if ("root".equals(pattern) || "root.".equals(pattern)) { + return; + } + try { + PathUtils.isLegalPath(pattern); + } catch (IllegalPathException e) { + try { + String[] pathNodes = StringUtils.split(pattern, "\\."); + PathUtils.splitPathToDetachedNodes( + String.join(".", Arrays.copyOfRange(pathNodes, 0, pathNodes.length - 1))); + String lastNode = pathNodes[pathNodes.length - 1]; + if (!"".equals(lastNode)) { + Double.parseDouble(lastNode); + } + } catch (Exception ignored) { + throw new IllegalArgumentException( + "The argument `extractor.pattern` or `source.pattern` is an illegal path."); + } + } + } + private void constructHistoricalExtractor() { // Enable historical extractor by default historicalExtractor = new PipeHistoricalDataRegionTsFileExtractor(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractorTest.java index e5df8cf786c4f..70fee9b3b73fb 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractorTest.java @@ -62,6 +62,25 @@ public void testIoTDBDataRegionExtractorWithPattern() { Assert.assertEquals( testIoTDBDataRegionExtractorWithPattern("root.1.a").getClass(), IllegalArgumentException.class); + Assert.assertEquals( + testIoTDBDataRegionExtractorWithPattern("r").getClass(), IllegalArgumentException.class); + Assert.assertEquals( + testIoTDBDataRegionExtractorWithPattern("").getClass(), IllegalArgumentException.class); + Assert.assertEquals( + testIoTDBDataRegionExtractorWithPattern("123").getClass(), IllegalArgumentException.class); + Assert.assertEquals( + testIoTDBDataRegionExtractorWithPattern("root.a b").getClass(), + IllegalArgumentException.class); + Assert.assertEquals( + testIoTDBDataRegionExtractorWithPattern("root.a+b").getClass(), + IllegalArgumentException.class); + Assert.assertNull(testIoTDBDataRegionExtractorWithPattern("root.a#b")); + Assert.assertNull(testIoTDBDataRegionExtractorWithPattern("root.一二三")); + Assert.assertNull(testIoTDBDataRegionExtractorWithPattern("root.一二。三")); + Assert.assertNull(testIoTDBDataRegionExtractorWithPattern("root.")); + Assert.assertNull(testIoTDBDataRegionExtractorWithPattern("root.ab")); + Assert.assertNull(testIoTDBDataRegionExtractorWithPattern("root.a.b.c.1e2")); + Assert.assertNull(testIoTDBDataRegionExtractorWithPattern("root")); Assert.assertNull(testIoTDBDataRegionExtractorWithPattern("root.`a-b`")); Assert.assertNull(testIoTDBDataRegionExtractorWithPattern("root.1")); } From 20d58077a37ee2276140eb5eb20718ed4363410c Mon Sep 17 00:00:00 2001 From: Xuan Ronaldo Date: Mon, 18 Dec 2023 13:42:51 +0800 Subject: [PATCH 8/9] update the method of validatePattern, and add some UTs for it. --- .../iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java | 8 ++++---- .../db/pipe/extractor/IoTDBDataRegionExtractorTest.java | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java index ef7212f07af3f..4a5d23b21a852 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java @@ -153,14 +153,14 @@ private void validatePattern(String pattern) { throw new IllegalArgumentException( "The argument `extractor.pattern` or `source.pattern` is an illegal path."); } - if ("root".equals(pattern) || "root.".equals(pattern)) { - return; - } try { PathUtils.isLegalPath(pattern); } catch (IllegalPathException e) { try { - String[] pathNodes = StringUtils.split(pattern, "\\."); + if ("root".equals(pattern) || "root.".equals(pattern)) { + return; + } + String[] pathNodes = StringUtils.splitPreserveAllTokens(pattern, "\\."); PathUtils.splitPathToDetachedNodes( String.join(".", Arrays.copyOfRange(pathNodes, 0, pathNodes.length - 1))); String lastNode = pathNodes[pathNodes.length - 1]; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractorTest.java index 70fee9b3b73fb..cdbff585ac601 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractorTest.java @@ -74,6 +74,7 @@ public void testIoTDBDataRegionExtractorWithPattern() { Assert.assertEquals( testIoTDBDataRegionExtractorWithPattern("root.a+b").getClass(), IllegalArgumentException.class); + Assert.assertNull(testIoTDBDataRegionExtractorWithPattern("root.ab.")); Assert.assertNull(testIoTDBDataRegionExtractorWithPattern("root.a#b")); Assert.assertNull(testIoTDBDataRegionExtractorWithPattern("root.一二三")); Assert.assertNull(testIoTDBDataRegionExtractorWithPattern("root.一二。三")); From 362bf137d32064144cb66fdc51c5d702fcddfc37 Mon Sep 17 00:00:00 2001 From: Xuan Ronaldo Date: Mon, 18 Dec 2023 14:00:43 +0800 Subject: [PATCH 9/9] add some comments. --- .../iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java index 4a5d23b21a852..80f4c8e5f19ec 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java @@ -153,6 +153,7 @@ private void validatePattern(String pattern) { throw new IllegalArgumentException( "The argument `extractor.pattern` or `source.pattern` is an illegal path."); } + try { PathUtils.isLegalPath(pattern); } catch (IllegalPathException e) { @@ -160,10 +161,16 @@ private void validatePattern(String pattern) { if ("root".equals(pattern) || "root.".equals(pattern)) { return; } + + // Split the pattern to nodes. String[] pathNodes = StringUtils.splitPreserveAllTokens(pattern, "\\."); + + // Check whether the pattern without last node is legal. PathUtils.splitPathToDetachedNodes( String.join(".", Arrays.copyOfRange(pathNodes, 0, pathNodes.length - 1))); String lastNode = pathNodes[pathNodes.length - 1]; + + // Check whether the last node is legal. if (!"".equals(lastNode)) { Double.parseDouble(lastNode); }