From f3cf7c69bdcaf2cc516f5e5081a822cc4450dbec Mon Sep 17 00:00:00 2001 From: Beasttt <64394711+Beasttt@users.noreply.github.com> Date: Mon, 9 Dec 2024 13:36:22 +0000 Subject: [PATCH 1/4] Modify UDF time precision --- .../iotdb/library/dprofile/UDTFResample.java | 11 ++- .../library/dquality/UDTFCompleteness.java | 5 +- .../library/dquality/UDTFConsistency.java | 5 +- .../library/dquality/UDTFTimeliness.java | 6 +- .../iotdb/library/dquality/UDTFValidity.java | 6 +- .../iotdb/library/frequency/UDTFIFFT.java | 13 +++- .../series/UDTFConsecutiveSequences.java | 11 ++- .../series/UDTFConsecutiveWindows.java | 15 +++- .../org/apache/iotdb/library/util/Util.java | 78 +++++++++++++++---- 9 files changed, 119 insertions(+), 31 deletions(-) diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFResample.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFResample.java index 91cbf24ae2c30..245d71525bf81 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFResample.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFResample.java @@ -34,19 +34,22 @@ /** This function does upsample or downsample of input series. */ public class UDTFResample implements UDTF { - + private static final String TIMESTAMP_PRECISION = "timestampPrecision"; + public static final String MS_PRECISION = "ms"; private static final String START_PARAM = "start"; private Resampler resampler; @Override public void validate(UDFParameterValidator validator) throws Exception { + String timestampPrecision = + validator.getParameters().getSystemStringOrDefault(TIMESTAMP_PRECISION, MS_PRECISION); validator .validateInputSeriesNumber(1) .validateInputSeriesDataType(0, Type.DOUBLE, Type.FLOAT, Type.INT32, Type.INT64) .validate( x -> (long) x > 0, "gap should be a time period whose unit is ms, s, m, h, d.", - Util.parseTime(validator.getParameters().getString("every"))) + Util.parseTime(validator.getParameters().getString("every"), timestampPrecision)) .validate( x -> "min".equals(x) @@ -80,7 +83,9 @@ public void validate(UDFParameterValidator validator) throws Exception { public void beforeStart(UDFParameters parameters, UDTFConfigurations configurations) throws Exception { configurations.setAccessStrategy(new RowByRowAccessStrategy()).setOutputDataType(Type.DOUBLE); - long newPeriod = Util.parseTime(parameters.getString("every")); + String timestampPrecision = + parameters.getSystemStringOrDefault(TIMESTAMP_PRECISION, MS_PRECISION); + long newPeriod = Util.parseTime(parameters.getString("every"), timestampPrecision); String aggregator = parameters.getStringOrDefault("aggr", "mean").toLowerCase(); String interpolator = parameters.getStringOrDefault("interp", "nan").toLowerCase(); SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFCompleteness.java b/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFCompleteness.java index e5241062d6cf4..4dbd06a9d7275 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFCompleteness.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFCompleteness.java @@ -36,6 +36,8 @@ /** This function calculates completeness of input series. */ public class UDTFCompleteness implements UDTF { + private static final String TIMESTAMP_PRECISION = "timestampPrecision"; + public static final String MS_PRECISION = "ms"; private boolean downtime; @Override @@ -44,7 +46,8 @@ public void beforeStart(UDFParameters udfp, UDTFConfigurations udtfc) throws Exc long window = Integer.MAX_VALUE; if (udfp.hasAttribute("window")) { String s = udfp.getString("window"); - window = Util.parseTime(s); + String timestampPrecision = udfp.getSystemStringOrDefault(TIMESTAMP_PRECISION, MS_PRECISION); + window = Util.parseTime(s, timestampPrecision); if (window > 0) { isTime = true; } else { diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFConsistency.java b/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFConsistency.java index 52f2787646899..ba5732d003eac 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFConsistency.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFConsistency.java @@ -37,6 +37,8 @@ /** This function calculates consistency of input series. */ public class UDTFConsistency implements UDTF { + private static final String TIMESTAMP_PRECISION = "timestampPrecision"; + public static final String MS_PRECISION = "ms"; @Override public void beforeStart(UDFParameters udfp, UDTFConfigurations udtfc) throws Exception { @@ -44,7 +46,8 @@ public void beforeStart(UDFParameters udfp, UDTFConfigurations udtfc) throws Exc long window = Integer.MAX_VALUE; if (udfp.hasAttribute("window")) { String s = udfp.getString("window"); - window = Util.parseTime(s); + String timestampPrecision = udfp.getSystemStringOrDefault(TIMESTAMP_PRECISION, MS_PRECISION); + window = Util.parseTime(s, timestampPrecision); if (window > 0) { isTime = true; } else { diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFTimeliness.java b/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFTimeliness.java index 8ac93e1bf5436..d1f12fcb11b59 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFTimeliness.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFTimeliness.java @@ -37,13 +37,17 @@ /** This function calculates timeliness of input series. */ public class UDTFTimeliness implements UDTF { + private static final String TIMESTAMP_PRECISION = "timestampPrecision"; + public static final String MS_PRECISION = "ms"; + @Override public void beforeStart(UDFParameters udfp, UDTFConfigurations udtfc) throws Exception { boolean isTime = false; long window = Integer.MAX_VALUE; if (udfp.hasAttribute("window")) { String s = udfp.getString("window"); - window = Util.parseTime(s); + String timestampPrecision = udfp.getSystemStringOrDefault(TIMESTAMP_PRECISION, MS_PRECISION); + window = Util.parseTime(s, timestampPrecision); if (window > 0) { isTime = true; } else { diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFValidity.java b/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFValidity.java index 7e7028d382fa7..a000b12ca0d05 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFValidity.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFValidity.java @@ -37,13 +37,17 @@ /** This function calculates validity of input series. */ public class UDTFValidity implements UDTF { + private static final String TIMESTAMP_PRECISION = "timestampPrecision"; + public static final String MS_PRECISION = "ms"; + @Override public void beforeStart(UDFParameters udfp, UDTFConfigurations udtfc) throws Exception { boolean isTime = false; long window = Integer.MAX_VALUE; if (udfp.hasAttribute("window")) { String s = udfp.getString("window"); - window = Util.parseTime(s); + String timestampPrecision = udfp.getSystemStringOrDefault(TIMESTAMP_PRECISION, MS_PRECISION); + window = Util.parseTime(s, timestampPrecision); if (window > 0) { isTime = true; } else { diff --git a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFIFFT.java b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFIFFT.java index 38a655f7b2cf5..e7636c1b71dd4 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFIFFT.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFIFFT.java @@ -42,18 +42,24 @@ public class UDTFIFFT implements UDTF { private final DoubleArrayList real = new DoubleArrayList(); private final DoubleArrayList imag = new DoubleArrayList(); private final IntArrayList time = new IntArrayList(); + private static final String TIMESTAMP_PRECISION = "timestampPrecision"; + public static final String MS_PRECISION = "ms"; private long start; private long interval; @Override public void validate(UDFParameterValidator validator) throws Exception { + String timestampPrecision = + validator.getParameters().getSystemStringOrDefault(TIMESTAMP_PRECISION, MS_PRECISION); validator .validateInputSeriesNumber(2) .validateInputSeriesDataType(0, Type.DOUBLE, Type.FLOAT, Type.INT32, Type.INT64) .validate( x -> (long) x > 0, "interval should be a time period whose unit is ms, s, m, h, d.", - Util.parseTime(validator.getParameters().getStringOrDefault("interval", "1s"))); + Util.parseTime( + validator.getParameters().getStringOrDefault("interval", "1s"), + timestampPrecision)); SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); if (validator.getParameters().hasAttribute(START_PARAM)) { validator.validate( @@ -67,7 +73,10 @@ public void validate(UDFParameterValidator validator) throws Exception { public void beforeStart(UDFParameters parameters, UDTFConfigurations configurations) throws Exception { configurations.setAccessStrategy(new RowByRowAccessStrategy()).setOutputDataType(Type.DOUBLE); - this.interval = Util.parseTime(parameters.getStringOrDefault("interval", "1s")); + String timestampPrecision = + parameters.getSystemStringOrDefault(TIMESTAMP_PRECISION, MS_PRECISION); + this.interval = + Util.parseTime(parameters.getStringOrDefault("interval", "1s"), timestampPrecision); SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); this.start = 0; if (parameters.hasAttribute(START_PARAM)) { diff --git a/library-udf/src/main/java/org/apache/iotdb/library/series/UDTFConsecutiveSequences.java b/library-udf/src/main/java/org/apache/iotdb/library/series/UDTFConsecutiveSequences.java index c58111fa0ada5..49e8b53b7b8cf 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/series/UDTFConsecutiveSequences.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/series/UDTFConsecutiveSequences.java @@ -35,20 +35,27 @@ /** This function searches for all longest consecutive subsequences of input sereis. */ public class UDTFConsecutiveSequences implements UDTF { private ConsecutiveUtil consUtil; + private static final String TIMESTAMP_PRECISION = "timestampPrecision"; + public static final String MS_PRECISION = "ms"; @Override public void validate(UDFParameterValidator validator) throws Exception { + String timestampPrecision = + validator.getParameters().getSystemStringOrDefault(TIMESTAMP_PRECISION, MS_PRECISION); validator.validate( x -> (long) x > 0, "gap should be a time period whose unit is ms, s, m, h.", - Util.parseTime(validator.getParameters().getStringOrDefault("gap", "1ms"))); + Util.parseTime( + validator.getParameters().getStringOrDefault("gap", "1ms"), timestampPrecision)); } @Override public void beforeStart(UDFParameters parameters, UDTFConfigurations configurations) throws Exception { configurations.setAccessStrategy(new RowByRowAccessStrategy()).setOutputDataType(Type.INT32); - long gap = Util.parseTime(parameters.getStringOrDefault("gap", "0ms")); + String timestampPrecision = + parameters.getSystemStringOrDefault(TIMESTAMP_PRECISION, MS_PRECISION); + long gap = Util.parseTime(parameters.getStringOrDefault("gap", "0ms"), timestampPrecision); consUtil = new ConsecutiveUtil(-gap, -gap, gap); } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/series/UDTFConsecutiveWindows.java b/library-udf/src/main/java/org/apache/iotdb/library/series/UDTFConsecutiveWindows.java index e7abedef87e9a..52c06283f474a 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/series/UDTFConsecutiveWindows.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/series/UDTFConsecutiveWindows.java @@ -36,27 +36,34 @@ public class UDTFConsecutiveWindows implements UDTF { private ConsecutiveUtil consUtil; private static final int MAX_LEN = 128; + private static final String TIMESTAMP_PRECISION = "timestampPrecision"; + public static final String MS_PRECISION = "ms"; private long len; @Override public void validate(UDFParameterValidator validator) throws Exception { + String timestampPrecision = + validator.getParameters().getSystemStringOrDefault(TIMESTAMP_PRECISION, MS_PRECISION); validator .validate( x -> (long) x > 0, "gap should be a time period whose unit is ms, s, m, h.", - Util.parseTime(validator.getParameters().getStringOrDefault("gap", "1ms"))) + Util.parseTime( + validator.getParameters().getStringOrDefault("gap", "1ms"), timestampPrecision)) .validate( x -> (long) x > 0, "length should be a time period whose unit is ms, s, m, h.", - Util.parseTime(validator.getParameters().getString("length"))); + Util.parseTime(validator.getParameters().getString("length"), timestampPrecision)); } @Override public void beforeStart(UDFParameters parameters, UDTFConfigurations configurations) throws Exception { configurations.setAccessStrategy(new RowByRowAccessStrategy()).setOutputDataType(Type.INT32); - long gap = Util.parseTime(parameters.getStringOrDefault("gap", "0ms")); - len = Util.parseTime(parameters.getString("length")); + String timestampPrecision = + parameters.getSystemStringOrDefault(TIMESTAMP_PRECISION, MS_PRECISION); + long gap = Util.parseTime(parameters.getStringOrDefault("gap", "0ms"), timestampPrecision); + len = Util.parseTime(parameters.getString("length"), timestampPrecision); int count = gap == 0 ? 0 : (int) (len / gap + 1); consUtil = new ConsecutiveUtil(-gap, -gap, gap); consUtil.setCount(count); diff --git a/library-udf/src/main/java/org/apache/iotdb/library/util/Util.java b/library-udf/src/main/java/org/apache/iotdb/library/util/Util.java index 5c65c4314ce9c..893223ee38c65 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/util/Util.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/util/Util.java @@ -300,26 +300,72 @@ public static long mode(long[] values) { * @param s input string * @return timestamp */ - public static long parseTime(String s) { + public static long parseTime(String s, String timestampPrecision) { long unit = 0; s = s.toLowerCase(); s = s.replace(" ", ""); - if (s.endsWith("ms")) { - unit = 1; - s = s.substring(0, s.length() - 2); - } else if (s.endsWith("s")) { - unit = 1000; - s = s.substring(0, s.length() - 1); - } else if (s.endsWith("m")) { - unit = 60 * 1000L; - s = s.substring(0, s.length() - 1); - } else if (s.endsWith("h")) { - unit = 60 * 60 * 1000L; - s = s.substring(0, s.length() - 1); - } else if (s.endsWith("d")) { - unit = 24 * 60 * 60 * 1000L; - s = s.substring(0, s.length() - 1); + if (timestampPrecision.equals("ms")) { + if (s.endsWith("ms")) { + unit = 1; + s = s.substring(0, s.length() - 2); + } else if (s.endsWith("s")) { + unit = 1000; + s = s.substring(0, s.length() - 1); + } else if (s.endsWith("m")) { + unit = 60 * 1000L; + s = s.substring(0, s.length() - 1); + } else if (s.endsWith("h")) { + unit = 60 * 60 * 1000L; + s = s.substring(0, s.length() - 1); + } else if (s.endsWith("d")) { + unit = 24 * 60 * 60 * 1000L; + s = s.substring(0, s.length() - 1); + } + } else if (timestampPrecision.equals("us")) { + if (s.endsWith("us")) { + unit = 1; + s = s.substring(0, s.length() - 2); + } else if (s.endsWith("ms")) { + unit = 1000; + s = s.substring(0, s.length() - 2); + } else if (s.endsWith("s")) { + unit = 1000 * 1000; + s = s.substring(0, s.length() - 1); + } else if (s.endsWith("m")) { + unit = 60 * 1000 * 1000L; + s = s.substring(0, s.length() - 1); + } else if (s.endsWith("h")) { + unit = 60 * 60 * 1000 * 1000L; + s = s.substring(0, s.length() - 1); + } else if (s.endsWith("d")) { + unit = 24 * 60 * 60 * 1000 * 1000L; + s = s.substring(0, s.length() - 1); + } + } else if (timestampPrecision.equals("ns")) { + if (s.endsWith("ns")) { + unit = 1; + s = s.substring(0, s.length() - 2); + } else if (s.endsWith("us")) { + unit = 1000; + s = s.substring(0, s.length() - 2); + } else if (s.endsWith("ms")) { + unit = 1000 * 1000; + s = s.substring(0, s.length() - 2); + } else if (s.endsWith("s")) { + unit = 1000 * 1000 * 1000; + s = s.substring(0, s.length() - 1); + } else if (s.endsWith("m")) { + unit = 60 * 1000 * 1000 * 1000L; + s = s.substring(0, s.length() - 1); + } else if (s.endsWith("h")) { + unit = 60 * 60 * 1000 * 1000 * 1000L; + s = s.substring(0, s.length() - 1); + } else if (s.endsWith("d")) { + unit = 24 * 60 * 60 * 1000 * 1000 * 1000L; + s = s.substring(0, s.length() - 1); + } } + double v = Double.parseDouble(s); return (long) (unit * v); } From 2d0d7de300e00ca5c47e00da0e616f09f5f10d2e Mon Sep 17 00:00:00 2001 From: Beasttt <64394711+Beasttt@users.noreply.github.com> Date: Thu, 12 Dec 2024 03:20:28 +0000 Subject: [PATCH 2/4] Modify UDF time precision_v1 --- .../iotdb/library/dprofile/UDTFResample.java | 11 +++-------- .../library/dquality/UDTFCompleteness.java | 6 ++---- .../library/dquality/UDTFConsistency.java | 5 +---- .../iotdb/library/dquality/UDTFTimeliness.java | 5 +---- .../iotdb/library/dquality/UDTFValidity.java | 5 +---- .../iotdb/library/frequency/UDTFIFFT.java | 12 +++--------- .../series/UDTFConsecutiveSequences.java | 10 ++-------- .../library/series/UDTFConsecutiveWindows.java | 16 ++++++---------- .../org/apache/iotdb/library/util/Util.java | 18 +++++++++++++++--- 9 files changed, 34 insertions(+), 54 deletions(-) diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFResample.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFResample.java index 245d71525bf81..15d2828c04f35 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFResample.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFResample.java @@ -34,22 +34,19 @@ /** This function does upsample or downsample of input series. */ public class UDTFResample implements UDTF { - private static final String TIMESTAMP_PRECISION = "timestampPrecision"; - public static final String MS_PRECISION = "ms"; private static final String START_PARAM = "start"; private Resampler resampler; @Override public void validate(UDFParameterValidator validator) throws Exception { - String timestampPrecision = - validator.getParameters().getSystemStringOrDefault(TIMESTAMP_PRECISION, MS_PRECISION); + validator .validateInputSeriesNumber(1) .validateInputSeriesDataType(0, Type.DOUBLE, Type.FLOAT, Type.INT32, Type.INT64) .validate( x -> (long) x > 0, "gap should be a time period whose unit is ms, s, m, h, d.", - Util.parseTime(validator.getParameters().getString("every"), timestampPrecision)) + Util.parseTime(validator.getParameters().getString("every"), validator.getParameters())) .validate( x -> "min".equals(x) @@ -83,9 +80,7 @@ public void validate(UDFParameterValidator validator) throws Exception { public void beforeStart(UDFParameters parameters, UDTFConfigurations configurations) throws Exception { configurations.setAccessStrategy(new RowByRowAccessStrategy()).setOutputDataType(Type.DOUBLE); - String timestampPrecision = - parameters.getSystemStringOrDefault(TIMESTAMP_PRECISION, MS_PRECISION); - long newPeriod = Util.parseTime(parameters.getString("every"), timestampPrecision); + long newPeriod = Util.parseTime(parameters.getString("every"), parameters); String aggregator = parameters.getStringOrDefault("aggr", "mean").toLowerCase(); String interpolator = parameters.getStringOrDefault("interp", "nan").toLowerCase(); SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFCompleteness.java b/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFCompleteness.java index 4dbd06a9d7275..8b349240386f9 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFCompleteness.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFCompleteness.java @@ -36,8 +36,7 @@ /** This function calculates completeness of input series. */ public class UDTFCompleteness implements UDTF { - private static final String TIMESTAMP_PRECISION = "timestampPrecision"; - public static final String MS_PRECISION = "ms"; + private boolean downtime; @Override @@ -46,8 +45,7 @@ public void beforeStart(UDFParameters udfp, UDTFConfigurations udtfc) throws Exc long window = Integer.MAX_VALUE; if (udfp.hasAttribute("window")) { String s = udfp.getString("window"); - String timestampPrecision = udfp.getSystemStringOrDefault(TIMESTAMP_PRECISION, MS_PRECISION); - window = Util.parseTime(s, timestampPrecision); + window = Util.parseTime(s, udfp); if (window > 0) { isTime = true; } else { diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFConsistency.java b/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFConsistency.java index ba5732d003eac..3178d33343cb2 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFConsistency.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFConsistency.java @@ -37,8 +37,6 @@ /** This function calculates consistency of input series. */ public class UDTFConsistency implements UDTF { - private static final String TIMESTAMP_PRECISION = "timestampPrecision"; - public static final String MS_PRECISION = "ms"; @Override public void beforeStart(UDFParameters udfp, UDTFConfigurations udtfc) throws Exception { @@ -46,8 +44,7 @@ public void beforeStart(UDFParameters udfp, UDTFConfigurations udtfc) throws Exc long window = Integer.MAX_VALUE; if (udfp.hasAttribute("window")) { String s = udfp.getString("window"); - String timestampPrecision = udfp.getSystemStringOrDefault(TIMESTAMP_PRECISION, MS_PRECISION); - window = Util.parseTime(s, timestampPrecision); + window = Util.parseTime(s, udfp); if (window > 0) { isTime = true; } else { diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFTimeliness.java b/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFTimeliness.java index d1f12fcb11b59..7a5fc98e18290 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFTimeliness.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFTimeliness.java @@ -37,8 +37,6 @@ /** This function calculates timeliness of input series. */ public class UDTFTimeliness implements UDTF { - private static final String TIMESTAMP_PRECISION = "timestampPrecision"; - public static final String MS_PRECISION = "ms"; @Override public void beforeStart(UDFParameters udfp, UDTFConfigurations udtfc) throws Exception { @@ -46,8 +44,7 @@ public void beforeStart(UDFParameters udfp, UDTFConfigurations udtfc) throws Exc long window = Integer.MAX_VALUE; if (udfp.hasAttribute("window")) { String s = udfp.getString("window"); - String timestampPrecision = udfp.getSystemStringOrDefault(TIMESTAMP_PRECISION, MS_PRECISION); - window = Util.parseTime(s, timestampPrecision); + window = Util.parseTime(s, udfp); if (window > 0) { isTime = true; } else { diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFValidity.java b/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFValidity.java index a000b12ca0d05..e3b0de6065bc8 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFValidity.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFValidity.java @@ -37,8 +37,6 @@ /** This function calculates validity of input series. */ public class UDTFValidity implements UDTF { - private static final String TIMESTAMP_PRECISION = "timestampPrecision"; - public static final String MS_PRECISION = "ms"; @Override public void beforeStart(UDFParameters udfp, UDTFConfigurations udtfc) throws Exception { @@ -46,8 +44,7 @@ public void beforeStart(UDFParameters udfp, UDTFConfigurations udtfc) throws Exc long window = Integer.MAX_VALUE; if (udfp.hasAttribute("window")) { String s = udfp.getString("window"); - String timestampPrecision = udfp.getSystemStringOrDefault(TIMESTAMP_PRECISION, MS_PRECISION); - window = Util.parseTime(s, timestampPrecision); + window = Util.parseTime(s, udfp); if (window > 0) { isTime = true; } else { diff --git a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFIFFT.java b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFIFFT.java index e7636c1b71dd4..6e5f52d7e6f1a 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFIFFT.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFIFFT.java @@ -42,15 +42,12 @@ public class UDTFIFFT implements UDTF { private final DoubleArrayList real = new DoubleArrayList(); private final DoubleArrayList imag = new DoubleArrayList(); private final IntArrayList time = new IntArrayList(); - private static final String TIMESTAMP_PRECISION = "timestampPrecision"; - public static final String MS_PRECISION = "ms"; + private long start; private long interval; @Override public void validate(UDFParameterValidator validator) throws Exception { - String timestampPrecision = - validator.getParameters().getSystemStringOrDefault(TIMESTAMP_PRECISION, MS_PRECISION); validator .validateInputSeriesNumber(2) .validateInputSeriesDataType(0, Type.DOUBLE, Type.FLOAT, Type.INT32, Type.INT64) @@ -59,7 +56,7 @@ public void validate(UDFParameterValidator validator) throws Exception { "interval should be a time period whose unit is ms, s, m, h, d.", Util.parseTime( validator.getParameters().getStringOrDefault("interval", "1s"), - timestampPrecision)); + validator.getParameters())); SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); if (validator.getParameters().hasAttribute(START_PARAM)) { validator.validate( @@ -73,10 +70,7 @@ public void validate(UDFParameterValidator validator) throws Exception { public void beforeStart(UDFParameters parameters, UDTFConfigurations configurations) throws Exception { configurations.setAccessStrategy(new RowByRowAccessStrategy()).setOutputDataType(Type.DOUBLE); - String timestampPrecision = - parameters.getSystemStringOrDefault(TIMESTAMP_PRECISION, MS_PRECISION); - this.interval = - Util.parseTime(parameters.getStringOrDefault("interval", "1s"), timestampPrecision); + this.interval = Util.parseTime(parameters.getStringOrDefault("interval", "1s"), parameters); SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); this.start = 0; if (parameters.hasAttribute(START_PARAM)) { diff --git a/library-udf/src/main/java/org/apache/iotdb/library/series/UDTFConsecutiveSequences.java b/library-udf/src/main/java/org/apache/iotdb/library/series/UDTFConsecutiveSequences.java index 49e8b53b7b8cf..a13caa6324319 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/series/UDTFConsecutiveSequences.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/series/UDTFConsecutiveSequences.java @@ -35,27 +35,21 @@ /** This function searches for all longest consecutive subsequences of input sereis. */ public class UDTFConsecutiveSequences implements UDTF { private ConsecutiveUtil consUtil; - private static final String TIMESTAMP_PRECISION = "timestampPrecision"; - public static final String MS_PRECISION = "ms"; @Override public void validate(UDFParameterValidator validator) throws Exception { - String timestampPrecision = - validator.getParameters().getSystemStringOrDefault(TIMESTAMP_PRECISION, MS_PRECISION); validator.validate( x -> (long) x > 0, "gap should be a time period whose unit is ms, s, m, h.", Util.parseTime( - validator.getParameters().getStringOrDefault("gap", "1ms"), timestampPrecision)); + validator.getParameters().getStringOrDefault("gap", "1ms"), validator.getParameters())); } @Override public void beforeStart(UDFParameters parameters, UDTFConfigurations configurations) throws Exception { configurations.setAccessStrategy(new RowByRowAccessStrategy()).setOutputDataType(Type.INT32); - String timestampPrecision = - parameters.getSystemStringOrDefault(TIMESTAMP_PRECISION, MS_PRECISION); - long gap = Util.parseTime(parameters.getStringOrDefault("gap", "0ms"), timestampPrecision); + long gap = Util.parseTime(parameters.getStringOrDefault("gap", "0ms"), parameters); consUtil = new ConsecutiveUtil(-gap, -gap, gap); } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/series/UDTFConsecutiveWindows.java b/library-udf/src/main/java/org/apache/iotdb/library/series/UDTFConsecutiveWindows.java index 52c06283f474a..d7dbe4148b3a8 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/series/UDTFConsecutiveWindows.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/series/UDTFConsecutiveWindows.java @@ -36,34 +36,30 @@ public class UDTFConsecutiveWindows implements UDTF { private ConsecutiveUtil consUtil; private static final int MAX_LEN = 128; - private static final String TIMESTAMP_PRECISION = "timestampPrecision"; - public static final String MS_PRECISION = "ms"; private long len; @Override public void validate(UDFParameterValidator validator) throws Exception { - String timestampPrecision = - validator.getParameters().getSystemStringOrDefault(TIMESTAMP_PRECISION, MS_PRECISION); validator .validate( x -> (long) x > 0, "gap should be a time period whose unit is ms, s, m, h.", Util.parseTime( - validator.getParameters().getStringOrDefault("gap", "1ms"), timestampPrecision)) + validator.getParameters().getStringOrDefault("gap", "1ms"), + validator.getParameters())) .validate( x -> (long) x > 0, "length should be a time period whose unit is ms, s, m, h.", - Util.parseTime(validator.getParameters().getString("length"), timestampPrecision)); + Util.parseTime( + validator.getParameters().getString("length"), validator.getParameters())); } @Override public void beforeStart(UDFParameters parameters, UDTFConfigurations configurations) throws Exception { configurations.setAccessStrategy(new RowByRowAccessStrategy()).setOutputDataType(Type.INT32); - String timestampPrecision = - parameters.getSystemStringOrDefault(TIMESTAMP_PRECISION, MS_PRECISION); - long gap = Util.parseTime(parameters.getStringOrDefault("gap", "0ms"), timestampPrecision); - len = Util.parseTime(parameters.getString("length"), timestampPrecision); + long gap = Util.parseTime(parameters.getStringOrDefault("gap", "0ms"), parameters); + len = Util.parseTime(parameters.getString("length"), parameters); int count = gap == 0 ? 0 : (int) (len / gap + 1); consUtil = new ConsecutiveUtil(-gap, -gap, gap); consUtil.setCount(count); diff --git a/library-udf/src/main/java/org/apache/iotdb/library/util/Util.java b/library-udf/src/main/java/org/apache/iotdb/library/util/Util.java index 893223ee38c65..d71555656d756 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/util/Util.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/util/Util.java @@ -21,6 +21,7 @@ import org.apache.iotdb.udf.api.access.Row; import org.apache.iotdb.udf.api.collector.PointCollector; +import org.apache.iotdb.udf.api.customizer.parameter.UDFParameters; import org.apache.iotdb.udf.api.type.Type; import org.apache.commons.math3.stat.descriptive.rank.Median; @@ -32,6 +33,9 @@ /** This class offers functions of getting and putting values from iotdb interface. */ public class Util { + private static final String TIMESTAMP_PRECISION = "timestampPrecision"; + public static final String MS_PRECISION = "ms"; + private Util() { throw new IllegalStateException("Utility class"); } @@ -300,12 +304,17 @@ public static long mode(long[] values) { * @param s input string * @return timestamp */ - public static long parseTime(String s, String timestampPrecision) { + public static long parseTime(String s, UDFParameters parameters) { long unit = 0; + String timestampPrecision = + parameters.getSystemStringOrDefault(TIMESTAMP_PRECISION, MS_PRECISION); s = s.toLowerCase(); s = s.replace(" ", ""); if (timestampPrecision.equals("ms")) { - if (s.endsWith("ms")) { + if (s.endsWith("ns") || s.endsWith("us")) { + throw new IllegalArgumentException( + "The provided time precision is lower than the system's minimum precision (ms). Please check your input."); + } else if (s.endsWith("ms")) { unit = 1; s = s.substring(0, s.length() - 2); } else if (s.endsWith("s")) { @@ -322,7 +331,10 @@ public static long parseTime(String s, String timestampPrecision) { s = s.substring(0, s.length() - 1); } } else if (timestampPrecision.equals("us")) { - if (s.endsWith("us")) { + if (s.endsWith("ns")) { + throw new IllegalArgumentException( + "The provided time precision is lower than the system's minimum precision (ms). Please check your input."); + } else if (s.endsWith("us")) { unit = 1; s = s.substring(0, s.length() - 2); } else if (s.endsWith("ms")) { From e105ec2d3bd79496bdd4ce1f4a2489c1da682264 Mon Sep 17 00:00:00 2001 From: Beasttt <64394711+Beasttt@users.noreply.github.com> Date: Thu, 12 Dec 2024 04:23:55 +0000 Subject: [PATCH 3/4] Modify UDF time precision_v2 --- .../src/main/java/org/apache/iotdb/library/util/Util.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/library-udf/src/main/java/org/apache/iotdb/library/util/Util.java b/library-udf/src/main/java/org/apache/iotdb/library/util/Util.java index d71555656d756..e30cc7d45c314 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/util/Util.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/util/Util.java @@ -313,7 +313,7 @@ public static long parseTime(String s, UDFParameters parameters) { if (timestampPrecision.equals("ms")) { if (s.endsWith("ns") || s.endsWith("us")) { throw new IllegalArgumentException( - "The provided time precision is lower than the system's minimum precision (ms). Please check your input."); + "The provided time precision is higher than the system's time precision (ms). Please check your input."); } else if (s.endsWith("ms")) { unit = 1; s = s.substring(0, s.length() - 2); @@ -333,7 +333,7 @@ public static long parseTime(String s, UDFParameters parameters) { } else if (timestampPrecision.equals("us")) { if (s.endsWith("ns")) { throw new IllegalArgumentException( - "The provided time precision is lower than the system's minimum precision (ms). Please check your input."); + "The provided time precision is higher than the system's time precision (us). Please check your input."); } else if (s.endsWith("us")) { unit = 1; s = s.substring(0, s.length() - 2); From 50bd005446dfb7f5526ac42c6e49615e607c9f28 Mon Sep 17 00:00:00 2001 From: Beasttt <64394711+Beasttt@users.noreply.github.com> Date: Sun, 15 Dec 2024 19:03:18 +0000 Subject: [PATCH 4/4] Modify UDF --- .../iotdb/library/dprofile/UDAFIntegral.java | 41 ++++--------------- .../library/drepair/UDTFTimestampRepair.java | 35 +++++++++++----- .../drepair/util/TimestampInterval.java | 22 ++++------ .../library/drepair/util/TimestampRepair.java | 2 +- 4 files changed, 42 insertions(+), 58 deletions(-) diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFIntegral.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFIntegral.java index 2bb938d5f327c..c293e929cd00b 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFIntegral.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFIntegral.java @@ -37,11 +37,8 @@ public class UDAFIntegral implements UDTF { private static final String TIME_UNIT_KEY = "unit"; - private static final String TIME_UNIT_MS = "1S"; private static final String TIME_UNIT_S = "1s"; - private static final String TIME_UNIT_M = "1m"; - private static final String TIME_UNIT_H = "1H"; - private static final String TIME_UNIT_D = "1d"; + long unitTime; long lastTime = -1; @@ -53,41 +50,19 @@ public void validate(UDFParameterValidator validator) throws Exception { validator .validateInputSeriesNumber(1) .validate( - unit -> - TIME_UNIT_D.equals(unit) - || TIME_UNIT_H.equals(unit) - || TIME_UNIT_M.equals(unit) - || TIME_UNIT_S.equals(unit) - || TIME_UNIT_MS.equals(unit), - "Unknown time unit input", - validator.getParameters().getStringOrDefault(TIME_UNIT_KEY, TIME_UNIT_S)); + x -> (long) x > 0, + "Unknown time unit input.", + Util.parseTime( + validator.getParameters().getStringOrDefault(TIME_UNIT_KEY, TIME_UNIT_S), + validator.getParameters())); } @Override public void beforeStart(UDFParameters parameters, UDTFConfigurations configurations) throws Exception { configurations.setAccessStrategy(new RowByRowAccessStrategy()).setOutputDataType(Type.DOUBLE); - switch (parameters.getStringOrDefault(TIME_UNIT_KEY, TIME_UNIT_S)) { - case TIME_UNIT_MS: - unitTime = 1L; - break; - case TIME_UNIT_S: - unitTime = 1000L; - break; - case TIME_UNIT_M: - unitTime = 60000L; - break; - case TIME_UNIT_H: - unitTime = 3600000L; - break; - case TIME_UNIT_D: - unitTime = 3600000L * 24L; - break; - default: - throw new UDFException( - "Unknown time unit input: " - + parameters.getStringOrDefault(TIME_UNIT_KEY, TIME_UNIT_S)); - } + unitTime = + Util.parseTime(parameters.getStringOrDefault(TIME_UNIT_KEY, TIME_UNIT_S), parameters); } @Override diff --git a/library-udf/src/main/java/org/apache/iotdb/library/drepair/UDTFTimestampRepair.java b/library-udf/src/main/java/org/apache/iotdb/library/drepair/UDTFTimestampRepair.java index 9554cff6d18c9..c9ec136dcf925 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/drepair/UDTFTimestampRepair.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/drepair/UDTFTimestampRepair.java @@ -33,18 +33,26 @@ /** This function is used for timestamp repair. */ public class UDTFTimestampRepair implements UDTF { String intervalMethod; - int interval; - int intervalMode; + long interval; + long intervalMode; @Override public void validate(UDFParameterValidator validator) throws Exception { validator .validateInputSeriesNumber(1) - .validateInputSeriesDataType(0, Type.DOUBLE, Type.FLOAT, Type.INT32, Type.INT64) - .validate( - x -> (Integer) x >= 0, - "Interval should be a positive integer.", - validator.getParameters().getIntOrDefault("interval", 0)); + .validateInputSeriesDataType(0, Type.DOUBLE, Type.FLOAT, Type.INT32, Type.INT64); + String intervalString = validator.getParameters().getStringOrDefault("interval", null); + if (intervalString != null) { + try { + long parsedInterval = Util.parseTime(intervalString, validator.getParameters()); + validator.validate( + x -> parsedInterval > 0, + "Invalid time unit input. Supported units are ns, us, ms, s, m, h, d."); + } catch (Exception e) { + throw new UDFException( + "Invalid time format for interval."); + } + } } @Override @@ -54,15 +62,20 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati .setAccessStrategy(new SlidingSizeWindowAccessStrategy(Integer.MAX_VALUE)) .setOutputDataType(parameters.getDataType(0)); intervalMethod = parameters.getStringOrDefault("method", "Median"); - interval = parameters.getIntOrDefault("interval", 0); + String intervalString = parameters.getStringOrDefault("interval", null); + if (intervalString != null) { + interval = Util.parseTime(intervalString, parameters); + } else { + interval = 0; + } if (interval > 0) { intervalMode = interval; } else if ("Median".equalsIgnoreCase(intervalMethod)) { - intervalMode = -1; + intervalMode = -1L; } else if ("Mode".equalsIgnoreCase(intervalMethod)) { - intervalMode = -2; + intervalMode = -2L; } else if ("Cluster".equalsIgnoreCase(intervalMethod)) { - intervalMode = -3; + intervalMode = -3L; } else { throw new UDFException("Illegal method."); } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/drepair/util/TimestampInterval.java b/library-udf/src/main/java/org/apache/iotdb/library/drepair/util/TimestampInterval.java index 57feae842299b..f596a3552db2c 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/drepair/util/TimestampInterval.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/drepair/util/TimestampInterval.java @@ -44,19 +44,15 @@ public TimestampInterval(long[] time, double[] original) { // get standard interval // -1 median -2 mode -3 cluster - public long getInterval(int mode) { - switch (mode) { - case -1: - this.deltaT = getIntervalByMedian(); - break; - case -2: - this.deltaT = getIntervalByMode(); - break; - case -3: - this.deltaT = getIntervalByCluster(); - break; - default: - this.deltaT = mode; + public long getInterval(long mode) { + if (mode == -1L) { + this.deltaT = getIntervalByMedian(); + } else if (mode == -2L) { + this.deltaT = getIntervalByMode(); + } else if (mode == -3L) { + this.deltaT = getIntervalByCluster(); + } else { + this.deltaT = mode; } return this.deltaT; } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/drepair/util/TimestampRepair.java b/library-udf/src/main/java/org/apache/iotdb/library/drepair/util/TimestampRepair.java index d1a856e56a9e4..4b98256436bef 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/drepair/util/TimestampRepair.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/drepair/util/TimestampRepair.java @@ -35,7 +35,7 @@ public class TimestampRepair { protected long deltaT; protected long start0; - public TimestampRepair(RowIterator dataIterator, int intervalMode, int startPointMode) + public TimestampRepair(RowIterator dataIterator, long intervalMode, int startPointMode) throws Exception { ArrayList timeList = new ArrayList<>(); ArrayList originList = new ArrayList<>();