From 7b1b6731b16d893f2489e1a50d88a7e37bda0060 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 9 Jun 2026 09:46:39 +0800 Subject: [PATCH 1/6] Fix library UDF edge cases --- .../apache/iotdb/library/anomaly/UDTFIQR.java | 15 +- .../iotdb/library/anomaly/UDTFKSigma.java | 6 +- .../apache/iotdb/library/anomaly/UDTFLOF.java | 7 +- .../iotdb/library/anomaly/UDTFRange.java | 9 +- .../apache/iotdb/library/dlearn/UDTFAR.java | 14 + .../iotdb/library/dlearn/UDTFCluster.java | 2 + .../apache/iotdb/library/dmatch/UDAFDtw.java | 10 +- .../iotdb/library/dprofile/UDAFMad.java | 14 +- .../iotdb/library/dprofile/UDAFMedian.java | 4 +- .../library/dprofile/UDAFPercentile.java | 139 ++- .../iotdb/library/dprofile/UDAFPeriod.java | 2 +- .../iotdb/library/dprofile/UDAFQuantile.java | 16 +- .../iotdb/library/dprofile/UDAFSkew.java | 7 + .../iotdb/library/dprofile/UDTFHistogram.java | 4 +- .../iotdb/library/dprofile/UDTFMinMax.java | 26 +- .../iotdb/library/dprofile/UDTFMvAvg.java | 3 +- .../iotdb/library/dprofile/UDTFQLB.java | 1 + .../iotdb/library/dprofile/UDTFResample.java | 1 + .../iotdb/library/dprofile/UDTFSample.java | 4 + .../iotdb/library/dprofile/UDTFSegment.java | 4 +- .../iotdb/library/dprofile/UDTFSpline.java | 15 +- .../iotdb/library/dprofile/UDTFZScore.java | 6 + .../dprofile/util/ExactOrderStatistics.java | 8 +- .../library/dquality/UDTFConsistency.java | 2 +- .../library/dquality/UDTFTimeliness.java | 2 +- .../iotdb/library/dquality/UDTFValidity.java | 2 +- .../frequency/UDFEnvelopeAnalysis.java | 3 + .../iotdb/library/frequency/UDTFConv.java | 3 + .../iotdb/library/frequency/UDTFDWT.java | 12 +- .../iotdb/library/frequency/UDTFDeconv.java | 20 +- .../iotdb/library/frequency/UDTFFFT.java | 3 + .../iotdb/library/frequency/UDTFHighPass.java | 3 + .../iotdb/library/frequency/UDTFIDWT.java | 12 +- .../iotdb/library/frequency/UDTFIFFT.java | 4 + .../iotdb/library/frequency/UDTFLowPass.java | 3 + .../iotdb/library/frequency/util/DWTUtil.java | 13 +- .../iotdb/library/match/model/DTWState.java | 25 - .../series/UDTFConsecutiveWindows.java | 1 + .../library/series/util/ConsecutiveUtil.java | 4 + .../iotdb/library/string/UDTFRegexMatch.java | 15 +- .../library/string/UDTFRegexReplace.java | 16 +- .../iotdb/library/string/UDTFRegexSplit.java | 16 +- .../iotdb/library/string/UDTFStrReplace.java | 2 + .../library/util/BooleanCircularQueue.java | 3 + .../iotdb/library/util/CircularQueue.java | 3 + .../library/util/DoubleCircularQueue.java | 3 + .../iotdb/library/util/LongCircularQueue.java | 3 + .../iotdb/library/UDFWindowAndQueueTest.java | 1088 +++++++++++++++++ 48 files changed, 1416 insertions(+), 162 deletions(-) create mode 100644 library-udf/src/test/java/org/apache/iotdb/library/UDFWindowAndQueueTest.java diff --git a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFIQR.java b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFIQR.java index ce13cd1296efe..4b800003dac82 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFIQR.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFIQR.java @@ -63,6 +63,12 @@ public void validate(UDFParameterValidator validator) throws Exception { "parameter $q1$ should be smaller than $q3$", validator.getParameters().getDoubleOrDefault("q1", -1), validator.getParameters().getDoubleOrDefault("q3", 1)); + if (validator + .getParameters() + .getStringOrDefault("compute", BATCH_COMPUTE) + .equalsIgnoreCase(STREAM_COMPUTE)) { + validator.validateRequiredAttribute("q1").validateRequiredAttribute("q3"); + } } @Override @@ -91,14 +97,19 @@ public void transform(Row row, PointCollector collector) throws Exception { } } else if (compute.equalsIgnoreCase(BATCH_COMPUTE)) { double v = Util.getValueAsDouble(row); - value.add(v); - timestamp.add(row.getTime()); + if (Double.isFinite(v)) { + value.add(v); + timestamp.add(row.getTime()); + } } } @Override public void terminate(PointCollector collector) throws Exception { if (compute.equalsIgnoreCase(BATCH_COMPUTE)) { + if (value.isEmpty()) { + return; + } q1 = Quantiles.quartiles().index(1).compute(value); q3 = Quantiles.quartiles().index(3).compute(value); iqr = q3 - q1; diff --git a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFKSigma.java b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFKSigma.java index 1d63d040d87fa..c19fd7a872272 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFKSigma.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFKSigma.java @@ -33,6 +33,8 @@ /** This function detects outliers which lies over average +/- k * sigma. */ public class UDTFKSigma implements UDTF { + private static final int DEFAULT_WINDOW_SIZE = 10; + private double mean = 0.0; private double variance = 0.0; private double sumX2 = 0.0; @@ -51,7 +53,7 @@ public void validate(UDFParameterValidator validator) throws Exception { .validate( x -> (int) x > 0, "Window size should be larger than 0.", - validator.getParameters().getIntOrDefault("window", 10)) + validator.getParameters().getIntOrDefault("window", DEFAULT_WINDOW_SIZE)) .validate( x -> (double) x > 0, "Parameter k should be larger than 0.", @@ -66,7 +68,7 @@ public void beforeStart(UDFParameters udfParameters, UDTFConfigurations udtfConf .setOutputDataType(udfParameters.getDataType(0)); this.multipleK = udfParameters.getDoubleOrDefault("k", 3); this.dataType = udfParameters.getDataType(0); - this.windowSize = udfParameters.getIntOrDefault("window", 10000); + this.windowSize = udfParameters.getIntOrDefault("window", DEFAULT_WINDOW_SIZE); this.v = new CircularQueue<>(windowSize); this.t = new LongCircularQueue(windowSize); } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFLOF.java b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFLOF.java index 22bc7d742376b..58078771b62a3 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFLOF.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFLOF.java @@ -119,7 +119,10 @@ private double dist(Double[] nnk, Double[] x) { @Override public void validate(UDFParameterValidator validator) throws Exception { - validator.validateInputSeriesDataType(0, Type.INT32, Type.INT64, Type.FLOAT, Type.DOUBLE); + validator.validateInputSeriesNumber(1, Integer.MAX_VALUE); + for (int i = 0; i < validator.getParameters().getChildExpressionsSize(); i++) { + validator.validateInputSeriesDataType(i, Type.INT32, Type.INT64, Type.FLOAT, Type.DOUBLE); + } } @Override @@ -147,7 +150,7 @@ public void transform(RowWindow rowWindow, PointCollector collector) throws Exce timestamp[i] = rowWindow.getRow(row).getTime(); for (int j = 0; j < dim; j++) { if (!rowWindow.getRow(row).isNull(j)) { - knn[i][j] = Util.getValueAsDouble(rowWindow.getRow(i), j); + knn[i][j] = Util.getValueAsDouble(rowWindow.getRow(row), j); } else { i--; size--; diff --git a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFRange.java b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFRange.java index 783162aab690c..dc75d575d9e07 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFRange.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFRange.java @@ -40,7 +40,14 @@ public class UDTFRange implements UDTF { public void validate(UDFParameterValidator validator) throws Exception { validator .validateInputSeriesNumber(1) - .validateInputSeriesDataType(0, Type.INT32, Type.INT64, Type.FLOAT, Type.DOUBLE); + .validateInputSeriesDataType(0, Type.INT32, Type.INT64, Type.FLOAT, Type.DOUBLE) + .validateRequiredAttribute("lower_bound") + .validateRequiredAttribute("upper_bound") + .validate( + params -> (double) params[0] < (double) params[1], + "parameter \"lower_bound\" should be smaller than \"upper_bound\".", + validator.getParameters().getDouble("lower_bound"), + validator.getParameters().getDouble("upper_bound")); } @Override diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dlearn/UDTFAR.java b/library-udf/src/main/java/org/apache/iotdb/library/dlearn/UDTFAR.java index 450b6359c36d1..cc4d496187660 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dlearn/UDTFAR.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dlearn/UDTFAR.java @@ -25,6 +25,7 @@ import org.apache.iotdb.udf.api.access.Row; import org.apache.iotdb.udf.api.collector.PointCollector; import org.apache.iotdb.udf.api.customizer.config.UDTFConfigurations; +import org.apache.iotdb.udf.api.customizer.parameter.UDFParameterValidator; import org.apache.iotdb.udf.api.customizer.parameter.UDFParameters; import org.apache.iotdb.udf.api.customizer.strategy.RowByRowAccessStrategy; import org.apache.iotdb.udf.api.exception.UDFException; @@ -38,12 +39,25 @@ public class UDTFAR implements UDTF { private List timeWindow = new ArrayList<>(); private List valueWindow = new ArrayList<>(); + @Override + public void validate(UDFParameterValidator validator) throws Exception { + validator + .validateInputSeriesNumber(1) + .validateInputSeriesDataType(0, Type.INT32, Type.INT64, Type.FLOAT, Type.DOUBLE) + .validate( + x -> (int) x > 0, + "Parameter p should be a positive integer.", + validator.getParameters().getIntOrDefault("p", 1)); + } + @Override public void beforeStart(UDFParameters udfParameters, UDTFConfigurations udtfConfigurations) throws Exception { udtfConfigurations .setAccessStrategy(new RowByRowAccessStrategy()) .setOutputDataType(udfParameters.getDataType(0)); + timeWindow.clear(); + valueWindow.clear(); this.p = udfParameters.getIntOrDefault("p", 1); udtfConfigurations.setAccessStrategy(new RowByRowAccessStrategy()); udtfConfigurations.setOutputDataType(Type.DOUBLE); diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dlearn/UDTFCluster.java b/library-udf/src/main/java/org/apache/iotdb/library/dlearn/UDTFCluster.java index 6faadf8c4a3e3..e349b188ad33d 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dlearn/UDTFCluster.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dlearn/UDTFCluster.java @@ -72,6 +72,8 @@ public void validate(UDFParameterValidator validator) throws Exception { validator .validateInputSeriesNumber(1) .validateInputSeriesDataType(0, Type.INT32, Type.INT64, Type.FLOAT, Type.DOUBLE) + .validateRequiredAttribute("l") + .validateRequiredAttribute("k") .validate( x -> (int) x > 0, "Parameter l must be a positive integer.", diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dmatch/UDAFDtw.java b/library-udf/src/main/java/org/apache/iotdb/library/dmatch/UDAFDtw.java index 37645f3d6ab83..66efda369a3d6 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dmatch/UDAFDtw.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dmatch/UDAFDtw.java @@ -52,6 +52,8 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati configurations .setAccessStrategy(new SlidingSizeWindowAccessStrategy(Integer.MAX_VALUE)) .setOutputDataType(Type.DOUBLE); + dp = null; + m = 0; } @Override @@ -68,6 +70,10 @@ public void transform(RowWindow rowWindow, PointCollector collector) throws Exce b.add(Util.getValueAsDouble(row, 1)); } m = a.size(); + if (m == 0) { + dp = null; + return; + } dp = new double[m + 1][m + 1]; for (int i = 1; i <= m; i++) { dp[0][i] = dp[i][0] = Double.MAX_VALUE; @@ -84,6 +90,8 @@ public void transform(RowWindow rowWindow, PointCollector collector) throws Exce @Override public void terminate(PointCollector collector) throws Exception { - collector.putDouble(0, dp[m][m]); + if (dp != null) { + collector.putDouble(0, dp[m][m]); + } } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFMad.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFMad.java index fb3ffca8043e8..08cbb3964bbb7 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFMad.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFMad.java @@ -31,6 +31,8 @@ import org.apache.iotdb.udf.api.customizer.strategy.RowByRowAccessStrategy; import org.apache.iotdb.udf.api.type.Type; +import java.util.NoSuchElementException; + /** calculate the exact or approximate median absolute deviation (mad). */ public class UDAFMad implements UDTF { @@ -73,10 +75,14 @@ public void transform(Row row, PointCollector collector) throws Exception { @Override public void terminate(PointCollector collector) throws Exception { - if (exact) { - collector.putDouble(0, statistics.getMad()); - } else { - collector.putDouble(0, sketch.getMad().result); + try { + if (exact) { + collector.putDouble(0, statistics.getMad()); + } else { + collector.putDouble(0, sketch.getMad().result); + } + } catch (NoSuchElementException e) { + // Empty inputs have no MAD to emit. } } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFMedian.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFMedian.java index 9c78bd316e5a7..425b4955df131 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFMedian.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFMedian.java @@ -81,8 +81,8 @@ public void terminate(PointCollector collector) throws Exception { } else { collector.putDouble(0, sketch.query(0.5)); } - } catch (NoSuchElementException e) { - // just ignore it + } catch (NoSuchElementException | ArithmeticException e) { + // Empty inputs have no median to emit. } } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFPercentile.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFPercentile.java index d3baa6a7bf0d9..9a58e1c0a500f 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFPercentile.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFPercentile.java @@ -33,13 +33,14 @@ import java.util.HashMap; import java.util.Map; +import java.util.NoSuchElementException; /** calculate the approximate percentile. */ public class UDAFPercentile implements UDTF { - protected static Map intDic; - protected static Map longDic; - protected static Map floatDic; - protected static Map doubleDic; + protected Map intDic; + protected Map longDic; + protected Map floatDic; + protected Map doubleDic; private ExactOrderStatistics statistics; private GKArray sketch; private boolean exact; @@ -106,16 +107,24 @@ public void transform(Row row, PointCollector collector) throws Exception { statistics.insert(row); switch (dataType) { case INT32: - intDic.put(row.getInt(0), row.getTime()); + if (!row.isNull(0)) { + intDic.put(row.getInt(0), row.getTime()); + } break; case INT64: - longDic.put(row.getLong(0), row.getTime()); + if (!row.isNull(0)) { + longDic.put(row.getLong(0), row.getTime()); + } break; case FLOAT: - floatDic.put(row.getFloat(0), row.getTime()); + if (!row.isNull(0) && Float.isFinite(row.getFloat(0))) { + floatDic.put(row.getFloat(0), row.getTime()); + } break; case DOUBLE: - doubleDic.put(row.getDouble(0), row.getTime()); + if (!row.isNull(0) && Double.isFinite(row.getDouble(0))) { + doubleDic.put(row.getDouble(0), row.getTime()); + } break; case BLOB: case BOOLEAN: @@ -134,62 +143,66 @@ public void transform(Row row, PointCollector collector) throws Exception { @Override public void terminate(PointCollector collector) throws Exception { - if (exact) { - long time; - switch (dataType) { - case INT32: - int ires = Integer.parseInt(statistics.getPercentile(rank)); - time = intDic.getOrDefault(ires, 0L); - collector.putInt(time, ires); - break; - case INT64: - long lres = Long.parseLong(statistics.getPercentile(rank)); - time = longDic.getOrDefault(lres, 0L); - collector.putLong(time, lres); - break; - case FLOAT: - float fres = Float.parseFloat(statistics.getPercentile(rank)); - time = floatDic.getOrDefault(fres, 0L); - collector.putFloat(time, fres); - break; - case DOUBLE: - double dres = Double.parseDouble(statistics.getPercentile(rank)); - time = doubleDic.getOrDefault(dres, 0L); - collector.putDouble(time, dres); - break; - case DATE: - case TIMESTAMP: - case TEXT: - case STRING: - case BOOLEAN: - case BLOB: - default: - break; - } - } else { - double res = sketch.query(rank); - switch (dataType) { - case INT32: - collector.putInt(0, (int) res); - break; - case INT64: - collector.putLong(0, (long) res); - break; - case FLOAT: - collector.putFloat(0, (float) res); - break; - case DOUBLE: - collector.putDouble(0, res); - break; - case BOOLEAN: - case BLOB: - case STRING: - case TEXT: - case TIMESTAMP: - case DATE: - default: - break; + try { + if (exact) { + long time; + switch (dataType) { + case INT32: + int ires = Integer.parseInt(statistics.getPercentile(rank)); + time = intDic.getOrDefault(ires, 0L); + collector.putInt(time, ires); + break; + case INT64: + long lres = Long.parseLong(statistics.getPercentile(rank)); + time = longDic.getOrDefault(lres, 0L); + collector.putLong(time, lres); + break; + case FLOAT: + float fres = Float.parseFloat(statistics.getPercentile(rank)); + time = floatDic.getOrDefault(fres, 0L); + collector.putFloat(time, fres); + break; + case DOUBLE: + double dres = Double.parseDouble(statistics.getPercentile(rank)); + time = doubleDic.getOrDefault(dres, 0L); + collector.putDouble(time, dres); + break; + case DATE: + case TIMESTAMP: + case TEXT: + case STRING: + case BOOLEAN: + case BLOB: + default: + break; + } + } else { + double res = sketch.query(rank); + switch (dataType) { + case INT32: + collector.putInt(0, (int) res); + break; + case INT64: + collector.putLong(0, (long) res); + break; + case FLOAT: + collector.putFloat(0, (float) res); + break; + case DOUBLE: + collector.putDouble(0, res); + break; + case BOOLEAN: + case BLOB: + case STRING: + case TEXT: + case TIMESTAMP: + case DATE: + default: + break; + } } + } catch (NoSuchElementException | ArithmeticException e) { + // Empty inputs have no percentile to emit. } } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFPeriod.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFPeriod.java index 5e189d1636788..6d17edea7569b 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFPeriod.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFPeriod.java @@ -61,7 +61,7 @@ public void transform(RowWindow rowWindow, PointCollector collector) throws Exce double v = Util.getValueAsDouble(row); if (Double.isFinite(v)) { value.add(v); - } else { + } else if (!value.isEmpty()) { value.add(value.getLast()); } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFQuantile.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFQuantile.java index 0518e7a3d768a..5e801b0728240 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFQuantile.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFQuantile.java @@ -83,16 +83,16 @@ public void transform(Row row, PointCollector collector) throws Exception { @Override public void terminate(PointCollector collector) throws Exception { long n = sketch.getN(); + if (n == 0) { + return; + } // Nearest-rank: k-th smallest uses getApproxRank (strictly-less-than count) in [0, n-1]; // rank=1 must map to k=n-1, not k=n which is unreachable and can overshoot the max sample. - long k = 0; - if (n > 0) { - k = (long) Math.ceil(rank * n) - 1; - if (k < 0) { - k = 0; - } else if (k >= n) { - k = n - 1; - } + long k = (long) Math.ceil(rank * n) - 1; + if (k < 0) { + k = 0; + } else if (k >= n) { + k = n - 1; } long result = sketch.findMinValueWithRank(k); switch (dataType) { diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFSkew.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFSkew.java index 7f583142967fc..3ee5954ed9cd9 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFSkew.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFSkew.java @@ -47,6 +47,10 @@ public void validate(UDFParameterValidator validator) throws Exception { public void beforeStart(UDFParameters parameters, UDTFConfigurations configurations) throws Exception { configurations.setAccessStrategy(new RowByRowAccessStrategy()).setOutputDataType(Type.DOUBLE); + count = 0; + sumX1 = 0.0; + sumX2 = 0.0; + sumX3 = 0.0; } @Override @@ -62,6 +66,9 @@ public void transform(Row row, PointCollector collector) throws Exception { @Override public void terminate(PointCollector collector) throws Exception { + if (count == 0) { + return; + } collector.putDouble( 0, (sumX3 / count - 3 * sumX1 / count * sumX2 / count + 2 * Math.pow(sumX1 / count, 3)) diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFHistogram.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFHistogram.java index 9976875b21ab5..d29f25d8717af 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFHistogram.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFHistogram.java @@ -50,8 +50,8 @@ public void validate(UDFParameterValidator validator) throws Exception { "parameter $count$ should be larger than 0", validator.getParameters().getIntOrDefault("count", 1)) .validate( - params -> (double) params[0] <= (double) params[1], - "parameter $end$ should be larger than or equal to $start$", + params -> (double) params[0] < (double) params[1], + "parameter $end$ should be larger than $start$", validator.getParameters().getDoubleOrDefault("min", -Double.MAX_VALUE), validator.getParameters().getDoubleOrDefault("max", Double.MAX_VALUE)); } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFMinMax.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFMinMax.java index a4eb892e778b9..b01cd79540241 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFMinMax.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFMinMax.java @@ -58,6 +58,12 @@ public void validate(UDFParameterValidator validator) throws Exception { "parameter $min$ should be smaller than $max$.", validator.getParameters().getDoubleOrDefault("min", -Double.MAX_VALUE), validator.getParameters().getDoubleOrDefault("max", Double.MAX_VALUE)); + if (validator + .getParameters() + .getStringOrDefault("compute", BATCH_COMPUTE) + .equalsIgnoreCase(STREAM_COMPUTE)) { + validator.validateRequiredAttribute("min").validateRequiredAttribute("max"); + } } @Override @@ -84,17 +90,17 @@ public void transform(Row row, PointCollector collector) throws Exception { double v = Util.getValueAsDouble(row); if (Double.isFinite(v)) { value.add(v); - } - timestamp.add(row.getTime()); - if (flag) { - min = v; - max = v; - flag = false; - } else { - if (v > max) { - max = v; - } else if (v < min) { + timestamp.add(row.getTime()); + if (flag) { min = v; + max = v; + flag = false; + } else { + if (v > max) { + max = v; + } else if (v < min) { + min = v; + } } } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFMvAvg.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFMvAvg.java index 7152e07ce80d9..9b93c7e223256 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFMvAvg.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFMvAvg.java @@ -35,6 +35,7 @@ public class UDTFMvAvg implements UDTF { int windowSize; Type dataType; DoubleCircularQueue v; + double windowSum; @Override public void validate(UDFParameterValidator validator) throws Exception { @@ -54,12 +55,12 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati dataType = parameters.getDataType(0); windowSize = parameters.getIntOrDefault("window", 10); v = new DoubleCircularQueue(windowSize); + windowSum = 0d; } @Override public void transform(Row row, PointCollector collector) throws Exception { long t = row.getTime(); - double windowSum = 0d; if (v.isFull()) { windowSum -= v.pop(); } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFQLB.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFQLB.java index 85ce3e7411e48..325360a89e76a 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFQLB.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFQLB.java @@ -55,6 +55,7 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati throws Exception { configurations.setAccessStrategy(new RowByRowAccessStrategy()).setOutputDataType(Type.DOUBLE); m = parameters.getIntOrDefault("lag", 0); + qlb = 0.0f; valueArrayList.clear(); } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFResample.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFResample.java index 15d2828c04f35..441067ceb6902 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFResample.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFResample.java @@ -43,6 +43,7 @@ public void validate(UDFParameterValidator validator) throws Exception { validator .validateInputSeriesNumber(1) .validateInputSeriesDataType(0, Type.DOUBLE, Type.FLOAT, Type.INT32, Type.INT64) + .validateRequiredAttribute("every") .validate( x -> (long) x > 0, "gap should be a time period whose unit is ms, s, m, h, d.", diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFSample.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFSample.java index 545d4096f51b3..f1a8f38d1922d 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFSample.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFSample.java @@ -80,6 +80,7 @@ public void validate(UDFParameterValidator validator) throws Exception { public void beforeStart(UDFParameters parameters, UDTFConfigurations configurations) throws Exception { this.k = parameters.getIntOrDefault("k", 1); + this.num = 0; this.dataType = parameters.getDataType(0); String methodIn = parameters.getStringOrDefault("method", METHOD_RESERVOIR); if ("triangle".equalsIgnoreCase(methodIn)) { @@ -99,7 +100,10 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati .setOutputDataType(parameters.getDataType(0)); this.samples = new Pair[this.k]; this.random = new Random(); + return; } + this.samples = null; + this.random = null; } @Override diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFSegment.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFSegment.java index aead60e7942e9..e941a7a5c7a34 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFSegment.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFSegment.java @@ -42,8 +42,8 @@ public class UDTFSegment implements UDTF { private static final String METHOD_BOTTOM_UP = "bottom-up"; private String output; private static final String OUTPUT_FIRST = "first"; - private static final List timestamp = new ArrayList<>(); - private static final List value = new ArrayList<>(); + private final List timestamp = new ArrayList<>(); + private final List value = new ArrayList<>(); @Override public void validate(UDFParameterValidator validator) throws Exception { diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFSpline.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFSpline.java index 34f213c571a34..62c4253db882b 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFSpline.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFSpline.java @@ -41,14 +41,19 @@ public class UDTFSpline implements UDTF { ArrayList timestamp = new ArrayList<>(); ArrayList yDouble = new ArrayList<>(); ArrayList xDouble = new ArrayList<>(); - Long minimumTimestamp = -1L; + Long minimumTimestamp; PolynomialSplineFunction psf; @Override public void validate(UDFParameterValidator validator) throws Exception { validator .validateInputSeriesNumber(1) - .validateInputSeriesDataType(0, Type.FLOAT, Type.DOUBLE, Type.INT32, Type.INT64); + .validateInputSeriesDataType(0, Type.FLOAT, Type.DOUBLE, Type.INT32, Type.INT64) + .validateRequiredAttribute("points") + .validate( + x -> (int) x >= 2, + "Parameter points should be at least 2.", + validator.getParameters().getInt("points")); } @Override @@ -59,6 +64,8 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati timestamp.clear(); xDouble.clear(); yDouble.clear(); + minimumTimestamp = null; + psf = null; } @Override @@ -66,7 +73,7 @@ public void transform(Row row, PointCollector collector) throws Exception { double v = Util.getValueAsDouble(row); if (Double.isFinite(v)) { Long t = row.getTime(); - if (minimumTimestamp < 0) { + if (minimumTimestamp == null) { minimumTimestamp = t; } timestamp.add(t); @@ -77,7 +84,7 @@ public void transform(Row row, PointCollector collector) throws Exception { @Override public void terminate(PointCollector collector) throws Exception { - if (yDouble.size() >= 4 && samplePoints >= 2) { // 4个点以上才进行插值 + if (yDouble.size() >= 5 && samplePoints >= 2) { asi = new AkimaSplineInterpolator(); double[] x = ArrayUtils.toPrimitive(xDouble.toArray(new Double[0])); double[] y = ArrayUtils.toPrimitive(yDouble.toArray(new Double[0])); diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFZScore.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFZScore.java index f8485af84614e..a040cc76f0f8a 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFZScore.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFZScore.java @@ -61,6 +61,12 @@ public void validate(UDFParameterValidator validator) throws Exception { x -> ((Double) x) > 0, "Parameter \"sd\" is illegal. It should be larger than 0.", validator.getParameters().getDoubleOrDefault("sd", 1.0)); + if (validator + .getParameters() + .getStringOrDefault("compute", BATCH_COMPUTE) + .equalsIgnoreCase(STREAM_COMPUTE)) { + validator.validateRequiredAttribute("avg").validateRequiredAttribute("sd"); + } } @Override diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/util/ExactOrderStatistics.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/util/ExactOrderStatistics.java index 47ca5e2b12fd8..91fe3fdd2ab6c 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/util/ExactOrderStatistics.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/util/ExactOrderStatistics.java @@ -172,7 +172,7 @@ public static double getMad(FloatArrayList nums) { double median = getMedian(nums); DoubleArrayList dal = new DoubleArrayList(); for (int i = 0; i < nums.size(); ++i) { - dal.set(i, Math.abs(nums.get(i) - median)); + dal.add(Math.abs(nums.get(i) - median)); } return getMedian(dal); } @@ -202,7 +202,7 @@ public static double getMad(DoubleArrayList nums) { double median = getMedian(nums); DoubleArrayList dal = new DoubleArrayList(); for (int i = 0; i < nums.size(); ++i) { - dal.set(i, Math.abs(nums.get(i) - median)); + dal.add(Math.abs(nums.get(i) - median)); } return getMedian(dal); } @@ -215,7 +215,7 @@ public static double getMad(IntArrayList nums) { double median = getMedian(nums); DoubleArrayList dal = new DoubleArrayList(); for (int i = 0; i < nums.size(); ++i) { - dal.set(i, Math.abs(nums.get(i) - median)); + dal.add(Math.abs(nums.get(i) - median)); } return getMedian(dal); } @@ -228,7 +228,7 @@ public static double getMad(LongArrayList nums) { double median = getMedian(nums); DoubleArrayList dal = new DoubleArrayList(); for (int i = 0; i < nums.size(); ++i) { - dal.set(i, Math.abs(nums.get(i) - median)); + dal.add(Math.abs(nums.get(i) - median)); } return getMedian(dal); } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFConsistency.java b/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFConsistency.java index 3178d33343cb2..f7e7af981c252 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFConsistency.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFConsistency.java @@ -68,7 +68,7 @@ public void transform(RowWindow rowWindow, PointCollector collector) throws Exce collector.putDouble(rowWindow.getRow(0).getTime(), tsq.getConsistency()); } } catch (IOException | NoNumberException ex) { - Logger.getLogger(UDTFCompleteness.class.getName()).log(Level.SEVERE, null, ex); + Logger.getLogger(UDTFConsistency.class.getName()).log(Level.SEVERE, null, ex); } } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFTimeliness.java b/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFTimeliness.java index 7a5fc98e18290..e1695d784a248 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFTimeliness.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFTimeliness.java @@ -68,7 +68,7 @@ public void transform(RowWindow rowWindow, PointCollector collector) throws Exce collector.putDouble(rowWindow.getRow(0).getTime(), tsq.getTimeliness()); } } catch (IOException | NoNumberException ex) { - Logger.getLogger(UDTFCompleteness.class.getName()).log(Level.SEVERE, null, ex); + Logger.getLogger(UDTFTimeliness.class.getName()).log(Level.SEVERE, null, ex); } } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFValidity.java b/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFValidity.java index e3b0de6065bc8..d642e557ec94b 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFValidity.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFValidity.java @@ -68,7 +68,7 @@ public void transform(RowWindow rowWindow, PointCollector collector) throws Exce collector.putDouble(rowWindow.getRow(0).getTime(), tsq.getValidity()); } } catch (IOException | NoNumberException ex) { - Logger.getLogger(UDTFCompleteness.class.getName()).log(Level.SEVERE, null, ex); + Logger.getLogger(UDTFValidity.class.getName()).log(Level.SEVERE, null, ex); } } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDFEnvelopeAnalysis.java b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDFEnvelopeAnalysis.java index b542c8a3d5184..1eaa1d18a686e 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDFEnvelopeAnalysis.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDFEnvelopeAnalysis.java @@ -87,6 +87,9 @@ public void transform(Row row, PointCollector collector) throws Exception { @Override public void terminate(PointCollector collector) throws Exception { + if (signals.isEmpty()) { + return; + } double[] envelopeValues = envelopeAnalyze(signals.toArray()); frequency = frequency != Double.MAX_VALUE ? frequency : calculateFrequency(timestamps); int signalSize = signals.size(); diff --git a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFConv.java b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFConv.java index 68e708bd607a8..a7cbfb6397692 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFConv.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFConv.java @@ -65,6 +65,9 @@ public void transform(Row row, PointCollector collector) throws Exception { @Override public void terminate(PointCollector collector) throws Exception { + if (list1.isEmpty() || list2.isEmpty()) { + return; + } double[] ans = new double[list1.size() + list2.size() - 1]; for (int i = 0; i < list1.size(); i++) { for (int j = 0; j < list2.size(); j++) { diff --git a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFDWT.java b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFDWT.java index c30877a126b4b..df756be5775f9 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFDWT.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFDWT.java @@ -76,12 +76,20 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati @Override public void transform(Row row, PointCollector pointCollector) throws Exception { - timestamp.add(row.getTime()); - value.add(Util.getValueAsDouble(row)); + if (!row.isNull(0)) { + double v = Util.getValueAsDouble(row); + if (Double.isFinite(v)) { + timestamp.add(row.getTime()); + value.add(v); + } + } } @Override public void terminate(PointCollector pointCollector) throws Exception { + if (value.isEmpty()) { + return; + } if (!s.equals("") || !method.equals("")) { // When user offers at least one parameter DWTUtil transformer = new DWTUtil(method, s, layer, value); transformer.waveletTransform(); diff --git a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFDeconv.java b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFDeconv.java index a8f74dca20257..542b1f628de50 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFDeconv.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFDeconv.java @@ -37,7 +37,8 @@ public class UDTFDeconv implements UDTF { private final ArrayList list1 = new ArrayList<>(); private final ArrayList list2 = new ArrayList<>(); - private static final String QUOTINENT_RESULT = "quotinent"; + private static final String QUOTIENT_RESULT = "quotient"; + private static final String LEGACY_QUOTINENT_RESULT = "quotinent"; private static final String REMAINDER_RESULT = "remainder"; private String result; @@ -48,11 +49,9 @@ public void validate(UDFParameterValidator validator) throws Exception { .validateInputSeriesDataType(0, Type.DOUBLE, Type.FLOAT, Type.INT32, Type.INT64) .validateInputSeriesDataType(1, Type.DOUBLE, Type.FLOAT, Type.INT32, Type.INT64) .validate( - x -> - ((String) x).equalsIgnoreCase(QUOTINENT_RESULT) - || ((String) x).equalsIgnoreCase(REMAINDER_RESULT), + x -> isQuotientResult((String) x) || ((String) x).equalsIgnoreCase(REMAINDER_RESULT), "Result should be 'quotient' or 'remainder'.", - validator.getParameters().getStringOrDefault("result", QUOTINENT_RESULT)); + validator.getParameters().getStringOrDefault("result", QUOTIENT_RESULT)); } @Override @@ -61,7 +60,7 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati configurations.setAccessStrategy(new RowByRowAccessStrategy()).setOutputDataType(Type.DOUBLE); list1.clear(); list2.clear(); - this.result = parameters.getStringOrDefault("result", QUOTINENT_RESULT); + this.result = parameters.getStringOrDefault("result", QUOTIENT_RESULT); } @Override @@ -79,7 +78,7 @@ public void terminate(PointCollector collector) throws Exception { if (list2.isEmpty()) { // Exception: divided by zero throw new ArithmeticException(LibraryUdfMessages.DIVIDED_BY_ZERO); } else if (list2.size() > list1.size()) { // order of divisor is larger than dividend - if (result.equalsIgnoreCase(QUOTINENT_RESULT)) { // quotient + if (isQuotientResult(result)) { // quotient collector.putDouble(0, 0); } else { // residue for (int i = 0; i < list1.size(); i++) { @@ -97,7 +96,7 @@ public void terminate(PointCollector collector) throws Exception { r[i + j] -= q[i] * list2.get(j); } } - if (result.equalsIgnoreCase(QUOTINENT_RESULT)) { // quotient + if (isQuotientResult(result)) { // quotient for (int i = 0; i < q.length; i++) { collector.putDouble(i, q[i]); } @@ -108,4 +107,9 @@ public void terminate(PointCollector collector) throws Exception { } } } + + private static boolean isQuotientResult(String result) { + return QUOTIENT_RESULT.equalsIgnoreCase(result) + || LEGACY_QUOTINENT_RESULT.equalsIgnoreCase(result); + } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFFFT.java b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFFFT.java index 0e69d0c50429a..d2d44bab2e399 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFFFT.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFFFT.java @@ -87,6 +87,9 @@ public void transform(Row row, PointCollector collector) throws Exception { @Override public void terminate(PointCollector collector) throws Exception { int n = list.size(); + if (n == 0) { + return; + } DoubleFFT_1D fft = new DoubleFFT_1D(n); // each data point count for 2 double values (re and im) double[] a = new double[2 * n]; diff --git a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFHighPass.java b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFHighPass.java index 34b509f9f21c4..567c20d1c50ae 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFHighPass.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFHighPass.java @@ -72,6 +72,9 @@ public void transform(Row row, PointCollector collector) throws Exception { @Override public void terminate(PointCollector collector) throws Exception { int n = valueList.size(); + if (n == 0) { + return; + } DoubleFFT_1D fft = new DoubleFFT_1D(n); // each data point count for 2 double values, same with UDTFFFT double[] a = new double[2 * n]; diff --git a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFIDWT.java b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFIDWT.java index 4b6feb7199ca7..0efb879985384 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFIDWT.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFIDWT.java @@ -77,12 +77,20 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati @Override public void transform(Row row, PointCollector pointCollector) throws Exception { - timestamp.add(row.getTime()); - value.add(Util.getValueAsDouble(row)); + if (!row.isNull(0)) { + double v = Util.getValueAsDouble(row); + if (Double.isFinite(v)) { + timestamp.add(row.getTime()); + value.add(v); + } + } } @Override public void terminate(PointCollector pointCollector) throws Exception { + if (value.isEmpty()) { + return; + } if (!s.equals("") || !method.equals("")) { // When user offers at least one parameter DWTUtil transformer = new DWTUtil(method, s, layer, value); transformer.inverse(); diff --git a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFIFFT.java b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFIFFT.java index 6e5f52d7e6f1a..8277aab834825 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFIFFT.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFIFFT.java @@ -51,6 +51,7 @@ public void validate(UDFParameterValidator validator) throws Exception { validator .validateInputSeriesNumber(2) .validateInputSeriesDataType(0, Type.DOUBLE, Type.FLOAT, Type.INT32, Type.INT64) + .validateInputSeriesDataType(1, Type.DOUBLE, Type.FLOAT, Type.INT32, Type.INT64) .validate( x -> (long) x > 0, "interval should be a time period whose unit is ms, s, m, h, d.", @@ -92,6 +93,9 @@ public void transform(Row row, PointCollector collector) throws Exception { @Override public void terminate(PointCollector collector) throws Exception { + if (time.isEmpty()) { + return; + } int n = time.get(time.size() - 1) + 1; double[] a = new double[n * 2]; for (int i = 0; i < time.size(); i++) { diff --git a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFLowPass.java b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFLowPass.java index b5d77cab4cb73..f5346b175eaac 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFLowPass.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFLowPass.java @@ -72,6 +72,9 @@ public void transform(Row row, PointCollector collector) throws Exception { @Override public void terminate(PointCollector collector) throws Exception { int n = valueList.size(); + if (n == 0) { + return; + } DoubleFFT_1D fft = new DoubleFFT_1D(n); // each data point count for 2 double values, same with UDTFFFT double[] a = new double[2 * n]; diff --git a/library-udf/src/main/java/org/apache/iotdb/library/frequency/util/DWTUtil.java b/library-udf/src/main/java/org/apache/iotdb/library/frequency/util/DWTUtil.java index d99b4dcca2ec1..67971e19c7370 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/frequency/util/DWTUtil.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/frequency/util/DWTUtil.java @@ -158,12 +158,11 @@ public void waveletTransform() { throw new IllegalArgumentException( "The data vector size is less than wavelet coefficient size."); } - int nn = n; for (int i = 0; i < layer; i++) { - if (nn < ncof) { + if (n < ncof) { break; } - forward(nn); + forward(n); n >>= 1; } } @@ -215,16 +214,16 @@ public void inverse() { throw new IllegalArgumentException( "The data vector size is less than wavelet coefficient size."); } - int nn = n; - for (int i = 0; i < layer - 1; i++) { - nn = n / 2; + int nn = n >> Math.max(layer - 1, 0); + if (nn == 0) { + nn = 1; } for (int i = 0; i < layer; i++) { if (nn > n) { break; } backward(nn); - n <<= 1; + nn <<= 1; } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/match/model/DTWState.java b/library-udf/src/main/java/org/apache/iotdb/library/match/model/DTWState.java index 45f103564e55f..add2d8f7661f0 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/match/model/DTWState.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/match/model/DTWState.java @@ -133,29 +133,4 @@ public void addMatchResult(DTWMatchResult matchResult) { public List getMatchResults() { return matchResults; } - - public static void main(String[] args) { - DTWState state = new DTWState(); - state.setSize(5); - state.reset(); - for (int i = 0; i < 4; i++) { - state.updateBuffer(i, i); - state.addMatchResult(new DTWMatchResult(i, i, i)); - } - for (int i = 0; i < state.getTimeBuffer().length; i++) { - System.out.println(state.getTimeBuffer()[i] + " " + state.getValueBuffer()[i]); - } - for (DTWMatchResult matchResult : state.matchResults) { - System.out.println(matchResult); - } - - DTWState newState = new DTWState(); - newState.setSize(5); - newState.reset(); - newState.deserialize(state.serialize()); - for (int i = 0; i < state.getTimeBuffer().length; i++) { - System.out.println(state.getTimeBuffer()[i] + " " + state.getValueBuffer()[i]); - } - System.out.println(newState.getMatchResults()); - } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/series/UDTFConsecutiveWindows.java b/library-udf/src/main/java/org/apache/iotdb/library/series/UDTFConsecutiveWindows.java index d7dbe4148b3a8..3208c76d5f499 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/series/UDTFConsecutiveWindows.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/series/UDTFConsecutiveWindows.java @@ -41,6 +41,7 @@ public class UDTFConsecutiveWindows implements UDTF { @Override public void validate(UDFParameterValidator validator) throws Exception { validator + .validateRequiredAttribute("length") .validate( x -> (long) x > 0, "gap should be a time period whose unit is ms, s, m, h.", diff --git a/library-udf/src/main/java/org/apache/iotdb/library/series/util/ConsecutiveUtil.java b/library-udf/src/main/java/org/apache/iotdb/library/series/util/ConsecutiveUtil.java index 2c9ea997c5fec..f11bd2a5128a3 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/series/util/ConsecutiveUtil.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/series/util/ConsecutiveUtil.java @@ -92,6 +92,10 @@ public boolean check(Row row) throws IOException { /** calculate standard timestamp gap in given window. */ public void calculateGap() { + if (window.size() < 2) { + gap = 0; + return; + } long[] time = new long[window.size() - 1]; for (int i = 0; i < time.length; i++) { time[i] = window.get(i + 1).getLeft() - window.get(i).getLeft(); diff --git a/library-udf/src/main/java/org/apache/iotdb/library/string/UDTFRegexMatch.java b/library-udf/src/main/java/org/apache/iotdb/library/string/UDTFRegexMatch.java index 7cd961c585e23..88cade9e76b1a 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/string/UDTFRegexMatch.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/string/UDTFRegexMatch.java @@ -35,6 +35,7 @@ import java.io.IOException; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; /** This function matches substring according to given regex from an input series. */ public class UDTFRegexMatch implements UDTF { @@ -46,16 +47,26 @@ public void validate(UDFParameterValidator validator) throws Exception { validator .validateInputSeriesNumber(1) .validateInputSeriesDataType(0, Type.TEXT) + .validateRequiredAttribute("regex") .validate( - regex -> ((String) regex).length() > 0, + regex -> ((String) regex).length() > 0 && isValidRegex((String) regex), "regexp has to be a valid regular expression.", - validator.getParameters().getStringOrDefault("regex", "")) + validator.getParameters().getString("regex")) .validate( group -> (int) group >= 0, "group index has to be a non-negative integer.", validator.getParameters().getIntOrDefault("group", 0)); } + private static boolean isValidRegex(String regex) { + try { + Pattern.compile(regex); + return true; + } catch (PatternSyntaxException e) { + return false; + } + } + @Override public void beforeStart(UDFParameters udfParameters, UDTFConfigurations udtfConfigurations) throws Exception { diff --git a/library-udf/src/main/java/org/apache/iotdb/library/string/UDTFRegexReplace.java b/library-udf/src/main/java/org/apache/iotdb/library/string/UDTFRegexReplace.java index b2b584c187e87..af1e5865be81a 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/string/UDTFRegexReplace.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/string/UDTFRegexReplace.java @@ -36,6 +36,7 @@ import java.io.IOException; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; /** This function replaces substring according to regex parameter from an input series. */ public class UDTFRegexReplace implements UDTF { @@ -52,9 +53,11 @@ public void validate(UDFParameterValidator validator) throws Exception { validator .validateInputSeriesNumber(1) .validateInputSeriesDataType(0, Type.TEXT) + .validateRequiredAttribute("regex") + .validateRequiredAttribute("replace") .validate( - regex -> ((String) regex).length() > 0, - "regex should not be empty", + regex -> ((String) regex).length() > 0 && isValidRegex((String) regex), + "regex should not be empty and has to be a valid regular expression.", validator.getParameters().getString("regex")) .validate( limit -> (int) limit >= -1, @@ -67,6 +70,15 @@ public void validate(UDFParameterValidator validator) throws Exception { validator.getParameters().getIntOrDefault("offset", 0)); } + private static boolean isValidRegex(String regex) { + try { + Pattern.compile(regex); + return true; + } catch (PatternSyntaxException e) { + return false; + } + } + @Override public void beforeStart(UDFParameters udfParameters, UDTFConfigurations udtfConfigurations) throws Exception { diff --git a/library-udf/src/main/java/org/apache/iotdb/library/string/UDTFRegexSplit.java b/library-udf/src/main/java/org/apache/iotdb/library/string/UDTFRegexSplit.java index d7df10f07b981..576711802fb91 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/string/UDTFRegexSplit.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/string/UDTFRegexSplit.java @@ -33,6 +33,8 @@ import org.apache.tsfile.utils.Binary; import java.io.IOException; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; /** This function splits string from an input series according to given regex. */ public class UDTFRegexSplit implements UDTF { @@ -125,13 +127,23 @@ public void validate(UDFParameterValidator validator) throws Exception { validator .validateInputSeriesNumber(1) .validateInputSeriesDataType(0, Type.TEXT) + .validateRequiredAttribute("regex") .validate( - regex -> ((String) regex).length() > 0, + regex -> ((String) regex).length() > 0 && isValidRegex((String) regex), "regexp has to be a valid regular expression.", - validator.getParameters().getStringOrDefault("regex", "")) + validator.getParameters().getString("regex")) .validate( index -> (int) index >= -1, "index must a non-negative integer to fetch split results or -1 to get length.", validator.getParameters().getIntOrDefault("index", -1)); } + + private static boolean isValidRegex(String regex) { + try { + Pattern.compile(regex); + return true; + } catch (PatternSyntaxException e) { + return false; + } + } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/string/UDTFStrReplace.java b/library-udf/src/main/java/org/apache/iotdb/library/string/UDTFStrReplace.java index 2a50b19fb7873..1490a19edf7ff 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/string/UDTFStrReplace.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/string/UDTFStrReplace.java @@ -48,6 +48,8 @@ public void validate(UDFParameterValidator validator) throws Exception { validator .validateInputSeriesNumber(1) .validateInputSeriesDataType(0, Type.TEXT) + .validateRequiredAttribute("target") + .validateRequiredAttribute("replace") .validate( target -> ((String) target).length() > 0, "target should not be empty", diff --git a/library-udf/src/main/java/org/apache/iotdb/library/util/BooleanCircularQueue.java b/library-udf/src/main/java/org/apache/iotdb/library/util/BooleanCircularQueue.java index edd0126f0d185..5a4d22c7c77b2 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/util/BooleanCircularQueue.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/util/BooleanCircularQueue.java @@ -33,6 +33,9 @@ public class BooleanCircularQueue { private boolean[] data; public BooleanCircularQueue(int capacity) { + if (capacity <= 0) { + throw new IllegalArgumentException("Capacity should be larger than 0."); + } head = tail = size = 0; data = new boolean[capacity]; minLen = Math.max(INITCAP, capacity); diff --git a/library-udf/src/main/java/org/apache/iotdb/library/util/CircularQueue.java b/library-udf/src/main/java/org/apache/iotdb/library/util/CircularQueue.java index cb869b86e1f5d..0b2512728d1af 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/util/CircularQueue.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/util/CircularQueue.java @@ -33,6 +33,9 @@ public class CircularQueue { private E[] data; public CircularQueue(int capacity) { + if (capacity <= 0) { + throw new IllegalArgumentException("Capacity should be larger than 0."); + } head = tail = size = 0; data = (E[]) new Object[capacity]; minLen = Math.max(INITCAP, capacity); diff --git a/library-udf/src/main/java/org/apache/iotdb/library/util/DoubleCircularQueue.java b/library-udf/src/main/java/org/apache/iotdb/library/util/DoubleCircularQueue.java index f387691c7f085..ee43146ab8d89 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/util/DoubleCircularQueue.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/util/DoubleCircularQueue.java @@ -33,6 +33,9 @@ public class DoubleCircularQueue { private double[] data; public DoubleCircularQueue(int capacity) { + if (capacity <= 0) { + throw new IllegalArgumentException("Capacity should be larger than 0."); + } head = tail = size = 0; data = new double[capacity]; minLen = Math.max(INITCAP, capacity); diff --git a/library-udf/src/main/java/org/apache/iotdb/library/util/LongCircularQueue.java b/library-udf/src/main/java/org/apache/iotdb/library/util/LongCircularQueue.java index 69716bd43b1df..bf7e9a613391b 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/util/LongCircularQueue.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/util/LongCircularQueue.java @@ -33,6 +33,9 @@ public class LongCircularQueue { private long[] data; public LongCircularQueue(int capacity) { + if (capacity <= 0) { + throw new IllegalArgumentException("Capacity should be larger than 0."); + } head = tail = size = 0; data = new long[capacity]; minLen = Math.max(INIT_CAP, capacity); diff --git a/library-udf/src/test/java/org/apache/iotdb/library/UDFWindowAndQueueTest.java b/library-udf/src/test/java/org/apache/iotdb/library/UDFWindowAndQueueTest.java new file mode 100644 index 0000000000000..ef6dd66572204 --- /dev/null +++ b/library-udf/src/test/java/org/apache/iotdb/library/UDFWindowAndQueueTest.java @@ -0,0 +1,1088 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.library; + +import org.apache.iotdb.library.anomaly.UDTFIQR; +import org.apache.iotdb.library.anomaly.UDTFKSigma; +import org.apache.iotdb.library.anomaly.UDTFLOF; +import org.apache.iotdb.library.anomaly.UDTFRange; +import org.apache.iotdb.library.dlearn.UDTFAR; +import org.apache.iotdb.library.dlearn.UDTFCluster; +import org.apache.iotdb.library.dmatch.UDAFDtw; +import org.apache.iotdb.library.dprofile.UDAFMad; +import org.apache.iotdb.library.dprofile.UDAFMedian; +import org.apache.iotdb.library.dprofile.UDAFPercentile; +import org.apache.iotdb.library.dprofile.UDAFPeriod; +import org.apache.iotdb.library.dprofile.UDAFQuantile; +import org.apache.iotdb.library.dprofile.UDAFSkew; +import org.apache.iotdb.library.dprofile.UDTFHistogram; +import org.apache.iotdb.library.dprofile.UDTFMinMax; +import org.apache.iotdb.library.dprofile.UDTFMvAvg; +import org.apache.iotdb.library.dprofile.UDTFQLB; +import org.apache.iotdb.library.dprofile.UDTFResample; +import org.apache.iotdb.library.dprofile.UDTFSample; +import org.apache.iotdb.library.dprofile.UDTFSegment; +import org.apache.iotdb.library.dprofile.UDTFSpline; +import org.apache.iotdb.library.dprofile.UDTFZScore; +import org.apache.iotdb.library.frequency.UDFEnvelopeAnalysis; +import org.apache.iotdb.library.frequency.UDTFConv; +import org.apache.iotdb.library.frequency.UDTFDWT; +import org.apache.iotdb.library.frequency.UDTFDeconv; +import org.apache.iotdb.library.frequency.UDTFFFT; +import org.apache.iotdb.library.frequency.UDTFHighPass; +import org.apache.iotdb.library.frequency.UDTFIDWT; +import org.apache.iotdb.library.frequency.UDTFIFFT; +import org.apache.iotdb.library.frequency.UDTFLowPass; +import org.apache.iotdb.library.series.UDTFConsecutiveSequences; +import org.apache.iotdb.library.series.UDTFConsecutiveWindows; +import org.apache.iotdb.library.string.UDTFRegexMatch; +import org.apache.iotdb.library.string.UDTFRegexReplace; +import org.apache.iotdb.library.string.UDTFRegexSplit; +import org.apache.iotdb.library.string.UDTFStrReplace; +import org.apache.iotdb.library.util.BooleanCircularQueue; +import org.apache.iotdb.library.util.CircularQueue; +import org.apache.iotdb.library.util.DoubleCircularQueue; +import org.apache.iotdb.library.util.LongCircularQueue; +import org.apache.iotdb.udf.api.access.Row; +import org.apache.iotdb.udf.api.access.RowIterator; +import org.apache.iotdb.udf.api.access.RowWindow; +import org.apache.iotdb.udf.api.collector.PointCollector; +import org.apache.iotdb.udf.api.customizer.config.UDTFConfigurations; +import org.apache.iotdb.udf.api.customizer.parameter.UDFParameterValidator; +import org.apache.iotdb.udf.api.customizer.parameter.UDFParameters; +import org.apache.iotdb.udf.api.exception.UDFAttributeNotProvidedException; +import org.apache.iotdb.udf.api.exception.UDFInputSeriesDataTypeNotValidException; +import org.apache.iotdb.udf.api.exception.UDFParameterNotValidException; +import org.apache.iotdb.udf.api.type.Binary; +import org.apache.iotdb.udf.api.type.Type; + +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class UDFWindowAndQueueTest { + + @Test + public void testKSigmaDefaultWindowIsConsistent() throws Exception { + UDTFKSigma kSigma = new UDTFKSigma(); + UDFParameters parameters = createSingleDoubleSeriesParameters(Collections.emptyMap()); + + kSigma.validate(new UDFParameterValidator(parameters)); + kSigma.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + + Assert.assertEquals(10, getWindowSize(kSigma)); + } + + @Test + public void testKSigmaExplicitWindowOverridesDefault() throws Exception { + Map attributes = new HashMap<>(); + attributes.put("window", "3"); + + UDTFKSigma kSigma = new UDTFKSigma(); + UDFParameters parameters = createSingleDoubleSeriesParameters(attributes); + + kSigma.validate(new UDFParameterValidator(parameters)); + kSigma.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + + Assert.assertEquals(3, getWindowSize(kSigma)); + } + + @Test + public void testMvAvgUsesRunningWindowSum() throws Exception { + Map attributes = new HashMap<>(); + attributes.put("window", "3"); + + UDTFMvAvg mvAvg = new UDTFMvAvg(); + UDFParameters parameters = createSingleDoubleSeriesParameters(attributes); + RecordingPointCollector collector = new RecordingPointCollector(); + + mvAvg.validate(new UDFParameterValidator(parameters)); + mvAvg.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + mvAvg.transform(new DoubleRow(1, 1.0), collector); + mvAvg.transform(new DoubleRow(2, 2.0), collector); + mvAvg.transform(new DoubleRow(3, 3.0), collector); + mvAvg.transform(new DoubleRow(4, 4.0), collector); + + Assert.assertEquals(Arrays.asList(3L, 4L), collector.timestamps); + Assert.assertEquals(2.0, collector.values.get(0), 0.0); + Assert.assertEquals(3.0, collector.values.get(1), 0.0); + } + + @Test + public void testLOFSkipsNullRowsWithoutReadingCompressedIndex() throws Exception { + Map attributes = new HashMap<>(); + attributes.put("k", "1"); + + UDTFLOF lof = new UDTFLOF(); + UDFParameters parameters = + new UDFParameters( + Arrays.asList("s1", "s2"), Arrays.asList(Type.DOUBLE, Type.DOUBLE), attributes); + RecordingPointCollector collector = new RecordingPointCollector(); + + lof.validate(new UDFParameterValidator(parameters)); + lof.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + lof.transform( + new SimpleRowWindow( + new DoubleRow(1, new double[] {0.0, 0.0}, new boolean[] {false, false}), + new DoubleRow(2, new double[] {0.0, 0.0}, new boolean[] {true, true}), + new DoubleRow(3, new double[] {10.0, 10.0}, new boolean[] {false, false}), + new DoubleRow(4, new double[] {20.0, 20.0}, new boolean[] {false, false})), + collector); + + Assert.assertEquals(Arrays.asList(1L, 3L, 4L), collector.timestamps); + Assert.assertEquals(3, collector.values.size()); + } + + @Test + public void testLOFValidatesAllInputSeriesTypes() { + UDFParameters parameters = + new UDFParameters( + Arrays.asList("s1", "s2"), + Arrays.asList(Type.DOUBLE, Type.TEXT), + Collections.emptyMap()); + + Assert.assertThrows( + UDFInputSeriesDataTypeNotValidException.class, + () -> new UDTFLOF().validate(new UDFParameterValidator(parameters))); + } + + @Test + public void testDeconvAcceptsQuotientResult() throws Exception { + Map attributes = new HashMap<>(); + attributes.put("result", "quotient"); + + UDTFDeconv deconv = new UDTFDeconv(); + UDFParameters parameters = createTwoDoubleSeriesParameters(attributes); + + deconv.validate(new UDFParameterValidator(parameters)); + deconv.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + } + + @Test + public void testConvIgnoresEmptyEffectiveInput() throws Exception { + UDTFConv conv = new UDTFConv(); + UDFParameters parameters = createTwoDoubleSeriesParameters(Collections.emptyMap()); + RecordingPointCollector collector = new RecordingPointCollector(); + + conv.validate(new UDFParameterValidator(parameters)); + conv.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + conv.transform( + new DoubleRow(1, new double[] {0.0, 0.0}, new boolean[] {true, true}), collector); + conv.terminate(collector); + + Assert.assertTrue(collector.timestamps.isEmpty()); + Assert.assertTrue(collector.values.isEmpty()); + } + + @Test + public void testConsecutiveSequencesIgnoresShortAutoGapWindow() throws Exception { + UDFParameters parameters = createSingleDoubleSeriesParameters(Collections.emptyMap()); + RecordingPointCollector collector = new RecordingPointCollector(); + + UDTFConsecutiveSequences sequences = new UDTFConsecutiveSequences(); + sequences.validate(new UDFParameterValidator(parameters)); + sequences.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + sequences.transform(new DoubleRow(1, 1.0), collector); + sequences.terminate(collector); + + Assert.assertTrue(collector.timestamps.isEmpty()); + Assert.assertTrue(collector.values.isEmpty()); + } + + @Test + public void testRangeRequiresBoundsAndValidatesOrder() { + UDTFRange range = new UDTFRange(); + + Assert.assertThrows( + UDFAttributeNotProvidedException.class, + () -> + range.validate( + new UDFParameterValidator( + createSingleDoubleSeriesParameters(Collections.emptyMap())))); + + Map attributes = new HashMap<>(); + attributes.put("lower_bound", "2"); + attributes.put("upper_bound", "1"); + Assert.assertThrows( + UDFParameterNotValidException.class, + () -> + range.validate( + new UDFParameterValidator(createSingleDoubleSeriesParameters(attributes)))); + } + + @Test + public void testStreamModeRequiresExplicitParameters() { + Map attributes = new HashMap<>(); + attributes.put("compute", "stream"); + + Assert.assertThrows( + UDFAttributeNotProvidedException.class, + () -> + new UDTFMinMax() + .validate( + new UDFParameterValidator(createSingleDoubleSeriesParameters(attributes)))); + Assert.assertThrows( + UDFAttributeNotProvidedException.class, + () -> + new UDTFZScore() + .validate( + new UDFParameterValidator(createSingleDoubleSeriesParameters(attributes)))); + Assert.assertThrows( + UDFAttributeNotProvidedException.class, + () -> + new UDTFIQR() + .validate( + new UDFParameterValidator(createSingleDoubleSeriesParameters(attributes)))); + } + + @Test + public void testMinMaxBatchKeepsTimestampsAlignedWhenSkippingInvalidValues() throws Exception { + UDTFMinMax minMax = new UDTFMinMax(); + UDFParameters parameters = createSingleDoubleSeriesParameters(Collections.emptyMap()); + RecordingPointCollector collector = new RecordingPointCollector(); + + minMax.validate(new UDFParameterValidator(parameters)); + minMax.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + minMax.transform(new DoubleRow(1, Double.NaN), collector); + minMax.transform(new DoubleRow(2, 10.0), collector); + minMax.transform(new DoubleRow(3, 20.0), collector); + minMax.terminate(collector); + + Assert.assertEquals(Arrays.asList(2L, 3L), collector.timestamps); + Assert.assertEquals(0.0, collector.values.get(0), 0.0); + Assert.assertEquals(1.0, collector.values.get(1), 0.0); + } + + @Test + public void testIQRIgnoresEmptyBatchInput() throws Exception { + UDTFIQR iqr = new UDTFIQR(); + UDFParameters parameters = createSingleDoubleSeriesParameters(Collections.emptyMap()); + RecordingPointCollector collector = new RecordingPointCollector(); + + iqr.validate(new UDFParameterValidator(parameters)); + iqr.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + iqr.terminate(collector); + + Assert.assertTrue(collector.timestamps.isEmpty()); + Assert.assertTrue(collector.values.isEmpty()); + } + + @Test + public void testStringReplaceRequiresParameters() { + Assert.assertThrows( + UDFAttributeNotProvidedException.class, + () -> + new UDTFStrReplace() + .validate( + new UDFParameterValidator( + createSingleTextSeriesParameters(Collections.emptyMap())))); + + Map attributes = new HashMap<>(); + attributes.put("target", "a"); + Assert.assertThrows( + UDFAttributeNotProvidedException.class, + () -> + new UDTFStrReplace() + .validate(new UDFParameterValidator(createSingleTextSeriesParameters(attributes)))); + + Assert.assertThrows( + UDFAttributeNotProvidedException.class, + () -> + new UDTFRegexReplace() + .validate( + new UDFParameterValidator( + createSingleTextSeriesParameters(Collections.emptyMap())))); + + attributes.clear(); + attributes.put("regex", "a+"); + Assert.assertThrows( + UDFAttributeNotProvidedException.class, + () -> + new UDTFRegexReplace() + .validate(new UDFParameterValidator(createSingleTextSeriesParameters(attributes)))); + + Assert.assertThrows( + UDFAttributeNotProvidedException.class, + () -> + new UDTFRegexSplit() + .validate( + new UDFParameterValidator( + createSingleTextSeriesParameters(Collections.emptyMap())))); + Assert.assertThrows( + UDFAttributeNotProvidedException.class, + () -> + new UDTFRegexMatch() + .validate( + new UDFParameterValidator( + createSingleTextSeriesParameters(Collections.emptyMap())))); + + attributes.clear(); + attributes.put("regex", "["); + attributes.put("replace", "x"); + Assert.assertThrows( + UDFParameterNotValidException.class, + () -> + new UDTFRegexReplace() + .validate(new UDFParameterValidator(createSingleTextSeriesParameters(attributes)))); + + attributes.remove("replace"); + Assert.assertThrows( + UDFParameterNotValidException.class, + () -> + new UDTFRegexSplit() + .validate(new UDFParameterValidator(createSingleTextSeriesParameters(attributes)))); + Assert.assertThrows( + UDFParameterNotValidException.class, + () -> + new UDTFRegexMatch() + .validate(new UDFParameterValidator(createSingleTextSeriesParameters(attributes)))); + } + + @Test + public void testAggregateFunctionsIgnoreEmptyInput() throws Exception { + UDFParameters parameters = createSingleDoubleSeriesParameters(Collections.emptyMap()); + RecordingPointCollector collector = new RecordingPointCollector(); + + UDAFMedian median = new UDAFMedian(); + median.validate(new UDFParameterValidator(parameters)); + median.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + median.terminate(collector); + + UDAFMad mad = new UDAFMad(); + mad.validate(new UDFParameterValidator(parameters)); + mad.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + mad.terminate(collector); + + UDAFPercentile percentile = new UDAFPercentile(); + percentile.validate(new UDFParameterValidator(parameters)); + percentile.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + percentile.terminate(collector); + + UDAFQuantile quantile = new UDAFQuantile(); + quantile.validate(new UDFParameterValidator(parameters)); + quantile.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + quantile.terminate(collector); + + Assert.assertTrue(collector.timestamps.isEmpty()); + Assert.assertTrue(collector.values.isEmpty()); + } + + @Test + public void testExactMadUsesAbsoluteDeviations() throws Exception { + UDFParameters parameters = createSingleDoubleSeriesParameters(Collections.emptyMap()); + UDAFMad mad = new UDAFMad(); + RecordingPointCollector collector = new RecordingPointCollector(); + + mad.validate(new UDFParameterValidator(parameters)); + mad.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + mad.transform(new DoubleRow(1, 1.0), collector); + mad.transform(new DoubleRow(2, 2.0), collector); + mad.transform(new DoubleRow(3, 4.0), collector); + mad.terminate(collector); + + Assert.assertEquals(Collections.singletonList(0L), collector.timestamps); + Assert.assertEquals(1.0, collector.values.get(0), 0.0); + } + + @Test + public void testPercentileInstancesKeepIndependentTimestamps() throws Exception { + UDFParameters parameters = createSingleDoubleSeriesParameters(Collections.emptyMap()); + UDAFPercentile first = new UDAFPercentile(); + UDAFPercentile second = new UDAFPercentile(); + RecordingPointCollector firstCollector = new RecordingPointCollector(); + + first.validate(new UDFParameterValidator(parameters)); + second.validate(new UDFParameterValidator(parameters)); + first.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + first.transform(new DoubleRow(1, 5.0), firstCollector); + + second.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + second.transform(new DoubleRow(2, 9.0), new RecordingPointCollector()); + first.terminate(firstCollector); + + Assert.assertEquals(Collections.singletonList(1L), firstCollector.timestamps); + Assert.assertEquals(5.0, firstCollector.values.get(0), 0.0); + } + + @Test + public void testPeriodSkipsLeadingInvalidValue() throws Exception { + UDAFPeriod period = new UDAFPeriod(); + UDFParameters parameters = createSingleDoubleSeriesParameters(Collections.emptyMap()); + RecordingPointCollector collector = new RecordingPointCollector(); + + period.validate(new UDFParameterValidator(parameters)); + period.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + period.transform( + new SimpleRowWindow(new DoubleRow(1, Double.NaN), new DoubleRow(2, 1.0)), collector); + + Assert.assertEquals(Collections.singletonList(0L), collector.timestamps); + Assert.assertEquals(0.0, collector.values.get(0), 0.0); + } + + @Test + public void testQLBResetsAccumulatedStatisticBetweenRuns() throws Exception { + UDFParameters parameters = createSingleDoubleSeriesParameters(Collections.emptyMap()); + UDTFQLB reused = new UDTFQLB(); + UDTFQLB fresh = new UDTFQLB(); + RecordingPointCollector firstRun = new RecordingPointCollector(); + RecordingPointCollector reusedSecondRun = new RecordingPointCollector(); + RecordingPointCollector freshRun = new RecordingPointCollector(); + + reused.validate(new UDFParameterValidator(parameters)); + fresh.validate(new UDFParameterValidator(parameters)); + runQLB(reused, parameters, firstRun); + runQLB(reused, parameters, reusedSecondRun); + runQLB(fresh, parameters, freshRun); + + Assert.assertEquals(freshRun.timestamps, reusedSecondRun.timestamps); + Assert.assertArrayEquals( + freshRun.values.stream().mapToDouble(Double::doubleValue).toArray(), + reusedSecondRun.values.stream().mapToDouble(Double::doubleValue).toArray(), + 1e-12); + } + + @Test + public void testSkewIgnoresEmptyInputAndResetsBetweenRuns() throws Exception { + UDFParameters parameters = createSingleDoubleSeriesParameters(Collections.emptyMap()); + UDAFSkew skew = new UDAFSkew(); + RecordingPointCollector emptyCollector = new RecordingPointCollector(); + RecordingPointCollector firstRun = new RecordingPointCollector(); + RecordingPointCollector secondRun = new RecordingPointCollector(); + + skew.validate(new UDFParameterValidator(parameters)); + skew.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + skew.terminate(emptyCollector); + Assert.assertTrue(emptyCollector.timestamps.isEmpty()); + + skew.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + skew.transform(new DoubleRow(1, 1.0), firstRun); + skew.transform(new DoubleRow(2, 2.0), firstRun); + skew.transform(new DoubleRow(3, 3.0), firstRun); + skew.terminate(firstRun); + + skew.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + skew.transform(new DoubleRow(4, 1.0), secondRun); + skew.transform(new DoubleRow(5, 2.0), secondRun); + skew.transform(new DoubleRow(6, 3.0), secondRun); + skew.terminate(secondRun); + + Assert.assertEquals(firstRun.values.get(0), secondRun.values.get(0), 0.0); + } + + @Test + public void testHistogramRejectsZeroWidthRange() { + Map attributes = new HashMap<>(); + attributes.put("min", "1"); + attributes.put("max", "1"); + + Assert.assertThrows( + UDFParameterNotValidException.class, + () -> + new UDTFHistogram() + .validate( + new UDFParameterValidator(createSingleDoubleSeriesParameters(attributes)))); + } + + @Test + public void testReservoirSampleResetsCountBetweenRuns() throws Exception { + Map attributes = new HashMap<>(); + attributes.put("k", "2"); + UDFParameters parameters = createSingleDoubleSeriesParameters(attributes); + UDTFSample sample = new UDTFSample(); + RecordingPointCollector firstRun = new RecordingPointCollector(); + RecordingPointCollector secondRun = new RecordingPointCollector(); + + sample.validate(new UDFParameterValidator(parameters)); + sample.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + sample.transform(new DoubleRow(1, 1.0), firstRun); + sample.terminate(firstRun); + + sample.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + sample.transform(new DoubleRow(2, 2.0), secondRun); + sample.terminate(secondRun); + + Assert.assertEquals(Collections.singletonList(2L), secondRun.timestamps); + Assert.assertEquals(2.0, secondRun.values.get(0), 0.0); + } + + @Test + public void testSampleClearsReservoirStateWhenSwitchingMethod() throws Exception { + Map reservoirAttributes = new HashMap<>(); + reservoirAttributes.put("k", "2"); + UDFParameters reservoirParameters = createSingleDoubleSeriesParameters(reservoirAttributes); + + Map isometricAttributes = new HashMap<>(); + isometricAttributes.put("k", "2"); + isometricAttributes.put("method", "isometric"); + UDFParameters isometricParameters = createSingleDoubleSeriesParameters(isometricAttributes); + + UDTFSample sample = new UDTFSample(); + RecordingPointCollector collector = new RecordingPointCollector(); + + sample.validate(new UDFParameterValidator(reservoirParameters)); + sample.beforeStart(reservoirParameters, new UDTFConfigurations(ZoneId.systemDefault())); + sample.transform(new DoubleRow(1, 1.0), collector); + + sample.validate(new UDFParameterValidator(isometricParameters)); + sample.beforeStart(isometricParameters, new UDTFConfigurations(ZoneId.systemDefault())); + sample.terminate(collector); + + Assert.assertTrue(collector.timestamps.isEmpty()); + } + + @Test + public void testDtwEmptyInputProducesNoOutput() throws Exception { + UDAFDtw dtw = new UDAFDtw(); + UDFParameters parameters = createTwoDoubleSeriesParameters(Collections.emptyMap()); + RecordingPointCollector collector = new RecordingPointCollector(); + + dtw.validate(new UDFParameterValidator(parameters)); + dtw.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + dtw.terminate(collector); + + Assert.assertTrue(collector.timestamps.isEmpty()); + } + + @Test + public void testDtwClearsPreviousResultForEmptyWindow() throws Exception { + UDAFDtw dtw = new UDAFDtw(); + UDFParameters parameters = createTwoDoubleSeriesParameters(Collections.emptyMap()); + RecordingPointCollector collector = new RecordingPointCollector(); + + dtw.validate(new UDFParameterValidator(parameters)); + dtw.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + dtw.transform( + new SimpleRowWindow( + new DoubleRow(1, new double[] {1.0, 1.0}, new boolean[] {false, false})), + collector); + dtw.terminate(collector); + Assert.assertFalse(collector.timestamps.isEmpty()); + + collector.timestamps.clear(); + collector.values.clear(); + dtw.transform( + new SimpleRowWindow(new DoubleRow(2, new double[] {0.0, 0.0}, new boolean[] {true, true})), + collector); + dtw.terminate(collector); + + Assert.assertTrue(collector.timestamps.isEmpty()); + } + + @Test + public void testFrequencyFunctionsIgnoreEmptyInput() throws Exception { + RecordingPointCollector collector = new RecordingPointCollector(); + UDFParameters singleSeries = createSingleDoubleSeriesParameters(Collections.emptyMap()); + + UDTFFFT fft = new UDTFFFT(); + fft.validate(new UDFParameterValidator(singleSeries)); + fft.beforeStart(singleSeries, new UDTFConfigurations(ZoneId.systemDefault())); + fft.terminate(collector); + + Map wpassAttributes = new HashMap<>(); + wpassAttributes.put("wpass", "0.5"); + UDFParameters wpassParameters = createSingleDoubleSeriesParameters(wpassAttributes); + UDTFLowPass lowPass = new UDTFLowPass(); + lowPass.validate(new UDFParameterValidator(wpassParameters)); + lowPass.beforeStart(wpassParameters, new UDTFConfigurations(ZoneId.systemDefault())); + lowPass.terminate(collector); + + UDTFHighPass highPass = new UDTFHighPass(); + highPass.validate(new UDFParameterValidator(wpassParameters)); + highPass.beforeStart(wpassParameters, new UDTFConfigurations(ZoneId.systemDefault())); + highPass.terminate(collector); + + Map waveletAttributes = new HashMap<>(); + waveletAttributes.put("method", "Haar"); + UDFParameters waveletParameters = createSingleDoubleSeriesParameters(waveletAttributes); + UDTFDWT dwt = new UDTFDWT(); + dwt.validate(new UDFParameterValidator(waveletParameters)); + dwt.beforeStart(waveletParameters, new UDTFConfigurations(ZoneId.systemDefault())); + dwt.terminate(collector); + + UDTFIDWT idwt = new UDTFIDWT(); + idwt.validate(new UDFParameterValidator(waveletParameters)); + idwt.beforeStart(waveletParameters, new UDTFConfigurations(ZoneId.systemDefault())); + idwt.terminate(collector); + + UDFEnvelopeAnalysis envelopeAnalysis = new UDFEnvelopeAnalysis(); + envelopeAnalysis.validate(new UDFParameterValidator(singleSeries)); + envelopeAnalysis.beforeStart(singleSeries, new UDTFConfigurations(ZoneId.systemDefault())); + envelopeAnalysis.terminate(collector); + + UDFParameters twoSeries = createTwoDoubleSeriesParameters(Collections.emptyMap()); + UDTFIFFT ifft = new UDTFIFFT(); + ifft.validate(new UDFParameterValidator(twoSeries)); + ifft.beforeStart(twoSeries, new UDTFConfigurations(ZoneId.systemDefault())); + ifft.terminate(collector); + + Assert.assertTrue(collector.timestamps.isEmpty()); + Assert.assertTrue(collector.values.isEmpty()); + } + + @Test + public void testMultiLayerDWTRoundTrip() throws Exception { + Map attributes = new HashMap<>(); + attributes.put("method", "Haar"); + attributes.put("layer", "2"); + UDFParameters parameters = createSingleDoubleSeriesParameters(attributes); + RecordingPointCollector dwtCollector = new RecordingPointCollector(); + + UDTFDWT dwt = new UDTFDWT(); + dwt.validate(new UDFParameterValidator(parameters)); + dwt.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + dwt.transform(new DoubleRow(1, 1.0), dwtCollector); + dwt.transform(new DoubleRow(2, 2.0), dwtCollector); + dwt.transform(new DoubleRow(3, 3.0), dwtCollector); + dwt.transform(new DoubleRow(4, 4.0), dwtCollector); + dwt.terminate(dwtCollector); + + UDTFIDWT idwt = new UDTFIDWT(); + RecordingPointCollector idwtCollector = new RecordingPointCollector(); + idwt.validate(new UDFParameterValidator(parameters)); + idwt.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + for (int i = 0; i < dwtCollector.values.size(); i++) { + idwt.transform( + new DoubleRow(dwtCollector.timestamps.get(i), dwtCollector.values.get(i)), idwtCollector); + } + idwt.terminate(idwtCollector); + + Assert.assertEquals(Arrays.asList(1L, 2L, 3L, 4L), idwtCollector.timestamps); + Assert.assertArrayEquals( + new double[] {1.0, 2.0, 3.0, 4.0}, + idwtCollector.values.stream().mapToDouble(Double::doubleValue).toArray(), + 1e-9); + } + + @Test + public void testIFFTValidatesImaginaryInputType() { + UDFParameters parameters = + new UDFParameters( + Arrays.asList("s1", "s2"), + Arrays.asList(Type.DOUBLE, Type.TEXT), + Collections.emptyMap()); + + Assert.assertThrows( + UDFInputSeriesDataTypeNotValidException.class, + () -> new UDTFIFFT().validate(new UDFParameterValidator(parameters))); + } + + @Test + public void testARValidatesPositiveOrder() { + Map attributes = new HashMap<>(); + attributes.put("p", "0"); + + Assert.assertThrows( + UDFParameterNotValidException.class, + () -> + new UDTFAR() + .validate( + new UDFParameterValidator(createSingleDoubleSeriesParameters(attributes)))); + } + + @Test + public void testRequiredParametersAreValidatedBeforeStart() { + UDFParameterValidator numericValidator = + new UDFParameterValidator(createSingleDoubleSeriesParameters(Collections.emptyMap())); + + Assert.assertThrows( + UDFAttributeNotProvidedException.class, () -> new UDTFCluster().validate(numericValidator)); + Assert.assertThrows( + UDFAttributeNotProvidedException.class, + () -> new UDTFResample().validate(numericValidator)); + Assert.assertThrows( + UDFAttributeNotProvidedException.class, () -> new UDTFSpline().validate(numericValidator)); + Assert.assertThrows( + UDFAttributeNotProvidedException.class, + () -> new UDTFConsecutiveWindows().validate(numericValidator)); + } + + @Test + public void testSplineRequiresAtLeastTwoPoints() { + Map attributes = new HashMap<>(); + attributes.put("points", "1"); + + Assert.assertThrows( + UDFParameterNotValidException.class, + () -> + new UDTFSpline() + .validate( + new UDFParameterValidator(createSingleDoubleSeriesParameters(attributes)))); + } + + @Test + public void testSplineSkipsTooFewPointsAndHandlesNegativeTimestamps() throws Exception { + Map attributes = new HashMap<>(); + attributes.put("points", "3"); + UDFParameters parameters = createSingleDoubleSeriesParameters(attributes); + + UDTFSpline spline = new UDTFSpline(); + RecordingPointCollector collector = new RecordingPointCollector(); + + spline.validate(new UDFParameterValidator(parameters)); + spline.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + spline.transform(new DoubleRow(1, 1.0), collector); + spline.transform(new DoubleRow(2, 2.0), collector); + spline.transform(new DoubleRow(3, 3.0), collector); + spline.transform(new DoubleRow(4, 4.0), collector); + spline.terminate(collector); + + Assert.assertTrue(collector.timestamps.isEmpty()); + + spline.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + spline.transform(new DoubleRow(-5, 1.0), collector); + spline.transform(new DoubleRow(-4, 2.0), collector); + spline.transform(new DoubleRow(-3, 3.0), collector); + spline.transform(new DoubleRow(-2, 4.0), collector); + spline.transform(new DoubleRow(-1, 5.0), collector); + spline.terminate(collector); + + Assert.assertEquals(Arrays.asList(-5L, -3L, -1L), collector.timestamps); + } + + @Test + public void testSegmentInstancesKeepIndependentBuffers() throws Exception { + UDFParameters parameters = createSingleDoubleSeriesParameters(Collections.emptyMap()); + UDTFSegment first = new UDTFSegment(); + UDTFSegment second = new UDTFSegment(); + RecordingPointCollector firstCollector = new RecordingPointCollector(); + + first.validate(new UDFParameterValidator(parameters)); + second.validate(new UDFParameterValidator(parameters)); + first.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + first.transform(new DoubleRow(1, 10.0), firstCollector); + + second.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + second.transform(new DoubleRow(2, 20.0), new RecordingPointCollector()); + first.terminate(firstCollector); + + Assert.assertEquals(Collections.singletonList(1L), firstCollector.timestamps); + Assert.assertEquals(10.0, firstCollector.values.get(0), 0.0); + } + + @Test + public void testCircularQueueRejectsNonPositiveCapacity() { + Assert.assertThrows(IllegalArgumentException.class, () -> new CircularQueue<>(0)); + Assert.assertThrows(IllegalArgumentException.class, () -> new CircularQueue<>(-1)); + Assert.assertThrows(IllegalArgumentException.class, () -> new DoubleCircularQueue(0)); + Assert.assertThrows(IllegalArgumentException.class, () -> new DoubleCircularQueue(-1)); + Assert.assertThrows(IllegalArgumentException.class, () -> new LongCircularQueue(0)); + Assert.assertThrows(IllegalArgumentException.class, () -> new LongCircularQueue(-1)); + Assert.assertThrows(IllegalArgumentException.class, () -> new BooleanCircularQueue(0)); + Assert.assertThrows(IllegalArgumentException.class, () -> new BooleanCircularQueue(-1)); + } + + @Test + public void testObjectCircularQueueMaintainsOrderAfterWrapAndResize() { + CircularQueue queue = new CircularQueue<>(2); + + queue.push("a"); + queue.push("b"); + Assert.assertTrue(queue.isFull()); + Assert.assertEquals("a", queue.pop()); + + queue.push("c"); + queue.push("d"); + + Assert.assertEquals(3, queue.getSize()); + Assert.assertEquals("b", queue.get(0)); + Assert.assertEquals("c", queue.get(1)); + Assert.assertEquals("d", queue.get(2)); + Assert.assertEquals("b", queue.pop()); + Assert.assertEquals("c", queue.pop()); + Assert.assertEquals("d", queue.pop()); + Assert.assertTrue(queue.isEmpty()); + } + + @Test + public void testPrimitiveCircularQueuesMaintainOrderAfterWrapAndResize() { + DoubleCircularQueue doubleQueue = new DoubleCircularQueue(2); + doubleQueue.push(1.5); + doubleQueue.push(2.5); + Assert.assertTrue(doubleQueue.isFull()); + Assert.assertEquals(1.5, doubleQueue.pop(), 0.0); + doubleQueue.push(3.5); + doubleQueue.push(4.5); + Assert.assertEquals(2.5, doubleQueue.get(0), 0.0); + Assert.assertEquals(3.5, doubleQueue.get(1), 0.0); + Assert.assertEquals(4.5, doubleQueue.get(2), 0.0); + + LongCircularQueue longQueue = new LongCircularQueue(2); + longQueue.push(1); + longQueue.push(2); + Assert.assertEquals(1, longQueue.pop()); + longQueue.push(3); + longQueue.push(4); + Assert.assertEquals(2, longQueue.get(0)); + Assert.assertEquals(3, longQueue.get(1)); + Assert.assertEquals(4, longQueue.get(2)); + + BooleanCircularQueue booleanQueue = new BooleanCircularQueue(2); + booleanQueue.push(true); + booleanQueue.push(false); + Assert.assertTrue(booleanQueue.pop()); + booleanQueue.push(true); + booleanQueue.push(false); + Assert.assertFalse(booleanQueue.get(0)); + Assert.assertTrue(booleanQueue.get(1)); + Assert.assertFalse(booleanQueue.get(2)); + } + + @Test + public void testCircularQueuesRejectEmptyPopAndHead() { + CircularQueue objectQueue = new CircularQueue<>(); + Assert.assertThrows(IllegalArgumentException.class, () -> objectQueue.pop()); + Assert.assertThrows(IllegalArgumentException.class, () -> objectQueue.getHead()); + + DoubleCircularQueue doubleQueue = new DoubleCircularQueue(); + Assert.assertThrows(IllegalArgumentException.class, () -> doubleQueue.pop()); + Assert.assertThrows(IllegalArgumentException.class, () -> doubleQueue.getHead()); + + LongCircularQueue longQueue = new LongCircularQueue(); + Assert.assertThrows(IllegalArgumentException.class, () -> longQueue.pop()); + Assert.assertThrows(IllegalArgumentException.class, () -> longQueue.getHead()); + + BooleanCircularQueue booleanQueue = new BooleanCircularQueue(); + Assert.assertThrows(IllegalArgumentException.class, () -> booleanQueue.pop()); + Assert.assertThrows(IllegalArgumentException.class, () -> booleanQueue.getHead()); + } + + private static UDFParameters createSingleDoubleSeriesParameters(Map attributes) { + return new UDFParameters( + Collections.singletonList("s1"), Collections.singletonList(Type.DOUBLE), attributes); + } + + private static UDFParameters createSingleTextSeriesParameters(Map attributes) { + return new UDFParameters( + Collections.singletonList("s1"), Collections.singletonList(Type.TEXT), attributes); + } + + private static UDFParameters createTwoDoubleSeriesParameters(Map attributes) { + return new UDFParameters( + Arrays.asList("s1", "s2"), Arrays.asList(Type.DOUBLE, Type.DOUBLE), attributes); + } + + private static int getWindowSize(UDTFKSigma kSigma) throws Exception { + Field windowSize = UDTFKSigma.class.getDeclaredField("windowSize"); + windowSize.setAccessible(true); + return (int) windowSize.get(kSigma); + } + + private static void runQLB( + UDTFQLB qlb, UDFParameters parameters, RecordingPointCollector collector) throws Exception { + qlb.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + qlb.transform(new DoubleRow(1, 0.1), collector); + qlb.transform(new DoubleRow(2, 0.2), collector); + qlb.transform(new DoubleRow(3, 0.3), collector); + qlb.terminate(collector); + } + + private static class DoubleRow implements Row { + + private final long time; + private final double[] values; + private final boolean[] nulls; + + private DoubleRow(long time, double value) { + this(time, new double[] {value}, new boolean[] {false}); + } + + private DoubleRow(long time, double[] values, boolean[] nulls) { + this.time = time; + this.values = values; + this.nulls = nulls; + } + + @Override + public long getTime() { + return time; + } + + @Override + public int getInt(int columnIndex) { + throw new UnsupportedOperationException(); + } + + @Override + public long getLong(int columnIndex) { + throw new UnsupportedOperationException(); + } + + @Override + public float getFloat(int columnIndex) { + throw new UnsupportedOperationException(); + } + + @Override + public double getDouble(int columnIndex) { + if (nulls[columnIndex]) { + throw new IllegalStateException("Null value should not be read"); + } + return values[columnIndex]; + } + + @Override + public boolean getBoolean(int columnIndex) { + throw new UnsupportedOperationException(); + } + + @Override + public Binary getBinary(int columnIndex) { + throw new UnsupportedOperationException(); + } + + @Override + public String getString(int columnIndex) { + throw new UnsupportedOperationException(); + } + + @Override + public Type getDataType(int columnIndex) { + return Type.DOUBLE; + } + + @Override + public boolean isNull(int columnIndex) { + return nulls[columnIndex]; + } + + @Override + public int size() { + return values.length; + } + } + + private static class SimpleRowWindow implements RowWindow { + + private final Row[] rows; + + private SimpleRowWindow(Row... rows) { + this.rows = rows; + } + + @Override + public int windowSize() { + return rows.length; + } + + @Override + public Row getRow(int rowIndex) { + return rows[rowIndex]; + } + + @Override + public Type getDataType(int columnIndex) { + return Type.DOUBLE; + } + + @Override + public RowIterator getRowIterator() { + return new SimpleRowIterator(rows); + } + + @Override + public long windowStartTime() { + throw new UnsupportedOperationException(); + } + + @Override + public long windowEndTime() { + throw new UnsupportedOperationException(); + } + } + + private static class SimpleRowIterator implements RowIterator { + + private final Row[] rows; + private int index; + + private SimpleRowIterator(Row[] rows) { + this.rows = rows; + this.index = 0; + } + + @Override + public boolean hasNextRow() { + return index < rows.length; + } + + @Override + public Row next() { + return rows[index++]; + } + + @Override + public void reset() { + index = 0; + } + } + + private static class RecordingPointCollector implements PointCollector { + + private final List timestamps = new ArrayList<>(); + private final List values = new ArrayList<>(); + + @Override + public void putInt(long timestamp, int value) { + timestamps.add(timestamp); + values.add((double) value); + } + + @Override + public void putLong(long timestamp, long value) { + timestamps.add(timestamp); + values.add((double) value); + } + + @Override + public void putFloat(long timestamp, float value) { + timestamps.add(timestamp); + values.add((double) value); + } + + @Override + public void putDouble(long timestamp, double value) { + timestamps.add(timestamp); + values.add(value); + } + + @Override + public void putBoolean(long timestamp, boolean value) { + throw new UnsupportedOperationException(); + } + + @Override + public void putBinary(long timestamp, Binary value) { + throw new UnsupportedOperationException(); + } + + @Override + public void putString(long timestamp, String value) { + throw new UnsupportedOperationException(); + } + } +} From 5288816450fe85eb51b6ba7595790e296e1a5447 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 9 Jun 2026 11:20:06 +0800 Subject: [PATCH 2/6] Fix library UDF edge cases --- .../apache/iotdb/library/anomaly/UDTFIQR.java | 5 +- .../iotdb/library/anomaly/UDTFKSigma.java | 7 + .../iotdb/library/anomaly/UDTFMissDetect.java | 7 +- .../iotdb/library/anomaly/UDTFOutlier.java | 59 +- .../iotdb/library/anomaly/UDTFRange.java | 3 + .../library/anomaly/util/MissDetector.java | 3 + .../anomaly/util/StreamMissDetector.java | 6 +- .../library/anomaly/util/WindowDetect.java | 6 +- .../iotdb/library/dprofile/UDAFIntegral.java | 12 +- .../library/dprofile/UDAFIntegralAvg.java | 22 +- .../iotdb/library/dprofile/UDAFMad.java | 8 +- .../iotdb/library/dprofile/UDAFMedian.java | 8 +- .../library/dprofile/UDAFPercentile.java | 24 +- .../iotdb/library/dprofile/UDAFPeriod.java | 6 + .../iotdb/library/dprofile/UDAFQuantile.java | 11 +- .../iotdb/library/dprofile/UDAFSkew.java | 3 + .../iotdb/library/dprofile/UDTFHistogram.java | 3 + .../iotdb/library/dprofile/UDTFMinMax.java | 8 +- .../iotdb/library/dprofile/UDTFMvAvg.java | 10 +- .../iotdb/library/dprofile/UDTFPACF.java | 2 +- .../iotdb/library/dprofile/UDTFResample.java | 7 +- .../iotdb/library/dprofile/UDTFSegment.java | 5 +- .../iotdb/library/dprofile/UDTFSpline.java | 5 +- .../iotdb/library/dprofile/UDTFZScore.java | 11 +- .../library/dprofile/util/Resampler.java | 5 +- .../frequency/UDFEnvelopeAnalysis.java | 14 +- .../iotdb/library/frequency/UDTFFFT.java | 4 + .../iotdb/library/frequency/UDTFHighPass.java | 5 + .../iotdb/library/frequency/UDTFIFFT.java | 17 +- .../iotdb/library/frequency/UDTFLowPass.java | 5 + .../iotdb/library/UDFWindowAndQueueTest.java | 544 +++++++++++++++++- 31 files changed, 770 insertions(+), 65 deletions(-) diff --git a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFIQR.java b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFIQR.java index 4b800003dac82..3278fed26a05c 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFIQR.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFIQR.java @@ -90,9 +90,12 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati @Override public void transform(Row row, PointCollector collector) throws Exception { + if (row.isNull(0)) { + return; + } if (compute.equalsIgnoreCase(STREAM_COMPUTE) && q3 > q1) { double v = Util.getValueAsDouble(row); - if (v < q1 - 1.5 * iqr || v > q3 + 1.5 * iqr) { + if (Double.isFinite(v) && (v < q1 - 1.5 * iqr || v > q3 + 1.5 * iqr)) { collector.putDouble(row.getTime(), v); } } else if (compute.equalsIgnoreCase(BATCH_COMPUTE)) { diff --git a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFKSigma.java b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFKSigma.java index c19fd7a872272..27e21406ed8b3 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFKSigma.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFKSigma.java @@ -69,12 +69,19 @@ public void beforeStart(UDFParameters udfParameters, UDTFConfigurations udtfConf this.multipleK = udfParameters.getDoubleOrDefault("k", 3); this.dataType = udfParameters.getDataType(0); this.windowSize = udfParameters.getIntOrDefault("window", DEFAULT_WINDOW_SIZE); + this.mean = 0.0; + this.variance = 0.0; + this.sumX1 = 0.0; + this.sumX2 = 0.0; this.v = new CircularQueue<>(windowSize); this.t = new LongCircularQueue(windowSize); } @Override public void transform(Row row, PointCollector collector) throws Exception { + if (row.isNull(0)) { + return; + } double value = Util.getValueAsDouble(row); long timestamp = row.getTime(); if (Double.isFinite(value) && !Double.isNaN(value)) { diff --git a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFMissDetect.java b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFMissDetect.java index c63d33c4a7941..31b3b017edc0e 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFMissDetect.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFMissDetect.java @@ -55,7 +55,12 @@ public void beforeStart(UDFParameters udfp, UDTFConfigurations udtfc) throws Exc @Override public void transform(Row row, PointCollector collector) throws Exception { - detector.insert(row.getTime(), Util.getValueAsDouble(row)); + if (!row.isNull(0)) { + double v = Util.getValueAsDouble(row); + if (Double.isFinite(v)) { + detector.insert(row.getTime(), v); + } + } while (detector.hasNext()) { collector.putBoolean(detector.getOutTime(), detector.getOutValue()); detector.next(); diff --git a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFOutlier.java b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFOutlier.java index bf80003f98e8b..308e5556cb7da 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFOutlier.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFOutlier.java @@ -24,6 +24,7 @@ import org.apache.iotdb.udf.api.access.Row; import org.apache.iotdb.udf.api.collector.PointCollector; import org.apache.iotdb.udf.api.customizer.config.UDTFConfigurations; +import org.apache.iotdb.udf.api.customizer.parameter.UDFParameterValidator; import org.apache.iotdb.udf.api.customizer.parameter.UDFParameters; import org.apache.iotdb.udf.api.customizer.strategy.RowByRowAccessStrategy; import org.apache.iotdb.udf.api.type.Type; @@ -47,38 +48,66 @@ public class UDTFOutlier implements UDTF { private ArrayList currentValueWindow = new ArrayList<>(); private Map outliers = new HashMap<>(); + @Override + public void validate(UDFParameterValidator validator) throws Exception { + validator + .validateInputSeriesNumber(1) + .validateInputSeriesDataType(0, Type.INT32, Type.INT64, Type.FLOAT, Type.DOUBLE) + .validate( + x -> (int) x > 0, + "Parameter k should be a positive integer.", + validator.getParameters().getIntOrDefault("k", 3)) + .validate( + x -> (double) x >= 0, + "Parameter r should be non-negative.", + validator.getParameters().getDoubleOrDefault("r", 5)) + .validate( + x -> (int) x > 0, + "Parameter w should be a positive integer.", + validator.getParameters().getIntOrDefault("w", 1000)) + .validate( + x -> (int) x > 0, + "Parameter s should be a positive integer.", + validator.getParameters().getIntOrDefault("s", 500)); + } + @Override public void beforeStart(UDFParameters udfParameters, UDTFConfigurations udtfConfigurations) throws Exception { udtfConfigurations .setAccessStrategy(new RowByRowAccessStrategy()) - .setOutputDataType(udfParameters.getDataType(0)); + .setOutputDataType(Type.DOUBLE); this.k = udfParameters.getIntOrDefault("k", 3); this.r = udfParameters.getDoubleOrDefault("r", 5); this.w = udfParameters.getIntOrDefault("w", 1000); this.s = udfParameters.getIntOrDefault("s", 500); this.i = 0; - - udtfConfigurations.setAccessStrategy(new RowByRowAccessStrategy()); - udtfConfigurations.setOutputDataType(Type.DOUBLE); + currentTimeWindow.clear(); + currentValueWindow.clear(); + outliers.clear(); } @Override public void transform(Row row, PointCollector collector) throws Exception { - if (!row.isNull(0)) { - if (i >= w && (i - w) % s == 0) { - detect(); - } + if (row.isNull(0)) { + return; + } + double v = Util.getValueAsDouble(row); + if (!Double.isFinite(v)) { + return; + } + if (i >= w && (i - w) % s == 0) { + detect(); + } - if (i >= w) { - currentValueWindow.remove(0); - currentTimeWindow.remove(0); - } - currentTimeWindow.add(row.getTime()); - currentValueWindow.add(Util.getValueAsDouble(row)); - i += 1; + if (i >= w) { + currentValueWindow.remove(0); + currentTimeWindow.remove(0); } + currentTimeWindow.add(row.getTime()); + currentValueWindow.add(v); + i += 1; } @Override diff --git a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFRange.java b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFRange.java index dc75d575d9e07..447ce16426779 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFRange.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFRange.java @@ -63,6 +63,9 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati @Override public void transform(Row row, PointCollector collector) throws Exception { + if (row.isNull(0)) { + return; + } int intValue; long longValue; float floatValue; diff --git a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/util/MissDetector.java b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/util/MissDetector.java index ab7f27722ad17..886481eae56c4 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/util/MissDetector.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/util/MissDetector.java @@ -49,6 +49,9 @@ public class MissDetector { public MissDetector(RowIterator iterator, int minLength) throws Exception { while (iterator.hasNextRow()) { Row row = iterator.next(); + if (row.isNull(0)) { + continue; + } double v = Util.getValueAsDouble(row); if (Double.isFinite(v)) { this.time.add(row.getTime()); diff --git a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/util/StreamMissDetector.java b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/util/StreamMissDetector.java index 892541ba8d960..21abae1051146 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/util/StreamMissDetector.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/util/StreamMissDetector.java @@ -37,21 +37,23 @@ public class StreamMissDetector { private int minLength; private int state; private long startTime; + private boolean hasStartTime; private int missingStartIndex; private boolean horizon; private double standard; public StreamMissDetector(int minLength) { this.state = 0; - this.startTime = -1; + this.hasStartTime = false; this.minLength = minLength; } public void insert(long time, double value) { timeWindow.push(time); valueWindow.push(value); - if (startTime < 0) { + if (!hasStartTime) { startTime = time; + hasStartTime = true; } switch (state) { case 0: diff --git a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/util/WindowDetect.java b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/util/WindowDetect.java index 0eeb69c15a617..0fba36379da5b 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/util/WindowDetect.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/util/WindowDetect.java @@ -41,12 +41,12 @@ public WindowDetect(RowIterator dataIterator, double l, double t) throws Excepti ArrayList originList = new ArrayList<>(); while (dataIterator.hasNextRow()) { Row row = dataIterator.next(); - Double v = Util.getValueAsDouble(row); timeList.add(row.getTime()); - if (v == null || !Double.isFinite(v)) { + if (row.isNull(0)) { originList.add(Double.NaN); } else { - originList.add(v); + double v = Util.getValueAsDouble(row); + originList.add(Double.isFinite(v) ? v : Double.NaN); } } time = Util.toLongArray(timeList); diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFIntegral.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFIntegral.java index 08a50fda2718d..6a5291ae943cd 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFIntegral.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFIntegral.java @@ -40,6 +40,7 @@ public class UDAFIntegral implements UDTF { long unitTime; long lastTime = -1; + boolean hasLastValue; double lastValue; double integralValue = 0; @@ -47,6 +48,7 @@ public class UDAFIntegral implements UDTF { public void validate(UDFParameterValidator validator) throws Exception { validator .validateInputSeriesNumber(1) + .validateInputSeriesDataType(0, Type.INT32, Type.INT64, Type.FLOAT, Type.DOUBLE) .validate( x -> (long) x > 0, "Unknown time unit input. Supported units are ns, us, ms, s, m, h, d.", @@ -61,20 +63,28 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati configurations.setAccessStrategy(new RowByRowAccessStrategy()).setOutputDataType(Type.DOUBLE); unitTime = Util.parseTime(parameters.getStringOrDefault(TIME_UNIT_KEY, TIME_UNIT_S), parameters); + lastTime = -1; + hasLastValue = false; + lastValue = 0; + integralValue = 0; } @Override public void transform(Row row, PointCollector collector) throws Exception { + if (row.isNull(0)) { + return; + } long nowTime = row.getTime(); double nowValue = Util.getValueAsDouble(row); if (Double.isFinite(nowValue)) { // calculate the ladder-shaped area between last point and this one // skip and initialize the memory if no existing previous point is available - if (lastTime >= 0) { + if (hasLastValue) { integralValue += (lastValue + nowValue) * (nowTime - lastTime) / 2.0 / unitTime; } lastTime = nowTime; lastValue = nowValue; + hasLastValue = true; } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFIntegralAvg.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFIntegralAvg.java index 76a5f31419f03..ab2ea0c953902 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFIntegralAvg.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFIntegralAvg.java @@ -37,6 +37,7 @@ public class UDAFIntegralAvg implements UDTF { long startTime = -1; long lastTime = -1; + boolean hasValue; double lastValue = 0; double integralValue = 0; @@ -51,29 +52,36 @@ public void validate(UDFParameterValidator validator) throws Exception { public void beforeStart(UDFParameters parameters, UDTFConfigurations configurations) throws Exception { configurations.setAccessStrategy(new RowByRowAccessStrategy()).setOutputDataType(Type.DOUBLE); + startTime = -1; + lastTime = -1; + hasValue = false; + lastValue = 0; + integralValue = 0; } @Override public void transform(Row row, PointCollector collector) throws Exception { + if (row.isNull(0)) { + return; + } long nowTime = row.getTime(); double nowValue = Util.getValueAsDouble(row); - if (startTime < 0) { - startTime = nowTime; - } if (Double.isFinite(nowValue)) { - // calculate the ladder-shaped area between last point and this one - // skip and initialize the memory if no existing previous point is available - if (lastTime >= 0) { + if (!hasValue) { + startTime = nowTime; + } else { + // calculate the ladder-shaped area between last point and this one integralValue += (lastValue + nowValue) * (nowTime - lastTime) / 2.0; } lastTime = nowTime; lastValue = nowValue; + hasValue = true; } } @Override public void terminate(PointCollector collector) throws Exception { - if (startTime < 0) { + if (!hasValue) { // empty input collector.putDouble(0, 0); } else if (startTime == lastTime) { diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFMad.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFMad.java index 08cbb3964bbb7..1833a0e4c8116 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFMad.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFMad.java @@ -66,10 +66,16 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati @Override public void transform(Row row, PointCollector collector) throws Exception { + if (row.isNull(0)) { + return; + } if (exact) { statistics.insert(row); } else { - sketch.insert(Util.getValueAsDouble(row)); + double v = Util.getValueAsDouble(row); + if (Double.isFinite(v)) { + sketch.insert(v); + } } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFMedian.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFMedian.java index 425b4955df131..9061d2f0721cf 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFMedian.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFMedian.java @@ -66,10 +66,16 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati @Override public void transform(Row row, PointCollector collector) throws Exception { + if (row.isNull(0)) { + return; + } if (exact) { statistics.insert(row); } else { - sketch.insert(Util.getValueAsDouble(row)); + double v = Util.getValueAsDouble(row); + if (Double.isFinite(v)) { + sketch.insert(v); + } } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFPercentile.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFPercentile.java index 9a58e1c0a500f..1eae1b061aa3b 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFPercentile.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFPercentile.java @@ -103,27 +103,28 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati @Override public void transform(Row row, PointCollector collector) throws Exception { + if (row.isNull(0)) { + return; + } if (exact) { statistics.insert(row); switch (dataType) { case INT32: - if (!row.isNull(0)) { - intDic.put(row.getInt(0), row.getTime()); - } + intDic.put(row.getInt(0), row.getTime()); break; case INT64: - if (!row.isNull(0)) { - longDic.put(row.getLong(0), row.getTime()); - } + longDic.put(row.getLong(0), row.getTime()); break; case FLOAT: - if (!row.isNull(0) && Float.isFinite(row.getFloat(0))) { - floatDic.put(row.getFloat(0), row.getTime()); + float fv = row.getFloat(0); + if (Float.isFinite(fv)) { + floatDic.put(fv, row.getTime()); } break; case DOUBLE: - if (!row.isNull(0) && Double.isFinite(row.getDouble(0))) { - doubleDic.put(row.getDouble(0), row.getTime()); + double dv = row.getDouble(0); + if (Double.isFinite(dv)) { + doubleDic.put(dv, row.getTime()); } break; case BLOB: @@ -137,6 +138,9 @@ public void transform(Row row, PointCollector collector) throws Exception { } } else { double res = Util.getValueAsDouble(row); + if (!Double.isFinite(res)) { + return; + } sketch.insert(res); } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFPeriod.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFPeriod.java index 6d17edea7569b..3c00fbd20f5d1 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFPeriod.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFPeriod.java @@ -58,6 +58,12 @@ public void transform(RowWindow rowWindow, PointCollector collector) throws Exce RowIterator iterator = rowWindow.getRowIterator(); while (iterator.hasNextRow()) { Row row = iterator.next(); + if (row.isNull(0)) { + if (!value.isEmpty()) { + value.add(value.getLast()); + } + continue; + } double v = Util.getValueAsDouble(row); if (Double.isFinite(v)) { value.add(v); diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFQuantile.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFQuantile.java index 5e801b0728240..649414fb361a7 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFQuantile.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFQuantile.java @@ -65,6 +65,9 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati @Override public void transform(Row row, PointCollector collector) throws Exception { + if (row.isNull(0)) { + return; + } final long encoded; switch (dataType) { case INT32: @@ -74,7 +77,11 @@ public void transform(Row row, PointCollector collector) throws Exception { encoded = row.getLong(0); break; default: - encoded = dataToLong(Util.getValueAsDouble(row)); + double v = Util.getValueAsDouble(row); + if (!Double.isFinite(v)) { + return; + } + encoded = dataToLong(v); break; } sketch.update(encoded); @@ -94,7 +101,7 @@ public void terminate(PointCollector collector) throws Exception { } else if (k >= n) { k = n - 1; } - long result = sketch.findMinValueWithRank(k); + long result = sketch.findMaxValueWithRank(k); switch (dataType) { case INT32: collector.putInt(0, (int) result); diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFSkew.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFSkew.java index 3ee5954ed9cd9..374659311350e 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFSkew.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFSkew.java @@ -55,6 +55,9 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati @Override public void transform(Row row, PointCollector collector) throws Exception { + if (row.isNull(0)) { + return; + } double value = Util.getValueAsDouble(row); if (Double.isFinite(value)) { this.count++; diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFHistogram.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFHistogram.java index d29f25d8717af..7a3700d551a87 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFHistogram.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFHistogram.java @@ -70,6 +70,9 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati @Override public void transform(Row row, PointCollector collector) throws Exception { + if (row.isNull(0)) { + return; + } double value = Util.getValueAsDouble(row); if (Double.isFinite(value)) { int id = Math.min(Math.max((int) Math.floor((value - start) / gap), 0), count - 1); diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFMinMax.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFMinMax.java index b01cd79540241..a4411a84244f4 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFMinMax.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFMinMax.java @@ -84,8 +84,14 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati @Override public void transform(Row row, PointCollector collector) throws Exception { + if (row.isNull(0)) { + return; + } if (compute.equalsIgnoreCase(STREAM_COMPUTE) && max > min) { - collector.putDouble(row.getTime(), (Util.getValueAsDouble(row) - min) / (max - min)); + double v = Util.getValueAsDouble(row); + if (Double.isFinite(v)) { + collector.putDouble(row.getTime(), (v - min) / (max - min)); + } } else if (compute.equalsIgnoreCase(BATCH_COMPUTE)) { double v = Util.getValueAsDouble(row); if (Double.isFinite(v)) { diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFMvAvg.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFMvAvg.java index 9b93c7e223256..953db950001e5 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFMvAvg.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFMvAvg.java @@ -60,16 +60,18 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati @Override public void transform(Row row, PointCollector collector) throws Exception { - long t = row.getTime(); - if (v.isFull()) { - windowSum -= v.pop(); + if (row.isNull(0)) { + return; } double value = Util.getValueAsDouble(row); if (Double.isFinite(value)) { + if (v.isFull()) { + windowSum -= v.pop(); + } v.push(value); windowSum += value; if (v.isFull()) { - collector.putDouble(t, windowSum / windowSize); + collector.putDouble(row.getTime(), windowSum / windowSize); } } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFPACF.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFPACF.java index 4eb180a541c6a..9256502dc421c 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFPACF.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFPACF.java @@ -60,7 +60,7 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati @Override public void transform(Row row, PointCollector collector) throws Exception { - double v = Util.getValueAsDouble(row); + double v = row.isNull(0) ? 0d : Util.getValueAsDouble(row); if (Double.isFinite(v)) { value.add(v); } else { diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFResample.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFResample.java index 441067ceb6902..c8df048ef45cb 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFResample.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFResample.java @@ -98,7 +98,12 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati @Override public void transform(Row row, PointCollector pc) throws Exception { - resampler.insert(row.getTime(), Util.getValueAsDouble(row)); + if (!row.isNull(0)) { + double v = Util.getValueAsDouble(row); + if (Double.isFinite(v)) { + resampler.insert(row.getTime(), v); + } + } while (resampler.hasNext()) { // output as early as possible pc.putDouble(resampler.getOutTime(), resampler.getOutValue()); resampler.next(); diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFSegment.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFSegment.java index e941a7a5c7a34..81308319122a3 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFSegment.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFSegment.java @@ -86,10 +86,13 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati @Override public void transform(Row row, PointCollector collector) throws Exception { + if (row.isNull(0)) { + return; + } double v = Util.getValueAsDouble(row); if (Double.isFinite(v)) { timestamp.add(row.getTime()); - value.add(Util.getValueAsDouble(row)); + value.add(v); } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFSpline.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFSpline.java index 62c4253db882b..432cb0e64fdef 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFSpline.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFSpline.java @@ -70,6 +70,9 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati @Override public void transform(Row row, PointCollector collector) throws Exception { + if (row.isNull(0)) { + return; + } double v = Util.getValueAsDouble(row); if (Double.isFinite(v)) { Long t = row.getTime(); @@ -78,7 +81,7 @@ public void transform(Row row, PointCollector collector) throws Exception { } timestamp.add(t); xDouble.add((Double.valueOf(Long.toString(t - minimumTimestamp)))); - yDouble.add(Util.getValueAsDouble(row)); + yDouble.add(v); } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFZScore.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFZScore.java index a040cc76f0f8a..4c53aa066752c 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFZScore.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFZScore.java @@ -86,8 +86,14 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati @Override public void transform(Row row, PointCollector collector) throws Exception { + if (row.isNull(0)) { + return; + } if (compute.equalsIgnoreCase(STREAM_COMPUTE) && sd > 0) { - collector.putDouble(row.getTime(), (Util.getValueAsDouble(row) - avg) / sd); + double v = Util.getValueAsDouble(row); + if (Double.isFinite(v)) { + collector.putDouble(row.getTime(), (v - avg) / sd); + } } else if (compute.equalsIgnoreCase(BATCH_COMPUTE)) { double v = Util.getValueAsDouble(row); if (Double.isFinite(v)) { @@ -102,6 +108,9 @@ public void transform(Row row, PointCollector collector) throws Exception { @Override public void terminate(PointCollector collector) throws Exception { if (compute.equalsIgnoreCase(BATCH_COMPUTE)) { + if (value.isEmpty()) { + return; + } avg = sum / value.size(); sd = Math.sqrt(squareSum / value.size() - avg * avg); for (int i = 0; i < value.size(); i++) { diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/util/Resampler.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/util/Resampler.java index ef69f0ad9c97c..11064c54f97f9 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/util/Resampler.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/util/Resampler.java @@ -43,6 +43,7 @@ public class Resampler { private final String aggregator; // method to aggregate private final String interpolator; // method to interpolate private long currentTime; // start time of the window, left close & right open + private boolean hasCurrentTime; private long startTime; private long endTime; // start time (contained) and end time (not contained) of resampling private boolean outer = true; // if to use outer interpolate @@ -59,6 +60,7 @@ public Resampler( this.startTime = startTime; this.endTime = endTime; this.currentTime = this.startTime; + this.hasCurrentTime = startTime >= 0; } /** 加入新的数据点 insert new datapoint. */ @@ -68,8 +70,9 @@ public void insert(long time, double value) { || (endTime > 0 && time >= endTime)) { return; } - if (currentTime < 0) { + if (!hasCurrentTime) { currentTime = time; + hasCurrentTime = true; } while (time >= currentTime + newPeriod) { downSample(); diff --git a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDFEnvelopeAnalysis.java b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDFEnvelopeAnalysis.java index 1eaa1d18a686e..4db051b27395e 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDFEnvelopeAnalysis.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDFEnvelopeAnalysis.java @@ -72,6 +72,8 @@ public void validate(UDFParameterValidator validator) throws Exception { public void beforeStart(UDFParameters parameters, UDTFConfigurations configurations) throws Exception { configurations.setAccessStrategy(new RowByRowAccessStrategy()).setOutputDataType(Type.DOUBLE); + signals.clear(); + timestamps.clear(); frequency = parameters.getDoubleOrDefault(FREQUENCY, Double.MAX_VALUE); amplification = parameters.getIntOrDefault(AMPLIFICATION, 1); timestampPrecision = parameters.getSystemStringOrDefault(TIMESTAMP_PRECISION, MS_PRECISION); @@ -79,9 +81,15 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati @Override public void transform(Row row, PointCollector collector) throws Exception { - signals.add(getValueAsDouble(row, 0)); - if (timestamps.size() < 10) { - timestamps.add(row.getTime()); + if (row.isNull(0)) { + return; + } + double value = getValueAsDouble(row, 0); + if (Double.isFinite(value)) { + signals.add(value); + if (timestamps.size() < 10) { + timestamps.add(row.getTime()); + } } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFFFT.java b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFFFT.java index d2d44bab2e399..004b7df602676 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFFFT.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFFFT.java @@ -70,6 +70,7 @@ public void validate(UDFParameterValidator validator) throws Exception { public void beforeStart(UDFParameters parameters, UDTFConfigurations configurations) throws Exception { configurations.setAccessStrategy(new RowByRowAccessStrategy()).setOutputDataType(Type.DOUBLE); + list.clear(); String result = parameters.getStringOrDefault("result", "abs"); this.compressed = parameters.hasAttribute(COMPRESS_PARAM); double compressRate = parameters.getDoubleOrDefault(COMPRESS_PARAM, 1); @@ -78,6 +79,9 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati @Override public void transform(Row row, PointCollector collector) throws Exception { + if (row.isNull(0)) { + return; + } double v = Util.getValueAsDouble(row); if (Double.isFinite(v)) { list.add(v); diff --git a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFHighPass.java b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFHighPass.java index 567c20d1c50ae..2220fa58698d8 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFHighPass.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFHighPass.java @@ -57,11 +57,16 @@ public void validate(UDFParameterValidator validator) throws Exception { public void beforeStart(UDFParameters parameters, UDTFConfigurations configurations) throws Exception { configurations.setAccessStrategy(new RowByRowAccessStrategy()).setOutputDataType(Type.DOUBLE); + valueList.clear(); + timeList.clear(); this.wpass = parameters.getDouble(WPASS_PARAM); } @Override public void transform(Row row, PointCollector collector) throws Exception { + if (row.isNull(0)) { + return; + } double v = Util.getValueAsDouble(row); if (Double.isFinite(v)) { valueList.add(v); diff --git a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFIFFT.java b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFIFFT.java index 8277aab834825..991d4c5a8f5d4 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFIFFT.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFIFFT.java @@ -71,6 +71,9 @@ public void validate(UDFParameterValidator validator) throws Exception { public void beforeStart(UDFParameters parameters, UDTFConfigurations configurations) throws Exception { configurations.setAccessStrategy(new RowByRowAccessStrategy()).setOutputDataType(Type.DOUBLE); + real.clear(); + imag.clear(); + time.clear(); this.interval = Util.parseTime(parameters.getStringOrDefault("interval", "1s"), parameters); SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); this.start = 0; @@ -81,13 +84,15 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati @Override public void transform(Row row, PointCollector collector) throws Exception { - if (!row.isNull(0) - && !row.isNull(1) - && Double.isFinite(Util.getValueAsDouble(row, 0)) - && Double.isFinite(Util.getValueAsDouble(row, 1))) { + if (!row.isNull(0) && !row.isNull(1)) { + double realValue = Util.getValueAsDouble(row, 0); + double imagValue = Util.getValueAsDouble(row, 1); + if (!Double.isFinite(realValue) || !Double.isFinite(imagValue)) { + return; + } time.add((int) row.getTime()); - real.add(Util.getValueAsDouble(row, 0)); - imag.add(Util.getValueAsDouble(row, 1)); + real.add(realValue); + imag.add(imagValue); } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFLowPass.java b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFLowPass.java index f5346b175eaac..dc5bc49c685a8 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFLowPass.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFLowPass.java @@ -57,11 +57,16 @@ public void validate(UDFParameterValidator validator) throws Exception { public void beforeStart(UDFParameters parameters, UDTFConfigurations configurations) throws Exception { configurations.setAccessStrategy(new RowByRowAccessStrategy()).setOutputDataType(Type.DOUBLE); + valueList.clear(); + timeList.clear(); this.wpass = parameters.getDouble(WPASS_PARAM); } @Override public void transform(Row row, PointCollector collector) throws Exception { + if (row.isNull(0)) { + return; + } double v = Util.getValueAsDouble(row); if (Double.isFinite(v)) { valueList.add(v); diff --git a/library-udf/src/test/java/org/apache/iotdb/library/UDFWindowAndQueueTest.java b/library-udf/src/test/java/org/apache/iotdb/library/UDFWindowAndQueueTest.java index ef6dd66572204..932ca43cf40dc 100644 --- a/library-udf/src/test/java/org/apache/iotdb/library/UDFWindowAndQueueTest.java +++ b/library-udf/src/test/java/org/apache/iotdb/library/UDFWindowAndQueueTest.java @@ -22,10 +22,16 @@ import org.apache.iotdb.library.anomaly.UDTFIQR; import org.apache.iotdb.library.anomaly.UDTFKSigma; import org.apache.iotdb.library.anomaly.UDTFLOF; +import org.apache.iotdb.library.anomaly.UDTFMissDetect; +import org.apache.iotdb.library.anomaly.UDTFOutlier; import org.apache.iotdb.library.anomaly.UDTFRange; +import org.apache.iotdb.library.anomaly.UDTFTwoSidedFilter; +import org.apache.iotdb.library.anomaly.util.StreamMissDetector; import org.apache.iotdb.library.dlearn.UDTFAR; import org.apache.iotdb.library.dlearn.UDTFCluster; import org.apache.iotdb.library.dmatch.UDAFDtw; +import org.apache.iotdb.library.dprofile.UDAFIntegral; +import org.apache.iotdb.library.dprofile.UDAFIntegralAvg; import org.apache.iotdb.library.dprofile.UDAFMad; import org.apache.iotdb.library.dprofile.UDAFMedian; import org.apache.iotdb.library.dprofile.UDAFPercentile; @@ -35,12 +41,14 @@ import org.apache.iotdb.library.dprofile.UDTFHistogram; import org.apache.iotdb.library.dprofile.UDTFMinMax; import org.apache.iotdb.library.dprofile.UDTFMvAvg; +import org.apache.iotdb.library.dprofile.UDTFPACF; import org.apache.iotdb.library.dprofile.UDTFQLB; import org.apache.iotdb.library.dprofile.UDTFResample; import org.apache.iotdb.library.dprofile.UDTFSample; import org.apache.iotdb.library.dprofile.UDTFSegment; import org.apache.iotdb.library.dprofile.UDTFSpline; import org.apache.iotdb.library.dprofile.UDTFZScore; +import org.apache.iotdb.library.dprofile.util.Resampler; import org.apache.iotdb.library.frequency.UDFEnvelopeAnalysis; import org.apache.iotdb.library.frequency.UDTFConv; import org.apache.iotdb.library.frequency.UDTFDWT; @@ -133,6 +141,27 @@ public void testMvAvgUsesRunningWindowSum() throws Exception { Assert.assertEquals(3.0, collector.values.get(1), 0.0); } + @Test + public void testMvAvgInvalidRowsDoNotShrinkWindow() throws Exception { + Map attributes = new HashMap<>(); + attributes.put("window", "3"); + + UDTFMvAvg mvAvg = new UDTFMvAvg(); + UDFParameters parameters = createSingleDoubleSeriesParameters(attributes); + RecordingPointCollector collector = new RecordingPointCollector(); + + mvAvg.validate(new UDFParameterValidator(parameters)); + mvAvg.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + mvAvg.transform(new DoubleRow(1, 1.0), collector); + mvAvg.transform(new DoubleRow(2, 2.0), collector); + mvAvg.transform(new DoubleRow(3, Double.NaN), collector); + mvAvg.transform(nullDoubleRow(4), collector); + mvAvg.transform(new DoubleRow(5, 3.0), collector); + + Assert.assertEquals(Collections.singletonList(5L), collector.timestamps); + Assert.assertEquals(2.0, collector.values.get(0), 0.0); + } + @Test public void testLOFSkipsNullRowsWithoutReadingCompressedIndex() throws Exception { Map attributes = new HashMap<>(); @@ -260,6 +289,120 @@ public void testStreamModeRequiresExplicitParameters() { new UDFParameterValidator(createSingleDoubleSeriesParameters(attributes)))); } + @Test + public void testRowByRowNumericFunctionsSkipNullAndInvalidValues() throws Exception { + UDFParameters defaultParameters = createSingleDoubleSeriesParameters(Collections.emptyMap()); + + Map rangeAttributes = new HashMap<>(); + rangeAttributes.put("lower_bound", "0"); + rangeAttributes.put("upper_bound", "10"); + UDFParameters rangeParameters = createSingleDoubleSeriesParameters(rangeAttributes); + UDTFRange range = new UDTFRange(); + RecordingPointCollector rangeCollector = new RecordingPointCollector(); + range.validate(new UDFParameterValidator(rangeParameters)); + range.beforeStart(rangeParameters, new UDTFConfigurations(ZoneId.systemDefault())); + range.transform(nullDoubleRow(1), rangeCollector); + range.transform(new DoubleRow(2, Double.NaN), rangeCollector); + Assert.assertTrue(rangeCollector.timestamps.isEmpty()); + + Map histogramAttributes = new HashMap<>(); + histogramAttributes.put("min", "0"); + histogramAttributes.put("max", "10"); + histogramAttributes.put("count", "2"); + UDFParameters histogramParameters = createSingleDoubleSeriesParameters(histogramAttributes); + UDTFHistogram histogram = new UDTFHistogram(); + RecordingPointCollector histogramCollector = new RecordingPointCollector(); + histogram.validate(new UDFParameterValidator(histogramParameters)); + histogram.beforeStart(histogramParameters, new UDTFConfigurations(ZoneId.systemDefault())); + histogram.transform(nullDoubleRow(1), histogramCollector); + histogram.transform(new DoubleRow(2, Double.NaN), histogramCollector); + histogram.transform(new DoubleRow(3, 8.0), histogramCollector); + histogram.terminate(histogramCollector); + Assert.assertEquals(Arrays.asList(0L, 1L), histogramCollector.timestamps); + Assert.assertEquals(0.0, histogramCollector.values.get(0), 0.0); + Assert.assertEquals(1.0, histogramCollector.values.get(1), 0.0); + + Map iqrAttributes = new HashMap<>(); + iqrAttributes.put("compute", "stream"); + iqrAttributes.put("q1", "1"); + iqrAttributes.put("q3", "3"); + UDFParameters iqrParameters = createSingleDoubleSeriesParameters(iqrAttributes); + UDTFIQR iqr = new UDTFIQR(); + RecordingPointCollector iqrCollector = new RecordingPointCollector(); + iqr.validate(new UDFParameterValidator(iqrParameters)); + iqr.beforeStart(iqrParameters, new UDTFConfigurations(ZoneId.systemDefault())); + iqr.transform(nullDoubleRow(1), iqrCollector); + iqr.transform(new DoubleRow(2, Double.POSITIVE_INFINITY), iqrCollector); + iqr.transform(new DoubleRow(3, 10.0), iqrCollector); + Assert.assertEquals(Collections.singletonList(3L), iqrCollector.timestamps); + Assert.assertEquals(10.0, iqrCollector.values.get(0), 0.0); + + Map minMaxAttributes = new HashMap<>(); + minMaxAttributes.put("compute", "stream"); + minMaxAttributes.put("min", "0"); + minMaxAttributes.put("max", "10"); + UDFParameters minMaxParameters = createSingleDoubleSeriesParameters(minMaxAttributes); + UDTFMinMax minMax = new UDTFMinMax(); + RecordingPointCollector minMaxCollector = new RecordingPointCollector(); + minMax.validate(new UDFParameterValidator(minMaxParameters)); + minMax.beforeStart(minMaxParameters, new UDTFConfigurations(ZoneId.systemDefault())); + minMax.transform(nullDoubleRow(1), minMaxCollector); + minMax.transform(new DoubleRow(2, Double.NaN), minMaxCollector); + minMax.transform(new DoubleRow(3, 5.0), minMaxCollector); + Assert.assertEquals(Collections.singletonList(3L), minMaxCollector.timestamps); + Assert.assertEquals(0.5, minMaxCollector.values.get(0), 0.0); + + Map zScoreAttributes = new HashMap<>(); + zScoreAttributes.put("compute", "stream"); + zScoreAttributes.put("avg", "0"); + zScoreAttributes.put("sd", "2"); + UDFParameters zScoreParameters = createSingleDoubleSeriesParameters(zScoreAttributes); + UDTFZScore zScore = new UDTFZScore(); + RecordingPointCollector zScoreCollector = new RecordingPointCollector(); + zScore.validate(new UDFParameterValidator(zScoreParameters)); + zScore.beforeStart(zScoreParameters, new UDTFConfigurations(ZoneId.systemDefault())); + zScore.transform(nullDoubleRow(1), zScoreCollector); + zScore.transform(new DoubleRow(2, Double.NEGATIVE_INFINITY), zScoreCollector); + zScore.transform(new DoubleRow(3, 4.0), zScoreCollector); + Assert.assertEquals(Collections.singletonList(3L), zScoreCollector.timestamps); + Assert.assertEquals(2.0, zScoreCollector.values.get(0), 0.0); + + UDTFKSigma kSigma = new UDTFKSigma(); + RecordingPointCollector kSigmaCollector = new RecordingPointCollector(); + kSigma.validate(new UDFParameterValidator(defaultParameters)); + kSigma.beforeStart(defaultParameters, new UDTFConfigurations(ZoneId.systemDefault())); + kSigma.transform(nullDoubleRow(1), kSigmaCollector); + kSigma.transform(new DoubleRow(2, 1.0), kSigmaCollector); + + UDTFPACF pacf = new UDTFPACF(); + RecordingPointCollector pacfCollector = new RecordingPointCollector(); + pacf.validate(new UDFParameterValidator(defaultParameters)); + pacf.beforeStart(defaultParameters, new UDTFConfigurations(ZoneId.systemDefault())); + pacf.transform(nullDoubleRow(1), pacfCollector); + pacf.transform(new DoubleRow(2, 1.0), pacfCollector); + pacf.terminate(pacfCollector); + Assert.assertEquals(Arrays.asList(1L, 2L), pacfCollector.timestamps); + + UDTFSegment segment = new UDTFSegment(); + RecordingPointCollector segmentCollector = new RecordingPointCollector(); + segment.validate(new UDFParameterValidator(defaultParameters)); + segment.beforeStart(defaultParameters, new UDTFConfigurations(ZoneId.systemDefault())); + segment.transform(nullDoubleRow(1), segmentCollector); + segment.transform(new DoubleRow(2, 1.0), segmentCollector); + + Map splineAttributes = new HashMap<>(); + splineAttributes.put("points", "2"); + UDFParameters splineParameters = createSingleDoubleSeriesParameters(splineAttributes); + UDTFSpline spline = new UDTFSpline(); + RecordingPointCollector splineCollector = new RecordingPointCollector(); + spline.validate(new UDFParameterValidator(splineParameters)); + spline.beforeStart(splineParameters, new UDTFConfigurations(ZoneId.systemDefault())); + spline.transform(nullDoubleRow(1), splineCollector); + spline.transform(new DoubleRow(2, 1.0), splineCollector); + spline.terminate(splineCollector); + Assert.assertTrue(splineCollector.timestamps.isEmpty()); + } + @Test public void testMinMaxBatchKeepsTimestampsAlignedWhenSkippingInvalidValues() throws Exception { UDTFMinMax minMax = new UDTFMinMax(); @@ -392,6 +535,102 @@ public void testAggregateFunctionsIgnoreEmptyInput() throws Exception { Assert.assertTrue(collector.values.isEmpty()); } + @Test + public void testAggregateFunctionsSkipNullAndInvalidRows() throws Exception { + UDFParameters parameters = createSingleDoubleSeriesParameters(Collections.emptyMap()); + + UDAFMedian median = new UDAFMedian(); + RecordingPointCollector medianCollector = new RecordingPointCollector(); + median.validate(new UDFParameterValidator(parameters)); + median.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + median.transform(nullDoubleRow(1), medianCollector); + median.transform(new DoubleRow(2, 2.0), medianCollector); + median.transform(new DoubleRow(3, 4.0), medianCollector); + median.terminate(medianCollector); + Assert.assertEquals(3.0, medianCollector.values.get(0), 0.0); + + UDAFMad mad = new UDAFMad(); + RecordingPointCollector madCollector = new RecordingPointCollector(); + mad.validate(new UDFParameterValidator(parameters)); + mad.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + mad.transform(nullDoubleRow(1), madCollector); + mad.transform(new DoubleRow(2, 1.0), madCollector); + mad.transform(new DoubleRow(3, 2.0), madCollector); + mad.transform(new DoubleRow(4, 4.0), madCollector); + mad.terminate(madCollector); + Assert.assertEquals(1.0, madCollector.values.get(0), 0.0); + + UDAFPercentile percentile = new UDAFPercentile(); + RecordingPointCollector percentileCollector = new RecordingPointCollector(); + percentile.validate(new UDFParameterValidator(parameters)); + percentile.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + percentile.transform(nullDoubleRow(1), percentileCollector); + percentile.transform(new DoubleRow(2, 2.0), percentileCollector); + percentile.transform(new DoubleRow(3, 4.0), percentileCollector); + percentile.terminate(percentileCollector); + Assert.assertEquals(Collections.singletonList(2L), percentileCollector.timestamps); + Assert.assertEquals(2.0, percentileCollector.values.get(0), 0.0); + + Map approximateAttributes = new HashMap<>(); + approximateAttributes.put("error", "0.1"); + UDFParameters approximateParameters = createSingleDoubleSeriesParameters(approximateAttributes); + + UDAFMedian approximateMedian = new UDAFMedian(); + RecordingPointCollector approximateMedianCollector = new RecordingPointCollector(); + approximateMedian.validate(new UDFParameterValidator(approximateParameters)); + approximateMedian.beforeStart( + approximateParameters, new UDTFConfigurations(ZoneId.systemDefault())); + approximateMedian.transform(nullDoubleRow(1), approximateMedianCollector); + approximateMedian.transform(new DoubleRow(2, Double.NaN), approximateMedianCollector); + approximateMedian.terminate(approximateMedianCollector); + Assert.assertTrue(approximateMedianCollector.timestamps.isEmpty()); + + UDAFMad approximateMad = new UDAFMad(); + RecordingPointCollector approximateMadCollector = new RecordingPointCollector(); + approximateMad.validate(new UDFParameterValidator(approximateParameters)); + approximateMad.beforeStart( + approximateParameters, new UDTFConfigurations(ZoneId.systemDefault())); + approximateMad.transform(nullDoubleRow(1), approximateMadCollector); + approximateMad.transform(new DoubleRow(2, Double.POSITIVE_INFINITY), approximateMadCollector); + approximateMad.terminate(approximateMadCollector); + Assert.assertTrue(approximateMadCollector.timestamps.isEmpty()); + + UDAFPercentile approximatePercentile = new UDAFPercentile(); + RecordingPointCollector approximatePercentileCollector = new RecordingPointCollector(); + approximatePercentile.validate(new UDFParameterValidator(approximateParameters)); + approximatePercentile.beforeStart( + approximateParameters, new UDTFConfigurations(ZoneId.systemDefault())); + approximatePercentile.transform(nullDoubleRow(1), approximatePercentileCollector); + approximatePercentile.transform( + new DoubleRow(2, Double.NEGATIVE_INFINITY), approximatePercentileCollector); + approximatePercentile.terminate(approximatePercentileCollector); + Assert.assertTrue(approximatePercentileCollector.timestamps.isEmpty()); + + UDAFQuantile quantile = new UDAFQuantile(); + RecordingPointCollector quantileCollector = new RecordingPointCollector(); + quantile.validate(new UDFParameterValidator(parameters)); + quantile.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + quantile.transform(nullDoubleRow(1), quantileCollector); + quantile.transform(new DoubleRow(2, Double.NaN), quantileCollector); + quantile.transform(new DoubleRow(3, 2.0), quantileCollector); + quantile.transform(new DoubleRow(4, 4.0), quantileCollector); + quantile.terminate(quantileCollector); + Assert.assertEquals(1, quantileCollector.values.size()); + Assert.assertEquals(2.0, quantileCollector.values.get(0), 0.0); + + UDAFSkew skew = new UDAFSkew(); + RecordingPointCollector skewCollector = new RecordingPointCollector(); + skew.validate(new UDFParameterValidator(parameters)); + skew.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + skew.transform(nullDoubleRow(1), skewCollector); + skew.transform(new DoubleRow(2, Double.NaN), skewCollector); + skew.transform(new DoubleRow(3, 1.0), skewCollector); + skew.transform(new DoubleRow(4, 2.0), skewCollector); + skew.transform(new DoubleRow(5, 3.0), skewCollector); + skew.terminate(skewCollector); + Assert.assertEquals(0.0, skewCollector.values.get(0), 0.0); + } + @Test public void testExactMadUsesAbsoluteDeviations() throws Exception { UDFParameters parameters = createSingleDoubleSeriesParameters(Collections.emptyMap()); @@ -444,6 +683,60 @@ public void testPeriodSkipsLeadingInvalidValue() throws Exception { Assert.assertEquals(0.0, collector.values.get(0), 0.0); } + @Test + public void testPeriodAndWindowDetectorsHandleNullRows() throws Exception { + UDFParameters parameters = createSingleDoubleSeriesParameters(Collections.emptyMap()); + + UDAFPeriod period = new UDAFPeriod(); + RecordingPointCollector periodCollector = new RecordingPointCollector(); + period.validate(new UDFParameterValidator(parameters)); + period.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + period.transform( + new SimpleRowWindow(nullDoubleRow(1), new DoubleRow(2, 1.0), nullDoubleRow(3)), + periodCollector); + Assert.assertEquals(Collections.singletonList(0L), periodCollector.timestamps); + Assert.assertEquals(0.0, periodCollector.values.get(0), 0.0); + + UDTFTwoSidedFilter twoSidedFilter = new UDTFTwoSidedFilter(); + RecordingPointCollector filterCollector = new RecordingPointCollector(); + twoSidedFilter.validate(new UDFParameterValidator(parameters)); + twoSidedFilter.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + twoSidedFilter.transform( + new SimpleRowWindow(new DoubleRow(1, 1.0), nullDoubleRow(2), new DoubleRow(3, 1.0)), + filterCollector); + Assert.assertFalse(filterCollector.timestamps.contains(2L)); + } + + @Test + public void testResampleAndMissDetectSkipNullRows() throws Exception { + Map resampleAttributes = new HashMap<>(); + resampleAttributes.put("every", "1ms"); + resampleAttributes.put("aggr", "first"); + resampleAttributes.put("interp", "nan"); + UDFParameters resampleParameters = createSingleDoubleSeriesParameters(resampleAttributes); + UDTFResample resample = new UDTFResample(); + RecordingPointCollector resampleCollector = new RecordingPointCollector(); + + resample.validate(new UDFParameterValidator(resampleParameters)); + resample.beforeStart(resampleParameters, new UDTFConfigurations(ZoneId.systemDefault())); + resample.transform(nullDoubleRow(0), resampleCollector); + resample.transform(new DoubleRow(1, 10.0), resampleCollector); + resample.transform(new DoubleRow(2, 20.0), resampleCollector); + resample.terminate(resampleCollector); + Assert.assertFalse(resampleCollector.timestamps.contains(0L)); + + UDFParameters missDetectParameters = createSingleDoubleSeriesParameters(Collections.emptyMap()); + UDTFMissDetect missDetect = new UDTFMissDetect(); + RecordingPointCollector missDetectCollector = new RecordingPointCollector(); + missDetect.validate(new UDFParameterValidator(missDetectParameters)); + missDetect.beforeStart(missDetectParameters, new UDTFConfigurations(ZoneId.systemDefault())); + missDetect.transform(nullDoubleRow(1), missDetectCollector); + missDetect.transform(new DoubleRow(2, 10.0), missDetectCollector); + missDetect.terminate(missDetectCollector); + Assert.assertEquals(Collections.singletonList(2L), missDetectCollector.timestamps); + Assert.assertEquals(0.0, missDetectCollector.values.get(0), 0.0); + } + @Test public void testQLBResetsAccumulatedStatisticBetweenRuns() throws Exception { UDFParameters parameters = createSingleDoubleSeriesParameters(Collections.emptyMap()); @@ -644,6 +937,107 @@ public void testFrequencyFunctionsIgnoreEmptyInput() throws Exception { Assert.assertTrue(collector.values.isEmpty()); } + @Test + public void testFrequencyFunctionsResetBuffersBetweenRuns() throws Exception { + UDFParameters singleSeries = createSingleDoubleSeriesParameters(Collections.emptyMap()); + + UDTFFFT fft = new UDTFFFT(); + fft.validate(new UDFParameterValidator(singleSeries)); + RecordingPointCollector fftCollector = new RecordingPointCollector(); + fft.beforeStart(singleSeries, new UDTFConfigurations(ZoneId.systemDefault())); + fft.transform(new DoubleRow(1, 1.0), fftCollector); + fft.transform(new DoubleRow(2, 2.0), fftCollector); + fft.terminate(fftCollector); + Assert.assertFalse(fftCollector.timestamps.isEmpty()); + + fftCollector.timestamps.clear(); + fftCollector.values.clear(); + fft.beforeStart(singleSeries, new UDTFConfigurations(ZoneId.systemDefault())); + fft.transform(nullDoubleRow(3), fftCollector); + fft.terminate(fftCollector); + Assert.assertTrue(fftCollector.timestamps.isEmpty()); + + Map wpassAttributes = new HashMap<>(); + wpassAttributes.put("wpass", "0.5"); + UDFParameters wpassParameters = createSingleDoubleSeriesParameters(wpassAttributes); + + UDTFLowPass lowPass = new UDTFLowPass(); + RecordingPointCollector lowPassCollector = new RecordingPointCollector(); + lowPass.validate(new UDFParameterValidator(wpassParameters)); + lowPass.beforeStart(wpassParameters, new UDTFConfigurations(ZoneId.systemDefault())); + lowPass.transform(new DoubleRow(1, 1.0), lowPassCollector); + lowPass.transform(new DoubleRow(2, 2.0), lowPassCollector); + lowPass.terminate(lowPassCollector); + Assert.assertFalse(lowPassCollector.timestamps.isEmpty()); + + lowPassCollector.timestamps.clear(); + lowPassCollector.values.clear(); + lowPass.beforeStart(wpassParameters, new UDTFConfigurations(ZoneId.systemDefault())); + lowPass.transform(nullDoubleRow(3), lowPassCollector); + lowPass.terminate(lowPassCollector); + Assert.assertTrue(lowPassCollector.timestamps.isEmpty()); + + UDTFHighPass highPass = new UDTFHighPass(); + RecordingPointCollector highPassCollector = new RecordingPointCollector(); + highPass.validate(new UDFParameterValidator(wpassParameters)); + highPass.beforeStart(wpassParameters, new UDTFConfigurations(ZoneId.systemDefault())); + highPass.transform(new DoubleRow(1, 1.0), highPassCollector); + highPass.transform(new DoubleRow(2, 2.0), highPassCollector); + highPass.terminate(highPassCollector); + Assert.assertFalse(highPassCollector.timestamps.isEmpty()); + + highPassCollector.timestamps.clear(); + highPassCollector.values.clear(); + highPass.beforeStart(wpassParameters, new UDTFConfigurations(ZoneId.systemDefault())); + highPass.transform(nullDoubleRow(3), highPassCollector); + highPass.terminate(highPassCollector); + Assert.assertTrue(highPassCollector.timestamps.isEmpty()); + + UDFEnvelopeAnalysis envelopeAnalysis = new UDFEnvelopeAnalysis(); + RecordingPointCollector envelopeCollector = new RecordingPointCollector(); + envelopeAnalysis.validate(new UDFParameterValidator(singleSeries)); + envelopeAnalysis.beforeStart(singleSeries, new UDTFConfigurations(ZoneId.systemDefault())); + envelopeAnalysis.transform(new DoubleRow(1, 1.0), envelopeCollector); + envelopeAnalysis.transform(new DoubleRow(2, 2.0), envelopeCollector); + envelopeAnalysis.terminate(envelopeCollector); + Assert.assertFalse(envelopeCollector.timestamps.isEmpty()); + + envelopeCollector.timestamps.clear(); + envelopeCollector.values.clear(); + envelopeAnalysis.beforeStart(singleSeries, new UDTFConfigurations(ZoneId.systemDefault())); + envelopeAnalysis.transform(nullDoubleRow(3), envelopeCollector); + envelopeAnalysis.terminate(envelopeCollector); + Assert.assertTrue(envelopeCollector.timestamps.isEmpty()); + } + + @Test + public void testIFFTClearsBuffersBetweenRuns() throws Exception { + UDFParameters parameters = createTwoDoubleSeriesParameters(Collections.emptyMap()); + UDTFIFFT ifft = new UDTFIFFT(); + RecordingPointCollector collector = new RecordingPointCollector(); + + ifft.validate(new UDFParameterValidator(parameters)); + ifft.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + ifft.transform( + new DoubleRow(0, new double[] {1.0, 0.0}, new boolean[] {false, false}), collector); + ifft.transform( + new DoubleRow(1, new double[] {0.0, 0.0}, new boolean[] {false, false}), collector); + ifft.terminate(collector); + Assert.assertFalse(collector.timestamps.isEmpty()); + + collector.timestamps.clear(); + collector.values.clear(); + ifft.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + ifft.transform( + new DoubleRow(0, new double[] {0.0, 0.0}, new boolean[] {true, false}), collector); + ifft.transform( + new DoubleRow(1, new double[] {0.0, Double.NaN}, new boolean[] {false, false}), collector); + ifft.terminate(collector); + + Assert.assertTrue(collector.timestamps.isEmpty()); + Assert.assertTrue(collector.values.isEmpty()); + } + @Test public void testMultiLayerDWTRoundTrip() throws Exception { Map attributes = new HashMap<>(); @@ -764,6 +1158,149 @@ public void testSplineSkipsTooFewPointsAndHandlesNegativeTimestamps() throws Exc Assert.assertEquals(Arrays.asList(-5L, -3L, -1L), collector.timestamps); } + @Test + public void testOutlierValidatesRuntimeSensitiveParameters() { + Map attributes = new HashMap<>(); + attributes.put("s", "0"); + + Assert.assertThrows( + UDFParameterNotValidException.class, + () -> + new UDTFOutlier() + .validate( + new UDFParameterValidator(createSingleDoubleSeriesParameters(attributes)))); + + Assert.assertThrows( + UDFInputSeriesDataTypeNotValidException.class, + () -> + new UDTFOutlier() + .validate( + new UDFParameterValidator( + createSingleTextSeriesParameters(Collections.emptyMap())))); + } + + @Test + public void testOutlierResetsBuffersBetweenRunsAndSkipsInvalidValues() throws Exception { + Map attributes = new HashMap<>(); + attributes.put("w", "3"); + attributes.put("s", "1"); + attributes.put("k", "3"); + attributes.put("r", "0"); + UDFParameters parameters = createSingleDoubleSeriesParameters(attributes); + UDTFOutlier outlier = new UDTFOutlier(); + RecordingPointCollector collector = new RecordingPointCollector(); + + outlier.validate(new UDFParameterValidator(parameters)); + outlier.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + outlier.transform(new DoubleRow(1, 1.0), collector); + outlier.transform(new DoubleRow(2, 100.0), collector); + outlier.transform(new DoubleRow(3, 1.0), collector); + outlier.transform(new DoubleRow(4, 2.0), collector); + outlier.terminate(collector); + + Assert.assertFalse(collector.timestamps.isEmpty()); + + collector.timestamps.clear(); + collector.values.clear(); + outlier.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + outlier.transform(new DoubleRow(5, new double[] {0.0}, new boolean[] {true}), collector); + outlier.transform(new DoubleRow(6, Double.NaN), collector); + outlier.terminate(collector); + + Assert.assertTrue(collector.timestamps.isEmpty()); + Assert.assertTrue(collector.values.isEmpty()); + } + + @Test + public void testIntegralResetsBetweenRunsAndIgnoresNullRows() throws Exception { + Map attributes = new HashMap<>(); + attributes.put("unit", "1ms"); + UDFParameters parameters = createSingleDoubleSeriesParameters(attributes); + UDAFIntegral integral = new UDAFIntegral(); + RecordingPointCollector collector = new RecordingPointCollector(); + + integral.validate(new UDFParameterValidator(parameters)); + integral.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + integral.transform(new DoubleRow(0, 1.0), collector); + integral.transform(new DoubleRow(1, new double[] {0.0}, new boolean[] {true}), collector); + integral.transform(new DoubleRow(2, 3.0), collector); + integral.terminate(collector); + + Assert.assertEquals(Collections.singletonList(0L), collector.timestamps); + Assert.assertEquals(4.0, collector.values.get(0), 0.0); + + collector.timestamps.clear(); + collector.values.clear(); + integral.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + integral.terminate(collector); + + Assert.assertEquals(Collections.singletonList(0L), collector.timestamps); + Assert.assertEquals(0.0, collector.values.get(0), 0.0); + + collector.timestamps.clear(); + collector.values.clear(); + integral.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + integral.transform(new DoubleRow(-2, 1.0), collector); + integral.transform(new DoubleRow(-1, 3.0), collector); + integral.terminate(collector); + + Assert.assertEquals(Collections.singletonList(0L), collector.timestamps); + Assert.assertEquals(2.0, collector.values.get(0), 0.0); + } + + @Test + public void testIntegralAvgUsesFirstFinitePointAndResetsBetweenRuns() throws Exception { + UDFParameters parameters = createSingleDoubleSeriesParameters(Collections.emptyMap()); + UDAFIntegralAvg integralAvg = new UDAFIntegralAvg(); + RecordingPointCollector collector = new RecordingPointCollector(); + + integralAvg.validate(new UDFParameterValidator(parameters)); + integralAvg.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + integralAvg.transform(new DoubleRow(1, new double[] {0.0}, new boolean[] {true}), collector); + integralAvg.transform(new DoubleRow(2, 5.0), collector); + integralAvg.terminate(collector); + + Assert.assertEquals(Collections.singletonList(0L), collector.timestamps); + Assert.assertEquals(5.0, collector.values.get(0), 0.0); + + collector.timestamps.clear(); + collector.values.clear(); + integralAvg.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + integralAvg.terminate(collector); + + Assert.assertEquals(Collections.singletonList(0L), collector.timestamps); + Assert.assertEquals(0.0, collector.values.get(0), 0.0); + + collector.timestamps.clear(); + collector.values.clear(); + integralAvg.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + integralAvg.transform(new DoubleRow(-2, 1.0), collector); + integralAvg.transform(new DoubleRow(-1, 3.0), collector); + integralAvg.terminate(collector); + + Assert.assertEquals(Collections.singletonList(0L), collector.timestamps); + Assert.assertEquals(2.0, collector.values.get(0), 0.0); + } + + @Test + public void testNegativeTimestampStateInitialization() throws Exception { + StreamMissDetector detector = new StreamMissDetector(10); + detector.insert(-5, 1.0); + detector.insert(-4, 2.0); + + Field startTime = StreamMissDetector.class.getDeclaredField("startTime"); + startTime.setAccessible(true); + Assert.assertEquals(-5L, startTime.getLong(detector)); + + Resampler resampler = new Resampler(2, "first", "nan"); + resampler.insert(-5, 1.0); + resampler.insert(-4, 2.0); + + Field currentTime = Resampler.class.getDeclaredField("currentTime"); + currentTime.setAccessible(true); + Assert.assertEquals(-5L, currentTime.getLong(resampler)); + } + @Test public void testSegmentInstancesKeepIndependentBuffers() throws Exception { UDFParameters parameters = createSingleDoubleSeriesParameters(Collections.emptyMap()); @@ -886,6 +1423,10 @@ private static UDFParameters createTwoDoubleSeriesParameters(Map Arrays.asList("s1", "s2"), Arrays.asList(Type.DOUBLE, Type.DOUBLE), attributes); } + private static DoubleRow nullDoubleRow(long time) { + return new DoubleRow(time, new double[] {0.0}, new boolean[] {true}); + } + private static int getWindowSize(UDTFKSigma kSigma) throws Exception { Field windowSize = UDTFKSigma.class.getDeclaredField("windowSize"); windowSize.setAccessible(true); @@ -1072,7 +1613,8 @@ public void putDouble(long timestamp, double value) { @Override public void putBoolean(long timestamp, boolean value) { - throw new UnsupportedOperationException(); + timestamps.add(timestamp); + values.add(value ? 1.0 : 0.0); } @Override From 00f0976fa5be06501cb5aef3d54e1aa29b0f4dff Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 9 Jun 2026 15:05:26 +0800 Subject: [PATCH 3/6] Fix library UDF edge cases --- .../apache/iotdb/library/anomaly/UDTFIQR.java | 7 +- .../iotdb/library/anomaly/UDTFKSigma.java | 4 +- .../apache/iotdb/library/anomaly/UDTFLOF.java | 64 ++- .../iotdb/library/anomaly/UDTFOutlier.java | 4 +- .../iotdb/library/anomaly/UDTFRange.java | 12 +- .../library/anomaly/UDTFTwoSidedFilter.java | 10 +- .../apache/iotdb/library/dlearn/UDTFAR.java | 7 +- .../iotdb/library/dlearn/UDTFCluster.java | 7 +- .../apache/iotdb/library/dmatch/UDAFDtw.java | 8 +- .../iotdb/library/dmatch/UDTFPtnSym.java | 13 +- .../iotdb/library/dprofile/UDAFSpread.java | 20 + .../iotdb/library/dprofile/UDTFDistinct.java | 3 + .../iotdb/library/dprofile/UDTFHistogram.java | 9 +- .../iotdb/library/dprofile/UDTFMinMax.java | 7 +- .../iotdb/library/dprofile/UDTFSample.java | 92 +++- .../iotdb/library/dprofile/UDTFSegment.java | 4 +- .../iotdb/library/dprofile/UDTFZScore.java | 8 +- .../dquality/util/TimeSeriesQuality.java | 8 +- .../iotdb/library/drepair/UDTFValueFill.java | 19 +- .../library/drepair/UDTFValueRepair.java | 74 ++- .../iotdb/library/drepair/util/ARFill.java | 11 +- .../iotdb/library/drepair/util/MeanFill.java | 4 +- .../library/drepair/util/ScreenFill.java | 7 +- .../drepair/util/TimestampInterval.java | 2 +- .../library/drepair/util/TimestampRepair.java | 13 +- .../iotdb/library/drepair/util/ValueFill.java | 15 +- .../library/drepair/util/ValueRepair.java | 6 +- .../frequency/UDFEnvelopeAnalysis.java | 4 +- .../iotdb/library/frequency/UDTFDWT.java | 10 +- .../iotdb/library/frequency/UDTFDeconv.java | 17 +- .../iotdb/library/frequency/UDTFIDWT.java | 10 +- .../iotdb/library/frequency/UDTFIFFT.java | 3 + .../iotdb/library/frequency/util/DWTUtil.java | 26 + .../iotdb/library/match/UDAFDTWMatch.java | 39 +- .../iotdb/library/match/UDAFPatternMatch.java | 39 +- .../iotdb/library/string/UDTFRegexMatch.java | 3 + .../library/string/UDTFRegexReplace.java | 3 + .../iotdb/library/string/UDTFRegexSplit.java | 3 + .../iotdb/library/string/UDTFStrReplace.java | 3 + .../apache/iotdb/library/UDAFPatternTest.java | 348 ++++++++++++ .../iotdb/library/UDFWindowAndQueueTest.java | 520 ++++++++++++++++++ 41 files changed, 1333 insertions(+), 133 deletions(-) diff --git a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFIQR.java b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFIQR.java index 3278fed26a05c..8aa63240ff328 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFIQR.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFIQR.java @@ -59,8 +59,11 @@ public void validate(UDFParameterValidator validator) throws Exception { "Parameter \"compute\" is illegal. Please use \"batch\" (for default) or \"stream\".", validator.getParameters().getStringOrDefault("compute", BATCH_COMPUTE)) .validate( - params -> (double) params[0] < (double) params[1], - "parameter $q1$ should be smaller than $q3$", + params -> + Double.isFinite((double) params[0]) + && Double.isFinite((double) params[1]) + && (double) params[0] < (double) params[1], + "parameter $q1$ and $q3$ should be finite, and $q1$ should be smaller than $q3$", validator.getParameters().getDoubleOrDefault("q1", -1), validator.getParameters().getDoubleOrDefault("q3", 1)); if (validator diff --git a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFKSigma.java b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFKSigma.java index 27e21406ed8b3..6b79b33e3406f 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFKSigma.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFKSigma.java @@ -55,8 +55,8 @@ public void validate(UDFParameterValidator validator) throws Exception { "Window size should be larger than 0.", validator.getParameters().getIntOrDefault("window", DEFAULT_WINDOW_SIZE)) .validate( - x -> (double) x > 0, - "Parameter k should be larger than 0.", + x -> Double.isFinite((double) x) && (double) x > 0, + "Parameter k should be finite and larger than 0.", validator.getParameters().getDoubleOrDefault("k", 3)); } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFLOF.java b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFLOF.java index 58078771b62a3..32ba5e4335662 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFLOF.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFLOF.java @@ -31,6 +31,9 @@ import org.apache.iotdb.udf.api.exception.UDFException; import org.apache.iotdb.udf.api.type.Type; +import java.util.ArrayList; +import java.util.List; + /** This function is used to detect density anomaly of time series. */ public class UDTFLOF implements UDTF { private int multipleK; @@ -141,23 +144,30 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati @Override public void transform(RowWindow rowWindow, PointCollector collector) throws Exception { if (this.method.equals(DEFAULT_METHOD)) { - int size = rowWindow.windowSize(); - Double[][] knn = new Double[size][dim]; - long[] timestamp = new long[size]; - int i = 0; + int size = 0; + Double[][] knn = new Double[rowWindow.windowSize()][dim]; + long[] timestamp = new long[rowWindow.windowSize()]; int row = 0; while (row < rowWindow.windowSize()) { - timestamp[i] = rowWindow.getRow(row).getTime(); + Double[] values = new Double[dim]; + boolean valid = true; for (int j = 0; j < dim; j++) { - if (!rowWindow.getRow(row).isNull(j)) { - knn[i][j] = Util.getValueAsDouble(rowWindow.getRow(row), j); - } else { - i--; - size--; + if (rowWindow.getRow(row).isNull(j)) { + valid = false; + break; + } + double value = Util.getValueAsDouble(rowWindow.getRow(row), j); + if (!Double.isFinite(value)) { + valid = false; break; } + values[j] = value; + } + if (valid) { + timestamp[size] = rowWindow.getRow(row).getTime(); + knn[size] = values; + size++; } - i++; row++; } if (size > multipleK) { @@ -174,36 +184,32 @@ public void transform(RowWindow rowWindow, PointCollector collector) throws Exce } else if (this.method.equals("series")) { int size = rowWindow.windowSize() - window + 1; if (size > 0) { - Double[][] knn = new Double[size][window]; - long[] timestamp = new long[rowWindow.windowSize()]; - double temp; - int i = 0; + List timestamp = new ArrayList<>(); + List values = new ArrayList<>(); int row = 0; while (row < rowWindow.windowSize()) { - timestamp[i] = rowWindow.getRow(row).getTime(); if (!rowWindow.getRow(row).isNull(0)) { - temp = Util.getValueAsDouble(rowWindow.getRow(row), 0); - for (int p = 0; p < window; p++) { - if (i - p < 0) { - break; - } - if (i - p < size) { - knn[i - p][p] = temp; - } + double value = Util.getValueAsDouble(rowWindow.getRow(row), 0); + if (Double.isFinite(value)) { + timestamp.add(rowWindow.getRow(row).getTime()); + values.add(value); } - } else { - i--; - size--; } - i++; row++; } + size = values.size() - window + 1; if (size > multipleK) { + Double[][] knn = new Double[size][window]; + for (int i = 0; i < size; i++) { + for (int p = 0; p < window; p++) { + knn[i][p] = values.get(i + p); + } + } double[] lof = new double[size]; for (int m = 0; m < size; m++) { try { lof[m] = getLOF(knn, knn[m], size); - collector.putDouble(timestamp[m], lof[m]); + collector.putDouble(timestamp.get(m), lof[m]); } catch (Exception e) { throw new UDFException(LibraryUdfMessages.FAIL_TO_GET_LOF + m, e); } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFOutlier.java b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFOutlier.java index 308e5556cb7da..2d6cacba968d3 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFOutlier.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFOutlier.java @@ -58,8 +58,8 @@ public void validate(UDFParameterValidator validator) throws Exception { "Parameter k should be a positive integer.", validator.getParameters().getIntOrDefault("k", 3)) .validate( - x -> (double) x >= 0, - "Parameter r should be non-negative.", + x -> Double.isFinite((double) x) && (double) x >= 0, + "Parameter r should be finite and non-negative.", validator.getParameters().getDoubleOrDefault("r", 5)) .validate( x -> (int) x > 0, diff --git a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFRange.java b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFRange.java index 447ce16426779..4f24e03c35942 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFRange.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFRange.java @@ -44,8 +44,11 @@ public void validate(UDFParameterValidator validator) throws Exception { .validateRequiredAttribute("lower_bound") .validateRequiredAttribute("upper_bound") .validate( - params -> (double) params[0] < (double) params[1], - "parameter \"lower_bound\" should be smaller than \"upper_bound\".", + params -> + Double.isFinite((double) params[0]) + && Double.isFinite((double) params[1]) + && (double) params[0] < (double) params[1], + "parameter \"lower_bound\" and \"upper_bound\" should be finite, and \"lower_bound\" should be smaller than \"upper_bound\".", validator.getParameters().getDouble("lower_bound"), validator.getParameters().getDouble("upper_bound")); } @@ -87,13 +90,14 @@ public void transform(Row row, PointCollector collector) throws Exception { break; case FLOAT: floatValue = row.getFloat(0); - if (floatValue > upperBound || floatValue < lowerBound) { + if (Float.isFinite(floatValue) && (floatValue > upperBound || floatValue < lowerBound)) { collector.putFloat(timestamp, floatValue); } break; case DOUBLE: doubleValue = row.getDouble(0); - if (doubleValue > upperBound || doubleValue < lowerBound) { + if (Double.isFinite(doubleValue) + && (doubleValue > upperBound || doubleValue < lowerBound)) { collector.putDouble(timestamp, doubleValue); } break; diff --git a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFTwoSidedFilter.java b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFTwoSidedFilter.java index a772861f2d8a8..4df9ff55b8209 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFTwoSidedFilter.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFTwoSidedFilter.java @@ -43,7 +43,15 @@ public class UDTFTwoSidedFilter implements UDTF { public void validate(UDFParameterValidator validator) throws Exception { validator .validateInputSeriesNumber(1) - .validateInputSeriesDataType(0, Type.INT32, Type.INT64, Type.FLOAT, Type.DOUBLE); + .validateInputSeriesDataType(0, Type.INT32, Type.INT64, Type.FLOAT, Type.DOUBLE) + .validate( + x -> Double.isFinite((double) x) && (double) x > 0, + "Parameter len should be finite and larger than 0.", + validator.getParameters().getDoubleOrDefault("len", 5)) + .validate( + x -> Double.isFinite((double) x) && (double) x >= 0, + "Parameter threshold should be finite and non-negative.", + validator.getParameters().getDoubleOrDefault("threshold", 0.4)); } @Override diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dlearn/UDTFAR.java b/library-udf/src/main/java/org/apache/iotdb/library/dlearn/UDTFAR.java index cc4d496187660..ed02daa82c1e9 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dlearn/UDTFAR.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dlearn/UDTFAR.java @@ -66,8 +66,11 @@ public void beforeStart(UDFParameters udfParameters, UDTFConfigurations udtfConf @Override public void transform(Row row, PointCollector collector) throws Exception { if (!row.isNull(0)) { - timeWindow.add(row.getTime()); - valueWindow.add(Util.getValueAsDouble(row)); + double value = Util.getValueAsDouble(row); + if (Double.isFinite(value)) { + timeWindow.add(row.getTime()); + valueWindow.add(value); + } } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dlearn/UDTFCluster.java b/library-udf/src/main/java/org/apache/iotdb/library/dlearn/UDTFCluster.java index e349b188ad33d..55aa0422bac04 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dlearn/UDTFCluster.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dlearn/UDTFCluster.java @@ -133,8 +133,11 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati @Override public void transform(Row row, PointCollector collector) throws Exception { if (!row.isNull(0)) { - timestamps.add(row.getTime()); - values.add(Util.getValueAsDouble(row)); + double value = Util.getValueAsDouble(row); + if (Double.isFinite(value)) { + timestamps.add(row.getTime()); + values.add(value); + } } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dmatch/UDAFDtw.java b/library-udf/src/main/java/org/apache/iotdb/library/dmatch/UDAFDtw.java index 66efda369a3d6..e8dc37fd1ccd5 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dmatch/UDAFDtw.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dmatch/UDAFDtw.java @@ -66,8 +66,12 @@ public void transform(RowWindow rowWindow, PointCollector collector) throws Exce if (row.isNull(0) || row.isNull(1)) { continue; } - a.add(Util.getValueAsDouble(row, 0)); - b.add(Util.getValueAsDouble(row, 1)); + double left = Util.getValueAsDouble(row, 0); + double right = Util.getValueAsDouble(row, 1); + if (Double.isFinite(left) && Double.isFinite(right)) { + a.add(left); + b.add(right); + } } m = a.size(); if (m == 0) { diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dmatch/UDTFPtnSym.java b/library-udf/src/main/java/org/apache/iotdb/library/dmatch/UDTFPtnSym.java index e3642dd1ead05..deeff2a200988 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dmatch/UDTFPtnSym.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dmatch/UDTFPtnSym.java @@ -48,8 +48,8 @@ public void validate(UDFParameterValidator validator) throws Exception { "window has to be a positive integer.", validator.getParameters().getIntOrDefault("window", 10)) .validate( - x -> (double) x >= 0.0d, - "threshold has to be non-negative.", + x -> Double.isFinite((double) x) && (double) x >= 0.0d, + "threshold has to be finite and non-negative.", validator.getParameters().getDoubleOrDefault("threshold", Double.MAX_VALUE)); } @@ -74,7 +74,14 @@ public void transform(RowWindow rowWindow, PointCollector collector) throws Exce long time = rowWindow.getRow(0).getTime(); for (int i = 0; i < n; i++) { Row row = rowWindow.getRow(i); - a.add(Util.getValueAsDouble(row, 0)); + if (row.isNull(0)) { + return; + } + double value = Util.getValueAsDouble(row, 0); + if (!Double.isFinite(value)) { + return; + } + a.add(value); } int m = a.size(); double[][] dp = new double[m + 1][m + 1]; diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFSpread.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFSpread.java index bc7d9ad7c8466..493995acf36f6 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFSpread.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFSpread.java @@ -46,6 +46,7 @@ public class UDAFSpread implements UDTF { double doubleMin = Double.MAX_VALUE; double doubleMax = -Double.MAX_VALUE; Type dataType; + boolean hasValue; @Override public void validate(UDFParameterValidator validator) throws Exception { @@ -59,10 +60,22 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati throws Exception { dataType = parameters.getDataType(0); configurations.setAccessStrategy(new RowByRowAccessStrategy()).setOutputDataType(dataType); + intMin = Integer.MAX_VALUE; + intMax = Integer.MIN_VALUE; + longMin = Long.MAX_VALUE; + longMax = Long.MIN_VALUE; + floatMin = Float.MAX_VALUE; + floatMax = -Float.MAX_VALUE; + doubleMin = Double.MAX_VALUE; + doubleMax = -Double.MAX_VALUE; + hasValue = false; } @Override public void transform(Row row, PointCollector pc) throws Exception { + if (row.isNull(0)) { + return; + } switch (dataType) { case INT32: transformInt(row); @@ -89,6 +102,9 @@ public void transform(Row row, PointCollector pc) throws Exception { @Override public void terminate(PointCollector pc) throws Exception { + if (!hasValue) { + return; + } switch (dataType) { case INT32: pc.putInt(0, intMax - intMin); @@ -117,12 +133,14 @@ private void transformInt(Row row) throws IOException { int v = row.getInt(0); intMin = Math.min(intMin, v); intMax = Math.max(intMax, v); + hasValue = true; } private void transformLong(Row row) throws IOException { long v = row.getLong(0); longMin = Math.min(longMin, v); longMax = Math.max(longMax, v); + hasValue = true; } private void transformFloat(Row row) throws IOException { @@ -130,6 +148,7 @@ private void transformFloat(Row row) throws IOException { if (Float.isFinite(v)) { floatMin = Math.min(floatMin, v); floatMax = Math.max(floatMax, v); + hasValue = true; } } @@ -138,6 +157,7 @@ private void transformDouble(Row row) throws IOException { if (Double.isFinite(v)) { doubleMin = Math.min(doubleMin, v); doubleMax = Math.max(doubleMax, v); + hasValue = true; } } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFDistinct.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFDistinct.java index 1698164a944fe..ee95a9e534f1d 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFDistinct.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFDistinct.java @@ -97,6 +97,9 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati @Override public void transform(Row row, PointCollector pc) throws Exception { + if (row.isNull(0)) { + return; + } switch (dataType) { case INT32: intSet.add(row.getInt(0)); diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFHistogram.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFHistogram.java index 7a3700d551a87..3d01b39eda125 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFHistogram.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFHistogram.java @@ -50,8 +50,11 @@ public void validate(UDFParameterValidator validator) throws Exception { "parameter $count$ should be larger than 0", validator.getParameters().getIntOrDefault("count", 1)) .validate( - params -> (double) params[0] < (double) params[1], - "parameter $end$ should be larger than $start$", + params -> + Double.isFinite((double) params[0]) + && Double.isFinite((double) params[1]) + && (double) params[0] < (double) params[1], + "parameter $end$ should be finite and larger than $start$", validator.getParameters().getDoubleOrDefault("min", -Double.MAX_VALUE), validator.getParameters().getDoubleOrDefault("max", Double.MAX_VALUE)); } @@ -65,7 +68,7 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati double end = parameters.getDoubleOrDefault("max", Double.MAX_VALUE); count = parameters.getIntOrDefault("count", 1); bucket = new int[count]; - gap = (end - start) / count; + gap = end / count - start / count; } @Override diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFMinMax.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFMinMax.java index a4411a84244f4..572fbc153d575 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFMinMax.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFMinMax.java @@ -54,8 +54,11 @@ public void validate(UDFParameterValidator validator) throws Exception { "Parameter \"compute\" is illegal. Please use \"batch\" (for default) or \"stream\".", validator.getParameters().getStringOrDefault("compute", BATCH_COMPUTE)) .validate( - params -> (double) params[0] < (double) params[1], - "parameter $min$ should be smaller than $max$.", + params -> + Double.isFinite((double) params[0]) + && Double.isFinite((double) params[1]) + && (double) params[0] < (double) params[1], + "parameter $min$ and $max$ should be finite, and $min$ should be smaller than $max$.", validator.getParameters().getDoubleOrDefault("min", -Double.MAX_VALUE), validator.getParameters().getDoubleOrDefault("max", Double.MAX_VALUE)); if (validator diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFSample.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFSample.java index f1a8f38d1922d..52ef11c7c055a 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFSample.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFSample.java @@ -36,6 +36,7 @@ import org.apache.commons.lang3.tuple.Pair; +import java.util.ArrayList; import java.util.Arrays; import java.util.LinkedList; import java.util.List; @@ -108,6 +109,9 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati @Override public void transform(Row row, PointCollector collector) throws Exception { + if (row.isNull(0)) { + return; + } // pool sampling int x; if (this.num < this.k) { @@ -127,17 +131,24 @@ public void transform(Row row, PointCollector collector) throws Exception { @SuppressWarnings("javabugs:S6320") public void transform(RowWindow rowWindow, PointCollector collector) throws Exception { // equal-distance sampling - int n = rowWindow.windowSize(); - - if (this.k < n) { - if (this.method == Method.TRIANGLE) { - List> input = new LinkedList<>(); - for (int i = 0; i < n; i++) { - Row row = rowWindow.getRow(i); - long time = row.getTime(); - double data = Util.getValueAsDouble(row); + if (this.method == Method.TRIANGLE) { + List> input = new LinkedList<>(); + for (int i = 0; i < rowWindow.windowSize(); i++) { + Row row = rowWindow.getRow(i); + if (row.isNull(0)) { + continue; + } + long time = row.getTime(); + double data = Util.getValueAsDouble(row); + if (Double.isFinite(data)) { input.add(Pair.of(time, data)); } + } + int n = input.size(); + if (n == 0) { + return; + } + if (this.k < n) { if (k > 2) { // The first and last element will always be sampled so the buckets is k - 2 List> output = LTThreeBuckets.sorted(input, k - 2); @@ -166,25 +177,41 @@ public void transform(RowWindow rowWindow, PointCollector collector) throws Exce } } } else { // For corner case of k == 1 and k == 2 - Row row = rowWindow.getRow(0); // Put first element - Util.putValue(collector, dataType, row.getTime(), Util.getValueAsObject(row)); + Pair row = input.get(0); // Put first element + putNumericValue(collector, row.getLeft(), row.getRight()); if (k == 2) { - row = rowWindow.getRow(n - 1); // Put last element - Util.putValue(collector, dataType, row.getTime(), Util.getValueAsObject(row)); + row = input.get(n - 1); // Put last element + putNumericValue(collector, row.getLeft(), row.getRight()); } } - } else { // Method.ISOMETRIC - for (long i = 0; i < this.k; i++) { - long j = Math.floorDiv(i * n, (long) k); // avoid intermediate result overflows - Row row = rowWindow.getRow((int) j); - Util.putValue(collector, dataType, row.getTime(), Util.getValueAsObject(row)); + } else { // when k is larger than series length, output all points + for (Pair row : input) { + putNumericValue(collector, row.getLeft(), row.getRight()); } } - } else { // when k is larger than series length, output all points + } else { + List validRows = new ArrayList<>(); RowIterator iterator = rowWindow.getRowIterator(); while (iterator.hasNextRow()) { Row row = iterator.next(); - Util.putValue(collector, dataType, row.getTime(), Util.getValueAsObject(row)); + if (!row.isNull(0)) { + validRows.add(row); + } + } + int n = validRows.size(); + if (n == 0) { + return; + } + if (this.k < n) { + for (long i = 0; i < this.k; i++) { + long j = Math.floorDiv(i * n, (long) k); // avoid intermediate result overflows + Row row = validRows.get((int) j); + Util.putValue(collector, dataType, row.getTime(), Util.getValueAsObject(row)); + } + } else { // when k is larger than series length, output all points + for (Row row : validRows) { + Util.putValue(collector, dataType, row.getTime(), Util.getValueAsObject(row)); + } } } } @@ -200,4 +227,29 @@ public void terminate(PointCollector pc) throws Exception { } } } + + private void putNumericValue(PointCollector collector, long time, double value) throws Exception { + switch (dataType) { + case INT32: + collector.putInt(time, (int) value); + break; + case INT64: + collector.putLong(time, (long) value); + break; + case FLOAT: + collector.putFloat(time, (float) value); + break; + case DOUBLE: + collector.putDouble(time, value); + break; + case TIMESTAMP: + case DATE: + case BLOB: + case BOOLEAN: + case STRING: + case TEXT: + default: + throw new NoNumberException(); + } + } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFSegment.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFSegment.java index 81308319122a3..55b469c30fe69 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFSegment.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFSegment.java @@ -54,8 +54,8 @@ public void validate(UDFParameterValidator validator) throws Exception { "Window size should be a positive integer.", validator.getParameters().getIntOrDefault("window", 10)) .validate( - x -> (double) x >= 0, - "Error bound should be no less than 0.", + x -> Double.isFinite((double) x) && (double) x >= 0, + "Error bound should be finite and no less than 0.", validator.getParameters().getDoubleOrDefault("error", 0.1)) .validate( x -> diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFZScore.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFZScore.java index 4c53aa066752c..b108efbd8ee56 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFZScore.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFZScore.java @@ -58,8 +58,12 @@ public void validate(UDFParameterValidator validator) throws Exception { "Parameter \"compute\" is illegal. Please use \"batch\" (for default) or \"stream\".", validator.getParameters().getStringOrDefault("compute", BATCH_COMPUTE)) .validate( - x -> ((Double) x) > 0, - "Parameter \"sd\" is illegal. It should be larger than 0.", + x -> Double.isFinite((double) x), + "Parameter \"avg\" is illegal. It should be finite.", + validator.getParameters().getDoubleOrDefault("avg", 0.0)) + .validate( + x -> Double.isFinite((double) x) && (double) x > 0, + "Parameter \"sd\" is illegal. It should be finite and larger than 0.", validator.getParameters().getDoubleOrDefault("sd", 1.0)); if (validator .getParameters() diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dquality/util/TimeSeriesQuality.java b/library-udf/src/main/java/org/apache/iotdb/library/dquality/util/TimeSeriesQuality.java index 78fd91acd6095..46ba4cefe43d6 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dquality/util/TimeSeriesQuality.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dquality/util/TimeSeriesQuality.java @@ -51,8 +51,14 @@ public TimeSeriesQuality(RowIterator dataIterator) throws Exception { while (dataIterator.hasNextRow()) { Row row = dataIterator.next(); cnt++; - double v = Util.getValueAsDouble(row); double t = row.getTime(); + if (row.isNull(0)) { + specialCnt++; + timeList.add(t); + originList.add(Double.NaN); + continue; + } + double v = Util.getValueAsDouble(row); if (Double.isFinite(v)) { timeList.add(t); originList.add(v); diff --git a/library-udf/src/main/java/org/apache/iotdb/library/drepair/UDTFValueFill.java b/library-udf/src/main/java/org/apache/iotdb/library/drepair/UDTFValueFill.java index 0b65001536895..dd3db8a869691 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/drepair/UDTFValueFill.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/drepair/UDTFValueFill.java @@ -75,28 +75,39 @@ public void transform(RowWindow rowWindow, PointCollector collector) throws Exce } else { throw new UDFException(LibraryUdfMessages.ILLEGAL_METHOD); } + if (!vf.hasValidValue()) { + return; + } vf.fill(); double[] repaired = vf.getFilled(); long[] time = vf.getTime(); switch (rowWindow.getDataType(0)) { case DOUBLE: for (int i = 0; i < time.length; i++) { - collector.putDouble(time[i], repaired[i]); + if (Double.isFinite(repaired[i])) { + collector.putDouble(time[i], repaired[i]); + } } break; case FLOAT: for (int i = 0; i < time.length; i++) { - collector.putFloat(time[i], (float) repaired[i]); + if (Double.isFinite(repaired[i])) { + collector.putFloat(time[i], (float) repaired[i]); + } } break; case INT32: for (int i = 0; i < time.length; i++) { - collector.putInt(time[i], (int) Math.round(repaired[i])); + if (Double.isFinite(repaired[i])) { + collector.putInt(time[i], (int) Math.round(repaired[i])); + } } break; case INT64: for (int i = 0; i < time.length; i++) { - collector.putLong(time[i], Math.round(repaired[i])); + if (Double.isFinite(repaired[i])) { + collector.putLong(time[i], Math.round(repaired[i])); + } } break; case TEXT: diff --git a/library-udf/src/main/java/org/apache/iotdb/library/drepair/UDTFValueRepair.java b/library-udf/src/main/java/org/apache/iotdb/library/drepair/UDTFValueRepair.java index a4d9b18ef60a2..6c2b5ca57114c 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/drepair/UDTFValueRepair.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/drepair/UDTFValueRepair.java @@ -47,14 +47,21 @@ public void validate(UDFParameterValidator validator) throws Exception { .validateInputSeriesNumber(1) .validateInputSeriesDataType(0, Type.FLOAT, Type.DOUBLE, Type.INT32, Type.INT64) .validate( - x -> (double) x > 0, - "Parameter $sigma$ should be larger than 0.", + x -> Double.isFinite((double) x) && (double) x > 0, + "Parameter $sigma$ should be finite and larger than 0.", validator.getParameters().getDoubleOrDefault("sigma", 1.0)) .validate( - params -> (double) params[0] < (double) params[1], - "parameter $minSpeed$ should be smaller than $maxSpeed$.", + params -> + Double.isFinite((double) params[0]) + && Double.isFinite((double) params[1]) + && (double) params[0] < (double) params[1], + "parameter $minSpeed$ and $maxSpeed$ should be finite, and $minSpeed$ should be smaller than $maxSpeed$.", validator.getParameters().getDoubleOrDefault("minSpeed", -1), - validator.getParameters().getDoubleOrDefault("maxSpeed", 1)); + validator.getParameters().getDoubleOrDefault("maxSpeed", 1)) + .validate( + x -> Double.isFinite((double) x), + "Parameter $center$ should be finite.", + validator.getParameters().getDoubleOrDefault("center", 0)); } @Override @@ -73,47 +80,62 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati @Override public void transform(RowWindow rowWindow, PointCollector collector) throws Exception { ValueRepair vr; - if ("screen".equalsIgnoreCase(method)) { - Screen screen = new Screen(rowWindow.getRowIterator()); - if (!Double.isNaN(minSpeed)) { - screen.setSmin(minSpeed); - } - if (!Double.isNaN(maxSpeed)) { - screen.setSmax(maxSpeed); + try { + if ("screen".equalsIgnoreCase(method)) { + Screen screen = new Screen(rowWindow.getRowIterator()); + if (!Double.isNaN(minSpeed)) { + screen.setSmin(minSpeed); + } + if (!Double.isNaN(maxSpeed)) { + screen.setSmax(maxSpeed); + } + vr = screen; + } else if ("lsgreedy".equalsIgnoreCase(method)) { + LsGreedy lsGreedy = new LsGreedy(rowWindow.getRowIterator()); + if (!Double.isNaN(sigma)) { + lsGreedy.setSigma(sigma); + } + lsGreedy.setCenter(center); + vr = lsGreedy; + } else { + throw new UDFException(LibraryUdfMessages.ILLEGAL_METHOD_WITH_DOT); } - vr = screen; - } else if ("lsgreedy".equalsIgnoreCase(method)) { - LsGreedy lsGreedy = new LsGreedy(rowWindow.getRowIterator()); - if (!Double.isNaN(sigma)) { - lsGreedy.setSigma(sigma); + vr.repair(); + } catch (UDFException e) { + if (LibraryUdfMessages.AT_LEAST_TWO_NON_NAN_VALUES_NEEDED.equals(e.getMessage())) { + return; } - lsGreedy.setCenter(center); - vr = lsGreedy; - } else { - throw new UDFException(LibraryUdfMessages.ILLEGAL_METHOD_WITH_DOT); + throw e; } - vr.repair(); double[] repaired = vr.getRepaired(); long[] time = vr.getTime(); switch (rowWindow.getDataType(0)) { case DOUBLE: for (int i = 0; i < time.length; i++) { - collector.putDouble(time[i], repaired[i]); + if (Double.isFinite(repaired[i])) { + collector.putDouble(time[i], repaired[i]); + } } break; case FLOAT: for (int i = 0; i < time.length; i++) { - collector.putFloat(time[i], (float) repaired[i]); + if (Double.isFinite(repaired[i])) { + collector.putFloat(time[i], (float) repaired[i]); + } } break; case INT32: for (int i = 0; i < time.length; i++) { - collector.putInt(time[i], (int) Math.round(repaired[i])); + if (Double.isFinite(repaired[i])) { + collector.putInt(time[i], (int) Math.round(repaired[i])); + } } break; case INT64: for (int i = 0; i < time.length; i++) { - collector.putLong(time[i], Math.round(repaired[i])); + if (Double.isFinite(repaired[i])) { + collector.putLong(time[i], Math.round(repaired[i])); + } } break; case TIMESTAMP: diff --git a/library-udf/src/main/java/org/apache/iotdb/library/drepair/util/ARFill.java b/library-udf/src/main/java/org/apache/iotdb/library/drepair/util/ARFill.java index 411b93f371674..2c6eca6dc67e1 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/drepair/util/ARFill.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/drepair/util/ARFill.java @@ -29,7 +29,9 @@ public class ARFill extends ValueFill { public ARFill(RowIterator dataIterator) throws Exception { super(dataIterator); - calMeanAndVar(); + if (hasValidValue()) { + calMeanAndVar(); + } } @Override @@ -49,12 +51,17 @@ public void fill() throws UDFException { acf += left * right; factor += left * left; } - if (factor == 0d || this.theta >= 1) { + if (factor == 0d) { this.time = new long[] {0}; this.repaired = new double[] {0D}; throw new UDFException(LibraryUdfMessages.CANNOT_FIT_AR1_MODEL); } this.theta = acf / factor; + if (!Double.isFinite(this.theta) || this.theta >= 1) { + this.time = new long[] {0}; + this.repaired = new double[] {0D}; + throw new UDFException(LibraryUdfMessages.CANNOT_FIT_AR1_MODEL); + } double meanEpsilon = 0; double cntEpsilon = 0; for (int i = 0; i < original.length - 1; i++) { diff --git a/library-udf/src/main/java/org/apache/iotdb/library/drepair/util/MeanFill.java b/library-udf/src/main/java/org/apache/iotdb/library/drepair/util/MeanFill.java index 8f96e87c2f431..0d4e4d1078eda 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/drepair/util/MeanFill.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/drepair/util/MeanFill.java @@ -24,7 +24,9 @@ public class MeanFill extends ValueFill { public MeanFill(RowIterator dataIterator) throws Exception { super(dataIterator); - calMeanAndVar(); + if (hasValidValue()) { + calMeanAndVar(); + } } @Override diff --git a/library-udf/src/main/java/org/apache/iotdb/library/drepair/util/ScreenFill.java b/library-udf/src/main/java/org/apache/iotdb/library/drepair/util/ScreenFill.java index b15ccf24a02c7..a5c341e049dba 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/drepair/util/ScreenFill.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/drepair/util/ScreenFill.java @@ -37,7 +37,9 @@ public class ScreenFill extends ValueFill { public ScreenFill(RowIterator dataIterator) throws Exception { super(dataIterator); - setParameters(); + if (hasValidValue()) { + setParameters(); + } } @Override @@ -101,6 +103,9 @@ private double getMedian(ArrayList> list, int index) { tempCount++; } } + if (count == 0) { + return Double.NaN; + } Arrays.sort(x); return x[count]; } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/drepair/util/TimestampInterval.java b/library-udf/src/main/java/org/apache/iotdb/library/drepair/util/TimestampInterval.java index f596a3552db2c..2924a1fd1fd05 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/drepair/util/TimestampInterval.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/drepair/util/TimestampInterval.java @@ -60,7 +60,7 @@ public long getInterval(long mode) { // median private long getIntervalByMedian() { ArrayList arrInterval = new ArrayList<>(); - for (int i = 0; i < n - 2; i++) { + for (int i = 0; i < n - 1; i++) { arrInterval.add(time[i + 1] - time[i]); } arrInterval.sort(Comparator.naturalOrder()); diff --git a/library-udf/src/main/java/org/apache/iotdb/library/drepair/util/TimestampRepair.java b/library-udf/src/main/java/org/apache/iotdb/library/drepair/util/TimestampRepair.java index 4b98256436bef..237893c129288 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/drepair/util/TimestampRepair.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/drepair/util/TimestampRepair.java @@ -41,8 +41,12 @@ public TimestampRepair(RowIterator dataIterator, long intervalMode, int startPoi ArrayList originList = new ArrayList<>(); while (dataIterator.hasNextRow()) { Row row = dataIterator.next(); - double v = Util.getValueAsDouble(row); timeList.add(row.getTime()); + if (row.isNull(0)) { + originList.add(Double.NaN); + continue; + } + double v = Util.getValueAsDouble(row); if (!Double.isFinite(v)) { originList.add(Double.NaN); } else { @@ -52,6 +56,13 @@ public TimestampRepair(RowIterator dataIterator, long intervalMode, int startPoi time = Util.toLongArray(timeList); original = Util.toDoubleArray(originList); n = time.length; + repaired = new long[n]; + repairedValue = new double[n]; + if (n <= 2) { + this.deltaT = intervalMode > 0 ? intervalMode : 1; + this.start0 = n == 0 ? 0 : time[0]; + return; + } TimestampInterval trParam = new TimestampInterval(time, original); this.deltaT = trParam.getInterval(intervalMode); this.start0 = trParam.getStart0(startPointMode); diff --git a/library-udf/src/main/java/org/apache/iotdb/library/drepair/util/ValueFill.java b/library-udf/src/main/java/org/apache/iotdb/library/drepair/util/ValueFill.java index 6d6017e5f76a7..798e529e255fa 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/drepair/util/ValueFill.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/drepair/util/ValueFill.java @@ -35,18 +35,24 @@ public abstract class ValueFill { protected double mean = 0; protected double variance = 0; protected int notNanNumber = 0; + private int validNumber = 0; protected ValueFill(RowIterator dataIterator) throws Exception { ArrayList timeList = new ArrayList<>(); ArrayList originList = new ArrayList<>(); while (dataIterator.hasNextRow()) { Row row = dataIterator.next(); - double v = Util.getValueAsDouble(row); timeList.add(row.getTime()); + if (row.isNull(0)) { + originList.add(Double.NaN); + continue; + } + double v = Util.getValueAsDouble(row); if (!Double.isFinite(v)) { originList.add(Double.NaN); } else { originList.add(v); + validNumber++; } } time = Util.toLongArray(timeList); @@ -57,6 +63,10 @@ protected ValueFill(RowIterator dataIterator) throws Exception { public abstract void fill() throws UDFException; + public boolean hasValidValue() { + return validNumber > 0; + } + public long[] getTime() { return time; } @@ -66,6 +76,9 @@ public double[] getFilled() { } public void calMeanAndVar() throws UDFException { + mean = 0; + variance = 0; + notNanNumber = 0; for (double v : original) { if (!Double.isNaN(v)) { mean += v; diff --git a/library-udf/src/main/java/org/apache/iotdb/library/drepair/util/ValueRepair.java b/library-udf/src/main/java/org/apache/iotdb/library/drepair/util/ValueRepair.java index a0f4213727d67..a171f8da7305d 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/drepair/util/ValueRepair.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/drepair/util/ValueRepair.java @@ -39,8 +39,12 @@ protected ValueRepair(RowIterator dataIterator) throws Exception { ArrayList originList = new ArrayList<>(); while (dataIterator.hasNextRow()) { Row row = dataIterator.next(); - Double v = Util.getValueAsDouble(row); timeList.add(row.getTime()); + if (row.isNull(0)) { + originList.add(Double.NaN); + continue; + } + Double v = Util.getValueAsDouble(row); if (!Double.isFinite(v)) { originList.add(Double.NaN); } else { diff --git a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDFEnvelopeAnalysis.java b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDFEnvelopeAnalysis.java index 4db051b27395e..8ca0cb3f1a024 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDFEnvelopeAnalysis.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDFEnvelopeAnalysis.java @@ -59,8 +59,8 @@ public void validate(UDFParameterValidator validator) throws Exception { .validateInputSeriesNumber(1) .validateInputSeriesDataType(0, Type.DOUBLE, Type.FLOAT, Type.INT32, Type.INT64) .validate( - x -> (double) x > 0, - "The param 'frequency' must > 0.", + x -> Double.isFinite((double) x) && (double) x > 0, + "The param 'frequency' must be finite and > 0.", validator.getParameters().getDoubleOrDefault(FREQUENCY, Double.MAX_VALUE)) .validate( x -> (int) x >= 1, diff --git a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFDWT.java b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFDWT.java index df756be5775f9..92a54903a6903 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFDWT.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFDWT.java @@ -60,7 +60,15 @@ public void validate(UDFParameterValidator validator) throws Exception { .validate( x -> (int) x > 0, "layer has to be a positive integer.", - validator.getParameters().getIntOrDefault("layer", 1)); + validator.getParameters().getIntOrDefault("layer", 1)) + .validate( + params -> + !((String) params[0]).equals("") + || ((String) params[1]).equals("") + || DWTUtil.isFiniteCoefficientList((String) params[1]), + "coef should be finite double,double... when method is blank.", + validator.getParameters().getStringOrDefault("method", ""), + validator.getParameters().getStringOrDefault("coef", "")); } @Override diff --git a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFDeconv.java b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFDeconv.java index 542b1f628de50..440c6388ae67c 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFDeconv.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFDeconv.java @@ -75,9 +75,20 @@ public void transform(Row row, PointCollector collector) throws Exception { @Override public void terminate(PointCollector collector) throws Exception { + if (list1.isEmpty() && list2.isEmpty()) { + return; + } if (list2.isEmpty()) { // Exception: divided by zero throw new ArithmeticException(LibraryUdfMessages.DIVIDED_BY_ZERO); - } else if (list2.size() > list1.size()) { // order of divisor is larger than dividend + } + int divisorDegree = list2.size() - 1; + while (divisorDegree >= 0 && list2.get(divisorDegree) == 0.0D) { + divisorDegree--; + } + if (divisorDegree < 0) { + throw new ArithmeticException(LibraryUdfMessages.DIVIDED_BY_ZERO); + } + if (divisorDegree + 1 > list1.size()) { // order of divisor is larger than dividend if (isQuotientResult(result)) { // quotient collector.putDouble(0, 0); } else { // residue @@ -86,9 +97,9 @@ public void terminate(PointCollector collector) throws Exception { } } } else { // order of divisor is no larger than dividend - double[] q = new double[list1.size() - list2.size() + 1]; + double[] q = new double[list1.size() - divisorDegree]; Double[] r = list1.toArray(new Double[0]); - int m = list2.size() - 1; + int m = divisorDegree; for (int i = q.length - 1; i >= 0; i--) { q[i] = r[i + m] / list2.get(m); r[i + m] = 0.0D; diff --git a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFIDWT.java b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFIDWT.java index 0efb879985384..b0a5aaf455b85 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFIDWT.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFIDWT.java @@ -61,7 +61,15 @@ public void validate(UDFParameterValidator validator) throws Exception { .validate( x -> (int) x > 0, "layer has to be a positive integer.", - validator.getParameters().getIntOrDefault("layer", 1)); + validator.getParameters().getIntOrDefault("layer", 1)) + .validate( + params -> + !((String) params[0]).equals("") + || ((String) params[1]).equals("") + || DWTUtil.isFiniteCoefficientList((String) params[1]), + "coef should be finite double,double... when method is blank.", + validator.getParameters().getStringOrDefault("method", ""), + validator.getParameters().getStringOrDefault("coef", "")); } @Override diff --git a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFIFFT.java b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFIFFT.java index 991d4c5a8f5d4..e01e580bd00f3 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFIFFT.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFIFFT.java @@ -84,6 +84,9 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati @Override public void transform(Row row, PointCollector collector) throws Exception { + if (row.getTime() < 0 || row.getTime() >= Integer.MAX_VALUE / 2) { + return; + } if (!row.isNull(0) && !row.isNull(1)) { double realValue = Util.getValueAsDouble(row, 0); double imagValue = Util.getValueAsDouble(row, 1); diff --git a/library-udf/src/main/java/org/apache/iotdb/library/frequency/util/DWTUtil.java b/library-udf/src/main/java/org/apache/iotdb/library/frequency/util/DWTUtil.java index 67971e19c7370..ad67cac8d6b74 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/frequency/util/DWTUtil.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/frequency/util/DWTUtil.java @@ -83,9 +83,15 @@ public DWTUtil(String method, String coef, int layer, List data) { } else { String[] coefString = coef.split(","); ncof = coefString.length; + if (ncof == 0) { + throw new IllegalArgumentException("Wavelet coefficients should not be empty."); + } cc = new double[ncof]; for (int i = 0; i < ncof; i++) { cc[i] = Double.parseDouble(coefString[i]); + if (!Double.isFinite(cc[i])) { + throw new IllegalArgumentException("Wavelet coefficients should be finite."); + } } } ncof = cc.length; @@ -102,6 +108,26 @@ public static boolean isPower2(int x) { return x > 0 && (x & (x - 1)) == 0; } + public static boolean isFiniteCoefficientList(String coef) { + if (coef == null || coef.isEmpty()) { + return false; + } + String[] coefString = coef.split(","); + if (coefString.length == 0) { + return false; + } + for (String coefficient : coefString) { + try { + if (!Double.isFinite(Double.parseDouble(coefficient))) { + return false; + } + } catch (NumberFormatException e) { + return false; + } + } + return true; + } + /** * Log of base 2. * diff --git a/library-udf/src/main/java/org/apache/iotdb/library/match/UDAFDTWMatch.java b/library-udf/src/main/java/org/apache/iotdb/library/match/UDAFDTWMatch.java index 47646fd5fe90a..f79b6b69acf71 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/match/UDAFDTWMatch.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/match/UDAFDTWMatch.java @@ -79,9 +79,12 @@ public void addInput(State state, Column[] columns, BitMap bitMap) { if (bitMap != null && !bitMap.isMarked(i)) { continue; } - if (!columns[1].isNull(i)) { - long timestamp = columns[1].getLong(i); + if (!columns[0].isNull(i) && !columns[1].isNull(i)) { double value = getValue(columns[0], i); + if (!Double.isFinite(value)) { + continue; + } + long timestamp = columns[1].getLong(i); DTWState.updateBuffer(timestamp, value); if (DTWState.getValueBuffer().length == pattern.length) { float dtw = calculateDTW(DTWState.getValueBuffer(), pattern); @@ -143,9 +146,12 @@ public void combineState(State state, State state1) { Long[] times = newDTWState.getTimeBuffer(); Double[] values = newDTWState.getValueBuffer(); + if (times.length == 0 || values.length == 0) { + return; + } for (int i = 0; i < times.length; i++) { - if (times[i] > dtwState.getFirstTime()) { + if (dtwState.getValueBuffer().length == 0 || times[i] > dtwState.getFirstTime()) { dtwState.updateBuffer(times[i], values[i]); if (dtwState.getValueBuffer().length == pattern.length) { float dtw = calculateDTW(dtwState.getValueBuffer(), pattern); @@ -198,6 +204,31 @@ public void validate(UDFParameterValidator validator) { .validateInputSeriesDataType( 0, Type.INT32, Type.INT64, Type.FLOAT, Type.DOUBLE, Type.BOOLEAN) .validateRequiredAttribute("pattern") - .validateRequiredAttribute("threshold"); + .validateRequiredAttribute("threshold") + .validate( + x -> isFiniteDoubleList((String) x), + "Illegal parameter, pattern must be finite double,double...", + validator.getParameters().getStringOrDefault("pattern", "")) + .validate( + x -> isFiniteNonNegativeFloat((String) x), + "Illegal parameter, threshold must be a finite non-negative number.", + validator.getParameters().getStringOrDefault("threshold", "")); + } + + private static boolean isFiniteDoubleList(String value) { + try { + return Arrays.stream(value.split(",")).map(Double::valueOf).allMatch(Double::isFinite); + } catch (Exception e) { + return false; + } + } + + private static boolean isFiniteNonNegativeFloat(String value) { + try { + float threshold = Float.parseFloat(value); + return Float.isFinite(threshold) && threshold >= 0; + } catch (NumberFormatException e) { + return false; + } } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/match/UDAFPatternMatch.java b/library-udf/src/main/java/org/apache/iotdb/library/match/UDAFPatternMatch.java index 89c902ab7f660..3def11e54b8bc 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/match/UDAFPatternMatch.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/match/UDAFPatternMatch.java @@ -77,9 +77,12 @@ public void addInput(State state, Column[] columns, BitMap bitMap) { if (bitMap != null && !bitMap.isMarked(i)) { continue; } - if (!columns[1].isNull(i)) { - long timestamp = columns[1].getLong(i); + if (!columns[0].isNull(i) && !columns[1].isNull(i)) { double value = getValue(columns[0], i); + if (!Double.isFinite(value)) { + continue; + } + long timestamp = columns[1].getLong(i); matchState.updateBuffer(timestamp, value); } } @@ -92,6 +95,9 @@ public void combineState(State state, State state1) { List times = newMatchState.getTimeBuffer(); List values = newMatchState.getValueBuffer(); + if (times == null || values == null || times.isEmpty() || values.isEmpty()) { + return; + } for (int i = 0; i < times.size(); i++) { matchState.updateBuffer(times.get(i), values.get(i)); @@ -101,10 +107,15 @@ public void combineState(State state, State state1) { @Override public void outputFinal(State state, ResultValue resultValue) { PatternState matchState = (PatternState) state; + List times = matchState.getTimeBuffer(); + List values = matchState.getValueBuffer(); + if (times == null || values == null || times.size() < 2 || values.size() < 2) { + resultValue.setNull(); + return; + } PatternExecutor executor = new PatternExecutor(); - List sourcePointsExtract = - executor.scalePoint(matchState.getTimeBuffer(), matchState.getValueBuffer()); + List sourcePointsExtract = executor.scalePoint(times, values); List queryPointsExtract = executor.extractPoints(timePattern, valuePattern); executor.setPoints(queryPointsExtract); @@ -173,7 +184,25 @@ public void validate(UDFParameterValidator validator) { payload -> ((Long[]) payload[0]).length == ((Double[]) payload[1]).length, "Illegal parameter, timePattern size must equals valuePattern size.", timePattern, - valuePattern); + valuePattern) + .validate( + (UDFParameterValidator.SingleObjectValidationRule) + payload -> Arrays.stream((Double[]) payload).allMatch(Double::isFinite), + "Illegal parameter, valuePattern values must be finite.", + valuePattern) + .validate( + x -> isFiniteNonNegativeFloat((String) x), + "Illegal parameter, threshold must be a finite non-negative number.", + validator.getParameters().getStringOrDefault(THRESHOLD_PARAM, "")); + } + + private static boolean isFiniteNonNegativeFloat(String value) { + try { + float threshold = Float.parseFloat(value); + return Float.isFinite(threshold) && threshold >= 0; + } catch (NumberFormatException e) { + return false; + } } private double getValue(Column column, int i) { diff --git a/library-udf/src/main/java/org/apache/iotdb/library/string/UDTFRegexMatch.java b/library-udf/src/main/java/org/apache/iotdb/library/string/UDTFRegexMatch.java index 88cade9e76b1a..42d0c182c199b 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/string/UDTFRegexMatch.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/string/UDTFRegexMatch.java @@ -79,6 +79,9 @@ public void beforeStart(UDFParameters udfParameters, UDTFConfigurations udtfConf @Override public void transform(Row row, PointCollector collector) throws Exception { + if (row.isNull(0)) { + return; + } Matcher matcher = pattern.matcher(row.getString(0)); if (matcher.find() && matcher.groupCount() >= group) { collector.putString(row.getTime(), matcher.group(group)); diff --git a/library-udf/src/main/java/org/apache/iotdb/library/string/UDTFRegexReplace.java b/library-udf/src/main/java/org/apache/iotdb/library/string/UDTFRegexReplace.java index af1e5865be81a..904d6763333ad 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/string/UDTFRegexReplace.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/string/UDTFRegexReplace.java @@ -95,6 +95,9 @@ public void beforeStart(UDFParameters udfParameters, UDTFConfigurations udtfConf @Override public void transform(Row row, PointCollector collector) throws Exception { + if (row.isNull(0)) { + return; + } String origin = row.getString(0); Matcher matcher = pattern.matcher(origin); String result = getResult(origin, matcher); diff --git a/library-udf/src/main/java/org/apache/iotdb/library/string/UDTFRegexSplit.java b/library-udf/src/main/java/org/apache/iotdb/library/string/UDTFRegexSplit.java index 576711802fb91..b97250c196e89 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/string/UDTFRegexSplit.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/string/UDTFRegexSplit.java @@ -57,6 +57,9 @@ public void beforeStart(UDFParameters udfParameters, UDTFConfigurations udtfConf @Override public void transform(Row row, PointCollector collector) throws Exception { + if (row.isNull(0)) { + return; + } String[] splitResult = row.getString(0).split(regex); if (index == -1) { collector.putInt(row.getTime(), splitResult.length); diff --git a/library-udf/src/main/java/org/apache/iotdb/library/string/UDTFStrReplace.java b/library-udf/src/main/java/org/apache/iotdb/library/string/UDTFStrReplace.java index 1490a19edf7ff..61df36869ea29 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/string/UDTFStrReplace.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/string/UDTFStrReplace.java @@ -80,6 +80,9 @@ public void beforeStart(UDFParameters udfParameters, UDTFConfigurations udtfConf @Override public void transform(Row row, PointCollector collector) throws Exception { + if (row.isNull(0)) { + return; + } String origin = row.getString(0); String result = getResult(origin); collector.putString(row.getTime(), result); diff --git a/library-udf/src/test/java/org/apache/iotdb/library/UDAFPatternTest.java b/library-udf/src/test/java/org/apache/iotdb/library/UDAFPatternTest.java index 6f1036ff0b588..0807c78fd063b 100644 --- a/library-udf/src/test/java/org/apache/iotdb/library/UDAFPatternTest.java +++ b/library-udf/src/test/java/org/apache/iotdb/library/UDAFPatternTest.java @@ -20,18 +20,30 @@ package org.apache.iotdb.library; import org.apache.iotdb.library.match.PatternExecutor; +import org.apache.iotdb.library.match.UDAFDTWMatch; import org.apache.iotdb.library.match.UDAFPatternMatch; +import org.apache.iotdb.library.match.model.DTWState; import org.apache.iotdb.library.match.model.PatternContext; import org.apache.iotdb.library.match.model.PatternResult; +import org.apache.iotdb.library.match.model.PatternState; import org.apache.iotdb.library.match.model.Point; +import org.apache.iotdb.udf.api.State; +import org.apache.iotdb.udf.api.customizer.config.UDAFConfigurations; import org.apache.iotdb.udf.api.customizer.parameter.UDFParameterValidator; import org.apache.iotdb.udf.api.customizer.parameter.UDFParameters; import org.apache.iotdb.udf.api.exception.UDFAttributeNotProvidedException; import org.apache.iotdb.udf.api.exception.UDFInputSeriesNumberNotValidException; import org.apache.iotdb.udf.api.exception.UDFParameterNotValidException; import org.apache.iotdb.udf.api.type.Type; +import org.apache.iotdb.udf.api.utils.ResultValue; import org.apache.commons.lang3.StringUtils; +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.block.column.ColumnBuilderStatus; +import org.apache.tsfile.block.column.ColumnEncoding; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Binary; import org.junit.Assert; import org.junit.Test; @@ -40,6 +52,8 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -154,5 +168,339 @@ public void testParameterValidator() { } catch (Exception e) { Assert.fail("Should not throw exception"); } + + userAttributes.put("threshold", "Infinity"); + Assert.assertThrows( + "Illegal parameter, threshold must be a finite non-negative number.", + UDFParameterNotValidException.class, + () -> patternMatch.validate(validator)); + + userAttributes.put("threshold", "100"); + userAttributes.put("valuePattern", "1.0,NaN,3.0"); + Assert.assertThrows( + "Illegal parameter, valuePattern values must be finite.", + UDFParameterNotValidException.class, + () -> patternMatch.validate(validator)); + } + + @Test + public void testDTWMatchValidatesFiniteParameters() { + Map attributes = new HashMap<>(); + attributes.put("pattern", "1.0,NaN"); + attributes.put("threshold", "100"); + + Assert.assertThrows( + UDFParameterNotValidException.class, + () -> + new UDAFDTWMatch() + .validate( + new UDFParameterValidator( + new UDFParameters( + Collections.singletonList("s1"), + Collections.singletonList(Type.DOUBLE), + attributes)))); + + attributes.put("pattern", "1.0,2.0"); + attributes.put("threshold", "Infinity"); + Assert.assertThrows( + UDFParameterNotValidException.class, + () -> + new UDAFDTWMatch() + .validate( + new UDFParameterValidator( + new UDFParameters( + Collections.singletonList("s1"), + Collections.singletonList(Type.DOUBLE), + attributes)))); + } + + @Test + public void testMatchUDAFsSkipNullAndInvalidRows() throws Exception { + Column[] columns = buildPatternInputColumns(); + + UDAFPatternMatch patternMatch = new UDAFPatternMatch(); + UDFParameters patternParameters = createPatternParameters(); + patternMatch.validate(new UDFParameterValidator(patternParameters)); + patternMatch.beforeStart(patternParameters, new UDAFConfigurations()); + PatternState patternState = (PatternState) patternMatch.createState(); + patternState.reset(); + patternMatch.addInput(patternState, columns, null); + + Assert.assertEquals(Arrays.asList(1L, 5L), patternState.getTimeBuffer()); + Assert.assertEquals(Arrays.asList(1.0, 3.0), patternState.getValueBuffer()); + + UDAFDTWMatch dtwMatch = new UDAFDTWMatch(); + UDFParameters dtwParameters = createDtwParameters(); + dtwMatch.validate(new UDFParameterValidator(dtwParameters)); + dtwMatch.beforeStart(dtwParameters, new UDAFConfigurations()); + DTWState dtwState = (DTWState) dtwMatch.createState(); + dtwState.reset(); + dtwMatch.addInput(dtwState, columns, null); + + Assert.assertArrayEquals(new Long[] {1L, 5L}, dtwState.getTimeBuffer()); + Assert.assertArrayEquals(new Double[] {1.0, 3.0}, dtwState.getValueBuffer()); + } + + @Test + public void testPatternMatchEmptyEffectiveInputOutputsNull() throws Exception { + UDAFPatternMatch patternMatch = new UDAFPatternMatch(); + UDFParameters parameters = createPatternParameters(); + patternMatch.validate(new UDFParameterValidator(parameters)); + patternMatch.beforeStart(parameters, new UDAFConfigurations()); + State state = patternMatch.createState(); + state.reset(); + + RecordingColumnBuilder resultBuilder = new RecordingColumnBuilder(TSDataType.TEXT); + patternMatch.outputFinal(state, new ResultValue(resultBuilder)); + + Assert.assertTrue(resultBuilder.build().isNull(0)); + } + + @Test + public void testMatchUDAFsCombineEmptyStates() throws Exception { + UDAFPatternMatch patternMatch = new UDAFPatternMatch(); + UDFParameters patternParameters = createPatternParameters(); + patternMatch.validate(new UDFParameterValidator(patternParameters)); + patternMatch.beforeStart(patternParameters, new UDAFConfigurations()); + PatternState patternTarget = (PatternState) patternMatch.createState(); + patternTarget.reset(); + PatternState patternEmptySource = (PatternState) patternMatch.createState(); + patternEmptySource.reset(); + + patternMatch.combineState(patternTarget, patternEmptySource); + + Assert.assertTrue(patternTarget.getTimeBuffer().isEmpty()); + Assert.assertTrue(patternTarget.getValueBuffer().isEmpty()); + + PatternState patternSource = (PatternState) patternMatch.createState(); + patternSource.reset(); + patternSource.updateBuffer(1L, 1.0); + patternMatch.combineState(patternTarget, patternSource); + + Assert.assertEquals(Collections.singletonList(1L), patternTarget.getTimeBuffer()); + Assert.assertEquals(Collections.singletonList(1.0), patternTarget.getValueBuffer()); + + UDAFDTWMatch dtwMatch = new UDAFDTWMatch(); + UDFParameters dtwParameters = createDtwParameters(); + dtwMatch.validate(new UDFParameterValidator(dtwParameters)); + dtwMatch.beforeStart(dtwParameters, new UDAFConfigurations()); + DTWState dtwTarget = (DTWState) dtwMatch.createState(); + dtwTarget.reset(); + DTWState dtwEmptySource = (DTWState) dtwMatch.createState(); + dtwEmptySource.reset(); + + dtwMatch.combineState(dtwTarget, dtwEmptySource); + + Assert.assertArrayEquals(new Long[0], dtwTarget.getTimeBuffer()); + Assert.assertArrayEquals(new Double[0], dtwTarget.getValueBuffer()); + + DTWState dtwSource = (DTWState) dtwMatch.createState(); + dtwSource.reset(); + dtwSource.updateBuffer(1L, 1.0); + dtwSource.updateBuffer(2L, 3.0); + dtwMatch.combineState(dtwTarget, dtwSource); + + Assert.assertArrayEquals(new Long[] {1L, 2L}, dtwTarget.getTimeBuffer()); + Assert.assertArrayEquals(new Double[] {1.0, 3.0}, dtwTarget.getValueBuffer()); + } + + private static UDFParameters createPatternParameters() { + Map attributes = new HashMap<>(); + attributes.put("timePattern", "1,2"); + attributes.put("valuePattern", "1.0,2.0"); + attributes.put("threshold", "100"); + return new UDFParameters( + Collections.singletonList("s1"), Collections.singletonList(Type.DOUBLE), attributes); + } + + private static UDFParameters createDtwParameters() { + Map attributes = new HashMap<>(); + attributes.put("pattern", "1.0,3.0"); + attributes.put("threshold", "100"); + return new UDFParameters( + Collections.singletonList("s1"), Collections.singletonList(Type.DOUBLE), attributes); + } + + private static Column[] buildPatternInputColumns() { + return new Column[] { + new TestColumn( + TSDataType.DOUBLE, + new long[0], + new double[] {1.0, 0.0, Double.NaN, Double.POSITIVE_INFINITY, 3.0}, + new boolean[] {false, true, false, false, false}), + new TestColumn( + TSDataType.INT64, + new long[] {1L, 2L, 3L, 4L, 5L}, + new double[0], + new boolean[] {false, false, false, false, false}) + }; + } + + private static class TestColumn implements Column { + + private final TSDataType dataType; + private final long[] longValues; + private final double[] doubleValues; + private final boolean[] nulls; + + private TestColumn( + TSDataType dataType, long[] longValues, double[] doubleValues, boolean[] nulls) { + this.dataType = dataType; + this.longValues = longValues; + this.doubleValues = doubleValues; + this.nulls = nulls; + } + + @Override + public TSDataType getDataType() { + return dataType; + } + + @Override + public ColumnEncoding getEncoding() { + return ColumnEncoding.INT32_ARRAY; + } + + @Override + public long getLong(int position) { + return longValues[position]; + } + + @Override + public double getDouble(int position) { + return doubleValues[position]; + } + + @Override + public boolean mayHaveNull() { + return true; + } + + @Override + public boolean isNull(int position) { + return nulls[position]; + } + + @Override + public boolean[] isNull() { + return nulls; + } + + @Override + public void setNull(int start, int end) { + throw new UnsupportedOperationException(); + } + + @Override + public int getPositionCount() { + return nulls.length; + } + + @Override + public long getRetainedSizeInBytes() { + return 0; + } + + @Override + public long getSizeInBytes() { + return 0; + } + + @Override + public Column getRegion(int positionOffset, int length) { + throw new UnsupportedOperationException(); + } + + @Override + public Column getRegionCopy(int positionOffset, int length) { + throw new UnsupportedOperationException(); + } + + @Override + public Column subColumn(int fromIndex) { + throw new UnsupportedOperationException(); + } + + @Override + public Column subColumnCopy(int fromIndex) { + throw new UnsupportedOperationException(); + } + + @Override + public Column getPositions(int[] positions, int offset, int length) { + throw new UnsupportedOperationException(); + } + + @Override + public Column copyPositions(int[] positions, int offset, int length) { + throw new UnsupportedOperationException(); + } + + @Override + public void reverse() { + throw new UnsupportedOperationException(); + } + + @Override + public int getInstanceSize() { + return 0; + } + + @Override + public void setPositionCount(int count) { + throw new UnsupportedOperationException(); + } + } + + private static class RecordingColumnBuilder implements ColumnBuilder { + + private final TSDataType dataType; + private boolean nullWritten; + + private RecordingColumnBuilder(TSDataType dataType) { + this.dataType = dataType; + } + + @Override + public int getPositionCount() { + return nullWritten ? 1 : 0; + } + + @Override + public ColumnBuilder write(Column column, int index) { + throw new UnsupportedOperationException(); + } + + @Override + public ColumnBuilder writeBinary(Binary value) { + nullWritten = false; + return this; + } + + @Override + public ColumnBuilder appendNull() { + nullWritten = true; + return this; + } + + @Override + public Column build() { + return new TestColumn(dataType, new long[0], new double[] {0.0}, new boolean[] {nullWritten}); + } + + @Override + public TSDataType getDataType() { + return dataType; + } + + @Override + public long getRetainedSizeInBytes() { + return 0; + } + + @Override + public ColumnBuilder newColumnBuilderLike(ColumnBuilderStatus status) { + return new RecordingColumnBuilder(dataType); + } } } diff --git a/library-udf/src/test/java/org/apache/iotdb/library/UDFWindowAndQueueTest.java b/library-udf/src/test/java/org/apache/iotdb/library/UDFWindowAndQueueTest.java index 932ca43cf40dc..4c05e7051cbf4 100644 --- a/library-udf/src/test/java/org/apache/iotdb/library/UDFWindowAndQueueTest.java +++ b/library-udf/src/test/java/org/apache/iotdb/library/UDFWindowAndQueueTest.java @@ -30,6 +30,7 @@ import org.apache.iotdb.library.dlearn.UDTFAR; import org.apache.iotdb.library.dlearn.UDTFCluster; import org.apache.iotdb.library.dmatch.UDAFDtw; +import org.apache.iotdb.library.dmatch.UDTFPtnSym; import org.apache.iotdb.library.dprofile.UDAFIntegral; import org.apache.iotdb.library.dprofile.UDAFIntegralAvg; import org.apache.iotdb.library.dprofile.UDAFMad; @@ -38,6 +39,8 @@ import org.apache.iotdb.library.dprofile.UDAFPeriod; import org.apache.iotdb.library.dprofile.UDAFQuantile; import org.apache.iotdb.library.dprofile.UDAFSkew; +import org.apache.iotdb.library.dprofile.UDAFSpread; +import org.apache.iotdb.library.dprofile.UDTFDistinct; import org.apache.iotdb.library.dprofile.UDTFHistogram; import org.apache.iotdb.library.dprofile.UDTFMinMax; import org.apache.iotdb.library.dprofile.UDTFMvAvg; @@ -49,6 +52,9 @@ import org.apache.iotdb.library.dprofile.UDTFSpline; import org.apache.iotdb.library.dprofile.UDTFZScore; import org.apache.iotdb.library.dprofile.util.Resampler; +import org.apache.iotdb.library.drepair.UDTFTimestampRepair; +import org.apache.iotdb.library.drepair.UDTFValueFill; +import org.apache.iotdb.library.drepair.UDTFValueRepair; import org.apache.iotdb.library.frequency.UDFEnvelopeAnalysis; import org.apache.iotdb.library.frequency.UDTFConv; import org.apache.iotdb.library.frequency.UDTFDWT; @@ -68,6 +74,7 @@ import org.apache.iotdb.library.util.CircularQueue; import org.apache.iotdb.library.util.DoubleCircularQueue; import org.apache.iotdb.library.util.LongCircularQueue; +import org.apache.iotdb.udf.api.UDTF; import org.apache.iotdb.udf.api.access.Row; import org.apache.iotdb.udf.api.access.RowIterator; import org.apache.iotdb.udf.api.access.RowWindow; @@ -187,6 +194,64 @@ public void testLOFSkipsNullRowsWithoutReadingCompressedIndex() throws Exception Assert.assertEquals(3, collector.values.size()); } + @Test + public void testLOFSkipsInvalidRows() throws Exception { + Map attributes = new HashMap<>(); + attributes.put("k", "1"); + UDTFLOF lof = new UDTFLOF(); + UDFParameters parameters = + new UDFParameters( + Arrays.asList("s1", "s2"), Arrays.asList(Type.DOUBLE, Type.DOUBLE), attributes); + RecordingPointCollector collector = new RecordingPointCollector(); + + lof.validate(new UDFParameterValidator(parameters)); + lof.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + lof.transform( + new SimpleRowWindow( + new DoubleRow(1, new double[] {0.0, 0.0}, new boolean[] {false, false}), + new DoubleRow(2, new double[] {Double.NaN, 0.0}, new boolean[] {false, false}), + new DoubleRow(3, new double[] {10.0, 10.0}, new boolean[] {false, false}), + new DoubleRow(4, new double[] {20.0, 20.0}, new boolean[] {false, false}), + new DoubleRow( + 5, new double[] {Double.POSITIVE_INFINITY, 1.0}, new boolean[] {false, false})), + collector); + + Assert.assertEquals(Arrays.asList(1L, 3L, 4L), collector.timestamps); + Assert.assertEquals(3, collector.values.size()); + for (double value : collector.values) { + Assert.assertTrue(Double.isFinite(value)); + } + } + + @Test + public void testLOFSeriesSkipsInvalidRows() throws Exception { + Map attributes = new HashMap<>(); + attributes.put("k", "1"); + attributes.put("method", "series"); + attributes.put("window", "2"); + UDTFLOF lof = new UDTFLOF(); + UDFParameters parameters = createSingleDoubleSeriesParameters(attributes); + RecordingPointCollector collector = new RecordingPointCollector(); + + lof.validate(new UDFParameterValidator(parameters)); + lof.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + lof.transform( + new SimpleRowWindow( + new DoubleRow(1, 1.0), + new DoubleRow(2, Double.NaN), + new DoubleRow(3, 2.0), + new DoubleRow(4, 3.0), + new DoubleRow(5, Double.POSITIVE_INFINITY), + new DoubleRow(6, 4.0)), + collector); + + Assert.assertEquals(Arrays.asList(1L, 3L, 4L), collector.timestamps); + Assert.assertEquals(3, collector.values.size()); + for (double value : collector.values) { + Assert.assertTrue(Double.isFinite(value)); + } + } + @Test public void testLOFValidatesAllInputSeriesTypes() { UDFParameters parameters = @@ -228,6 +293,38 @@ public void testConvIgnoresEmptyEffectiveInput() throws Exception { Assert.assertTrue(collector.values.isEmpty()); } + @Test + public void testDeconvHandlesEmptyInputAndZeroDivisor() throws Exception { + UDTFDeconv deconv = new UDTFDeconv(); + UDFParameters parameters = createTwoDoubleSeriesParameters(Collections.emptyMap()); + RecordingPointCollector collector = new RecordingPointCollector(); + + deconv.validate(new UDFParameterValidator(parameters)); + deconv.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + deconv.transform( + new DoubleRow(1, new double[] {0.0, 0.0}, new boolean[] {true, true}), collector); + deconv.terminate(collector); + + Assert.assertTrue(collector.timestamps.isEmpty()); + Assert.assertTrue(collector.values.isEmpty()); + + deconv.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + deconv.transform( + new DoubleRow(1, new double[] {1.0, 0.0}, new boolean[] {false, false}), collector); + Assert.assertThrows(ArithmeticException.class, () -> deconv.terminate(collector)); + + deconv.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + deconv.transform( + new DoubleRow(1, new double[] {4.0, 2.0}, new boolean[] {false, false}), collector); + deconv.transform( + new DoubleRow(2, new double[] {6.0, 0.0}, new boolean[] {false, false}), collector); + deconv.terminate(collector); + + Assert.assertEquals(Arrays.asList(0L, 1L), collector.timestamps); + Assert.assertEquals(2.0, collector.values.get(0), 0.0); + Assert.assertEquals(3.0, collector.values.get(1), 0.0); + } + @Test public void testConsecutiveSequencesIgnoresShortAutoGapWindow() throws Exception { UDFParameters parameters = createSingleDoubleSeriesParameters(Collections.emptyMap()); @@ -289,6 +386,70 @@ public void testStreamModeRequiresExplicitParameters() { new UDFParameterValidator(createSingleDoubleSeriesParameters(attributes)))); } + @Test + public void testNonFiniteNumericParametersAreRejected() { + Map attributes = new HashMap<>(); + attributes.put("lower_bound", "-Infinity"); + attributes.put("upper_bound", "Infinity"); + assertInvalidSingleDoubleParameters(new UDTFRange(), attributes); + + attributes = new HashMap<>(); + attributes.put("min", "-Infinity"); + attributes.put("max", "Infinity"); + assertInvalidSingleDoubleParameters(new UDTFHistogram(), attributes); + + attributes = new HashMap<>(); + attributes.put("compute", "stream"); + attributes.put("q1", "-Infinity"); + attributes.put("q3", "Infinity"); + assertInvalidSingleDoubleParameters(new UDTFIQR(), attributes); + + attributes = new HashMap<>(); + attributes.put("compute", "stream"); + attributes.put("min", "-Infinity"); + attributes.put("max", "Infinity"); + assertInvalidSingleDoubleParameters(new UDTFMinMax(), attributes); + + attributes = new HashMap<>(); + attributes.put("compute", "stream"); + attributes.put("avg", "Infinity"); + attributes.put("sd", "1"); + assertInvalidSingleDoubleParameters(new UDTFZScore(), attributes); + + attributes = new HashMap<>(); + attributes.put("k", "Infinity"); + assertInvalidSingleDoubleParameters(new UDTFKSigma(), attributes); + + attributes = new HashMap<>(); + attributes.put("r", "Infinity"); + assertInvalidSingleDoubleParameters(new UDTFOutlier(), attributes); + + attributes = new HashMap<>(); + attributes.put("len", "NaN"); + assertInvalidSingleDoubleParameters(new UDTFTwoSidedFilter(), attributes); + + attributes = new HashMap<>(); + attributes.put("error", "Infinity"); + assertInvalidSingleDoubleParameters(new UDTFSegment(), attributes); + + attributes = new HashMap<>(); + attributes.put("threshold", "Infinity"); + assertInvalidSingleDoubleParameters(new UDTFPtnSym(), attributes); + + attributes = new HashMap<>(); + attributes.put("sigma", "Infinity"); + assertInvalidSingleDoubleParameters(new UDTFValueRepair(), attributes); + + attributes = new HashMap<>(); + attributes.put("frequency", "Infinity"); + assertInvalidSingleDoubleParameters(new UDFEnvelopeAnalysis(), attributes); + + attributes = new HashMap<>(); + attributes.put("coef", "NaN,1"); + assertInvalidSingleDoubleParameters(new UDTFDWT(), attributes); + assertInvalidSingleDoubleParameters(new UDTFIDWT(), attributes); + } + @Test public void testRowByRowNumericFunctionsSkipNullAndInvalidValues() throws Exception { UDFParameters defaultParameters = createSingleDoubleSeriesParameters(Collections.emptyMap()); @@ -303,6 +464,7 @@ public void testRowByRowNumericFunctionsSkipNullAndInvalidValues() throws Except range.beforeStart(rangeParameters, new UDTFConfigurations(ZoneId.systemDefault())); range.transform(nullDoubleRow(1), rangeCollector); range.transform(new DoubleRow(2, Double.NaN), rangeCollector); + range.transform(new DoubleRow(3, Double.POSITIVE_INFINITY), rangeCollector); Assert.assertTrue(rangeCollector.timestamps.isEmpty()); Map histogramAttributes = new HashMap<>(); @@ -506,6 +668,41 @@ public void testStringReplaceRequiresParameters() { .validate(new UDFParameterValidator(createSingleTextSeriesParameters(attributes)))); } + @Test + public void testStringFunctionsSkipNullRows() throws Exception { + Map strReplaceAttributes = new HashMap<>(); + strReplaceAttributes.put("target", "a"); + strReplaceAttributes.put("replace", "b"); + UDFParameters strReplaceParameters = createSingleTextSeriesParameters(strReplaceAttributes); + UDTFStrReplace strReplace = new UDTFStrReplace(); + strReplace.validate(new UDFParameterValidator(strReplaceParameters)); + strReplace.beforeStart(strReplaceParameters, new UDTFConfigurations(ZoneId.systemDefault())); + strReplace.transform(nullTextRow(1), new RecordingPointCollector()); + + Map regexReplaceAttributes = new HashMap<>(); + regexReplaceAttributes.put("regex", "a+"); + regexReplaceAttributes.put("replace", "b"); + UDFParameters regexReplaceParameters = createSingleTextSeriesParameters(regexReplaceAttributes); + UDTFRegexReplace regexReplace = new UDTFRegexReplace(); + regexReplace.validate(new UDFParameterValidator(regexReplaceParameters)); + regexReplace.beforeStart( + regexReplaceParameters, new UDTFConfigurations(ZoneId.systemDefault())); + regexReplace.transform(nullTextRow(2), new RecordingPointCollector()); + + Map regexAttributes = new HashMap<>(); + regexAttributes.put("regex", "a+"); + UDFParameters regexParameters = createSingleTextSeriesParameters(regexAttributes); + UDTFRegexSplit regexSplit = new UDTFRegexSplit(); + regexSplit.validate(new UDFParameterValidator(regexParameters)); + regexSplit.beforeStart(regexParameters, new UDTFConfigurations(ZoneId.systemDefault())); + regexSplit.transform(nullTextRow(3), new RecordingPointCollector()); + + UDTFRegexMatch regexMatch = new UDTFRegexMatch(); + regexMatch.validate(new UDFParameterValidator(regexParameters)); + regexMatch.beforeStart(regexParameters, new UDTFConfigurations(ZoneId.systemDefault())); + regexMatch.transform(nullTextRow(4), new RecordingPointCollector()); + } + @Test public void testAggregateFunctionsIgnoreEmptyInput() throws Exception { UDFParameters parameters = createSingleDoubleSeriesParameters(Collections.emptyMap()); @@ -823,6 +1020,38 @@ public void testReservoirSampleResetsCountBetweenRuns() throws Exception { Assert.assertEquals(2.0, secondRun.values.get(0), 0.0); } + @Test + public void testSampleSkipsNullRows() throws Exception { + Map attributes = new HashMap<>(); + attributes.put("k", "3"); + UDFParameters parameters = createSingleDoubleSeriesParameters(attributes); + UDTFSample sample = new UDTFSample(); + RecordingPointCollector collector = new RecordingPointCollector(); + + sample.validate(new UDFParameterValidator(parameters)); + sample.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + sample.transform(nullDoubleRow(1), collector); + sample.transform(new DoubleRow(2, 2.0), collector); + sample.terminate(collector); + + Assert.assertEquals(Collections.singletonList(2L), collector.timestamps); + Assert.assertEquals(2.0, collector.values.get(0), 0.0); + + attributes.put("method", "isometric"); + UDFParameters isometricParameters = createSingleDoubleSeriesParameters(attributes); + collector.timestamps.clear(); + collector.values.clear(); + + sample.validate(new UDFParameterValidator(isometricParameters)); + sample.beforeStart(isometricParameters, new UDTFConfigurations(ZoneId.systemDefault())); + sample.transform( + new SimpleRowWindow(nullDoubleRow(3), new DoubleRow(4, 4.0), new DoubleRow(5, 5.0)), + collector); + + Assert.assertEquals(Arrays.asList(4L, 5L), collector.timestamps); + Assert.assertEquals(Arrays.asList(4.0, 5.0), collector.values); + } + @Test public void testSampleClearsReservoirStateWhenSwitchingMethod() throws Exception { Map reservoirAttributes = new HashMap<>(); @@ -848,6 +1077,25 @@ public void testSampleClearsReservoirStateWhenSwitchingMethod() throws Exception Assert.assertTrue(collector.timestamps.isEmpty()); } + @Test + public void testPatternSymmetrySkipsInvalidWindows() throws Exception { + Map attributes = new HashMap<>(); + attributes.put("window", "2"); + UDFParameters parameters = createSingleDoubleSeriesParameters(attributes); + UDTFPtnSym ptnSym = new UDTFPtnSym(); + RecordingPointCollector collector = new RecordingPointCollector(); + + ptnSym.validate(new UDFParameterValidator(parameters)); + ptnSym.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + ptnSym.transform(new SimpleRowWindow(nullDoubleRow(1), new DoubleRow(2, 2.0)), collector); + ptnSym.transform( + new SimpleRowWindow(new DoubleRow(3, Double.POSITIVE_INFINITY), new DoubleRow(4, 4.0)), + collector); + + Assert.assertTrue(collector.timestamps.isEmpty()); + Assert.assertTrue(collector.values.isEmpty()); + } + @Test public void testDtwEmptyInputProducesNoOutput() throws Exception { UDAFDtw dtw = new UDAFDtw(); @@ -861,6 +1109,28 @@ public void testDtwEmptyInputProducesNoOutput() throws Exception { Assert.assertTrue(collector.timestamps.isEmpty()); } + @Test + public void testDtwSkipsInvalidRows() throws Exception { + UDAFDtw dtw = new UDAFDtw(); + UDFParameters parameters = createTwoDoubleSeriesParameters(Collections.emptyMap()); + RecordingPointCollector collector = new RecordingPointCollector(); + + dtw.validate(new UDFParameterValidator(parameters)); + dtw.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + dtw.transform( + new SimpleRowWindow( + new DoubleRow(1, new double[] {1.0, 1.0}, new boolean[] {false, false}), + new DoubleRow(2, new double[] {Double.NaN, 2.0}, new boolean[] {false, false}), + new DoubleRow( + 3, new double[] {3.0, Double.POSITIVE_INFINITY}, new boolean[] {false, false}), + new DoubleRow(4, new double[] {2.0, 2.0}, new boolean[] {false, false})), + collector); + dtw.terminate(collector); + + Assert.assertEquals(Collections.singletonList(0L), collector.timestamps); + Assert.assertEquals(0.0, collector.values.get(0), 0.0); + } + @Test public void testDtwClearsPreviousResultForEmptyWindow() throws Exception { UDAFDtw dtw = new UDAFDtw(); @@ -1032,6 +1302,11 @@ public void testIFFTClearsBuffersBetweenRuns() throws Exception { new DoubleRow(0, new double[] {0.0, 0.0}, new boolean[] {true, false}), collector); ifft.transform( new DoubleRow(1, new double[] {0.0, Double.NaN}, new boolean[] {false, false}), collector); + ifft.transform( + new DoubleRow(-1, new double[] {1.0, 0.0}, new boolean[] {false, false}), collector); + ifft.transform( + new DoubleRow(Integer.MAX_VALUE / 2, new double[] {1.0, 0.0}, new boolean[] {false, false}), + collector); ifft.terminate(collector); Assert.assertTrue(collector.timestamps.isEmpty()); @@ -1098,6 +1373,44 @@ public void testARValidatesPositiveOrder() { new UDFParameterValidator(createSingleDoubleSeriesParameters(attributes)))); } + @Test + public void testARSkipsInvalidRows() throws Exception { + UDFParameters parameters = createSingleDoubleSeriesParameters(Collections.emptyMap()); + UDTFAR ar = new UDTFAR(); + RecordingPointCollector collector = new RecordingPointCollector(); + + ar.validate(new UDFParameterValidator(parameters)); + ar.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + ar.transform(new DoubleRow(1, 1.0), collector); + ar.transform(new DoubleRow(2, Double.NaN), collector); + ar.transform(new DoubleRow(3, 2.0), collector); + ar.transform(new DoubleRow(4, 3.0), collector); + ar.terminate(collector); + + Assert.assertEquals(Collections.singletonList(1L), collector.timestamps); + Assert.assertTrue(Double.isFinite(collector.values.get(0))); + } + + @Test + public void testClusterSkipsInvalidRows() throws Exception { + Map attributes = new HashMap<>(); + attributes.put("l", "1"); + attributes.put("k", "2"); + UDFParameters parameters = createSingleDoubleSeriesParameters(attributes); + UDTFCluster cluster = new UDTFCluster(); + RecordingPointCollector collector = new RecordingPointCollector(); + + cluster.validate(new UDFParameterValidator(parameters)); + cluster.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + cluster.transform(new DoubleRow(1, 1.0), collector); + cluster.transform(new DoubleRow(2, Double.POSITIVE_INFINITY), collector); + cluster.transform(new DoubleRow(3, 2.0), collector); + cluster.terminate(collector); + + Assert.assertEquals(Arrays.asList(1L, 3L), collector.timestamps); + Assert.assertEquals(2, collector.values.size()); + } + @Test public void testRequiredParametersAreValidatedBeforeStart() { UDFParameterValidator numericValidator = @@ -1282,6 +1595,129 @@ public void testIntegralAvgUsesFirstFinitePointAndResetsBetweenRuns() throws Exc Assert.assertEquals(2.0, collector.values.get(0), 0.0); } + @Test + public void testSpreadSkipsInvalidRowsAndResetsBetweenRuns() throws Exception { + UDFParameters parameters = createSingleDoubleSeriesParameters(Collections.emptyMap()); + UDAFSpread spread = new UDAFSpread(); + RecordingPointCollector collector = new RecordingPointCollector(); + + spread.validate(new UDFParameterValidator(parameters)); + spread.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + spread.transform(nullDoubleRow(1), collector); + spread.transform(new DoubleRow(2, Double.NaN), collector); + spread.transform(new DoubleRow(3, 1.0), collector); + spread.transform(new DoubleRow(4, 4.0), collector); + spread.terminate(collector); + + Assert.assertEquals(Collections.singletonList(0L), collector.timestamps); + Assert.assertEquals(3.0, collector.values.get(0), 0.0); + + collector.timestamps.clear(); + collector.values.clear(); + spread.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + spread.terminate(collector); + + Assert.assertTrue(collector.timestamps.isEmpty()); + Assert.assertTrue(collector.values.isEmpty()); + } + + @Test + public void testDistinctSkipsNullRows() throws Exception { + UDFParameters parameters = createSingleDoubleSeriesParameters(Collections.emptyMap()); + UDTFDistinct distinct = new UDTFDistinct(); + RecordingPointCollector collector = new RecordingPointCollector(); + + distinct.validate(new UDFParameterValidator(parameters)); + distinct.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + distinct.transform(nullDoubleRow(1), collector); + distinct.transform(new DoubleRow(2, 2.0), collector); + distinct.terminate(collector); + + Assert.assertEquals(Collections.singletonList(0L), collector.timestamps); + Assert.assertEquals(2.0, collector.values.get(0), 0.0); + } + + @Test + public void testRepairFunctionsHandleNullAndShortInputs() throws Exception { + UDFParameters parameters = createSingleDoubleSeriesParameters(Collections.emptyMap()); + RecordingPointCollector collector = new RecordingPointCollector(); + + UDTFValueFill valueFill = new UDTFValueFill(); + valueFill.validate(new UDFParameterValidator(parameters)); + valueFill.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + valueFill.transform( + new SimpleRowWindow(nullDoubleRow(1), new DoubleRow(2, 5.0), new DoubleRow(3, 7.0)), + collector); + Assert.assertEquals(Arrays.asList(1L, 2L, 3L), collector.timestamps); + Assert.assertEquals(5.0, collector.values.get(0), 0.0); + + collector.timestamps.clear(); + collector.values.clear(); + valueFill.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + valueFill.transform(new SimpleRowWindow(nullDoubleRow(1), nullDoubleRow(2)), collector); + Assert.assertTrue(collector.timestamps.isEmpty()); + Assert.assertTrue(collector.values.isEmpty()); + + collector.timestamps.clear(); + collector.values.clear(); + Map screenFillAttributes = new HashMap<>(); + screenFillAttributes.put("method", "screen"); + UDFParameters screenFillParameters = createSingleDoubleSeriesParameters(screenFillAttributes); + UDTFValueFill screenFill = new UDTFValueFill(); + screenFill.validate(new UDFParameterValidator(screenFillParameters)); + screenFill.beforeStart(screenFillParameters, new UDTFConfigurations(ZoneId.systemDefault())); + screenFill.transform(new SimpleRowWindow(nullDoubleRow(1), nullDoubleRow(2)), collector); + Assert.assertTrue(collector.timestamps.isEmpty()); + Assert.assertTrue(collector.values.isEmpty()); + + collector.timestamps.clear(); + collector.values.clear(); + Map arFillAttributes = new HashMap<>(); + arFillAttributes.put("method", "ar"); + UDFParameters arFillParameters = createSingleDoubleSeriesParameters(arFillAttributes); + UDTFValueFill arFill = new UDTFValueFill(); + arFill.validate(new UDFParameterValidator(arFillParameters)); + arFill.beforeStart(arFillParameters, new UDTFConfigurations(ZoneId.systemDefault())); + arFill.transform( + new SimpleRowWindow(new DoubleRow(1, 4.0), new DoubleRow(2, 2.0), nullDoubleRow(3)), + collector); + Assert.assertEquals(Arrays.asList(1L, 2L, 3L), collector.timestamps); + Assert.assertEquals(1.2, collector.values.get(2), 1e-9); + + collector.timestamps.clear(); + collector.values.clear(); + UDTFValueRepair valueRepair = new UDTFValueRepair(); + valueRepair.validate(new UDFParameterValidator(parameters)); + valueRepair.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + valueRepair.transform( + new SimpleRowWindow(nullDoubleRow(1), new DoubleRow(2, 5.0), new DoubleRow(3, 7.0)), + collector); + Assert.assertEquals(Arrays.asList(1L, 2L, 3L), collector.timestamps); + + collector.timestamps.clear(); + collector.values.clear(); + valueRepair.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + valueRepair.transform(new SimpleRowWindow(nullDoubleRow(1), nullDoubleRow(2)), collector); + Assert.assertTrue(collector.timestamps.isEmpty()); + Assert.assertTrue(collector.values.isEmpty()); + + collector.timestamps.clear(); + collector.values.clear(); + UDTFTimestampRepair timestampRepair = new UDTFTimestampRepair(); + timestampRepair.validate(new UDFParameterValidator(parameters)); + timestampRepair.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + timestampRepair.transform(new SimpleRowWindow(new DoubleRow(1, 1.0)), collector); + Assert.assertEquals(Collections.singletonList(1L), collector.timestamps); + Assert.assertEquals(1.0, collector.values.get(0), 0.0); + + collector.timestamps.clear(); + collector.values.clear(); + timestampRepair.transform( + new SimpleRowWindow(new DoubleRow(1, 1.0), new DoubleRow(3, 2.0), new DoubleRow(5, 3.0)), + collector); + Assert.assertEquals(Arrays.asList(1L, 3L, 5L), collector.timestamps); + } + @Test public void testNegativeTimestampStateInitialization() throws Exception { StreamMissDetector detector = new StreamMissDetector(10); @@ -1423,10 +1859,23 @@ private static UDFParameters createTwoDoubleSeriesParameters(Map Arrays.asList("s1", "s2"), Arrays.asList(Type.DOUBLE, Type.DOUBLE), attributes); } + private static void assertInvalidSingleDoubleParameters( + UDTF function, Map attributes) { + Assert.assertThrows( + UDFParameterNotValidException.class, + () -> + function.validate( + new UDFParameterValidator(createSingleDoubleSeriesParameters(attributes)))); + } + private static DoubleRow nullDoubleRow(long time) { return new DoubleRow(time, new double[] {0.0}, new boolean[] {true}); } + private static TextRow nullTextRow(long time) { + return new TextRow(time, null, true); + } + private static int getWindowSize(UDTFKSigma kSigma) throws Exception { Field windowSize = UDTFKSigma.class.getDeclaredField("windowSize"); windowSize.setAccessible(true); @@ -1517,6 +1966,77 @@ public int size() { } } + private static class TextRow implements Row { + + private final long time; + private final String value; + private final boolean isNull; + + private TextRow(long time, String value, boolean isNull) { + this.time = time; + this.value = value; + this.isNull = isNull; + } + + @Override + public long getTime() { + return time; + } + + @Override + public int getInt(int columnIndex) { + throw new UnsupportedOperationException(); + } + + @Override + public long getLong(int columnIndex) { + throw new UnsupportedOperationException(); + } + + @Override + public float getFloat(int columnIndex) { + throw new UnsupportedOperationException(); + } + + @Override + public double getDouble(int columnIndex) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean getBoolean(int columnIndex) { + throw new UnsupportedOperationException(); + } + + @Override + public Binary getBinary(int columnIndex) { + throw new UnsupportedOperationException(); + } + + @Override + public String getString(int columnIndex) { + if (isNull) { + throw new IllegalStateException("Null value should not be read"); + } + return value; + } + + @Override + public Type getDataType(int columnIndex) { + return Type.TEXT; + } + + @Override + public boolean isNull(int columnIndex) { + return isNull; + } + + @Override + public int size() { + return 1; + } + } + private static class SimpleRowWindow implements RowWindow { private final Row[] rows; From 61792b4cd7af98c7854021df0ebdaad5921918d6 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 9 Jun 2026 16:16:50 +0800 Subject: [PATCH 4/6] Fix more library UDF edge cases --- .../apache/iotdb/library/anomaly/UDTFLOF.java | 22 +- .../apache/iotdb/library/dlearn/UDTFAR.java | 13 +- .../apache/iotdb/library/dmatch/UDAFCov.java | 8 +- .../iotdb/library/dmatch/UDAFPearson.java | 20 +- .../iotdb/library/dprofile/UDAFIntegral.java | 6 +- .../iotdb/library/dprofile/UDAFSkew.java | 15 +- .../iotdb/library/dprofile/UDTFPACF.java | 6 +- .../iotdb/library/dprofile/UDTFResample.java | 20 +- .../iotdb/library/dprofile/UDTFSegment.java | 3 + .../iotdb/library/dprofile/UDTFSpline.java | 16 +- .../iotdb/library/dprofile/UDTFZScore.java | 14 +- .../library/dquality/QualityUDTFConfigs.java | 104 ++++ .../library/dquality/UDTFCompleteness.java | 34 +- .../library/dquality/UDTFConsistency.java | 31 +- .../library/dquality/UDTFTimeliness.java | 31 +- .../iotdb/library/dquality/UDTFValidity.java | 31 +- .../library/drepair/UDTFTimestampRepair.java | 40 +- .../iotdb/library/drepair/UDTFValueFill.java | 36 +- .../library/drepair/UDTFValueRepair.java | 19 +- .../frequency/UDFEnvelopeAnalysis.java | 26 +- .../iotdb/library/frequency/UDTFConv.java | 4 +- .../iotdb/library/frequency/UDTFDWT.java | 4 +- .../iotdb/library/frequency/UDTFDeconv.java | 12 +- .../iotdb/library/frequency/UDTFFFT.java | 4 +- .../iotdb/library/frequency/UDTFHighPass.java | 5 +- .../iotdb/library/frequency/UDTFIDWT.java | 4 +- .../iotdb/library/frequency/UDTFIFFT.java | 25 +- .../iotdb/library/frequency/UDTFLowPass.java | 5 +- .../iotdb/library/frequency/util/FFTUtil.java | 6 +- .../series/UDTFConsecutiveSequences.java | 5 +- .../series/UDTFConsecutiveWindows.java | 11 +- .../org/apache/iotdb/library/util/Util.java | 41 +- .../iotdb/library/UDFWindowAndQueueTest.java | 488 ++++++++++++++++++ 33 files changed, 928 insertions(+), 181 deletions(-) create mode 100644 library-udf/src/main/java/org/apache/iotdb/library/dquality/QualityUDTFConfigs.java diff --git a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFLOF.java b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFLOF.java index 32ba5e4335662..333d20906e9c3 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFLOF.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFLOF.java @@ -39,6 +39,7 @@ public class UDTFLOF implements UDTF { private int multipleK; private int dim; private static final String DEFAULT_METHOD = "default"; + private static final String METHOD_SERIES = "series"; private String method = DEFAULT_METHOD; private int window; @@ -126,6 +127,23 @@ public void validate(UDFParameterValidator validator) throws Exception { for (int i = 0; i < validator.getParameters().getChildExpressionsSize(); i++) { validator.validateInputSeriesDataType(i, Type.INT32, Type.INT64, Type.FLOAT, Type.DOUBLE); } + validator + .validate( + k -> (int) k > 0, + "Parameter k should be a positive integer.", + validator.getParameters().getIntOrDefault("k", 3)) + .validate( + window -> (int) window > 0, + "Parameter window should be a positive integer.", + validator.getParameters().getIntOrDefault("window", 10000)) + .validate( + method -> isValidMethod((String) method), + "Method should be default or series.", + validator.getParameters().getStringOrDefault("method", DEFAULT_METHOD)); + } + + private static boolean isValidMethod(String method) { + return DEFAULT_METHOD.equalsIgnoreCase(method) || METHOD_SERIES.equalsIgnoreCase(method); } @Override @@ -143,7 +161,7 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati @Override public void transform(RowWindow rowWindow, PointCollector collector) throws Exception { - if (this.method.equals(DEFAULT_METHOD)) { + if (this.method.equalsIgnoreCase(DEFAULT_METHOD)) { int size = 0; Double[][] knn = new Double[rowWindow.windowSize()][dim]; long[] timestamp = new long[rowWindow.windowSize()]; @@ -181,7 +199,7 @@ public void transform(RowWindow rowWindow, PointCollector collector) throws Exce } } } - } else if (this.method.equals("series")) { + } else if (this.method.equalsIgnoreCase(METHOD_SERIES)) { int size = rowWindow.windowSize() - window + 1; if (size > 0) { List timestamp = new ArrayList<>(); diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dlearn/UDTFAR.java b/library-udf/src/main/java/org/apache/iotdb/library/dlearn/UDTFAR.java index ed02daa82c1e9..6a24e901d023f 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dlearn/UDTFAR.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dlearn/UDTFAR.java @@ -96,6 +96,9 @@ public void terminate(PointCollector collector) throws Exception { } } long interval = maxFreqInterval; + if (interval <= 0) { + return; + } List imputedTimeWindow = new ArrayList<>(); List imputedValueWindow = new ArrayList<>(); @@ -137,7 +140,13 @@ public void terminate(PointCollector collector) throws Exception { for (int j = 1; j <= i - 1; j++) { tmpSum += alphas[j][i - 1] * resultCovariances[i - j]; } + if (!Double.isFinite(epsilons[i - 1]) || Math.abs(epsilons[i - 1]) < 1e-12) { + return; + } kappas[i] = (resultCovariances[i] - tmpSum) / epsilons[i - 1]; + if (!Double.isFinite(kappas[i])) { + return; + } alphas[i][i] = kappas[i]; if (i > 1) { for (int j = 1; j <= i - 1; j++) { @@ -147,7 +156,9 @@ public void terminate(PointCollector collector) throws Exception { epsilons[i] = (1 - kappas[i] * kappas[i]) * epsilons[i - 1]; } for (int i = 1; i <= p; i++) { - collector.putDouble(i, alphas[i][p]); + if (Double.isFinite(alphas[i][p])) { + collector.putDouble(i, alphas[i][p]); + } } } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dmatch/UDAFCov.java b/library-udf/src/main/java/org/apache/iotdb/library/dmatch/UDAFCov.java index 219c4d8de576f..4ac1435d3deb4 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dmatch/UDAFCov.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dmatch/UDAFCov.java @@ -72,11 +72,11 @@ public void transform(Row row, PointCollector collector) throws Exception { @Override public void terminate(PointCollector collector) throws Exception { - if (count > 0) { // calculate Cov only when there is more than 1 point + if (count > 0) { double cov = (sumXY - sumX * sumY / count) / count; - collector.putDouble(0, cov); - } else { - collector.putDouble(0, Double.NaN); + if (Double.isFinite(cov)) { + collector.putDouble(0, cov); + } } } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dmatch/UDAFPearson.java b/library-udf/src/main/java/org/apache/iotdb/library/dmatch/UDAFPearson.java index 30ef838b84b80..9eba802ff5ff9 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dmatch/UDAFPearson.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dmatch/UDAFPearson.java @@ -78,14 +78,20 @@ public void transform(Row row, PointCollector collector) throws Exception { @Override public void terminate(PointCollector collector) throws Exception { - if (count > 0) { // calculate R only when there is more than 1 point - double pearson = - (count * sumXY - sumX * sumY) - / Math.sqrt(count * sumXX - sumX * sumX) - / Math.sqrt(count * sumYY - sumY * sumY); + if (count == 0) { + return; + } + double xVariance = count * sumXX - sumX * sumX; + double yVariance = count * sumYY - sumY * sumY; + if (!Double.isFinite(xVariance) + || !Double.isFinite(yVariance) + || xVariance <= 0 + || yVariance <= 0) { + return; + } + double pearson = (count * sumXY - sumX * sumY) / Math.sqrt(xVariance) / Math.sqrt(yVariance); + if (Double.isFinite(pearson)) { collector.putDouble(0, pearson); - } else { - collector.putDouble(0, Double.NaN); } } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFIntegral.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFIntegral.java index 6a5291ae943cd..57604c8924338 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFIntegral.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFIntegral.java @@ -50,11 +50,9 @@ public void validate(UDFParameterValidator validator) throws Exception { .validateInputSeriesNumber(1) .validateInputSeriesDataType(0, Type.INT32, Type.INT64, Type.FLOAT, Type.DOUBLE) .validate( - x -> (long) x > 0, + x -> Util.isPositiveTime((String) x, validator.getParameters()), "Unknown time unit input. Supported units are ns, us, ms, s, m, h, d.", - Util.parseTime( - validator.getParameters().getStringOrDefault(TIME_UNIT_KEY, TIME_UNIT_S), - validator.getParameters())); + validator.getParameters().getStringOrDefault(TIME_UNIT_KEY, TIME_UNIT_S)); } @Override diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFSkew.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFSkew.java index 374659311350e..3708ea56d89e0 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFSkew.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFSkew.java @@ -72,9 +72,16 @@ public void terminate(PointCollector collector) throws Exception { if (count == 0) { return; } - collector.putDouble( - 0, - (sumX3 / count - 3 * sumX1 / count * sumX2 / count + 2 * Math.pow(sumX1 / count, 3)) - / Math.pow(sumX2 / count - sumX1 / count * sumX1 / count, 1.5)); + double mean = sumX1 / count; + double variance = sumX2 / count - mean * mean; + if (!Double.isFinite(variance) || variance <= 0) { + return; + } + double skew = + (sumX3 / count - 3 * mean * sumX2 / count + 2 * Math.pow(mean, 3)) + / Math.pow(variance, 1.5); + if (Double.isFinite(skew)) { + collector.putDouble(0, skew); + } } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFPACF.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFPACF.java index 9256502dc421c..98b357b95f991 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFPACF.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFPACF.java @@ -84,8 +84,12 @@ public void terminate(PointCollector collector) throws Exception { x[i] -= xmean; } collector.putDouble(timestamp.get(0), 1.0d); + YuleWalker yuleWalker = new YuleWalker(); for (int k = 1; k <= lag; k++) { - collector.putDouble(timestamp.get(k), new YuleWalker().yuleWalker(x, k, method, n)); + double partialCorrelation = yuleWalker.yuleWalker(x, k, method, n); + if (Double.isFinite(partialCorrelation)) { + collector.putDouble(timestamp.get(k), partialCorrelation); + } } } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFResample.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFResample.java index c8df048ef45cb..dd9a1c802216c 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFResample.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFResample.java @@ -30,8 +30,6 @@ import org.apache.iotdb.udf.api.customizer.strategy.RowByRowAccessStrategy; import org.apache.iotdb.udf.api.type.Type; -import java.text.SimpleDateFormat; - /** This function does upsample or downsample of input series. */ public class UDTFResample implements UDTF { private static final String START_PARAM = "start"; @@ -45,9 +43,9 @@ public void validate(UDFParameterValidator validator) throws Exception { .validateInputSeriesDataType(0, Type.DOUBLE, Type.FLOAT, Type.INT32, Type.INT64) .validateRequiredAttribute("every") .validate( - x -> (long) x > 0, + x -> Util.isPositiveTime((String) x, validator.getParameters()), "gap should be a time period whose unit is ms, s, m, h, d.", - Util.parseTime(validator.getParameters().getString("every"), validator.getParameters())) + validator.getParameters().getString("every")) .validate( x -> "min".equals(x) @@ -62,18 +60,17 @@ public void validate(UDFParameterValidator validator) throws Exception { x -> "nan".equals(x) || "ffill".equals(x) || "bfill".equals(x) || "linear".equals(x), "aggr should be min, max, mean, median, first, last.", validator.getParameters().getStringOrDefault("interp", "nan").toLowerCase()); - SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); if (validator.getParameters().hasAttribute(START_PARAM)) { validator.validate( - x -> (long) x > 0, + x -> Util.isPositiveDateTime((String) x), "start should conform to the format yyyy-MM-dd HH:mm:ss.", - format.parse(validator.getParameters().getString(START_PARAM)).getTime()); + validator.getParameters().getString(START_PARAM)); } if (validator.getParameters().hasAttribute("end")) { validator.validate( - x -> (long) x > 0, + x -> Util.isPositiveDateTime((String) x), "end should conform to the format yyyy-MM-dd HH:mm:ss.", - format.parse(validator.getParameters().getString("end")).getTime()); + validator.getParameters().getString("end")); } } @@ -84,14 +81,13 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati long newPeriod = Util.parseTime(parameters.getString("every"), parameters); String aggregator = parameters.getStringOrDefault("aggr", "mean").toLowerCase(); String interpolator = parameters.getStringOrDefault("interp", "nan").toLowerCase(); - SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); long startTime = -1; long endTime = -1; if (parameters.hasAttribute(START_PARAM)) { - startTime = format.parse(parameters.getString(START_PARAM)).getTime(); + startTime = Util.parseDateTime(parameters.getString(START_PARAM)); } if (parameters.hasAttribute("end")) { - endTime = format.parse(parameters.getString("end")).getTime(); + endTime = Util.parseDateTime(parameters.getString("end")); } resampler = new Resampler(newPeriod, aggregator, interpolator, startTime, endTime); } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFSegment.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFSegment.java index 55b469c30fe69..89b1b94cb9c0d 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFSegment.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFSegment.java @@ -98,6 +98,9 @@ public void transform(Row row, PointCollector collector) throws Exception { @Override public void terminate(PointCollector collector) throws Exception { + if (value.isEmpty()) { + return; + } long[] ts = timestamp.stream().mapToLong(Long::valueOf).toArray(); double[] v = value.stream().mapToDouble(Double::valueOf).toArray(); List seg = new ArrayList<>(); diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFSpline.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFSpline.java index 432cb0e64fdef..d6387cca6067c 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFSpline.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFSpline.java @@ -78,9 +78,11 @@ public void transform(Row row, PointCollector collector) throws Exception { Long t = row.getTime(); if (minimumTimestamp == null) { minimumTimestamp = t; + } else if (t <= timestamp.get(timestamp.size() - 1)) { + return; } timestamp.add(t); - xDouble.add((Double.valueOf(Long.toString(t - minimumTimestamp)))); + xDouble.add((double) t - minimumTimestamp); yDouble.add(v); } } @@ -93,14 +95,12 @@ public void terminate(PointCollector collector) throws Exception { double[] y = ArrayUtils.toPrimitive(yDouble.toArray(new Double[0])); psf = asi.interpolate(x, y); for (int i = 0; i < samplePoints; i++) { - int approximation = - (int) - Math.floor( - (x[0] * (samplePoints - 1 - i) + x[yDouble.size() - 1] * (i)) - / (samplePoints - 1) - + 0.5); + double approximation = + Math.floor( + (x[0] * (samplePoints - 1 - i) + x[yDouble.size() - 1] * (i)) / (samplePoints - 1) + + 0.5); double yhead = psf.value(approximation); - collector.putDouble(minimumTimestamp + approximation, yhead); + collector.putDouble(minimumTimestamp + Math.round(approximation), yhead); } } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFZScore.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFZScore.java index b108efbd8ee56..7d19bd949395b 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFZScore.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDTFZScore.java @@ -116,9 +116,19 @@ public void terminate(PointCollector collector) throws Exception { return; } avg = sum / value.size(); - sd = Math.sqrt(squareSum / value.size() - avg * avg); + double variance = squareSum / value.size() - avg * avg; + if (!Double.isFinite(variance) || variance <= 0) { + return; + } + sd = Math.sqrt(variance); + if (!Double.isFinite(sd) || sd <= 0) { + return; + } for (int i = 0; i < value.size(); i++) { - collector.putDouble(timestamp.get(i), (value.get(i) - avg) / sd); + double zScore = (value.get(i) - avg) / sd; + if (Double.isFinite(zScore)) { + collector.putDouble(timestamp.get(i), zScore); + } } } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dquality/QualityUDTFConfigs.java b/library-udf/src/main/java/org/apache/iotdb/library/dquality/QualityUDTFConfigs.java new file mode 100644 index 0000000000000..725b2d1bdaf82 --- /dev/null +++ b/library-udf/src/main/java/org/apache/iotdb/library/dquality/QualityUDTFConfigs.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.library.dquality; + +import org.apache.iotdb.library.i18n.LibraryUdfMessages; +import org.apache.iotdb.library.util.Util; +import org.apache.iotdb.udf.api.customizer.config.UDTFConfigurations; +import org.apache.iotdb.udf.api.customizer.parameter.UDFParameterValidator; +import org.apache.iotdb.udf.api.customizer.parameter.UDFParameters; +import org.apache.iotdb.udf.api.customizer.strategy.SlidingSizeWindowAccessStrategy; +import org.apache.iotdb.udf.api.customizer.strategy.SlidingTimeWindowAccessStrategy; +import org.apache.iotdb.udf.api.exception.UDFException; +import org.apache.iotdb.udf.api.type.Type; + +final class QualityUDTFConfigs { + private static final String WINDOW_KEY = "window"; + + private QualityUDTFConfigs() { + throw new IllegalStateException(LibraryUdfMessages.UTILITY_CLASS); + } + + static void validate(UDFParameterValidator validator) throws Exception { + validator + .validateInputSeriesNumber(1) + .validateInputSeriesDataType(0, Type.INT32, Type.INT64, Type.FLOAT, Type.DOUBLE); + + if (validator.getParameters().hasAttribute(WINDOW_KEY)) { + validator.validate( + window -> isValidWindow((String) window, validator.getParameters()), + "Parameter window should be a positive duration or a positive integer.", + validator.getParameters().getString(WINDOW_KEY)); + } + } + + static void configureAccessStrategy(UDFParameters parameters, UDTFConfigurations configurations) { + WindowConfig windowConfig = parseWindow(parameters); + if (windowConfig.isTimeWindow) { + configurations.setAccessStrategy(new SlidingTimeWindowAccessStrategy(windowConfig.window)); + } else { + configurations.setAccessStrategy( + new SlidingSizeWindowAccessStrategy((int) windowConfig.window)); + } + } + + static boolean isInsufficientValidData(UDFException exception) { + return LibraryUdfMessages.AT_LEAST_TWO_NON_NAN_VALUES_NEEDED.equals(exception.getMessage()); + } + + private static boolean isValidWindow(String window, UDFParameters parameters) { + try { + parseWindow(window, parameters); + return true; + } catch (RuntimeException e) { + return false; + } + } + + private static WindowConfig parseWindow(UDFParameters parameters) { + if (!parameters.hasAttribute(WINDOW_KEY)) { + return new WindowConfig(false, Integer.MAX_VALUE); + } + return parseWindow(parameters.getString(WINDOW_KEY), parameters); + } + + private static WindowConfig parseWindow(String window, UDFParameters parameters) { + long parsedWindow = Util.parseTime(window, parameters); + if (parsedWindow > 0) { + return new WindowConfig(true, parsedWindow); + } + + long sizeWindow = Long.parseLong(window.replace(" ", "")); + if (sizeWindow <= 0 || sizeWindow > Integer.MAX_VALUE) { + throw new IllegalArgumentException("Window size should be in (0, Integer.MAX_VALUE]."); + } + return new WindowConfig(false, sizeWindow); + } + + private static class WindowConfig { + private final boolean isTimeWindow; + private final long window; + + private WindowConfig(boolean isTimeWindow, long window) { + this.isTimeWindow = isTimeWindow; + this.window = window; + } + } +} diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFCompleteness.java b/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFCompleteness.java index 8b349240386f9..a8cc9d1a75c3f 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFCompleteness.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFCompleteness.java @@ -20,14 +20,14 @@ package org.apache.iotdb.library.dquality; import org.apache.iotdb.library.dquality.util.TimeSeriesQuality; -import org.apache.iotdb.library.util.Util; +import org.apache.iotdb.library.util.NoNumberException; import org.apache.iotdb.udf.api.UDTF; import org.apache.iotdb.udf.api.access.RowWindow; import org.apache.iotdb.udf.api.collector.PointCollector; import org.apache.iotdb.udf.api.customizer.config.UDTFConfigurations; +import org.apache.iotdb.udf.api.customizer.parameter.UDFParameterValidator; import org.apache.iotdb.udf.api.customizer.parameter.UDFParameters; -import org.apache.iotdb.udf.api.customizer.strategy.SlidingSizeWindowAccessStrategy; -import org.apache.iotdb.udf.api.customizer.strategy.SlidingTimeWindowAccessStrategy; +import org.apache.iotdb.udf.api.exception.UDFException; import org.apache.iotdb.udf.api.type.Type; import java.io.IOException; @@ -39,24 +39,14 @@ public class UDTFCompleteness implements UDTF { private boolean downtime; + @Override + public void validate(UDFParameterValidator validator) throws Exception { + QualityUDTFConfigs.validate(validator); + } + @Override public void beforeStart(UDFParameters udfp, UDTFConfigurations udtfc) throws Exception { - boolean isTime = false; - long window = Integer.MAX_VALUE; - if (udfp.hasAttribute("window")) { - String s = udfp.getString("window"); - window = Util.parseTime(s, udfp); - if (window > 0) { - isTime = true; - } else { - window = Long.parseLong(s); - } - } - if (isTime) { - udtfc.setAccessStrategy(new SlidingTimeWindowAccessStrategy(window)); - } else { - udtfc.setAccessStrategy(new SlidingSizeWindowAccessStrategy((int) window)); - } + QualityUDTFConfigs.configureAccessStrategy(udfp, udtfc); udtfc.setOutputDataType(Type.DOUBLE); downtime = udfp.getBooleanOrDefault("downtime", true); } @@ -70,8 +60,12 @@ public void transform(RowWindow rowWindow, PointCollector collector) throws Exce tsq.timeDetect(); collector.putDouble(rowWindow.getRow(0).getTime(), tsq.getCompleteness()); } - } catch (IOException ex) { + } catch (IOException | NoNumberException ex) { Logger.getLogger(UDTFCompleteness.class.getName()).log(Level.SEVERE, null, ex); + } catch (UDFException ex) { + if (!QualityUDTFConfigs.isInsufficientValidData(ex)) { + throw ex; + } } } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFConsistency.java b/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFConsistency.java index f7e7af981c252..8de906b8570e9 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFConsistency.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFConsistency.java @@ -21,14 +21,13 @@ import org.apache.iotdb.library.dquality.util.TimeSeriesQuality; import org.apache.iotdb.library.util.NoNumberException; -import org.apache.iotdb.library.util.Util; import org.apache.iotdb.udf.api.UDTF; import org.apache.iotdb.udf.api.access.RowWindow; import org.apache.iotdb.udf.api.collector.PointCollector; import org.apache.iotdb.udf.api.customizer.config.UDTFConfigurations; +import org.apache.iotdb.udf.api.customizer.parameter.UDFParameterValidator; import org.apache.iotdb.udf.api.customizer.parameter.UDFParameters; -import org.apache.iotdb.udf.api.customizer.strategy.SlidingSizeWindowAccessStrategy; -import org.apache.iotdb.udf.api.customizer.strategy.SlidingTimeWindowAccessStrategy; +import org.apache.iotdb.udf.api.exception.UDFException; import org.apache.iotdb.udf.api.type.Type; import java.io.IOException; @@ -38,24 +37,14 @@ /** This function calculates consistency of input series. */ public class UDTFConsistency implements UDTF { + @Override + public void validate(UDFParameterValidator validator) throws Exception { + QualityUDTFConfigs.validate(validator); + } + @Override public void beforeStart(UDFParameters udfp, UDTFConfigurations udtfc) throws Exception { - boolean isTime = false; - long window = Integer.MAX_VALUE; - if (udfp.hasAttribute("window")) { - String s = udfp.getString("window"); - window = Util.parseTime(s, udfp); - if (window > 0) { - isTime = true; - } else { - window = Long.parseLong(s); - } - } - if (isTime) { - udtfc.setAccessStrategy(new SlidingTimeWindowAccessStrategy(window)); - } else { - udtfc.setAccessStrategy(new SlidingSizeWindowAccessStrategy((int) window)); - } + QualityUDTFConfigs.configureAccessStrategy(udfp, udtfc); udtfc.setOutputDataType(Type.DOUBLE); } @@ -69,6 +58,10 @@ public void transform(RowWindow rowWindow, PointCollector collector) throws Exce } } catch (IOException | NoNumberException ex) { Logger.getLogger(UDTFConsistency.class.getName()).log(Level.SEVERE, null, ex); + } catch (UDFException ex) { + if (!QualityUDTFConfigs.isInsufficientValidData(ex)) { + throw ex; + } } } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFTimeliness.java b/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFTimeliness.java index e1695d784a248..fc0af472c903b 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFTimeliness.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFTimeliness.java @@ -21,14 +21,13 @@ import org.apache.iotdb.library.dquality.util.TimeSeriesQuality; import org.apache.iotdb.library.util.NoNumberException; -import org.apache.iotdb.library.util.Util; import org.apache.iotdb.udf.api.UDTF; import org.apache.iotdb.udf.api.access.RowWindow; import org.apache.iotdb.udf.api.collector.PointCollector; import org.apache.iotdb.udf.api.customizer.config.UDTFConfigurations; +import org.apache.iotdb.udf.api.customizer.parameter.UDFParameterValidator; import org.apache.iotdb.udf.api.customizer.parameter.UDFParameters; -import org.apache.iotdb.udf.api.customizer.strategy.SlidingSizeWindowAccessStrategy; -import org.apache.iotdb.udf.api.customizer.strategy.SlidingTimeWindowAccessStrategy; +import org.apache.iotdb.udf.api.exception.UDFException; import org.apache.iotdb.udf.api.type.Type; import java.io.IOException; @@ -38,24 +37,14 @@ /** This function calculates timeliness of input series. */ public class UDTFTimeliness implements UDTF { + @Override + public void validate(UDFParameterValidator validator) throws Exception { + QualityUDTFConfigs.validate(validator); + } + @Override public void beforeStart(UDFParameters udfp, UDTFConfigurations udtfc) throws Exception { - boolean isTime = false; - long window = Integer.MAX_VALUE; - if (udfp.hasAttribute("window")) { - String s = udfp.getString("window"); - window = Util.parseTime(s, udfp); - if (window > 0) { - isTime = true; - } else { - window = Long.parseLong(s); - } - } - if (isTime) { - udtfc.setAccessStrategy(new SlidingTimeWindowAccessStrategy(window)); - } else { - udtfc.setAccessStrategy(new SlidingSizeWindowAccessStrategy((int) window)); - } + QualityUDTFConfigs.configureAccessStrategy(udfp, udtfc); udtfc.setOutputDataType(Type.DOUBLE); } @@ -69,6 +58,10 @@ public void transform(RowWindow rowWindow, PointCollector collector) throws Exce } } catch (IOException | NoNumberException ex) { Logger.getLogger(UDTFTimeliness.class.getName()).log(Level.SEVERE, null, ex); + } catch (UDFException ex) { + if (!QualityUDTFConfigs.isInsufficientValidData(ex)) { + throw ex; + } } } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFValidity.java b/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFValidity.java index d642e557ec94b..9a10506cef848 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFValidity.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dquality/UDTFValidity.java @@ -21,14 +21,13 @@ import org.apache.iotdb.library.dquality.util.TimeSeriesQuality; import org.apache.iotdb.library.util.NoNumberException; -import org.apache.iotdb.library.util.Util; import org.apache.iotdb.udf.api.UDTF; import org.apache.iotdb.udf.api.access.RowWindow; import org.apache.iotdb.udf.api.collector.PointCollector; import org.apache.iotdb.udf.api.customizer.config.UDTFConfigurations; +import org.apache.iotdb.udf.api.customizer.parameter.UDFParameterValidator; import org.apache.iotdb.udf.api.customizer.parameter.UDFParameters; -import org.apache.iotdb.udf.api.customizer.strategy.SlidingSizeWindowAccessStrategy; -import org.apache.iotdb.udf.api.customizer.strategy.SlidingTimeWindowAccessStrategy; +import org.apache.iotdb.udf.api.exception.UDFException; import org.apache.iotdb.udf.api.type.Type; import java.io.IOException; @@ -38,24 +37,14 @@ /** This function calculates validity of input series. */ public class UDTFValidity implements UDTF { + @Override + public void validate(UDFParameterValidator validator) throws Exception { + QualityUDTFConfigs.validate(validator); + } + @Override public void beforeStart(UDFParameters udfp, UDTFConfigurations udtfc) throws Exception { - boolean isTime = false; - long window = Integer.MAX_VALUE; - if (udfp.hasAttribute("window")) { - String s = udfp.getString("window"); - window = Util.parseTime(s, udfp); - if (window > 0) { - isTime = true; - } else { - window = Long.parseLong(s); - } - } - if (isTime) { - udtfc.setAccessStrategy(new SlidingTimeWindowAccessStrategy(window)); - } else { - udtfc.setAccessStrategy(new SlidingSizeWindowAccessStrategy((int) window)); - } + QualityUDTFConfigs.configureAccessStrategy(udfp, udtfc); udtfc.setOutputDataType(Type.DOUBLE); } @@ -69,6 +58,10 @@ public void transform(RowWindow rowWindow, PointCollector collector) throws Exce } } catch (IOException | NoNumberException ex) { Logger.getLogger(UDTFValidity.class.getName()).log(Level.SEVERE, null, ex); + } catch (UDFException ex) { + if (!QualityUDTFConfigs.isInsufficientValidData(ex)) { + throw ex; + } } } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/drepair/UDTFTimestampRepair.java b/library-udf/src/main/java/org/apache/iotdb/library/drepair/UDTFTimestampRepair.java index 873754a11943a..d7be52d869df1 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/drepair/UDTFTimestampRepair.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/drepair/UDTFTimestampRepair.java @@ -34,6 +34,10 @@ /** This function is used for timestamp repair. */ public class UDTFTimestampRepair implements UDTF { + private static final String METHOD_MEDIAN = "Median"; + private static final String METHOD_MODE = "Mode"; + private static final String METHOD_CLUSTER = "Cluster"; + String intervalMethod; long interval; long intervalMode; @@ -42,7 +46,11 @@ public class UDTFTimestampRepair implements UDTF { public void validate(UDFParameterValidator validator) throws Exception { validator .validateInputSeriesNumber(1) - .validateInputSeriesDataType(0, Type.DOUBLE, Type.FLOAT, Type.INT32, Type.INT64); + .validateInputSeriesDataType(0, Type.DOUBLE, Type.FLOAT, Type.INT32, Type.INT64) + .validate( + method -> isValidMethod((String) method), + "Method should be Median, Mode, or Cluster.", + validator.getParameters().getStringOrDefault("method", METHOD_MEDIAN)); String intervalString = validator.getParameters().getStringOrDefault("interval", null); if (intervalString != null) { @@ -61,6 +69,12 @@ public void validate(UDFParameterValidator validator) throws Exception { } } + private static boolean isValidMethod(String method) { + return METHOD_MEDIAN.equalsIgnoreCase(method) + || METHOD_MODE.equalsIgnoreCase(method) + || METHOD_CLUSTER.equalsIgnoreCase(method); + } + @Override public void beforeStart(UDFParameters parameters, UDTFConfigurations configurations) throws Exception { @@ -68,7 +82,7 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati .setAccessStrategy(new SlidingSizeWindowAccessStrategy(Integer.MAX_VALUE)) .setOutputDataType(parameters.getDataType(0)); - intervalMethod = parameters.getStringOrDefault("method", "Median"); + intervalMethod = parameters.getStringOrDefault("method", METHOD_MEDIAN); String intervalString = parameters.getStringOrDefault("interval", null); if (intervalString != null) { @@ -83,11 +97,11 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati if (interval > 0) { intervalMode = interval; - } else if ("Median".equalsIgnoreCase(intervalMethod)) { + } else if (METHOD_MEDIAN.equalsIgnoreCase(intervalMethod)) { intervalMode = -1L; - } else if ("Mode".equalsIgnoreCase(intervalMethod)) { + } else if (METHOD_MODE.equalsIgnoreCase(intervalMethod)) { intervalMode = -2L; - } else if ("Cluster".equalsIgnoreCase(intervalMethod)) { + } else if (METHOD_CLUSTER.equalsIgnoreCase(intervalMethod)) { intervalMode = -3L; } else { throw new UDFException(LibraryUdfMessages.ILLEGAL_METHOD_WITH_DOT); @@ -103,22 +117,30 @@ public void transform(RowWindow rowWindow, PointCollector collector) throws Exce switch (rowWindow.getDataType(0)) { case DOUBLE: for (int i = 0; i < timestamp.length; i++) { - collector.putDouble(timestamp[i], value[i]); + if (Double.isFinite(value[i])) { + collector.putDouble(timestamp[i], value[i]); + } } break; case FLOAT: for (int i = 0; i < timestamp.length; i++) { - collector.putFloat(timestamp[i], (float) value[i]); + if (Double.isFinite(value[i])) { + collector.putFloat(timestamp[i], (float) value[i]); + } } break; case INT32: for (int i = 0; i < timestamp.length; i++) { - collector.putInt(timestamp[i], (int) value[i]); + if (Double.isFinite(value[i])) { + collector.putInt(timestamp[i], (int) value[i]); + } } break; case INT64: for (int i = 0; i < timestamp.length; i++) { - collector.putLong(timestamp[i], (long) value[i]); + if (Double.isFinite(value[i])) { + collector.putLong(timestamp[i], (long) value[i]); + } } break; case DATE: diff --git a/library-udf/src/main/java/org/apache/iotdb/library/drepair/UDTFValueFill.java b/library-udf/src/main/java/org/apache/iotdb/library/drepair/UDTFValueFill.java index dd3db8a869691..d16bd1f1c6c8f 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/drepair/UDTFValueFill.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/drepair/UDTFValueFill.java @@ -39,13 +39,33 @@ /** This function is used to interpolate time series. */ public class UDTFValueFill implements UDTF { + private static final String METHOD_PREVIOUS = "previous"; + private static final String METHOD_LINEAR = "linear"; + private static final String METHOD_MEAN = "mean"; + private static final String METHOD_AR = "ar"; + private static final String METHOD_SCREEN = "screen"; + private static final String METHOD_LIKELIHOOD = "likelihood"; + private String method; @Override public void validate(UDFParameterValidator validator) throws Exception { validator .validateInputSeriesNumber(1) - .validateInputSeriesDataType(0, Type.FLOAT, Type.DOUBLE, Type.INT32, Type.INT64); + .validateInputSeriesDataType(0, Type.FLOAT, Type.DOUBLE, Type.INT32, Type.INT64) + .validate( + method -> isValidMethod((String) method), + "Illegal fill method.", + validator.getParameters().getStringOrDefault("method", METHOD_LINEAR)); + } + + private static boolean isValidMethod(String method) { + return METHOD_PREVIOUS.equalsIgnoreCase(method) + || METHOD_LINEAR.equalsIgnoreCase(method) + || METHOD_MEAN.equalsIgnoreCase(method) + || METHOD_AR.equalsIgnoreCase(method) + || METHOD_SCREEN.equalsIgnoreCase(method) + || METHOD_LIKELIHOOD.equalsIgnoreCase(method); } @Override @@ -54,23 +74,23 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati configurations .setAccessStrategy(new SlidingSizeWindowAccessStrategy(Integer.MAX_VALUE)) .setOutputDataType(parameters.getDataType(0)); - method = parameters.getStringOrDefault("method", "linear"); + method = parameters.getStringOrDefault("method", METHOD_LINEAR); } @Override public void transform(RowWindow rowWindow, PointCollector collector) throws Exception { ValueFill vf; - if ("previous".equalsIgnoreCase(method)) { + if (METHOD_PREVIOUS.equalsIgnoreCase(method)) { vf = new PreviousFill(rowWindow.getRowIterator()); - } else if ("linear".equalsIgnoreCase(method)) { + } else if (METHOD_LINEAR.equalsIgnoreCase(method)) { vf = new LinearFill(rowWindow.getRowIterator()); - } else if ("mean".equalsIgnoreCase(method)) { + } else if (METHOD_MEAN.equalsIgnoreCase(method)) { vf = new MeanFill(rowWindow.getRowIterator()); - } else if ("ar".equalsIgnoreCase(method)) { + } else if (METHOD_AR.equalsIgnoreCase(method)) { vf = new ARFill(rowWindow.getRowIterator()); - } else if ("screen".equalsIgnoreCase(method)) { + } else if (METHOD_SCREEN.equalsIgnoreCase(method)) { vf = new ScreenFill(rowWindow.getRowIterator()); - } else if ("likelihood".equalsIgnoreCase(method)) { + } else if (METHOD_LIKELIHOOD.equalsIgnoreCase(method)) { vf = new LikelihoodFill(rowWindow.getRowIterator()); } else { throw new UDFException(LibraryUdfMessages.ILLEGAL_METHOD); diff --git a/library-udf/src/main/java/org/apache/iotdb/library/drepair/UDTFValueRepair.java b/library-udf/src/main/java/org/apache/iotdb/library/drepair/UDTFValueRepair.java index 6c2b5ca57114c..ff95f47854e28 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/drepair/UDTFValueRepair.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/drepair/UDTFValueRepair.java @@ -35,6 +35,9 @@ /** This function is used to repair the value of the time series. */ public class UDTFValueRepair implements UDTF { + private static final String METHOD_SCREEN = "screen"; + private static final String METHOD_LS_GREEDY = "lsgreedy"; + String method; double minSpeed; double maxSpeed; @@ -61,7 +64,15 @@ public void validate(UDFParameterValidator validator) throws Exception { .validate( x -> Double.isFinite((double) x), "Parameter $center$ should be finite.", - validator.getParameters().getDoubleOrDefault("center", 0)); + validator.getParameters().getDoubleOrDefault("center", 0)) + .validate( + method -> isValidMethod((String) method), + "Method should be screen or lsgreedy.", + validator.getParameters().getStringOrDefault("method", METHOD_SCREEN)); + } + + private static boolean isValidMethod(String method) { + return METHOD_SCREEN.equalsIgnoreCase(method) || METHOD_LS_GREEDY.equalsIgnoreCase(method); } @Override @@ -70,7 +81,7 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati configurations .setAccessStrategy(new SlidingSizeWindowAccessStrategy(Integer.MAX_VALUE)) .setOutputDataType(parameters.getDataType(0)); - method = parameters.getStringOrDefault("method", "screen"); + method = parameters.getStringOrDefault("method", METHOD_SCREEN); minSpeed = parameters.getDoubleOrDefault("minSpeed", Double.NaN); maxSpeed = parameters.getDoubleOrDefault("maxSpeed", Double.NaN); center = parameters.getDoubleOrDefault("center", 0); @@ -81,7 +92,7 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati public void transform(RowWindow rowWindow, PointCollector collector) throws Exception { ValueRepair vr; try { - if ("screen".equalsIgnoreCase(method)) { + if (METHOD_SCREEN.equalsIgnoreCase(method)) { Screen screen = new Screen(rowWindow.getRowIterator()); if (!Double.isNaN(minSpeed)) { screen.setSmin(minSpeed); @@ -90,7 +101,7 @@ public void transform(RowWindow rowWindow, PointCollector collector) throws Exce screen.setSmax(maxSpeed); } vr = screen; - } else if ("lsgreedy".equalsIgnoreCase(method)) { + } else if (METHOD_LS_GREEDY.equalsIgnoreCase(method)) { LsGreedy lsGreedy = new LsGreedy(rowWindow.getRowIterator()); if (!Double.isNaN(sigma)) { lsGreedy.setSigma(sigma); diff --git a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDFEnvelopeAnalysis.java b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDFEnvelopeAnalysis.java index 8ca0cb3f1a024..4ce429829c235 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDFEnvelopeAnalysis.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDFEnvelopeAnalysis.java @@ -42,6 +42,7 @@ public class UDFEnvelopeAnalysis implements UDTF { private double frequency; + private boolean hasFrequency; private int amplification; private String timestampPrecision; private final DoubleArrayList signals = new DoubleArrayList(); @@ -61,7 +62,7 @@ public void validate(UDFParameterValidator validator) throws Exception { .validate( x -> Double.isFinite((double) x) && (double) x > 0, "The param 'frequency' must be finite and > 0.", - validator.getParameters().getDoubleOrDefault(FREQUENCY, Double.MAX_VALUE)) + validator.getParameters().getDoubleOrDefault(FREQUENCY, 1.0d)) .validate( x -> (int) x >= 1, "The param 'amplification' must >= 1.", @@ -74,7 +75,8 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati configurations.setAccessStrategy(new RowByRowAccessStrategy()).setOutputDataType(Type.DOUBLE); signals.clear(); timestamps.clear(); - frequency = parameters.getDoubleOrDefault(FREQUENCY, Double.MAX_VALUE); + hasFrequency = parameters.hasAttribute(FREQUENCY); + frequency = parameters.getDoubleOrDefault(FREQUENCY, 0.0d); amplification = parameters.getIntOrDefault(AMPLIFICATION, 1); timestampPrecision = parameters.getSystemStringOrDefault(TIMESTAMP_PRECISION, MS_PRECISION); } @@ -99,14 +101,27 @@ public void terminate(PointCollector collector) throws Exception { return; } double[] envelopeValues = envelopeAnalyze(signals.toArray()); - frequency = frequency != Double.MAX_VALUE ? frequency : calculateFrequency(timestamps); + frequency = hasFrequency ? frequency : calculateFrequency(timestamps); + if (!Double.isFinite(frequency) || frequency <= 0) { + return; + } int signalSize = signals.size(); double[] frequencies = new double[signalSize / 2]; + double frequencyStep = frequency * amplification / signalSize; + if (!Double.isFinite(frequencyStep)) { + return; + } for (int i = 0; i < signalSize / 2; i++) { - frequencies[i] = i * (frequency * amplification / signalSize); + frequencies[i] = i * frequencyStep; } for (int i = 0; i < envelopeValues.length; i++) { + if (!Double.isFinite(frequencies[i]) + || !Double.isFinite(envelopeValues[i]) + || frequencies[i] < 0 + || frequencies[i] > Long.MAX_VALUE) { + continue; + } collector.putDouble((long) frequencies[i], envelopeValues[i]); } } @@ -252,6 +267,9 @@ public double getValueAsDouble(Row row, int index) throws IOException { } public static double calculateFrequencyByTimeUnit(long time, String timeUnit) { + if (time <= 0) { + return Double.NaN; + } switch (timeUnit) { case MS_PRECISION: return 1000.0 / time; diff --git a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFConv.java b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFConv.java index a7cbfb6397692..54932568f7ab6 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFConv.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFConv.java @@ -75,7 +75,9 @@ public void terminate(PointCollector collector) throws Exception { } } for (int i = 0; i < ans.length; i++) { - collector.putDouble(i, ans[i]); + if (Double.isFinite(ans[i])) { + collector.putDouble(i, ans[i]); + } } } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFDWT.java b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFDWT.java index 92a54903a6903..c3945892a5372 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFDWT.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFDWT.java @@ -103,7 +103,9 @@ public void terminate(PointCollector pointCollector) throws Exception { transformer.waveletTransform(); double[] r = transformer.getData(); for (int i = 0; i < r.length; i++) { - pointCollector.putDouble(timestamp.get(i), r[i]); + if (Double.isFinite(r[i])) { + pointCollector.putDouble(timestamp.get(i), r[i]); + } } } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFDeconv.java b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFDeconv.java index 440c6388ae67c..0e3c033698b92 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFDeconv.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFDeconv.java @@ -93,7 +93,9 @@ public void terminate(PointCollector collector) throws Exception { collector.putDouble(0, 0); } else { // residue for (int i = 0; i < list1.size(); i++) { - collector.putDouble(i, list1.get(i)); + if (Double.isFinite(list1.get(i))) { + collector.putDouble(i, list1.get(i)); + } } } } else { // order of divisor is no larger than dividend @@ -109,11 +111,15 @@ public void terminate(PointCollector collector) throws Exception { } if (isQuotientResult(result)) { // quotient for (int i = 0; i < q.length; i++) { - collector.putDouble(i, q[i]); + if (Double.isFinite(q[i])) { + collector.putDouble(i, q[i]); + } } } else { // residue for (int i = 0; i < r.length; i++) { - collector.putDouble(i, r[i]); + if (Double.isFinite(r[i])) { + collector.putDouble(i, r[i]); + } } } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFFFT.java b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFFFT.java index 004b7df602676..2389e55231509 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFFFT.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFFFT.java @@ -33,6 +33,8 @@ import org.eclipse.collections.impl.list.mutable.primitive.DoubleArrayList; import org.jtransforms.fft.DoubleFFT_1D; +import java.util.Locale; + /** This function does Fast Fourier Transform for input series. */ public class UDTFFFT implements UDTF { @@ -71,7 +73,7 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati throws Exception { configurations.setAccessStrategy(new RowByRowAccessStrategy()).setOutputDataType(Type.DOUBLE); list.clear(); - String result = parameters.getStringOrDefault("result", "abs"); + String result = parameters.getStringOrDefault("result", "abs").toLowerCase(Locale.ROOT); this.compressed = parameters.hasAttribute(COMPRESS_PARAM); double compressRate = parameters.getDoubleOrDefault(COMPRESS_PARAM, 1); this.fftutil = new FFTUtil(result, compressRate); diff --git a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFHighPass.java b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFHighPass.java index 2220fa58698d8..7e83c95cfee62 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFHighPass.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFHighPass.java @@ -99,7 +99,10 @@ public void terminate(PointCollector collector) throws Exception { fft.complexInverse(a, true); // output for (int i = 0; i < n; i++) { - collector.putDouble(timeList.get(i), a[i * 2]); + double value = a[i * 2]; + if (Double.isFinite(value)) { + collector.putDouble(timeList.get(i), value); + } } } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFIDWT.java b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFIDWT.java index b0a5aaf455b85..6ac1ad11672b7 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFIDWT.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFIDWT.java @@ -104,7 +104,9 @@ public void terminate(PointCollector pointCollector) throws Exception { transformer.inverse(); double[] r = transformer.getData(); for (int i = 0; i < r.length; i++) { - pointCollector.putDouble(timestamp.get(i), r[i]); + if (Double.isFinite(r[i])) { + pointCollector.putDouble(timestamp.get(i), r[i]); + } } } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFIFFT.java b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFIFFT.java index e01e580bd00f3..c89d2b0cfc510 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFIFFT.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFIFFT.java @@ -33,8 +33,6 @@ import org.eclipse.collections.impl.list.mutable.primitive.IntArrayList; import org.jtransforms.fft.DoubleFFT_1D; -import java.text.SimpleDateFormat; - /** This function does Inverse Fast Fourier Transform for input series. */ public class UDTFIFFT implements UDTF { @@ -53,17 +51,14 @@ public void validate(UDFParameterValidator validator) throws Exception { .validateInputSeriesDataType(0, Type.DOUBLE, Type.FLOAT, Type.INT32, Type.INT64) .validateInputSeriesDataType(1, Type.DOUBLE, Type.FLOAT, Type.INT32, Type.INT64) .validate( - x -> (long) x > 0, + x -> Util.isPositiveTime((String) x, validator.getParameters()), "interval should be a time period whose unit is ms, s, m, h, d.", - Util.parseTime( - validator.getParameters().getStringOrDefault("interval", "1s"), - validator.getParameters())); - SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + validator.getParameters().getStringOrDefault("interval", "1s")); if (validator.getParameters().hasAttribute(START_PARAM)) { validator.validate( - x -> (long) x > 0, + x -> Util.isPositiveDateTime((String) x), "start should conform to the format yyyy-MM-dd HH:mm:ss.", - format.parse(validator.getParameters().getString(START_PARAM)).getTime()); + validator.getParameters().getString(START_PARAM)); } } @@ -75,10 +70,9 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati imag.clear(); time.clear(); this.interval = Util.parseTime(parameters.getStringOrDefault("interval", "1s"), parameters); - SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); this.start = 0; if (parameters.hasAttribute(START_PARAM)) { - this.start = format.parse(parameters.getString(START_PARAM)).getTime(); + this.start = Util.parseDateTime(parameters.getString(START_PARAM)); } } @@ -120,7 +114,14 @@ public void terminate(PointCollector collector) throws Exception { DoubleFFT_1D fft = new DoubleFFT_1D(n); fft.complexInverse(a, true); for (int i = 0; i < n; i++) { - collector.putDouble(start + i * interval, a[2 * i]); + double value = a[2 * i]; + if (Double.isFinite(value)) { + try { + collector.putDouble(Math.addExact(start, Math.multiplyExact((long) i, interval)), value); + } catch (ArithmeticException ignored) { + // skip timestamps that overflow the long range + } + } } } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFLowPass.java b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFLowPass.java index dc5bc49c685a8..f7ee03e1c6c3f 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFLowPass.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/frequency/UDTFLowPass.java @@ -96,7 +96,10 @@ public void terminate(PointCollector collector) throws Exception { fft.complexInverse(a, true); // output for (int i = 0; i < n; i++) { - collector.putDouble(timeList.get(i), a[i * 2]); + double value = a[i * 2]; + if (Double.isFinite(value)) { + collector.putDouble(timeList.get(i), value); + } } } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/frequency/util/FFTUtil.java b/library-udf/src/main/java/org/apache/iotdb/library/frequency/util/FFTUtil.java index f4f92cef6f6e1..d65852850f4ff 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/frequency/util/FFTUtil.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/frequency/util/FFTUtil.java @@ -71,7 +71,7 @@ public void add(PointCollector collector, double[] a, int i) throws Exception { ans = a[i * 2 + 1]; break; case "abs": - ans = Math.sqrt(a[i * 2] * a[i * 2] + a[2 * i + 1] * a[2 * i + 1]); + ans = Math.hypot(a[i * 2], a[2 * i + 1]); break; case "angle": ans = Math.atan2(a[2 * i + 1], a[2 * i]); @@ -79,7 +79,9 @@ public void add(PointCollector collector, double[] a, int i) throws Exception { default: throw new UDFException(LibraryUdfMessages.ITS_IMPOSSIBLE); } - collector.putDouble(i, ans); + if (Double.isFinite(ans)) { + collector.putDouble(i, ans); + } } public void outputUncompressed(PointCollector collector, double[] a) throws Exception { diff --git a/library-udf/src/main/java/org/apache/iotdb/library/series/UDTFConsecutiveSequences.java b/library-udf/src/main/java/org/apache/iotdb/library/series/UDTFConsecutiveSequences.java index a13caa6324319..1b92db0a3201f 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/series/UDTFConsecutiveSequences.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/series/UDTFConsecutiveSequences.java @@ -39,10 +39,9 @@ public class UDTFConsecutiveSequences implements UDTF { @Override public void validate(UDFParameterValidator validator) throws Exception { validator.validate( - x -> (long) x > 0, + x -> Util.isPositiveTime((String) x, validator.getParameters()), "gap should be a time period whose unit is ms, s, m, h.", - Util.parseTime( - validator.getParameters().getStringOrDefault("gap", "1ms"), validator.getParameters())); + validator.getParameters().getStringOrDefault("gap", "1ms")); } @Override diff --git a/library-udf/src/main/java/org/apache/iotdb/library/series/UDTFConsecutiveWindows.java b/library-udf/src/main/java/org/apache/iotdb/library/series/UDTFConsecutiveWindows.java index 3208c76d5f499..6f0392c381d13 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/series/UDTFConsecutiveWindows.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/series/UDTFConsecutiveWindows.java @@ -43,16 +43,13 @@ public void validate(UDFParameterValidator validator) throws Exception { validator .validateRequiredAttribute("length") .validate( - x -> (long) x > 0, + x -> Util.isPositiveTime((String) x, validator.getParameters()), "gap should be a time period whose unit is ms, s, m, h.", - Util.parseTime( - validator.getParameters().getStringOrDefault("gap", "1ms"), - validator.getParameters())) + validator.getParameters().getStringOrDefault("gap", "1ms")) .validate( - x -> (long) x > 0, + x -> Util.isPositiveTime((String) x, validator.getParameters()), "length should be a time period whose unit is ms, s, m, h.", - Util.parseTime( - validator.getParameters().getString("length"), validator.getParameters())); + validator.getParameters().getString("length")); } @Override diff --git a/library-udf/src/main/java/org/apache/iotdb/library/util/Util.java b/library-udf/src/main/java/org/apache/iotdb/library/util/Util.java index c493445c19bf4..20f7e65812f43 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/util/Util.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/util/Util.java @@ -30,11 +30,16 @@ import org.eclipse.collections.impl.map.mutable.primitive.LongIntHashMap; import java.io.IOException; +import java.text.ParseException; +import java.text.ParsePosition; +import java.text.SimpleDateFormat; +import java.util.Date; import java.util.List; /** This class offers functions of getting and putting values from iotdb interface. */ public class Util { private static final String TIMESTAMP_PRECISION = "timestampPrecision"; + private static final String DATE_TIME_FORMAT = "yyyy-MM-dd HH:mm:ss"; public static final String MS_PRECISION = "ms"; private Util() { @@ -380,6 +385,40 @@ public static long parseTime(String s, UDFParameters parameters) { } double v = Double.parseDouble(s); - return (long) (unit * v); + double duration = unit * v; + if (!Double.isFinite(v) + || !Double.isFinite(duration) + || duration > Long.MAX_VALUE + || duration < Long.MIN_VALUE) { + throw new IllegalArgumentException("Time value should be finite."); + } + return (long) duration; + } + + public static boolean isPositiveTime(String s, UDFParameters parameters) { + try { + return parseTime(s, parameters) > 0; + } catch (RuntimeException e) { + return false; + } + } + + public static boolean isPositiveDateTime(String s) { + try { + return parseDateTime(s) > 0; + } catch (ParseException e) { + return false; + } + } + + public static long parseDateTime(String s) throws ParseException { + SimpleDateFormat format = new SimpleDateFormat(DATE_TIME_FORMAT); + format.setLenient(false); + ParsePosition position = new ParsePosition(0); + Date date = format.parse(s, position); + if (date == null || position.getIndex() != s.length()) { + throw new ParseException("Invalid datetime format.", position.getErrorIndex()); + } + return date.getTime(); } } diff --git a/library-udf/src/test/java/org/apache/iotdb/library/UDFWindowAndQueueTest.java b/library-udf/src/test/java/org/apache/iotdb/library/UDFWindowAndQueueTest.java index 4c05e7051cbf4..6dcfe595d1189 100644 --- a/library-udf/src/test/java/org/apache/iotdb/library/UDFWindowAndQueueTest.java +++ b/library-udf/src/test/java/org/apache/iotdb/library/UDFWindowAndQueueTest.java @@ -29,7 +29,9 @@ import org.apache.iotdb.library.anomaly.util.StreamMissDetector; import org.apache.iotdb.library.dlearn.UDTFAR; import org.apache.iotdb.library.dlearn.UDTFCluster; +import org.apache.iotdb.library.dmatch.UDAFCov; import org.apache.iotdb.library.dmatch.UDAFDtw; +import org.apache.iotdb.library.dmatch.UDAFPearson; import org.apache.iotdb.library.dmatch.UDTFPtnSym; import org.apache.iotdb.library.dprofile.UDAFIntegral; import org.apache.iotdb.library.dprofile.UDAFIntegralAvg; @@ -52,6 +54,10 @@ import org.apache.iotdb.library.dprofile.UDTFSpline; import org.apache.iotdb.library.dprofile.UDTFZScore; import org.apache.iotdb.library.dprofile.util.Resampler; +import org.apache.iotdb.library.dquality.UDTFCompleteness; +import org.apache.iotdb.library.dquality.UDTFConsistency; +import org.apache.iotdb.library.dquality.UDTFTimeliness; +import org.apache.iotdb.library.dquality.UDTFValidity; import org.apache.iotdb.library.drepair.UDTFTimestampRepair; import org.apache.iotdb.library.drepair.UDTFValueFill; import org.apache.iotdb.library.drepair.UDTFValueRepair; @@ -74,6 +80,7 @@ import org.apache.iotdb.library.util.CircularQueue; import org.apache.iotdb.library.util.DoubleCircularQueue; import org.apache.iotdb.library.util.LongCircularQueue; +import org.apache.iotdb.library.util.Util; import org.apache.iotdb.udf.api.UDTF; import org.apache.iotdb.udf.api.access.Row; import org.apache.iotdb.udf.api.access.RowIterator; @@ -450,6 +457,69 @@ public void testNonFiniteNumericParametersAreRejected() { assertInvalidSingleDoubleParameters(new UDTFIDWT(), attributes); } + @Test + public void testRuntimeMethodParametersAreValidated() { + Map attributes = new HashMap<>(); + attributes.put("method", "unknown"); + assertInvalidSingleDoubleParameters(new UDTFValueFill(), attributes); + assertInvalidSingleDoubleParameters(new UDTFValueRepair(), attributes); + assertInvalidSingleDoubleParameters(new UDTFTimestampRepair(), attributes); + assertInvalidSingleDoubleParameters(new UDTFLOF(), attributes); + + attributes = new HashMap<>(); + attributes.put("k", "0"); + assertInvalidSingleDoubleParameters(new UDTFLOF(), attributes); + + attributes = new HashMap<>(); + attributes.put("window", "0"); + assertInvalidSingleDoubleParameters(new UDTFLOF(), attributes); + } + + @Test + public void testParseTimeRejectsNonFiniteDurations() { + UDFParameters parameters = createSingleDoubleSeriesParameters(Collections.emptyMap()); + + Assert.assertThrows( + IllegalArgumentException.class, () -> Util.parseTime("Infinityms", parameters)); + Assert.assertThrows(IllegalArgumentException.class, () -> Util.parseTime("NaNs", parameters)); + Assert.assertThrows( + IllegalArgumentException.class, () -> Util.parseTime("1e100ms", parameters)); + } + + @Test + public void testTimeParameterValidatorsRejectInvalidDurations() { + Map attributes = new HashMap<>(); + attributes.put("unit", "Infinityms"); + assertInvalidSingleDoubleParameters(new UDAFIntegral(), attributes); + + attributes = new HashMap<>(); + attributes.put("gap", "NaNms"); + assertInvalidSingleDoubleParameters(new UDTFConsecutiveSequences(), attributes); + + attributes = new HashMap<>(); + attributes.put("gap", "1ms"); + attributes.put("length", "1e100ms"); + assertInvalidSingleDoubleParameters(new UDTFConsecutiveWindows(), attributes); + + attributes = new HashMap<>(); + attributes.put("every", "Infinityms"); + assertInvalidSingleDoubleParameters(new UDTFResample(), attributes); + + attributes = new HashMap<>(); + attributes.put("every", "1ms"); + attributes.put("start", "invalid datetime"); + assertInvalidSingleDoubleParameters(new UDTFResample(), attributes); + + attributes = new HashMap<>(); + attributes.put("interval", "Infinityms"); + assertInvalidTwoDoubleParameters(new UDTFIFFT(), attributes); + + attributes = new HashMap<>(); + attributes.put("interval", "1ms"); + attributes.put("start", "invalid datetime"); + assertInvalidTwoDoubleParameters(new UDTFIFFT(), attributes); + } + @Test public void testRowByRowNumericFunctionsSkipNullAndInvalidValues() throws Exception { UDFParameters defaultParameters = createSingleDoubleSeriesParameters(Collections.emptyMap()); @@ -545,6 +615,16 @@ public void testRowByRowNumericFunctionsSkipNullAndInvalidValues() throws Except pacf.terminate(pacfCollector); Assert.assertEquals(Arrays.asList(1L, 2L), pacfCollector.timestamps); + pacfCollector.timestamps.clear(); + pacfCollector.values.clear(); + pacf.beforeStart(defaultParameters, new UDTFConfigurations(ZoneId.systemDefault())); + pacf.transform(new DoubleRow(1, 1.0), pacfCollector); + pacf.transform(new DoubleRow(2, 1.0), pacfCollector); + pacf.transform(new DoubleRow(3, 1.0), pacfCollector); + pacf.terminate(pacfCollector); + Assert.assertEquals(Collections.singletonList(1L), pacfCollector.timestamps); + Assert.assertEquals(1.0, pacfCollector.values.get(0), 0.0); + UDTFSegment segment = new UDTFSegment(); RecordingPointCollector segmentCollector = new RecordingPointCollector(); segment.validate(new UDFParameterValidator(defaultParameters)); @@ -583,6 +663,38 @@ public void testMinMaxBatchKeepsTimestampsAlignedWhenSkippingInvalidValues() thr Assert.assertEquals(1.0, collector.values.get(1), 0.0); } + @Test + public void testMinMaxBatchSkipsZeroRange() throws Exception { + UDTFMinMax minMax = new UDTFMinMax(); + UDFParameters parameters = createSingleDoubleSeriesParameters(Collections.emptyMap()); + RecordingPointCollector collector = new RecordingPointCollector(); + + minMax.validate(new UDFParameterValidator(parameters)); + minMax.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + minMax.transform(new DoubleRow(1, 5.0), collector); + minMax.transform(new DoubleRow(2, 5.0), collector); + minMax.terminate(collector); + + Assert.assertTrue(collector.timestamps.isEmpty()); + Assert.assertTrue(collector.values.isEmpty()); + } + + @Test + public void testZScoreBatchSkipsZeroVariance() throws Exception { + UDTFZScore zScore = new UDTFZScore(); + UDFParameters parameters = createSingleDoubleSeriesParameters(Collections.emptyMap()); + RecordingPointCollector collector = new RecordingPointCollector(); + + zScore.validate(new UDFParameterValidator(parameters)); + zScore.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + zScore.transform(new DoubleRow(1, 1.0), collector); + zScore.transform(new DoubleRow(2, 1.0), collector); + zScore.terminate(collector); + + Assert.assertTrue(collector.timestamps.isEmpty()); + Assert.assertTrue(collector.values.isEmpty()); + } + @Test public void testIQRIgnoresEmptyBatchInput() throws Exception { UDTFIQR iqr = new UDTFIQR(); @@ -904,6 +1016,52 @@ public void testPeriodAndWindowDetectorsHandleNullRows() throws Exception { Assert.assertFalse(filterCollector.timestamps.contains(2L)); } + @Test + public void testQualityFunctionsValidateInputsAndWindows() { + Assert.assertThrows( + UDFInputSeriesDataTypeNotValidException.class, + () -> + new UDTFCompleteness() + .validate( + new UDFParameterValidator( + createSingleTextSeriesParameters(Collections.emptyMap())))); + + Map attributes = new HashMap<>(); + attributes.put("window", "0"); + Assert.assertThrows( + UDFParameterNotValidException.class, + () -> + new UDTFConsistency() + .validate( + new UDFParameterValidator(createSingleDoubleSeriesParameters(attributes)))); + + attributes.put("window", String.valueOf(((long) Integer.MAX_VALUE) + 1)); + Assert.assertThrows( + UDFParameterNotValidException.class, + () -> + new UDTFTimeliness() + .validate( + new UDFParameterValidator(createSingleDoubleSeriesParameters(attributes)))); + } + + @Test + public void testQualityFunctionsSkipInsufficientValidData() throws Exception { + UDFParameters parameters = createSingleDoubleSeriesParameters(Collections.emptyMap()); + Row[] invalidRows = new Row[11]; + for (int i = 0; i < invalidRows.length; i++) { + invalidRows[i] = i % 2 == 0 ? nullDoubleRow(i) : new DoubleRow(i, Double.NaN); + } + + assertQualityFunctionSkipsInsufficientValidData( + new UDTFCompleteness(), parameters, new SimpleRowWindow(invalidRows)); + assertQualityFunctionSkipsInsufficientValidData( + new UDTFConsistency(), parameters, new SimpleRowWindow(invalidRows)); + assertQualityFunctionSkipsInsufficientValidData( + new UDTFTimeliness(), parameters, new SimpleRowWindow(invalidRows)); + assertQualityFunctionSkipsInsufficientValidData( + new UDTFValidity(), parameters, new SimpleRowWindow(invalidRows)); + } + @Test public void testResampleAndMissDetectSkipNullRows() throws Exception { Map resampleAttributes = new HashMap<>(); @@ -984,6 +1142,23 @@ public void testSkewIgnoresEmptyInputAndResetsBetweenRuns() throws Exception { Assert.assertEquals(firstRun.values.get(0), secondRun.values.get(0), 0.0); } + @Test + public void testSkewSkipsZeroVarianceInput() throws Exception { + UDFParameters parameters = createSingleDoubleSeriesParameters(Collections.emptyMap()); + UDAFSkew skew = new UDAFSkew(); + RecordingPointCollector collector = new RecordingPointCollector(); + + skew.validate(new UDFParameterValidator(parameters)); + skew.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + skew.transform(new DoubleRow(1, 1.0), collector); + skew.transform(new DoubleRow(2, 1.0), collector); + skew.transform(new DoubleRow(3, 1.0), collector); + skew.terminate(collector); + + Assert.assertTrue(collector.timestamps.isEmpty()); + Assert.assertTrue(collector.values.isEmpty()); + } + @Test public void testHistogramRejectsZeroWidthRange() { Map attributes = new HashMap<>(); @@ -1156,6 +1331,51 @@ public void testDtwClearsPreviousResultForEmptyWindow() throws Exception { Assert.assertTrue(collector.timestamps.isEmpty()); } + @Test + public void testCovarianceAndPearsonSkipUndefinedOutputs() throws Exception { + UDFParameters parameters = createTwoDoubleSeriesParameters(Collections.emptyMap()); + + UDAFCov cov = new UDAFCov(); + RecordingPointCollector covCollector = new RecordingPointCollector(); + cov.validate(new UDFParameterValidator(parameters)); + cov.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + cov.terminate(covCollector); + Assert.assertTrue(covCollector.timestamps.isEmpty()); + + cov.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + cov.transform( + new DoubleRow(1, new double[] {1.0, 2.0}, new boolean[] {false, false}), covCollector); + cov.transform( + new DoubleRow(2, new double[] {3.0, 4.0}, new boolean[] {false, false}), covCollector); + cov.terminate(covCollector); + Assert.assertEquals(Collections.singletonList(0L), covCollector.timestamps); + Assert.assertEquals(1.0, covCollector.values.get(0), 0.0); + + UDAFPearson pearson = new UDAFPearson(); + RecordingPointCollector pearsonCollector = new RecordingPointCollector(); + pearson.validate(new UDFParameterValidator(parameters)); + pearson.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + pearson.transform( + new DoubleRow(1, new double[] {1.0, 1.0}, new boolean[] {false, false}), pearsonCollector); + pearson.transform( + new DoubleRow(2, new double[] {1.0, 2.0}, new boolean[] {false, false}), pearsonCollector); + pearson.transform( + new DoubleRow(3, new double[] {1.0, 3.0}, new boolean[] {false, false}), pearsonCollector); + pearson.terminate(pearsonCollector); + Assert.assertTrue(pearsonCollector.timestamps.isEmpty()); + + pearson.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + pearson.transform( + new DoubleRow(1, new double[] {1.0, 1.0}, new boolean[] {false, false}), pearsonCollector); + pearson.transform( + new DoubleRow(2, new double[] {2.0, 2.0}, new boolean[] {false, false}), pearsonCollector); + pearson.transform( + new DoubleRow(3, new double[] {3.0, 3.0}, new boolean[] {false, false}), pearsonCollector); + pearson.terminate(pearsonCollector); + Assert.assertEquals(Collections.singletonList(0L), pearsonCollector.timestamps); + Assert.assertEquals(1.0, pearsonCollector.values.get(0), 1e-12); + } + @Test public void testFrequencyFunctionsIgnoreEmptyInput() throws Exception { RecordingPointCollector collector = new RecordingPointCollector(); @@ -1207,6 +1427,154 @@ public void testFrequencyFunctionsIgnoreEmptyInput() throws Exception { Assert.assertTrue(collector.values.isEmpty()); } + @Test + public void testFFTAcceptsCaseInsensitiveResultParameter() throws Exception { + Map attributes = new HashMap<>(); + attributes.put("result", "ABS"); + UDFParameters parameters = createSingleDoubleSeriesParameters(attributes); + RecordingPointCollector collector = new RecordingPointCollector(); + UDTFFFT fft = new UDTFFFT(); + + fft.validate(new UDFParameterValidator(parameters)); + fft.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + fft.transform(new DoubleRow(1, 1.0), collector); + fft.transform(new DoubleRow(2, 2.0), collector); + fft.terminate(collector); + + Assert.assertFalse(collector.timestamps.isEmpty()); + } + + @Test + public void testEnvelopeAnalysisSkipsInvalidAutoFrequency() throws Exception { + UDFParameters parameters = createSingleDoubleSeriesParameters(Collections.emptyMap()); + RecordingPointCollector collector = new RecordingPointCollector(); + UDFEnvelopeAnalysis envelopeAnalysis = new UDFEnvelopeAnalysis(); + + envelopeAnalysis.validate(new UDFParameterValidator(parameters)); + envelopeAnalysis.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + envelopeAnalysis.transform(new DoubleRow(1, 1.0), collector); + envelopeAnalysis.transform(new DoubleRow(1, 2.0), collector); + envelopeAnalysis.transform(new DoubleRow(1, 3.0), collector); + envelopeAnalysis.transform(new DoubleRow(1, 4.0), collector); + envelopeAnalysis.terminate(collector); + + Assert.assertTrue(collector.timestamps.isEmpty()); + Assert.assertTrue(collector.values.isEmpty()); + } + + @Test + public void testEnvelopeAnalysisSkipsNonFiniteEnvelopeValues() throws Exception { + Map attributes = new HashMap<>(); + attributes.put("frequency", "1"); + UDFParameters parameters = createSingleDoubleSeriesParameters(attributes); + RecordingPointCollector collector = new RecordingPointCollector(); + UDFEnvelopeAnalysis envelopeAnalysis = new UDFEnvelopeAnalysis(); + + envelopeAnalysis.validate(new UDFParameterValidator(parameters)); + envelopeAnalysis.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + envelopeAnalysis.transform(new DoubleRow(1, Double.MAX_VALUE), collector); + envelopeAnalysis.transform(new DoubleRow(2, Double.MAX_VALUE), collector); + envelopeAnalysis.terminate(collector); + + Assert.assertTrue(collector.timestamps.isEmpty()); + Assert.assertTrue(collector.values.isEmpty()); + } + + @Test + public void testFrequencyFunctionsSkipNonFiniteComputedOutputs() throws Exception { + UDFParameters singleSeries = createSingleDoubleSeriesParameters(Collections.emptyMap()); + UDFParameters twoSeries = createTwoDoubleSeriesParameters(Collections.emptyMap()); + + UDTFFFT fft = new UDTFFFT(); + RecordingPointCollector fftCollector = new RecordingPointCollector(); + fft.validate(new UDFParameterValidator(singleSeries)); + fft.beforeStart(singleSeries, new UDTFConfigurations(ZoneId.systemDefault())); + fft.transform(new DoubleRow(1, Double.MAX_VALUE), fftCollector); + fft.transform(new DoubleRow(2, Double.MAX_VALUE), fftCollector); + fft.terminate(fftCollector); + assertAllValuesFinite(fftCollector); + + UDTFConv conv = new UDTFConv(); + RecordingPointCollector convCollector = new RecordingPointCollector(); + conv.validate(new UDFParameterValidator(twoSeries)); + conv.beforeStart(twoSeries, new UDTFConfigurations(ZoneId.systemDefault())); + conv.transform( + new DoubleRow( + 1, new double[] {Double.MAX_VALUE, Double.MAX_VALUE}, new boolean[] {false, false}), + convCollector); + conv.terminate(convCollector); + assertAllValuesFinite(convCollector); + + UDTFDeconv deconv = new UDTFDeconv(); + RecordingPointCollector deconvCollector = new RecordingPointCollector(); + deconv.validate(new UDFParameterValidator(twoSeries)); + deconv.beforeStart(twoSeries, new UDTFConfigurations(ZoneId.systemDefault())); + deconv.transform( + new DoubleRow(1, new double[] {1.0, Double.MIN_VALUE}, new boolean[] {false, false}), + deconvCollector); + deconv.terminate(deconvCollector); + assertAllValuesFinite(deconvCollector); + + Map wpassAttributes = new HashMap<>(); + wpassAttributes.put("wpass", "0.5"); + UDFParameters wpassParameters = createSingleDoubleSeriesParameters(wpassAttributes); + + UDTFLowPass lowPass = new UDTFLowPass(); + RecordingPointCollector lowPassCollector = new RecordingPointCollector(); + lowPass.validate(new UDFParameterValidator(wpassParameters)); + lowPass.beforeStart(wpassParameters, new UDTFConfigurations(ZoneId.systemDefault())); + lowPass.transform(new DoubleRow(1, Double.MAX_VALUE), lowPassCollector); + lowPass.transform(new DoubleRow(2, Double.MAX_VALUE), lowPassCollector); + lowPass.terminate(lowPassCollector); + assertAllValuesFinite(lowPassCollector); + + UDTFHighPass highPass = new UDTFHighPass(); + RecordingPointCollector highPassCollector = new RecordingPointCollector(); + highPass.validate(new UDFParameterValidator(wpassParameters)); + highPass.beforeStart(wpassParameters, new UDTFConfigurations(ZoneId.systemDefault())); + highPass.transform(new DoubleRow(1, Double.MAX_VALUE), highPassCollector); + highPass.transform(new DoubleRow(2, Double.MAX_VALUE), highPassCollector); + highPass.terminate(highPassCollector); + assertAllValuesFinite(highPassCollector); + + Map dwtAttributes = new HashMap<>(); + dwtAttributes.put("method", "Haar"); + UDFParameters dwtParameters = createSingleDoubleSeriesParameters(dwtAttributes); + + UDTFDWT dwt = new UDTFDWT(); + RecordingPointCollector dwtCollector = new RecordingPointCollector(); + dwt.validate(new UDFParameterValidator(dwtParameters)); + dwt.beforeStart(dwtParameters, new UDTFConfigurations(ZoneId.systemDefault())); + dwt.transform(new DoubleRow(1, Double.MAX_VALUE), dwtCollector); + dwt.transform(new DoubleRow(2, Double.MAX_VALUE), dwtCollector); + dwt.terminate(dwtCollector); + assertAllValuesFinite(dwtCollector); + + UDTFIDWT idwt = new UDTFIDWT(); + RecordingPointCollector idwtCollector = new RecordingPointCollector(); + idwt.validate(new UDFParameterValidator(dwtParameters)); + idwt.beforeStart(dwtParameters, new UDTFConfigurations(ZoneId.systemDefault())); + idwt.transform(new DoubleRow(1, Double.MAX_VALUE), idwtCollector); + idwt.transform(new DoubleRow(2, Double.MAX_VALUE), idwtCollector); + idwt.terminate(idwtCollector); + assertAllValuesFinite(idwtCollector); + + UDTFIFFT ifft = new UDTFIFFT(); + RecordingPointCollector ifftCollector = new RecordingPointCollector(); + ifft.validate(new UDFParameterValidator(twoSeries)); + ifft.beforeStart(twoSeries, new UDTFConfigurations(ZoneId.systemDefault())); + ifft.transform( + new DoubleRow( + 0, new double[] {Double.MAX_VALUE, Double.MAX_VALUE}, new boolean[] {false, false}), + ifftCollector); + ifft.transform( + new DoubleRow( + 1, new double[] {Double.MAX_VALUE, Double.MAX_VALUE}, new boolean[] {false, false}), + ifftCollector); + ifft.terminate(ifftCollector); + assertAllValuesFinite(ifftCollector); + } + @Test public void testFrequencyFunctionsResetBuffersBetweenRuns() throws Exception { UDFParameters singleSeries = createSingleDoubleSeriesParameters(Collections.emptyMap()); @@ -1391,6 +1759,26 @@ public void testARSkipsInvalidRows() throws Exception { Assert.assertTrue(Double.isFinite(collector.values.get(0))); } + @Test + public void testARSkipsDegenerateCoefficients() throws Exception { + Map attributes = new HashMap<>(); + attributes.put("p", "2"); + UDFParameters parameters = createSingleDoubleSeriesParameters(attributes); + UDTFAR ar = new UDTFAR(); + RecordingPointCollector collector = new RecordingPointCollector(); + + ar.validate(new UDFParameterValidator(parameters)); + ar.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + ar.transform(new DoubleRow(1, 1.0), collector); + ar.transform(new DoubleRow(2, 1.0), collector); + ar.transform(new DoubleRow(3, 1.0), collector); + ar.transform(new DoubleRow(4, 1.0), collector); + ar.terminate(collector); + + Assert.assertTrue(collector.timestamps.isEmpty()); + Assert.assertTrue(collector.values.isEmpty()); + } + @Test public void testClusterSkipsInvalidRows() throws Exception { Map attributes = new HashMap<>(); @@ -1471,6 +1859,55 @@ public void testSplineSkipsTooFewPointsAndHandlesNegativeTimestamps() throws Exc Assert.assertEquals(Arrays.asList(-5L, -3L, -1L), collector.timestamps); } + @Test + public void testSplineHandlesLargeTimestampSpan() throws Exception { + Map attributes = new HashMap<>(); + attributes.put("points", "3"); + UDFParameters parameters = createSingleDoubleSeriesParameters(attributes); + + UDTFSpline spline = new UDTFSpline(); + RecordingPointCollector collector = new RecordingPointCollector(); + + spline.validate(new UDFParameterValidator(parameters)); + spline.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + spline.transform(new DoubleRow(0, 0.0), collector); + spline.transform(new DoubleRow(1_500_000_000L, 1.0), collector); + spline.transform(new DoubleRow(3_000_000_000L, 2.0), collector); + spline.transform(new DoubleRow(4_500_000_000L, 3.0), collector); + spline.transform(new DoubleRow(6_000_000_000L, 4.0), collector); + spline.terminate(collector); + + Assert.assertEquals(Arrays.asList(0L, 3_000_000_000L, 6_000_000_000L), collector.timestamps); + Assert.assertEquals(0.0, collector.values.get(0), 1e-9); + Assert.assertEquals(2.0, collector.values.get(1), 1e-9); + Assert.assertEquals(4.0, collector.values.get(2), 1e-9); + } + + @Test + public void testSplineSkipsNonIncreasingTimestamps() throws Exception { + Map attributes = new HashMap<>(); + attributes.put("points", "3"); + UDFParameters parameters = createSingleDoubleSeriesParameters(attributes); + + UDTFSpline spline = new UDTFSpline(); + RecordingPointCollector collector = new RecordingPointCollector(); + + spline.validate(new UDFParameterValidator(parameters)); + spline.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + spline.transform(new DoubleRow(0, 0.0), collector); + spline.transform(new DoubleRow(10, 1.0), collector); + spline.transform(new DoubleRow(10, 100.0), collector); + spline.transform(new DoubleRow(20, 2.0), collector); + spline.transform(new DoubleRow(30, 3.0), collector); + spline.transform(new DoubleRow(40, 4.0), collector); + spline.terminate(collector); + + Assert.assertEquals(Arrays.asList(0L, 20L, 40L), collector.timestamps); + Assert.assertEquals(0.0, collector.values.get(0), 1e-9); + Assert.assertEquals(2.0, collector.values.get(1), 1e-9); + Assert.assertEquals(4.0, collector.values.get(2), 1e-9); + } + @Test public void testOutlierValidatesRuntimeSensitiveParameters() { Map attributes = new HashMap<>(); @@ -1710,6 +2147,15 @@ public void testRepairFunctionsHandleNullAndShortInputs() throws Exception { Assert.assertEquals(Collections.singletonList(1L), collector.timestamps); Assert.assertEquals(1.0, collector.values.get(0), 0.0); + collector.timestamps.clear(); + collector.values.clear(); + timestampRepair.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + timestampRepair.transform( + new SimpleRowWindow(nullDoubleRow(1), new DoubleRow(2, Double.POSITIVE_INFINITY)), + collector); + Assert.assertTrue(collector.timestamps.isEmpty()); + Assert.assertTrue(collector.values.isEmpty()); + collector.timestamps.clear(); collector.values.clear(); timestampRepair.transform( @@ -1757,6 +2203,22 @@ public void testSegmentInstancesKeepIndependentBuffers() throws Exception { Assert.assertEquals(10.0, firstCollector.values.get(0), 0.0); } + @Test + public void testSegmentSkipsEmptyValidInput() throws Exception { + UDFParameters parameters = createSingleDoubleSeriesParameters(Collections.emptyMap()); + UDTFSegment segment = new UDTFSegment(); + RecordingPointCollector collector = new RecordingPointCollector(); + + segment.validate(new UDFParameterValidator(parameters)); + segment.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + segment.transform(nullDoubleRow(1), collector); + segment.transform(new DoubleRow(2, Double.NaN), collector); + segment.terminate(collector); + + Assert.assertTrue(collector.timestamps.isEmpty()); + Assert.assertTrue(collector.values.isEmpty()); + } + @Test public void testCircularQueueRejectsNonPositiveCapacity() { Assert.assertThrows(IllegalArgumentException.class, () -> new CircularQueue<>(0)); @@ -1868,6 +2330,32 @@ private static void assertInvalidSingleDoubleParameters( new UDFParameterValidator(createSingleDoubleSeriesParameters(attributes)))); } + private static void assertInvalidTwoDoubleParameters( + UDTF function, Map attributes) { + Assert.assertThrows( + UDFParameterNotValidException.class, + () -> + function.validate( + new UDFParameterValidator(createTwoDoubleSeriesParameters(attributes)))); + } + + private static void assertAllValuesFinite(RecordingPointCollector collector) { + for (double value : collector.values) { + Assert.assertTrue(Double.isFinite(value)); + } + } + + private static void assertQualityFunctionSkipsInsufficientValidData( + UDTF function, UDFParameters parameters, RowWindow invalidWindow) throws Exception { + RecordingPointCollector collector = new RecordingPointCollector(); + function.validate(new UDFParameterValidator(parameters)); + function.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + function.transform(invalidWindow, collector); + + Assert.assertTrue(collector.timestamps.isEmpty()); + Assert.assertTrue(collector.values.isEmpty()); + } + private static DoubleRow nullDoubleRow(long time) { return new DoubleRow(time, new double[] {0.0}, new boolean[] {true}); } From 28919e971806675cbd9bacf179414b15cd90c8e2 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 9 Jun 2026 16:58:31 +0800 Subject: [PATCH 5/6] Fix library UDF matching edge cases --- .../iotdb/library/dmatch/UDTFPtnSym.java | 4 +- .../iotdb/library/match/UDAFDTWMatch.java | 20 +++-- .../iotdb/library/match/UDAFPatternMatch.java | 46 +++++++--- .../apache/iotdb/library/UDAFPatternTest.java | 84 +++++++++++++++++-- .../iotdb/library/UDFWindowAndQueueTest.java | 39 +++++++++ 5 files changed, 167 insertions(+), 26 deletions(-) diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dmatch/UDTFPtnSym.java b/library-udf/src/main/java/org/apache/iotdb/library/dmatch/UDTFPtnSym.java index deeff2a200988..9e99eef8053ae 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dmatch/UDTFPtnSym.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dmatch/UDTFPtnSym.java @@ -92,9 +92,9 @@ public void transform(RowWindow rowWindow, PointCollector collector) throws Exce } } for (int len = 3; len <= m; len++) { - for (int i = 1, j = len; j <= m; j++) { + for (int i = 1, j = len; j <= m; i++, j++) { dp[i][j] = - Math.pow(Math.abs(a.get(0) - a.get(j - 1)), 2) + Math.pow(Math.abs(a.get(i - 1) - a.get(j - 1)), 2) + Math.min(Math.min(dp[i + 1][j], dp[i][j - 1]), dp[i + 1][j - 1]); } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/match/UDAFDTWMatch.java b/library-udf/src/main/java/org/apache/iotdb/library/match/UDAFDTWMatch.java index f79b6b69acf71..42e001ffbc01c 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/match/UDAFDTWMatch.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/match/UDAFDTWMatch.java @@ -118,25 +118,27 @@ private double getValue(Column column, int i) { private float calculateDTW(Double[] series1, Double[] series2) { int n = series1.length; - double[][] dtw = new double[n][n]; + int m = series2.length; + if (n == 0 || m == 0) { + return Float.POSITIVE_INFINITY; + } - // Initialize the DTW matrix - for (int i = 0; i < n; i++) { - for (int j = 0; j < n; j++) { + double[][] dtw = new double[n + 1][m + 1]; + for (int i = 0; i <= n; i++) { + for (int j = 0; j <= m; j++) { dtw[i][j] = Double.POSITIVE_INFINITY; } } dtw[0][0] = 0; - // Compute the DTW distance - for (int i = 1; i < n; i++) { - for (int j = 1; j < n; j++) { - double cost = Math.abs(series1[i] - series2[j]); + for (int i = 1; i <= n; i++) { + for (int j = 1; j <= m; j++) { + double cost = Math.abs(series1[i - 1] - series2[j - 1]); dtw[i][j] = cost + Math.min(Math.min(dtw[i - 1][j], dtw[i][j - 1]), dtw[i - 1][j - 1]); } } - return (float) dtw[n - 1][n - 1]; + return (float) dtw[n][m]; } @Override diff --git a/library-udf/src/main/java/org/apache/iotdb/library/match/UDAFPatternMatch.java b/library-udf/src/main/java/org/apache/iotdb/library/match/UDAFPatternMatch.java index 3def11e54b8bc..fc26d6730dbc8 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/match/UDAFPatternMatch.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/match/UDAFPatternMatch.java @@ -40,6 +40,7 @@ import java.nio.charset.Charset; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.stream.IntStream; @@ -113,17 +114,20 @@ public void outputFinal(State state, ResultValue resultValue) { resultValue.setNull(); return; } - PatternExecutor executor = new PatternExecutor(); - List sourcePointsExtract = executor.scalePoint(times, values); - List queryPointsExtract = executor.extractPoints(timePattern, valuePattern); - - executor.setPoints(queryPointsExtract); - PatternContext ctx = new PatternContext(); - ctx.setThreshold(threshold); - ctx.setDataPoints(sourcePointsExtract); - // State only records time and recorded values, and the final result is calculated - List results = executor.executeQuery(ctx); + List results = Collections.emptyList(); + if (hasPositiveTimeRange(times) && hasPositiveValueRange(values)) { + PatternExecutor executor = new PatternExecutor(); + List sourcePointsExtract = executor.scalePoint(times, values); + List queryPointsExtract = executor.extractPoints(timePattern, valuePattern); + + executor.setPoints(queryPointsExtract); + PatternContext ctx = new PatternContext(); + ctx.setThreshold(threshold); + ctx.setDataPoints(sourcePointsExtract); + // State only records time and recorded values, and the final result is calculated + results = executor.executeQuery(ctx); + } if (!results.isEmpty()) { resultValue.setBinary(new Binary(results.toString(), Charset.defaultCharset())); } else { @@ -140,6 +144,28 @@ public void outputFinal(State state, ResultValue resultValue) { } } + private static boolean hasPositiveTimeRange(List times) { + long previous = times.get(0); + for (int i = 1; i < times.size(); i++) { + long time = times.get(i); + if (time <= previous) { + return false; + } + previous = time; + } + return true; + } + + private static boolean hasPositiveValueRange(List values) { + double min = Double.POSITIVE_INFINITY; + double max = Double.NEGATIVE_INFINITY; + for (double value : values) { + min = Math.min(min, value); + max = Math.max(max, value); + } + return min < max; + } + @Override public void validate(UDFParameterValidator validator) { diff --git a/library-udf/src/test/java/org/apache/iotdb/library/UDAFPatternTest.java b/library-udf/src/test/java/org/apache/iotdb/library/UDAFPatternTest.java index 0807c78fd063b..1ad217c6ed3d1 100644 --- a/library-udf/src/test/java/org/apache/iotdb/library/UDAFPatternTest.java +++ b/library-udf/src/test/java/org/apache/iotdb/library/UDAFPatternTest.java @@ -214,6 +214,36 @@ public void testDTWMatchValidatesFiniteParameters() { attributes)))); } + @Test + public void testDTWMatchUsesSinglePointDistance() throws Exception { + UDAFDTWMatch dtwMatch = new UDAFDTWMatch(); + UDFParameters parameters = createDtwParameters("10.0", "1"); + dtwMatch.validate(new UDFParameterValidator(parameters)); + dtwMatch.beforeStart(parameters, new UDAFConfigurations()); + DTWState state = (DTWState) dtwMatch.createState(); + state.reset(); + + Column[] columns = + new Column[] { + new TestColumn(TSDataType.DOUBLE, new long[0], new double[] {1.0}, new boolean[] {false}), + new TestColumn(TSDataType.INT64, new long[] {1L}, new double[0], new boolean[] {false}) + }; + dtwMatch.addInput(state, columns, null); + + Assert.assertTrue(state.getMatchResults().isEmpty()); + + UDAFDTWMatch exactMatch = new UDAFDTWMatch(); + UDFParameters exactParameters = createDtwParameters("1.0", "0"); + exactMatch.validate(new UDFParameterValidator(exactParameters)); + exactMatch.beforeStart(exactParameters, new UDAFConfigurations()); + DTWState exactState = (DTWState) exactMatch.createState(); + exactState.reset(); + + exactMatch.addInput(exactState, columns, null); + + Assert.assertEquals(1, exactState.getMatchResults().size()); + } + @Test public void testMatchUDAFsSkipNullAndInvalidRows() throws Exception { Column[] columns = buildPatternInputColumns(); @@ -256,6 +286,41 @@ public void testPatternMatchEmptyEffectiveInputOutputsNull() throws Exception { Assert.assertTrue(resultBuilder.build().isNull(0)); } + @Test + public void testPatternMatchFlatInputFallsBackToDtw() throws Exception { + UDAFPatternMatch patternMatch = new UDAFPatternMatch(); + UDFParameters parameters = createPatternParameters("1,2", "5.0,5.0", "0"); + patternMatch.validate(new UDFParameterValidator(parameters)); + patternMatch.beforeStart(parameters, new UDAFConfigurations()); + PatternState state = (PatternState) patternMatch.createState(); + state.reset(); + state.updateBuffer(1L, 5.0); + state.updateBuffer(2L, 5.0); + + RecordingColumnBuilder resultBuilder = new RecordingColumnBuilder(TSDataType.TEXT); + patternMatch.outputFinal(state, new ResultValue(resultBuilder)); + + Assert.assertFalse(resultBuilder.build().isNull(0)); + } + + @Test + public void testPatternMatchNonMonotonicInputFallsBackToDtw() throws Exception { + UDAFPatternMatch patternMatch = new UDAFPatternMatch(); + UDFParameters parameters = createPatternParameters("1,2,3", "1.0,2.0,3.0", "0"); + patternMatch.validate(new UDFParameterValidator(parameters)); + patternMatch.beforeStart(parameters, new UDAFConfigurations()); + PatternState state = (PatternState) patternMatch.createState(); + state.reset(); + state.updateBuffer(1L, 1.0); + state.updateBuffer(3L, 2.0); + state.updateBuffer(2L, 3.0); + + RecordingColumnBuilder resultBuilder = new RecordingColumnBuilder(TSDataType.TEXT); + patternMatch.outputFinal(state, new ResultValue(resultBuilder)); + + Assert.assertFalse(resultBuilder.build().isNull(0)); + } + @Test public void testMatchUDAFsCombineEmptyStates() throws Exception { UDAFPatternMatch patternMatch = new UDAFPatternMatch(); @@ -305,18 +370,27 @@ public void testMatchUDAFsCombineEmptyStates() throws Exception { } private static UDFParameters createPatternParameters() { + return createPatternParameters("1,2", "1.0,2.0", "100"); + } + + private static UDFParameters createPatternParameters( + String timePattern, String valuePattern, String threshold) { Map attributes = new HashMap<>(); - attributes.put("timePattern", "1,2"); - attributes.put("valuePattern", "1.0,2.0"); - attributes.put("threshold", "100"); + attributes.put("timePattern", timePattern); + attributes.put("valuePattern", valuePattern); + attributes.put("threshold", threshold); return new UDFParameters( Collections.singletonList("s1"), Collections.singletonList(Type.DOUBLE), attributes); } private static UDFParameters createDtwParameters() { + return createDtwParameters("1.0,3.0", "100"); + } + + private static UDFParameters createDtwParameters(String pattern, String threshold) { Map attributes = new HashMap<>(); - attributes.put("pattern", "1.0,3.0"); - attributes.put("threshold", "100"); + attributes.put("pattern", pattern); + attributes.put("threshold", threshold); return new UDFParameters( Collections.singletonList("s1"), Collections.singletonList(Type.DOUBLE), attributes); } diff --git a/library-udf/src/test/java/org/apache/iotdb/library/UDFWindowAndQueueTest.java b/library-udf/src/test/java/org/apache/iotdb/library/UDFWindowAndQueueTest.java index 6dcfe595d1189..58459e9bf7a53 100644 --- a/library-udf/src/test/java/org/apache/iotdb/library/UDFWindowAndQueueTest.java +++ b/library-udf/src/test/java/org/apache/iotdb/library/UDFWindowAndQueueTest.java @@ -1271,6 +1271,45 @@ public void testPatternSymmetrySkipsInvalidWindows() throws Exception { Assert.assertTrue(collector.values.isEmpty()); } + @Test + public void testPatternSymmetryUsesFullWindowDtwCost() throws Exception { + Map attributes = new HashMap<>(); + attributes.put("window", "4"); + attributes.put("threshold", "1650"); + UDFParameters parameters = createSingleDoubleSeriesParameters(attributes); + UDTFPtnSym ptnSym = new UDTFPtnSym(); + RecordingPointCollector collector = new RecordingPointCollector(); + + ptnSym.validate(new UDFParameterValidator(parameters)); + ptnSym.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + ptnSym.transform( + new SimpleRowWindow( + new DoubleRow(1, 0.0), + new DoubleRow(2, 10.0), + new DoubleRow(3, 20.0), + new DoubleRow(4, 40.0)), + collector); + + Assert.assertTrue(collector.timestamps.isEmpty()); + Assert.assertTrue(collector.values.isEmpty()); + + attributes.put("threshold", "0"); + parameters = createSingleDoubleSeriesParameters(attributes); + ptnSym = new UDTFPtnSym(); + ptnSym.validate(new UDFParameterValidator(parameters)); + ptnSym.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + ptnSym.transform( + new SimpleRowWindow( + new DoubleRow(5, 1.0), + new DoubleRow(6, 2.0), + new DoubleRow(7, 2.0), + new DoubleRow(8, 1.0)), + collector); + + Assert.assertEquals(Collections.singletonList(5L), collector.timestamps); + Assert.assertEquals(0.0, collector.values.get(0), 0.0); + } + @Test public void testDtwEmptyInputProducesNoOutput() throws Exception { UDAFDtw dtw = new UDAFDtw(); From 8fe96a5a12569d7a01cc7aac0004562daea56f8c Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 9 Jun 2026 18:51:21 +0800 Subject: [PATCH 6/6] Fix K-Sigma default window size --- .../iotdb/library/anomaly/UDTFKSigma.java | 2 +- .../iotdb/library/UDFWindowAndQueueTest.java | 22 ++++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFKSigma.java b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFKSigma.java index 6b79b33e3406f..c1c55460e9e2d 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFKSigma.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFKSigma.java @@ -33,7 +33,7 @@ /** This function detects outliers which lies over average +/- k * sigma. */ public class UDTFKSigma implements UDTF { - private static final int DEFAULT_WINDOW_SIZE = 10; + private static final int DEFAULT_WINDOW_SIZE = 10000; private double mean = 0.0; private double variance = 0.0; diff --git a/library-udf/src/test/java/org/apache/iotdb/library/UDFWindowAndQueueTest.java b/library-udf/src/test/java/org/apache/iotdb/library/UDFWindowAndQueueTest.java index 58459e9bf7a53..e334d412a67e2 100644 --- a/library-udf/src/test/java/org/apache/iotdb/library/UDFWindowAndQueueTest.java +++ b/library-udf/src/test/java/org/apache/iotdb/library/UDFWindowAndQueueTest.java @@ -117,7 +117,7 @@ public void testKSigmaDefaultWindowIsConsistent() throws Exception { kSigma.validate(new UDFParameterValidator(parameters)); kSigma.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); - Assert.assertEquals(10, getWindowSize(kSigma)); + Assert.assertEquals(10000, getWindowSize(kSigma)); } @Test @@ -134,6 +134,26 @@ public void testKSigmaExplicitWindowOverridesDefault() throws Exception { Assert.assertEquals(3, getWindowSize(kSigma)); } + @Test + public void testKSigmaDefaultWindowMatchesAnomalyITExpectation() throws Exception { + Map attributes = new HashMap<>(); + attributes.put("k", "1.0"); + UDFParameters parameters = createSingleDoubleSeriesParameters(attributes); + UDTFKSigma kSigma = new UDTFKSigma(); + RecordingPointCollector collector = new RecordingPointCollector(); + + kSigma.validate(new UDFParameterValidator(parameters)); + kSigma.beforeStart(parameters, new UDTFConfigurations(ZoneId.systemDefault())); + double[] values = {0, 50, 100, 150, 200, 200, 200, 200, 200, 200, 150, 100, 50, 0}; + for (int i = 0; i < values.length; i++) { + kSigma.transform(new DoubleRow((i + 1) * 100L, values[i]), collector); + } + kSigma.terminate(collector); + + Assert.assertEquals(Arrays.asList(100L, 200L, 1300L, 1400L), collector.timestamps); + Assert.assertEquals(Arrays.asList(0.0, 50.0, 50.0, 0.0), collector.values); + } + @Test public void testMvAvgUsesRunningWindowSum() throws Exception { Map attributes = new HashMap<>();