diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/fill/filter/FixedIntervalFillFilter.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/fill/filter/FixedIntervalFillFilter.java index e7dd783b38722..6f37c41726f07 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/fill/filter/FixedIntervalFillFilter.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/fill/filter/FixedIntervalFillFilter.java @@ -37,11 +37,11 @@ public boolean needFill(long time, long previousTime) { return isTimeDistanceLessThanOrEqual(time, previousTime, timeInterval); } - private boolean isTimeDistanceLessThanOrEqual(long left, long right, long maxDistance) { + private static boolean isTimeDistanceLessThanOrEqual(long left, long right, long maxDistance) { if (maxDistance < 0) { return false; } - long distance = left >= right ? left - right : right - left; + final long distance = left >= right ? left - right : right - left; return Long.compareUnsigned(distance, maxDistance) <= 0; } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/fill/linear/LinearFill.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/fill/linear/LinearFill.java index 2231b4f029b0c..4e3d4ec61cdd4 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/fill/linear/LinearFill.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/fill/linear/LinearFill.java @@ -156,9 +156,8 @@ private boolean fill( } private double getFactor(long currentTime) { - return nextTimeInCurrentColumn - previousTime == 0 - ? 0.0 - : ((double) (currentTime - previousTime)) / (nextTimeInCurrentColumn - previousTime); + double timeRange = (double) nextTimeInCurrentColumn - (double) previousTime; + return timeRange == 0 ? 0.0 : ((double) currentTime - (double) previousTime) / timeRange; } /** diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/gapfill/AbstractGapFillOperator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/gapfill/AbstractGapFillOperator.java index 52fb6b59bace7..c14acc8f09ad6 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/gapfill/AbstractGapFillOperator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/gapfill/AbstractGapFillOperator.java @@ -38,6 +38,7 @@ import java.util.List; import static com.google.common.base.Preconditions.checkArgument; +import static org.apache.iotdb.calc.transformation.dag.column.unary.scalar.DateBinFunctionColumnTransformer.saturatingAdd; abstract class AbstractGapFillOperator implements ProcessOperator { @@ -125,7 +126,9 @@ public TsBlock next() throws Exception { // -1 because we should not include current row, current row will be appended in // writeCurrentRow long currentEndTime = - timeColumn.isNull(i) ? endTime : block.getColumn(timeColumnIndex).getLong(i) - 1; + timeColumn.isNull(i) + ? endTime + : saturatingAdd(block.getColumn(timeColumnIndex).getLong(i), -1); fillGaps(block, i, currentEndTime); writeCurrentRow(block, i); } @@ -155,8 +158,12 @@ private void writeCurrentRow(TsBlock block, int rowIndex) { private void fillGaps(TsBlock block, int rowIndex, long currentEndTime) { while (currentTime <= currentEndTime) { + long previousTime = currentTime; gapFillRow(currentTime, block, rowIndex); nextTime(); + if (currentTime <= previousTime) { + break; + } } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/DateBinFunctionColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/DateBinFunctionColumnTransformer.java index 71abdf569fad4..c2b441b746874 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/DateBinFunctionColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/DateBinFunctionColumnTransformer.java @@ -27,6 +27,7 @@ import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.type.Type; +import java.math.BigInteger; import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneId; @@ -37,6 +38,8 @@ public class DateBinFunctionColumnTransformer extends UnaryColumnTransformer { private static final long NANOSECONDS_IN_MILLISECOND = 1_000_000; private static final long NANOSECONDS_IN_MICROSECOND = 1_000; + private static final BigInteger BIG_LONG_MIN = BigInteger.valueOf(Long.MIN_VALUE); + private static final BigInteger BIG_LONG_MAX = BigInteger.valueOf(Long.MAX_VALUE); private final int monthDuration; private final long nonMonthDuration; @@ -149,12 +152,18 @@ public static long dateBin( return convertToTimestamp(binStart, zoneId); } - long diff = source - origin; - long n = diff >= 0 ? diff / nonMonthDuration : (diff - nonMonthDuration + 1) / nonMonthDuration; - return origin + (n * nonMonthDuration); + return saturateToLong(getNonMonthDateBinStart(source, origin, nonMonthDuration)); } public long[] dateBinStartEnd(long source) { + return dateBinStartEnd(source, false); + } + + public long[] dateBinStartEndClosed(long source) { + return dateBinStartEnd(source, true); + } + + private long[] dateBinStartEnd(long source, boolean closedEnd) { // return source if interval is 0 if (monthDuration == 0 && nonMonthDuration == 0) { return new long[] {source, source}; @@ -180,17 +189,17 @@ public long[] dateBinStartEnd(long source) { binStart.minusMonths(monthDuration).minusNanos(getNanoTimeStamp(nonMonthDuration)); } - return new long[] { - convertToTimestamp(binStart, zoneId), - convertToTimestamp(binStart.plusMonths(monthDuration), zoneId) - }; + long startTime = convertToTimestamp(binStart, zoneId); + long endTime = convertToTimestamp(binStart.plusMonths(monthDuration), zoneId); + return new long[] {startTime, closedEnd ? saturatingAdd(endTime, -1) : endTime}; } - long diff = source - origin; - long n = diff >= 0 ? diff / nonMonthDuration : (diff - nonMonthDuration + 1) / nonMonthDuration; - return new long[] { - origin + (n * nonMonthDuration), origin + (n * nonMonthDuration) + nonMonthDuration - }; + BigInteger startTime = getNonMonthDateBinStart(source, origin, nonMonthDuration); + BigInteger endTime = startTime.add(BigInteger.valueOf(nonMonthDuration)); + if (closedEnd) { + endTime = endTime.subtract(BigInteger.ONE); + } + return new long[] {saturateToLong(startTime), saturateToLong(endTime)}; } public static long nextDateBin(int monthDuration, ZoneId zoneId, long currentTime) { @@ -200,7 +209,33 @@ public static long nextDateBin(int monthDuration, ZoneId zoneId, long currentTim } public static long nextDateBin(long nonMonthDuration, long currentTime) { - return currentTime + nonMonthDuration; + return saturatingAdd(currentTime, nonMonthDuration); + } + + public static long saturatingAdd(long left, long right) { + return saturateToLong(BigInteger.valueOf(left).add(BigInteger.valueOf(right))); + } + + private static BigInteger getNonMonthDateBinStart( + long source, long origin, long nonMonthDuration) { + BigInteger diff = BigInteger.valueOf(source).subtract(BigInteger.valueOf(origin)); + BigInteger duration = BigInteger.valueOf(nonMonthDuration); + BigInteger[] quotientAndRemainder = diff.divideAndRemainder(duration); + BigInteger n = quotientAndRemainder[0]; + if (diff.signum() < 0 && quotientAndRemainder[1].signum() != 0) { + n = n.subtract(BigInteger.ONE); + } + return BigInteger.valueOf(origin).add(n.multiply(duration)); + } + + private static long saturateToLong(BigInteger value) { + if (value.compareTo(BIG_LONG_MAX) > 0) { + return Long.MAX_VALUE; + } + if (value.compareTo(BIG_LONG_MIN) < 0) { + return Long.MIN_VALUE; + } + return value.longValue(); } @Override diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/subscription/SubscriptionWalRetentionCalculator.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/subscription/SubscriptionWalRetentionCalculator.java index 3bd2be60f3742..101176b4507a3 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/subscription/SubscriptionWalRetentionCalculator.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/subscription/SubscriptionWalRetentionCalculator.java @@ -118,9 +118,15 @@ private SubscriptionRetentionBound buildTimeRetentionBound(final long retentionM return SubscriptionRetentionBound.retainAll(); } - final long cutoffTimeMs = System.currentTimeMillis() - retentionMs; + final long cutoffTimeMs = getTimeRetentionCutoffTimeMs(System.currentTimeMillis(), retentionMs); final Pair deletionBound = consensusReqReader.getDeletionBoundBeforeTimestamp(cutoffTimeMs); return SubscriptionRetentionBound.of(deletionBound.left, deletionBound.right); } + + static long getTimeRetentionCutoffTimeMs(final long currentTimeMs, final long retentionMs) { + return retentionMs > 0 && currentTimeMs < Long.MIN_VALUE + retentionMs + ? Long.MIN_VALUE + : currentTimeMs - retentionMs; + } } diff --git a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/subscription/SubscriptionWalRetentionCalculatorTest.java b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/subscription/SubscriptionWalRetentionCalculatorTest.java new file mode 100644 index 0000000000000..d7568076d98f2 --- /dev/null +++ b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/subscription/SubscriptionWalRetentionCalculatorTest.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.consensus.iot.subscription; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class SubscriptionWalRetentionCalculatorTest { + + @Test + public void testTimeRetentionCutoffUsesNormalSubtractionWhenSafe() { + assertEquals( + 700L, SubscriptionWalRetentionCalculator.getTimeRetentionCutoffTimeMs(1000L, 300L)); + } + + @Test + public void testTimeRetentionCutoffSaturatesOnUnderflow() { + assertEquals( + Long.MIN_VALUE, + SubscriptionWalRetentionCalculator.getTimeRetentionCutoffTimeMs(Long.MIN_VALUE, 1L)); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/auth/LoginLockManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/auth/LoginLockManager.java index 6015d1d538f91..3ea85c3a53b7c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/auth/LoginLockManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/auth/LoginLockManager.java @@ -36,6 +36,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; public class LoginLockManager { private static final Logger LOGGER = LoggerFactory.getLogger(LoginLockManager.class); @@ -152,7 +153,7 @@ public boolean checkLock(long userId, String ip) { UserLockInfo userIpLock = userIpLocks.get(userIpKey); if (userIpLock != null) { long now = System.currentTimeMillis(); - long cutoffTime = now - (passwordLockTimeMinutes * 60 * 1000L); + long cutoffTime = getLockWindowCutoffTime(now); userIpLock.removeOldFailures(cutoffTime); if (userIpLock.getFailureCount() >= failedLoginAttempts) { return true; @@ -165,7 +166,7 @@ public boolean checkLock(long userId, String ip) { UserLockInfo userLock = userLocks.get(userId); if (userLock != null) { long now = System.currentTimeMillis(); - long cutoffTime = now - (passwordLockTimeMinutes * 60 * 1000L); + long cutoffTime = getLockWindowCutoffTime(now); userLock.removeOldFailures(cutoffTime); return userLock.getFailureCount() >= failedLoginAttemptsPerUser; } @@ -196,7 +197,7 @@ public void recordFailure(long userId, String ip) { } long now = System.currentTimeMillis(); - long cutoffTime = now - (passwordLockTimeMinutes * 60 * 1000L); + long cutoffTime = getLockWindowCutoffTime(now); // Handle user@ip failures in sliding window if (failedLoginAttempts != -1) { @@ -288,7 +289,7 @@ public void unlock(long userId, String ip) { /** Clean up expired locks (no failures in the sliding window) */ public void cleanExpiredLocks() { long now = System.currentTimeMillis(); - long cutoffTime = now - (passwordLockTimeMinutes * 60 * 1000L); + long cutoffTime = getLockWindowCutoffTime(now); // Clean expired user locks userLocks @@ -330,6 +331,17 @@ private String buildUserIpKey(long userId, String ip) { return userId + "@" + ip; } + private long getLockWindowCutoffTime(long currentTimeMillis) { + return getLockWindowCutoffTime(currentTimeMillis, passwordLockTimeMinutes); + } + + static long getLockWindowCutoffTime(long currentTimeMillis, int passwordLockTimeMinutes) { + final long lockWindowMs = TimeUnit.MINUTES.toMillis(passwordLockTimeMinutes); + return currentTimeMillis < Long.MIN_VALUE + lockWindowMs + ? Long.MIN_VALUE + : currentTimeMillis - lockWindowMs; + } + private void checkForPotentialAttacks(long userId, String ip) { // Check if IP is locked by many users Set usersForIp = new HashSet<>(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/query/QueryTimeoutRuntimeException.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/query/QueryTimeoutRuntimeException.java index b333f287a21f8..32349c6132a06 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/query/QueryTimeoutRuntimeException.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/query/QueryTimeoutRuntimeException.java @@ -30,8 +30,23 @@ public class QueryTimeoutRuntimeException extends IoTDBRuntimeException { public QueryTimeoutRuntimeException(long startTime, long currentTime, long timeout) { super( - String.format(QUERY_TIMEOUT_EXCEPTION_MESSAGE, startTime, startTime + timeout, currentTime), + String.format( + QUERY_TIMEOUT_EXCEPTION_MESSAGE, + startTime, + saturatingAdd(startTime, timeout), + currentTime), QUERY_TIMEOUT.getStatusCode(), true); } + + private static long saturatingAdd(long left, long right) { + long result = left + right; + if (right > 0 && result < left) { + return Long.MAX_VALUE; + } + if (right < 0 && result > left) { + return Long.MIN_VALUE; + } + return result; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/window/processor/TumblingWindowingProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/window/processor/TumblingWindowingProcessor.java index 59aae9625f6d0..0a92513178230 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/window/processor/TumblingWindowingProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/window/processor/TumblingWindowingProcessor.java @@ -31,6 +31,7 @@ import org.apache.tsfile.utils.Pair; +import java.math.BigInteger; import java.util.Collections; import java.util.List; import java.util.Set; @@ -80,10 +81,12 @@ public Set mayAddWindow( ? slidingBoundaryTime : windowList.get(windowList.size() - 1).getTimestamp(); - if (timeStamp >= (windowList.isEmpty() ? lastTime : lastTime + slidingInterval)) { + if (windowList.isEmpty() + ? timeStamp >= lastTime + : isTimestampAtOrAfterWindowEnd(timeStamp, lastTime, slidingInterval)) { final TimeSeriesWindow window = new TimeSeriesWindow(this, null); // Align to the last time + k * slidingInterval, k is a natural number - window.setTimestamp(((timeStamp - lastTime) / slidingInterval) * slidingInterval + lastTime); + window.setTimestamp(alignWindowStart(timeStamp, lastTime, slidingInterval)); windowList.add(window); return Collections.singleton(window); } @@ -96,12 +99,12 @@ public Pair updateAndMaySetWindowState( if (timeStamp < window.getTimestamp()) { return new Pair<>(WindowState.IGNORE_VALUE, null); } - if (timeStamp >= window.getTimestamp() + slidingInterval) { + if (isTimestampAtOrAfterWindowEnd(timeStamp, window.getTimestamp(), slidingInterval)) { return new Pair<>( WindowState.EMIT_AND_PURGE_WITHOUT_COMPUTE, new WindowOutput() .setTimestamp(window.getTimestamp()) - .setProgressTime(window.getTimestamp() + slidingInterval)); + .setProgressTime(saturatingAdd(window.getTimestamp(), slidingInterval))); } return new Pair<>(WindowState.COMPUTE, null); } @@ -110,6 +113,33 @@ public Pair updateAndMaySetWindowState( public WindowOutput forceOutput(final TimeSeriesWindow window) { return new WindowOutput() .setTimestamp(window.getTimestamp()) - .setProgressTime(window.getTimestamp() + slidingInterval); + .setProgressTime(saturatingAdd(window.getTimestamp(), slidingInterval)); + } + + private static boolean isTimestampAtOrAfterWindowEnd( + final long timestamp, final long windowStart, final long interval) { + return windowStart <= Long.MAX_VALUE - interval && timestamp >= windowStart + interval; + } + + private static long alignWindowStart( + final long timestamp, final long baseTime, final long interval) { + final BigInteger base = BigInteger.valueOf(baseTime); + final BigInteger intervalValue = BigInteger.valueOf(interval); + return base.add( + BigInteger.valueOf(timestamp) + .subtract(base) + .divide(intervalValue) + .multiply(intervalValue)) + .longValueExact(); + } + + private static long saturatingAdd(final long left, final long right) { + if (right > 0 && left > Long.MAX_VALUE - right) { + return Long.MAX_VALUE; + } + if (right < 0 && left < Long.MIN_VALUE - right) { + return Long.MIN_VALUE; + } + return left + right; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/DownSamplingTimeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/DownSamplingTimeUtils.java new file mode 100644 index 0000000000000..ecc37d09f5d8f --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/DownSamplingTimeUtils.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.processor.downsampling; + +public class DownSamplingTimeUtils { + + private DownSamplingTimeUtils() { + // Utility class. + } + + public static boolean isTimeDistanceLessThanOrEqualTo(long left, long right, long distance) { + if (distance < 0) { + return false; + } + final long difference = left >= right ? left - right : right - left; + return Long.compareUnsigned(difference, distance) <= 0; + } + + public static boolean isTimeDistanceGreaterThanOrEqualTo(long left, long right, long distance) { + if (distance < 0) { + return true; + } + final long difference = left >= right ? left - right : right - left; + return Long.compareUnsigned(difference, distance) >= 0; + } + + public static double timeDifferenceAsDouble(long left, long right) { + return (double) left - (double) right; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/changing/ChangingValueFilter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/changing/ChangingValueFilter.java index 9bb5dfcb9a761..81fbff2bfe902 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/changing/ChangingValueFilter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/changing/ChangingValueFilter.java @@ -24,6 +24,9 @@ import java.time.LocalDate; import java.util.Objects; +import static org.apache.iotdb.db.pipe.processor.downsampling.DownSamplingTimeUtils.isTimeDistanceGreaterThanOrEqualTo; +import static org.apache.iotdb.db.pipe.processor.downsampling.DownSamplingTimeUtils.isTimeDistanceLessThanOrEqualTo; + public class ChangingValueFilter { private final ChangingValueSamplingProcessor processor; @@ -59,12 +62,12 @@ public boolean filter(final long timestamp, final T value) { } private boolean tryFilter(final long timestamp, final T value) { - if (isTimeDistanceLessThanOrEqual( + if (isTimeDistanceLessThanOrEqualTo( timestamp, lastStoredTimestamp, processor.getCompressionMinTimeInterval())) { return false; } - if (isTimeDistanceGreaterThanOrEqual( + if (isTimeDistanceGreaterThanOrEqualTo( timestamp, lastStoredTimestamp, processor.getCompressionMaxTimeInterval())) { reset(timestamp, value); return true; @@ -94,18 +97,6 @@ private boolean tryFilter(final long timestamp, final T value) { return false; } - private boolean isTimeDistanceLessThanOrEqual( - final long left, final long right, final long maxDistance) { - final long distance = left >= right ? left - right : right - left; - return Long.compareUnsigned(distance, maxDistance) <= 0; - } - - private boolean isTimeDistanceGreaterThanOrEqual( - final long left, final long right, final long minDistance) { - final long distance = left >= right ? left - right : right - left; - return Long.compareUnsigned(distance, minDistance) >= 0; - } - private void reset(final long timestamp, final T value) { lastStoredTimestamp = timestamp; lastStoredValue = value; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/sdt/SwingingDoorTrendingFilter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/sdt/SwingingDoorTrendingFilter.java index bfc0bc5d3f791..ab86a80065997 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/sdt/SwingingDoorTrendingFilter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/sdt/SwingingDoorTrendingFilter.java @@ -24,6 +24,10 @@ import java.time.LocalDate; import java.util.Objects; +import static org.apache.iotdb.db.pipe.processor.downsampling.DownSamplingTimeUtils.isTimeDistanceGreaterThanOrEqualTo; +import static org.apache.iotdb.db.pipe.processor.downsampling.DownSamplingTimeUtils.isTimeDistanceLessThanOrEqualTo; +import static org.apache.iotdb.db.pipe.processor.downsampling.DownSamplingTimeUtils.timeDifferenceAsDouble; + public class SwingingDoorTrendingFilter { private final SwingingDoorTrendingSamplingProcessor processor; @@ -85,14 +89,12 @@ public boolean filter(final long timestamp, final T value) { } private boolean tryFilter(final long timestamp, final T value) { - final long timeDiff = timestamp - lastStoredTimestamp; - - if (isTimeDistanceLessThanOrEqual( + if (isTimeDistanceLessThanOrEqualTo( timestamp, lastStoredTimestamp, processor.getCompressionMinTimeInterval())) { return false; } - if (isTimeDistanceGreaterThanOrEqual( + if (isTimeDistanceGreaterThanOrEqualTo( timestamp, lastStoredTimestamp, processor.getCompressionMaxTimeInterval())) { reset(timestamp, value); return true; @@ -115,6 +117,7 @@ private boolean tryFilter(final long timestamp, final T value) { final double doubleValue = Double.parseDouble(value.toString()); final double lastStoredDoubleValue = Double.parseDouble(lastStoredValue.toString()); final double valueDiff = doubleValue - lastStoredDoubleValue; + final double timeDiff = timeDifferenceAsDouble(timestamp, lastStoredTimestamp); final double currentUpperSlope = (valueDiff - processor.getCompressionDeviation()) / timeDiff; if (currentUpperSlope > upperDoor) { @@ -145,18 +148,6 @@ private boolean tryFilter(final long timestamp, final T value) { return false; } - private boolean isTimeDistanceLessThanOrEqual( - final long left, final long right, final long maxDistance) { - final long distance = left >= right ? left - right : right - left; - return Long.compareUnsigned(distance, maxDistance) <= 0; - } - - private boolean isTimeDistanceGreaterThanOrEqual( - final long left, final long right, final long minDistance) { - final long distance = left >= right ? left - right : right - left; - return Long.compareUnsigned(distance, minDistance) >= 0; - } - private void reset(final long timestamp, final T value) { upperDoor = Double.MIN_VALUE; lowerDoor = Double.MAX_VALUE; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/tumbling/TumblingTimeSamplingProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/tumbling/TumblingTimeSamplingProcessor.java index 8beab1343888c..47b8d84c99768 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/tumbling/TumblingTimeSamplingProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/tumbling/TumblingTimeSamplingProcessor.java @@ -41,6 +41,7 @@ import static org.apache.iotdb.commons.pipe.config.constant.PipeProcessorConstant.PROCESSOR_DOWN_SAMPLING_SPLIT_FILE_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeProcessorConstant.PROCESSOR_TUMBLING_TIME_INTERVAL_SECONDS_DEFAULT_VALUE; import static org.apache.iotdb.commons.pipe.config.constant.PipeProcessorConstant.PROCESSOR_TUMBLING_TIME_INTERVAL_SECONDS_KEY; +import static org.apache.iotdb.db.pipe.processor.downsampling.DownSamplingTimeUtils.isTimeDistanceGreaterThanOrEqualTo; @TreeModel public class TumblingTimeSamplingProcessor extends DownSamplingProcessor { @@ -116,7 +117,8 @@ protected void processRow( final Long lastSampleTime = pathLastObjectCache.getPartialPathLastObject(timeSeriesSuffix); if (lastSampleTime == null - || isTimeDistanceGreaterThanOrEqual(currentRowTime, lastSampleTime)) { + || isTimeDistanceGreaterThanOrEqualTo( + currentRowTime, lastSampleTime, intervalInCurrentPrecision)) { try { rowCollector.collectRow(row); @@ -135,9 +137,4 @@ protected void processRow( } } } - - private boolean isTimeDistanceGreaterThanOrEqual(long left, long right) { - long distance = left >= right ? left - right : right - left; - return Long.compareUnsigned(distance, intervalInCurrentPrecision) >= 0; - } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/DataRegionWatermarkInjector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/DataRegionWatermarkInjector.java index 33a5698e4c2ad..e608f3470fc29 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/DataRegionWatermarkInjector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/DataRegionWatermarkInjector.java @@ -67,7 +67,21 @@ public PipeWatermarkEvent inject() { } private static long calculateNextInjectionTime(long injectionIntervalInMs) { - final long currentTime = System.currentTimeMillis(); - return currentTime / injectionIntervalInMs * injectionIntervalInMs + injectionIntervalInMs; + return calculateNextInjectionTime(System.currentTimeMillis(), injectionIntervalInMs); + } + + static long calculateNextInjectionTime(long currentTime, long injectionIntervalInMs) { + return saturatingAdd( + currentTime / injectionIntervalInMs * injectionIntervalInMs, injectionIntervalInMs); + } + + private static long saturatingAdd(final long left, final long right) { + if (right > 0 && left > Long.MAX_VALUE - right) { + return Long.MAX_VALUE; + } + if (right < 0 && left < Long.MIN_VALUE - right) { + return Long.MIN_VALUE; + } + return left + right; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSource.java index b13b204001651..2038766ded207 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSource.java @@ -251,11 +251,11 @@ public void customize( } startTimePartitionIdLowerBound = - (realtimeDataExtractionStartTime % TimePartitionUtils.getTimePartitionInterval() == 0) + TimePartitionUtils.isTimePartitionStartTime(realtimeDataExtractionStartTime) ? TimePartitionUtils.getTimePartitionId(realtimeDataExtractionStartTime) : TimePartitionUtils.getTimePartitionId(realtimeDataExtractionStartTime) + 1; endTimePartitionIdUpperBound = - (realtimeDataExtractionEndTime % TimePartitionUtils.getTimePartitionInterval() == 0) + TimePartitionUtils.isTimePartitionStartTime(realtimeDataExtractionEndTime) ? TimePartitionUtils.getTimePartitionId(realtimeDataExtractionEndTime) : TimePartitionUtils.getTimePartitionId(realtimeDataExtractionEndTime) - 1; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/TimeDurationAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/TimeDurationAccumulator.java index 8e9917d89d101..480bf8dcde040 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/TimeDurationAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/TimeDurationAccumulator.java @@ -89,7 +89,7 @@ public void outputFinal(ColumnBuilder tsBlockBuilder) { if (!initResult) { tsBlockBuilder.appendNull(); } else { - tsBlockBuilder.writeLong(maxTime - minTime); + tsBlockBuilder.writeLong(saturatingTimeDifference(maxTime, minTime)); } } @@ -129,4 +129,9 @@ protected void updateMinTime(long curTime) { initResult = true; minTime = Math.min(minTime, curTime); } + + private static long saturatingTimeDifference(long maxTime, long minTime) { + long timeDifference = maxTime - minTime; + return timeDifference < 0 ? Long.MAX_VALUE : timeDifference; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/timerangeiterator/AggrWindowIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/timerangeiterator/AggrWindowIterator.java index 5e82e01151b7f..77c0b85dd1e0b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/timerangeiterator/AggrWindowIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/timerangeiterator/AggrWindowIterator.java @@ -86,7 +86,8 @@ private TimeRange getLeftmostTimeRange() { retEndTime = Math.min(DateTimeUtils.calcPositiveIntervalByMonth(startTime, interval, zoneId), endTime); } else { - retEndTime = Math.min(startTime + interval.nonMonthDuration, endTime); + retEndTime = + Math.min(ITimeRangeIterator.saturatingAdd(startTime, interval.nonMonthDuration), endTime); } return new TimeRange(startTime, retEndTime); } @@ -94,15 +95,14 @@ private TimeRange getLeftmostTimeRange() { private TimeRange getRightmostTimeRange() { long retStartTime; long retEndTime; - long queryRange = endTime - startTime; long intervalNum; if (slidingStep.containsMonth()) { intervalNum = - (long) - Math.ceil( - (double) queryRange - / (slidingStep.getMaxTotalDuration(TimestampPrecisionUtils.currPrecision))); + ITimeRangeIterator.ceilDivTimeRange( + startTime, + endTime, + slidingStep.getMaxTotalDuration(TimestampPrecisionUtils.currPrecision)); long tempRetStartTime = DateTimeUtils.calcPositiveIntervalByMonth( startTime, slidingStep.multiple(intervalNum - 1), zoneId); @@ -116,8 +116,11 @@ private TimeRange getRightmostTimeRange() { } intervalNum -= 1; } else { - intervalNum = (long) Math.ceil(queryRange / (double) slidingStep.nonMonthDuration); - retStartTime = slidingStep.nonMonthDuration * (intervalNum - 1) + startTime; + intervalNum = + ITimeRangeIterator.ceilDivTimeRange(startTime, endTime, slidingStep.nonMonthDuration); + retStartTime = + ITimeRangeIterator.rightmostTimeRangeStart( + startTime, endTime, slidingStep.nonMonthDuration); } if (interval.containsMonth()) { @@ -129,7 +132,9 @@ private TimeRange getRightmostTimeRange() { startTime, interval.merge(slidingStep.multiple(intervalNum - 1)), zoneId), endTime); } else { - retEndTime = Math.min(retStartTime + interval.nonMonthDuration, endTime); + retEndTime = + Math.min( + ITimeRangeIterator.saturatingAdd(retStartTime, interval.nonMonthDuration), endTime); } return new TimeRange(retStartTime, retEndTime); } @@ -155,6 +160,10 @@ public boolean hasNextTimeRange() { DateTimeUtils.calcPositiveIntervalByMonth( startTime, slidingStep.multiple(timeRangeCount), zoneId); } else { + if (!ITimeRangeIterator.canMoveForward( + curStartTime, slidingStep.nonMonthDuration, endTime)) { + return false; + } retStartTime = curStartTime + slidingStep.nonMonthDuration; } // This is an open interval , [0-100) @@ -167,6 +176,10 @@ public boolean hasNextTimeRange() { throw new UnsupportedOperationException( "Ascending is not supported when sliding step contains month."); } else { + if (!ITimeRangeIterator.canMoveBackward( + curStartTime, slidingStep.nonMonthDuration, startTime)) { + return false; + } retStartTime = curStartTime - slidingStep.nonMonthDuration; } if (retStartTime < startTime) { @@ -179,7 +192,7 @@ public boolean hasNextTimeRange() { DateTimeUtils.calcPositiveIntervalByMonth( startTime, slidingStep.multiple(timeRangeCount).merge(interval), zoneId); } else { - retEndTime = retStartTime + interval.nonMonthDuration; + retEndTime = ITimeRangeIterator.saturatingAdd(retStartTime, interval.nonMonthDuration); } retEndTime = Math.min(retEndTime, endTime); curTimeRange = new TimeRange(retStartTime, retEndTime); @@ -209,15 +222,14 @@ public long currentOutputTime() { @Override public long getTotalIntervalNum() { - long queryRange = endTime - startTime; long intervalNum; if (slidingStep.containsMonth()) { intervalNum = - (long) - Math.ceil( - (double) queryRange - / (slidingStep.getMaxTotalDuration(TimestampPrecisionUtils.currPrecision))); + ITimeRangeIterator.ceilDivTimeRange( + startTime, + endTime, + slidingStep.getMaxTotalDuration(TimestampPrecisionUtils.currPrecision)); long retStartTime = DateTimeUtils.calcPositiveIntervalByMonth( startTime, slidingStep.multiple(intervalNum), zoneId); @@ -228,7 +240,8 @@ public long getTotalIntervalNum() { startTime, slidingStep.multiple(intervalNum), zoneId); } } else { - intervalNum = (long) Math.ceil(queryRange / (double) slidingStep.nonMonthDuration); + intervalNum = + ITimeRangeIterator.ceilDivTimeRange(startTime, endTime, slidingStep.nonMonthDuration); } return intervalNum; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/timerangeiterator/ITimeRangeIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/timerangeiterator/ITimeRangeIterator.java index 9dbd2ce77a309..78436ee6cbec4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/timerangeiterator/ITimeRangeIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/timerangeiterator/ITimeRangeIterator.java @@ -21,6 +21,8 @@ import org.apache.tsfile.read.common.TimeRange; +import java.math.BigInteger; + /** * This interface used for iteratively generating aggregated time windows in GROUP BY query. * @@ -47,8 +49,60 @@ public interface ITimeRangeIterator { default TimeRange getFinalTimeRange(TimeRange timeRange, boolean leftCRightO) { return leftCRightO - ? new TimeRange(timeRange.getMin(), timeRange.getMax() - 1) - : new TimeRange(timeRange.getMin() + 1, timeRange.getMax()); + ? new TimeRange(timeRange.getMin(), saturatingAdd(timeRange.getMax(), -1)) + : new TimeRange(saturatingAdd(timeRange.getMin(), 1), timeRange.getMax()); + } + + static long saturatingAdd(long left, long right) { + if (right > 0 && left > Long.MAX_VALUE - right) { + return Long.MAX_VALUE; + } + if (right < 0 && left < Long.MIN_VALUE - right) { + return Long.MIN_VALUE; + } + return left + right; + } + + static boolean canMoveForward(long current, long step, long upperBound) { + return step > 0 && current <= Long.MAX_VALUE - step && current + step < upperBound; + } + + static boolean canMoveBackward(long current, long step, long lowerBound) { + return step > 0 && current >= Long.MIN_VALUE + step && current - step >= lowerBound; + } + + static long ceilDivTimeRange(long startTime, long endTime, long divisor) { + BigInteger range = + BigInteger.valueOf(endTime) + .subtract(BigInteger.valueOf(startTime)) + .add(BigInteger.valueOf(divisor).subtract(BigInteger.ONE)); + return saturateToLong(range.divide(BigInteger.valueOf(divisor))); + } + + static long rightmostTimeRangeStart(long startTime, long endTime, long slidingStep) { + BigInteger distanceMinusOne = + BigInteger.valueOf(endTime) + .subtract(BigInteger.valueOf(startTime)) + .subtract(BigInteger.ONE); + long remainder = distanceMinusOne.mod(BigInteger.valueOf(slidingStep)).longValue(); + return saturatingAdd(saturatingAdd(endTime, -1), -remainder); + } + + static boolean isTimeRangeDistanceGreaterThan(long startTime, long endTime, long distance) { + return BigInteger.valueOf(endTime) + .subtract(BigInteger.valueOf(startTime)) + .compareTo(BigInteger.valueOf(distance)) + > 0; + } + + static long saturateToLong(BigInteger value) { + if (value.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0) { + return Long.MAX_VALUE; + } + if (value.compareTo(BigInteger.valueOf(Long.MIN_VALUE)) < 0) { + return Long.MIN_VALUE; + } + return value.longValue(); } /** diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/timerangeiterator/PreAggrWindowIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/timerangeiterator/PreAggrWindowIterator.java index df07e15d032ef..8bf86c550c9ef 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/timerangeiterator/PreAggrWindowIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/timerangeiterator/PreAggrWindowIterator.java @@ -21,6 +21,8 @@ import org.apache.tsfile.read.common.TimeRange; +import java.math.BigInteger; + /** * This class iteratively generates pre-aggregated time windows. * @@ -71,7 +73,7 @@ public TimeRange getFirstTimeRange() { } private TimeRange getLeftmostTimeRange() { - long retEndTime = Math.min(startTime + curInterval, endTime); + long retEndTime = Math.min(ITimeRangeIterator.saturatingAdd(startTime, curInterval), endTime); updateIntervalAndStep(); return new TimeRange(startTime, retEndTime); } @@ -79,13 +81,14 @@ private TimeRange getLeftmostTimeRange() { private TimeRange getRightmostTimeRange() { long retStartTime; long retEndTime; - long intervalNum = (long) Math.ceil((endTime - startTime) / (double) slidingStep); - retStartTime = slidingStep * (intervalNum - 1) + startTime; - if (isIntervalCyclicChange && endTime - retStartTime > interval % slidingStep) { - retStartTime += interval % slidingStep; + retStartTime = ITimeRangeIterator.rightmostTimeRangeStart(startTime, endTime, slidingStep); + if (isIntervalCyclicChange + && ITimeRangeIterator.isTimeRangeDistanceGreaterThan( + retStartTime, endTime, interval % slidingStep)) { + retStartTime = ITimeRangeIterator.saturatingAdd(retStartTime, interval % slidingStep); updateIntervalAndStep(); } - retEndTime = Math.min(retStartTime + curInterval, endTime); + retEndTime = Math.min(ITimeRangeIterator.saturatingAdd(retStartTime, curInterval), endTime); updateIntervalAndStep(); return new TimeRange(retStartTime, retEndTime); } @@ -105,18 +108,24 @@ public boolean hasNextTimeRange() { long retEndTime; long curStartTime = curTimeRange.getMin(); if (isAscending) { + if (!ITimeRangeIterator.canMoveForward(curStartTime, curSlidingStep, endTime)) { + return false; + } retStartTime = curStartTime + curSlidingStep; // This is an open interval , [0-100) if (retStartTime >= endTime) { return false; } } else { + if (!ITimeRangeIterator.canMoveBackward(curStartTime, curSlidingStep, startTime)) { + return false; + } retStartTime = curStartTime - curSlidingStep; if (retStartTime < startTime) { return false; } } - retEndTime = Math.min(retStartTime + curInterval, endTime); + retEndTime = Math.min(ITimeRangeIterator.saturatingAdd(retStartTime, curInterval), endTime); updateIntervalAndStep(); curTimeRange = new TimeRange(retStartTime, retEndTime); hasCachedTimeRange = true; @@ -177,19 +186,24 @@ public long currentOutputTime() { @Override public long getTotalIntervalNum() { - long queryRange = endTime - startTime; if (slidingStep >= interval || interval % slidingStep == 0) { - return (long) Math.ceil(queryRange / (double) slidingStep); + return ITimeRangeIterator.ceilDivTimeRange(startTime, endTime, slidingStep); } long interval1 = interval % slidingStep; long interval2 = slidingStep - interval % slidingStep; - long intervalNum = Math.floorDiv(queryRange, interval1 + interval2); - long tmpStartTime = startTime + intervalNum * (interval1 + interval2); - if (tmpStartTime + interval1 > endTime) { - return intervalNum * 2 + 1; - } else { - return intervalNum * 2 + 2; - } + BigInteger queryRange = BigInteger.valueOf(endTime).subtract(BigInteger.valueOf(startTime)); + BigInteger intervalNum = queryRange.divide(BigInteger.valueOf(interval1 + interval2)); + BigInteger result = + intervalNum + .multiply(BigInteger.valueOf(2)) + .add( + queryRange + .remainder(BigInteger.valueOf(interval1 + interval2)) + .compareTo(BigInteger.valueOf(interval1)) + < 0 + ? BigInteger.ONE + : BigInteger.valueOf(2)); + return ITimeRangeIterator.saturateToLong(result); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/timerangeiterator/PreAggrWindowWithNaturalMonthIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/timerangeiterator/PreAggrWindowWithNaturalMonthIterator.java index a4a0aa6261143..0674564906f9b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/timerangeiterator/PreAggrWindowWithNaturalMonthIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/timerangeiterator/PreAggrWindowWithNaturalMonthIterator.java @@ -109,12 +109,12 @@ private void initHeap() { TimeRange firstTimeRange = aggrWindowIterator.nextTimeRange(); if (leftCRightO) { timeBoundaryHeap.add(firstTimeRange.getMin()); - timeBoundaryHeap.add(firstTimeRange.getMax() + 1); + timeBoundaryHeap.add(ITimeRangeIterator.saturatingAdd(firstTimeRange.getMax(), 1)); curStartTimeForIterator = firstTimeRange.getMin(); } else { - timeBoundaryHeap.add(firstTimeRange.getMin() - 1); + timeBoundaryHeap.add(ITimeRangeIterator.saturatingAdd(firstTimeRange.getMin(), -1)); timeBoundaryHeap.add(firstTimeRange.getMax()); - curStartTimeForIterator = firstTimeRange.getMin() - 1; + curStartTimeForIterator = ITimeRangeIterator.saturatingAdd(firstTimeRange.getMin(), -1); } tryToExpandHeap(); } @@ -125,12 +125,12 @@ private void tryToExpandHeap() { timeRangeToExpand = aggrWindowIterator.nextTimeRange(); if (leftCRightO) { timeBoundaryHeap.add(timeRangeToExpand.getMin()); - timeBoundaryHeap.add(timeRangeToExpand.getMax() + 1); + timeBoundaryHeap.add(ITimeRangeIterator.saturatingAdd(timeRangeToExpand.getMax(), 1)); curStartTimeForIterator = timeRangeToExpand.getMin(); } else { - timeBoundaryHeap.add(timeRangeToExpand.getMin() - 1); + timeBoundaryHeap.add(ITimeRangeIterator.saturatingAdd(timeRangeToExpand.getMin(), -1)); timeBoundaryHeap.add(timeRangeToExpand.getMax()); - curStartTimeForIterator = timeRangeToExpand.getMin() - 1; + curStartTimeForIterator = ITimeRangeIterator.saturatingAdd(timeRangeToExpand.getMin(), -1); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/timerangeiterator/TableDateBinTimeRangeIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/timerangeiterator/TableDateBinTimeRangeIterator.java index 148d0cde40917..33b2ec2c49704 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/timerangeiterator/TableDateBinTimeRangeIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/timerangeiterator/TableDateBinTimeRangeIterator.java @@ -47,15 +47,15 @@ public boolean canFinishCurrentTimeRange(long startTime) { @Override public void updateCurTimeRange(long startTime) { - long[] timeArray = dateBinTransformer.dateBinStartEnd(startTime); + long[] timeArray = dateBinTransformer.dateBinStartEndClosed(startTime); if (curTimeRange != null) { // meet new time range, remove old time range if (timeArray[0] != curTimeRange.getMin()) { - this.curTimeRange = new TimeRange(timeArray[0], timeArray[1] - 1); + this.curTimeRange = new TimeRange(timeArray[0], timeArray[1]); } } else { - this.curTimeRange = new TimeRange(timeArray[0], timeArray[1] - 1); + this.curTimeRange = new TimeRange(timeArray[0], timeArray[1]); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/ai/InferenceOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/ai/InferenceOperator.java index 8db754768a34d..251843e700980 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/ai/InferenceOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/ai/InferenceOperator.java @@ -45,6 +45,7 @@ import org.apache.tsfile.read.common.block.column.TsBlockSerde; import org.apache.tsfile.utils.RamUsageEstimator; +import java.math.BigInteger; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; @@ -155,7 +156,7 @@ private void fillTimeColumn(TsBlock tsBlock) { Column timeColumn = tsBlock.getTimeColumn(); long[] time = timeColumn.getLongs(); for (int i = 0; i < time.length; i++) { - time[i] = maxTimestamp + interval * currentRowIndex; + time[i] = calculateGeneratedTime(maxTimestamp, interval, currentRowIndex); currentRowIndex++; } } @@ -242,7 +243,7 @@ private void appendTsBlockToBuilder(TsBlock inputTsBlock) { private void submitInferenceTask() { if (generateTimeColumn) { - interval = (maxTimestamp - minTimestamp) / totalRow; + interval = calculateGeneratedTimeInterval(minTimestamp, maxTimestamp, totalRow); } TsBlock inputTsBlock = inputTsBlockBuilder.build(); @@ -300,4 +301,27 @@ public long ramBytesUsed() { + inputTsBlockBuilder.getRetainedSizeInBytes() + (long) columnIndexes.length * Integer.BYTES; } + + static long calculateGeneratedTimeInterval(long minTimestamp, long maxTimestamp, long totalRow) { + try { + BigInteger interval = + BigInteger.valueOf(maxTimestamp) + .subtract(BigInteger.valueOf(minTimestamp)) + .divide(BigInteger.valueOf(totalRow)); + if (interval.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0) { + throw new ArithmeticException(); + } + return interval.longValue(); + } catch (ArithmeticException e) { + throw new ModelInferenceProcessException("Generated time column is out of range."); + } + } + + static long calculateGeneratedTime(long maxTimestamp, long interval, long currentRowIndex) { + try { + return Math.addExact(maxTimestamp, Math.multiplyExact(interval, currentRowIndex)); + } catch (ArithmeticException e) { + throw new ModelInferenceProcessException("Generated time column is out of range."); + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java index d00ee54428bf0..8229034de8ad1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java @@ -532,12 +532,20 @@ private static IDPredicate parseComparison( switch (comparisonExpression.getOperator()) { case LESS_THAN: + if (rightHandValue == Long.MIN_VALUE) { + throw new SemanticException( + "The time predicate does not select any time range: " + comparisonExpression); + } timeRange.setEndTime(Math.min(timeRange.getEndTime(), rightHandValue - 1)); break; case LESS_THAN_OR_EQUAL: timeRange.setEndTime(Math.min(timeRange.getEndTime(), rightHandValue)); break; case GREATER_THAN: + if (rightHandValue == Long.MAX_VALUE) { + throw new SemanticException( + "The time predicate does not select any time range: " + comparisonExpression); + } timeRange.setStartTime(Math.max(timeRange.getStartTime(), rightHandValue + 1)); break; case GREATER_THAN_OR_EQUAL: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java index 22d5b0518ac55..3b5781a3e92dd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java @@ -2095,7 +2095,7 @@ static void analyzeGroupByTime(Analysis analysis, QueryStatement queryStatement) analysis.setGroupByTimeParameter(groupByTimeParameter); Expression globalTimePredicate = analysis.getGlobalTimePredicate(); - Expression groupByTimePredicate = ExpressionFactory.groupByTime(groupByTimeParameter); + Expression groupByTimePredicate = getGroupByTimePredicate(groupByTimeParameter); if (globalTimePredicate == null) { globalTimePredicate = groupByTimePredicate; } else { @@ -2104,6 +2104,24 @@ static void analyzeGroupByTime(Analysis analysis, QueryStatement queryStatement) analysis.setGlobalTimePredicate(globalTimePredicate); } + private static Expression getGroupByTimePredicate(GroupByTimeParameter groupByTimeParameter) { + if (groupByTimeParameter.isLeftCRightO() + || groupByTimeParameter.getEndTime() != Long.MAX_VALUE) { + return ExpressionFactory.groupByTime(groupByTimeParameter); + } + GroupByTimeParameter rightOpenParameter = + new GroupByTimeParameter( + groupByTimeParameter.getStartTime() + 1, + groupByTimeParameter.getEndTime(), + groupByTimeParameter.getInterval(), + groupByTimeParameter.getSlidingStep(), + true); + return ExpressionFactory.or( + ExpressionFactory.groupByTime(rightOpenParameter), + ExpressionFactory.eq( + ExpressionFactory.time(), ExpressionFactory.longValue(Long.MAX_VALUE))); + } + static void analyzeFill(Analysis analysis, QueryStatement queryStatement) { if (queryStatement.getFillComponent() == null) { return; @@ -2221,10 +2239,16 @@ public static Pair, Pair> getTimePart List result = new ArrayList<>(); TimeRange currentTimeRange = timeRangeList.get(index); reserveMemoryForTimePartitionSlot( - currentTimeRange.getMax(), currentTimeRange.getMin(), context); + currentTimeRange.getMax(), currentTimeRange.getMin(), needRightAll, context); + boolean compressedRightUnclosedRange = false; while (index < size) { long curLeft = timeRangeList.get(index).getMin(); long curRight = timeRangeList.get(index).getMax(); + if (isRangeCoveringRightUnclosedPartitions(curLeft, curRight, needRightAll)) { + timePartitionSlot = TimePartitionUtils.getTimePartitionSlot(curLeft); + compressedRightUnclosedRange = true; + break; + } if (curLeft >= endTime) { result.add(timePartitionSlot); // next init @@ -2234,23 +2258,19 @@ public static Pair, Pair> getTimePart result.add(timePartitionSlot); // next init timePartitionSlot = new TTimePartitionSlot(endTime); - // beware of overflow - endTime = - endTime + TimePartitionUtils.getTimePartitionInterval() > endTime - ? endTime + TimePartitionUtils.getTimePartitionInterval() - : Long.MAX_VALUE; + endTime = TimePartitionUtils.getTimePartitionUpperBound(endTime); } else { index++; if (index < size) { currentTimeRange = timeRangeList.get(index); reserveMemoryForTimePartitionSlot( - currentTimeRange.getMax(), currentTimeRange.getMin(), context); + currentTimeRange.getMax(), currentTimeRange.getMin(), needRightAll, context); } } } result.add(timePartitionSlot); - if (needRightAll) { + if (needRightAll && !compressedRightUnclosedRange) { TTimePartitionSlot lastTimePartitionSlot = TimePartitionUtils.getTimePartitionSlot( timeRangeList.get(timeRangeList.size() - 1).getMin()); @@ -2262,13 +2282,29 @@ public static Pair, Pair> getTimePart } private static void reserveMemoryForTimePartitionSlot( - long maxTime, long minTime, MPPQueryContext context) { - if (maxTime == Long.MAX_VALUE || minTime == Long.MIN_VALUE) { + long maxTime, long minTime, boolean needRightAll, MPPQueryContext context) { + if (maxTime == Long.MAX_VALUE + || minTime == Long.MIN_VALUE + || isRangeCoveringRightUnclosedPartitions(minTime, maxTime, needRightAll)) { return; } long size = TimePartitionUtils.getEstimateTimePartitionSize(minTime, maxTime); - context.reserveMemoryForFrontEnd( - RamUsageEstimator.shallowSizeOfInstance(TTimePartitionSlot.class) * size); + context.reserveMemoryForFrontEnd(estimateTimePartitionSlotMemory(size)); + } + + private static boolean isRangeCoveringRightUnclosedPartitions( + long minTime, long maxTime, boolean needRightAll) { + return needRightAll + && maxTime == Long.MAX_VALUE - 1 + && TimePartitionUtils.getTimePartitionSlot(minTime).getStartTime() + != TimePartitionUtils.getTimePartitionSlot(Long.MAX_VALUE).getStartTime(); + } + + static long estimateTimePartitionSlotMemory(long timePartitionSlotCount) { + long timePartitionSlotSize = RamUsageEstimator.shallowSizeOfInstance(TTimePartitionSlot.class); + return timePartitionSlotCount > Long.MAX_VALUE / timePartitionSlotSize + ? Long.MAX_VALUE + : timePartitionSlotSize * timePartitionSlotCount; } private void analyzeInto( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/ExpressionFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/ExpressionFactory.java index f0d8a2cd0b805..0e56041ed3802 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/ExpressionFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/ExpressionFactory.java @@ -217,13 +217,26 @@ public static BetweenExpression notBetween( } public static GroupByTimeExpression groupByTime(GroupByTimeParameter parameter) { + if (!parameter.isLeftCRightO() && parameter.getEndTime() == Long.MAX_VALUE) { + throw new IllegalArgumentException( + "Right-closed GROUP BY TIME with Long.MAX_VALUE end time cannot be represented as a single right-open time filter."); + } long startTime = - parameter.isLeftCRightO() ? parameter.getStartTime() : parameter.getStartTime() + 1; - long endTime = parameter.isLeftCRightO() ? parameter.getEndTime() : parameter.getEndTime() + 1; + parameter.isLeftCRightO() + ? parameter.getStartTime() + : saturatingIncrement(parameter.getStartTime()); + long endTime = + parameter.isLeftCRightO() + ? parameter.getEndTime() + : saturatingIncrement(parameter.getEndTime()); return new GroupByTimeExpression( startTime, endTime, parameter.getInterval(), parameter.getSlidingStep()); } + private static long saturatingIncrement(long value) { + return value == Long.MAX_VALUE ? Long.MAX_VALUE : value + 1; + } + public static GroupByTimeExpression groupByTime( long startTime, long endTime, long interval, long slidingStep) { return new GroupByTimeExpression( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/optimization/LimitOffsetPushDown.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/optimization/LimitOffsetPushDown.java index c291a196b5ac3..3f5df7823aad8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/optimization/LimitOffsetPushDown.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/optimization/LimitOffsetPushDown.java @@ -48,6 +48,7 @@ import org.apache.tsfile.utils.TimeDuration; +import java.math.BigInteger; import java.time.ZoneId; import java.util.ArrayList; import java.util.Collections; @@ -69,6 +70,11 @@ */ public class LimitOffsetPushDown implements PlanOptimizer { + private static final BigInteger BIG_LONG_MIN = BigInteger.valueOf(Long.MIN_VALUE); + private static final BigInteger BIG_LONG_MAX = BigInteger.valueOf(Long.MAX_VALUE); + private static final BigInteger BIG_INTEGER_MAX = BigInteger.valueOf(Integer.MAX_VALUE); + private static final BigInteger BIG_INTEGER_MIN = BigInteger.valueOf(Integer.MIN_VALUE); + @Override public PlanNode optimize(PlanNode plan, Analysis analysis, MPPQueryContext context) { StatementType statementType = analysis.getTreeStatement().getType(); @@ -320,8 +326,8 @@ private static void pushDownLimitOffsetToTimeParameterContainingMonth( // Evaluate the day of month as 28 days long totalStep = slidingStep.getMinTotalDuration(TimeUnit.MILLISECONDS); - long size = (endTime - startTime + totalStep - 1) / totalStep; - if (size > offsetSize) { + BigInteger size = ceilDivTimeRange(startTime, endTime, totalStep); + if (size.compareTo(BigInteger.valueOf(offsetSize)) > 0) { TimeZone timeZone = TimeZone.getTimeZone(zoneId); // ordering in group by month must be ascending long newStartTime = @@ -351,10 +357,19 @@ private static void pushDownLimitOffsetToTimeParameterContainingMonth( private static TimeDuration calculateEndTimeDuration( TimeDuration slidingStep, TimeDuration interval, long limitSize, long offsetSize) { - long length = offsetSize + limitSize - 1; + BigInteger length = + BigInteger.valueOf(offsetSize).add(BigInteger.valueOf(limitSize)).subtract(BigInteger.ONE); // startTime + offsetSize * step + (limitSize - 1) * step + interval - int monthDuration = (int) (length * slidingStep.monthDuration + interval.monthDuration); - long nonMonthDuration = length * slidingStep.nonMonthDuration + interval.nonMonthDuration; + int monthDuration = + saturateToInt( + length + .multiply(BigInteger.valueOf(slidingStep.monthDuration)) + .add(BigInteger.valueOf(interval.monthDuration))); + long nonMonthDuration = + saturateToLong( + length + .multiply(BigInteger.valueOf(slidingStep.nonMonthDuration)) + .add(BigInteger.valueOf(interval.nonMonthDuration))); return new TimeDuration(monthDuration, nonMonthDuration); } @@ -373,18 +388,26 @@ public static void pushDownLimitOffsetToTimeParameter( long interval = groupByTimeComponent.getInterval().nonMonthDuration; long limitSize = queryStatement.getRowLimit(); long offsetSize = queryStatement.getRowOffset(); - long size = (endTime - startTime + step - 1) / step; - if (size > offsetSize) { + BigInteger size = ceilDivTimeRange(startTime, endTime, step); + if (size.compareTo(BigInteger.valueOf(offsetSize)) > 0) { if (queryStatement.getResultTimeOrder() == Ordering.ASC) { - startTime = startTime + offsetSize * step; + startTime = addTimeDuration(startTime, BigInteger.valueOf(offsetSize), step, 0); } else { - long startTimeInterval = size - offsetSize - limitSize; - startTime = startTime + (startTimeInterval < 0 ? 0 : startTimeInterval) * step; + BigInteger startTimeInterval = + size.subtract(BigInteger.valueOf(offsetSize)).subtract(BigInteger.valueOf(limitSize)); + startTime = + addTimeDuration( + startTime, + startTimeInterval.signum() < 0 ? BigInteger.ZERO : startTimeInterval, + step, + 0); } endTime = limitSize == 0 ? endTime - : Math.min(endTime, startTime + (limitSize - 1) * step + interval); + : Math.min( + endTime, + addTimeDuration(startTime, BigInteger.valueOf(limitSize - 1), step, interval)); groupByTimeComponent.setEndTime(endTime); groupByTimeComponent.setStartTime(startTime); } else { @@ -423,8 +446,11 @@ public static List pushDownLimitOffsetInGroupByTimeForDevice( long startTime = groupByTimeComponent.getStartTime(); long endTime = groupByTimeComponent.getEndTime(); long slidingStep = groupByTimeComponent.getSlidingStep().nonMonthDuration; - long size = (endTime - startTime + slidingStep - 1) / slidingStep; - if (size == 0 || size * deviceNames.size() <= queryStatement.getRowOffset()) { + BigInteger size = ceilDivTimeRange(startTime, endTime, slidingStep); + if (size.signum() == 0 + || size.multiply(BigInteger.valueOf(deviceNames.size())) + .compareTo(BigInteger.valueOf(queryStatement.getRowOffset())) + <= 0) { // resultSet is empty queryStatement.setResultSetEmpty(true); return deviceNames; @@ -433,19 +459,20 @@ public static List pushDownLimitOffsetInGroupByTimeForDevice( long limitSize = queryStatement.getRowLimit(); long offsetSize = queryStatement.getRowOffset(); List optimizedDeviceNames = new ArrayList<>(); - int startDeviceIndex = (int) (offsetSize / size); + int startDeviceIndex = saturateToInt(BigInteger.valueOf(offsetSize).divide(size)); int endDeviceIndex = limitSize == 0 ? deviceNames.size() - 1 - : (int) - ((limitSize - ((startDeviceIndex + 1) * size - offsetSize) + size - 1) / size - + startDeviceIndex); + : calculateEndDeviceIndex(size, limitSize, offsetSize, startDeviceIndex); int index = 0; while (index < startDeviceIndex) { index++; } - queryStatement.setRowOffset(offsetSize - startDeviceIndex * size); + queryStatement.setRowOffset( + saturateToLong( + BigInteger.valueOf(offsetSize) + .subtract(BigInteger.valueOf(startDeviceIndex).multiply(size)))); // if only refer to one device, optimize the time parameter if (startDeviceIndex == endDeviceIndex) { @@ -465,4 +492,54 @@ public static List pushDownLimitOffsetInGroupByTimeForDevice( private static boolean hasLimitOffset(QueryStatement queryStatement) { return queryStatement.hasLimit() || queryStatement.hasOffset(); } + + private static BigInteger ceilDivTimeRange(long startTime, long endTime, long divisor) { + return BigInteger.valueOf(endTime) + .subtract(BigInteger.valueOf(startTime)) + .add(BigInteger.valueOf(divisor).subtract(BigInteger.ONE)) + .divide(BigInteger.valueOf(divisor)); + } + + private static long addTimeDuration( + long startTime, BigInteger stepCount, long step, long interval) { + return saturateToLong( + BigInteger.valueOf(startTime) + .add(stepCount.multiply(BigInteger.valueOf(step))) + .add(BigInteger.valueOf(interval))); + } + + private static int calculateEndDeviceIndex( + BigInteger size, long limitSize, long offsetSize, int startDeviceIndex) { + BigInteger firstDeviceRemaining = + BigInteger.valueOf(startDeviceIndex + 1L) + .multiply(size) + .subtract(BigInteger.valueOf(offsetSize)); + return saturateToInt( + BigInteger.valueOf(limitSize) + .subtract(firstDeviceRemaining) + .add(size) + .subtract(BigInteger.ONE) + .divide(size) + .add(BigInteger.valueOf(startDeviceIndex))); + } + + private static long saturateToLong(BigInteger value) { + if (value.compareTo(BIG_LONG_MAX) > 0) { + return Long.MAX_VALUE; + } + if (value.compareTo(BIG_LONG_MIN) < 0) { + return Long.MIN_VALUE; + } + return value.longValue(); + } + + private static int saturateToInt(BigInteger value) { + if (value.compareTo(BIG_INTEGER_MAX) > 0) { + return Integer.MAX_VALUE; + } + if (value.compareTo(BIG_INTEGER_MIN) < 0) { + return Integer.MIN_VALUE; + } + return value.intValue(); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java index 49f79a6a923c0..f5d812b884de5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java @@ -1920,7 +1920,8 @@ private GroupByTimeComponent parseGroupByTimeClause( TimeDuration slidingStep = groupByTimeComponent.getSlidingStep(); if (slidingStep.containsMonth() && Math.ceil( - ((groupByTimeComponent.getEndTime() - groupByTimeComponent.getStartTime()) + (((double) groupByTimeComponent.getEndTime() + - (double) groupByTimeComponent.getStartTime()) / (double) slidingStep.getMinTotalDuration(currPrecision))) >= 10000) { throw new SemanticException( @@ -3156,10 +3157,16 @@ private TimeRange parseTimeRangeForDeleteTimeRange( long time = Long.parseLong(((ConstantOperand) valueExpression).getValueString()); switch (expressionType) { case LESS_THAN: + if (time == Long.MIN_VALUE) { + throw new SemanticException(DELETE_RANGE_COMPARISON_ERROR_MSG); + } return new TimeRange(Long.MIN_VALUE, time - 1); case LESS_EQUAL: return new TimeRange(Long.MIN_VALUE, time); case GREATER_THAN: + if (time == Long.MAX_VALUE) { + throw new SemanticException(DELETE_RANGE_COMPARISON_ERROR_MSG); + } return new TimeRange(time + 1, Long.MAX_VALUE); case GREATER_EQUAL: return new TimeRange(time, Long.MAX_VALUE); @@ -3613,14 +3620,16 @@ public Long parseDateExpression(IoTDBSqlParser.DateExpressionContext ctx, String long time; time = parseDateTimeFormat(ctx.getChild(0).getText()); for (int i = 1; i < ctx.getChildCount(); i = i + 2) { - if ("+".equals(ctx.getChild(i).getText())) { - time += - DataNodeDateTimeUtils.convertDurationStrToLong( - time, ctx.getChild(i + 1).getText(), precision, false); - } else { - time -= + try { + long duration = DataNodeDateTimeUtils.convertDurationStrToLong( time, ctx.getChild(i + 1).getText(), precision, false); + time = + "+".equals(ctx.getChild(i).getText()) + ? Math.addExact(time, duration) + : Math.subtractExact(time, duration); + } catch (ArithmeticException e) { + throw new SemanticException("Date expression is out of range: " + ctx.getText()); } } return time; @@ -3630,14 +3639,16 @@ private Long parseDateExpression(IoTDBSqlParser.DateExpressionContext ctx, long long time; time = parseDateTimeFormat(ctx.getChild(0).getText(), currentTime, zoneId); for (int i = 1; i < ctx.getChildCount(); i = i + 2) { - if ("+".equals(ctx.getChild(i).getText())) { - time += - DataNodeDateTimeUtils.convertDurationStrToLong( - time, ctx.getChild(i + 1).getText(), false); - } else { - time -= + try { + long duration = DataNodeDateTimeUtils.convertDurationStrToLong( time, ctx.getChild(i + 1).getText(), false); + time = + "+".equals(ctx.getChild(i).getText()) + ? Math.addExact(time, duration) + : Math.subtractExact(time, duration); + } catch (ArithmeticException e) { + throw new SemanticException("Date expression is out of range: " + ctx.getText()); } } return time; @@ -4545,6 +4556,10 @@ public GetRegionIdStatement parseTimeRangeExpression( Math.max(getRegionIdStatement.getStartTimeStamp(), timestamp)); break; case GREATER_THAN: + if (timestamp == Long.MAX_VALUE) { + throw new SemanticException( + "The time predicate does not select any time range: " + timeRangeExpression); + } getRegionIdStatement.setStartTimeStamp( Math.max(getRegionIdStatement.getStartTimeStamp(), timestamp + 1)); break; @@ -4553,6 +4568,10 @@ public GetRegionIdStatement parseTimeRangeExpression( Math.min(getRegionIdStatement.getEndTimeStamp(), timestamp)); break; case LESS_THAN: + if (timestamp == Long.MIN_VALUE) { + throw new SemanticException( + "The time predicate does not select any time range: " + timeRangeExpression); + } getRegionIdStatement.setEndTimeStamp( Math.min(getRegionIdStatement.getEndTimeStamp(), timestamp - 1)); break; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java index 98d0eca98b0c3..7fabdcea18316 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java @@ -29,7 +29,6 @@ import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.commons.queryengine.utils.DateTimeUtils; -import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.exception.DataTypeInconsistentException; @@ -77,6 +76,7 @@ import java.util.Map.Entry; import java.util.Objects; +import static org.apache.iotdb.db.utils.CommonUtils.getTTLLowerBound; import static org.apache.iotdb.db.utils.CommonUtils.isAlive; public class InsertTabletNode extends InsertNode implements WALEntryValue { @@ -254,7 +254,9 @@ private Map collectSplitRanges() { for (int i = 1; i < rowCount; i++) { // times are sorted in session API. IDeviceID nextDeviceId = getDeviceID(i); - if (times[i] >= upperBoundOfTimePartition || !currDeviceId.equals(nextDeviceId)) { + if (TimePartitionUtils.isAfterOrEqualToTimePartitionUpperBound( + times[i], timePartitionSlot.getStartTime(), upperBoundOfTimePartition) + || !currDeviceId.equals(nextDeviceId)) { final PartitionSplitInfo splitInfo = deviceIDSplitInfoMap.computeIfAbsent( currDeviceId, deviceID1 -> new PartitionSplitInfo()); @@ -393,7 +395,8 @@ public List getTimePartitionSlots() { long upperBoundOfTimePartition = TimePartitionUtils.getTimePartitionUpperBound(times[0]); TTimePartitionSlot timePartitionSlot = TimePartitionUtils.getTimePartitionSlot(times[0]); for (int i = 1; i < times.length; i++) { // times are sorted in session API. - if (times[i] >= upperBoundOfTimePartition) { + if (TimePartitionUtils.isAfterOrEqualToTimePartitionUpperBound( + times[i], timePartitionSlot.getStartTime(), upperBoundOfTimePartition)) { result.add(timePartitionSlot); // next init upperBoundOfTimePartition = TimePartitionUtils.getTimePartitionUpperBound(times[i]); @@ -1388,7 +1391,7 @@ protected int checkTTLInternal(TSStatus[] results, long ttl, boolean breakOnFirs String.format( "Insertion time [%s] is less than ttl time bound [%s]", DateTimeUtils.convertLongToDate(currTime), - DateTimeUtils.convertLongToDate(CommonDateTimeUtils.currentTime() - ttl))); + DateTimeUtils.convertLongToDate(getTTLLowerBound(ttl)))); } else { if (firstAliveLoc == -1) { firstAliveLoc = loc; @@ -1402,8 +1405,7 @@ protected int checkTTLInternal(TSStatus[] results, long ttl, boolean breakOnFirs if (firstAliveLoc == -1) { // no alive data - throw new OutOfTTLException( - getTimes()[getTimes().length - 1], (CommonDateTimeUtils.currentTime() - ttl)); + throw new OutOfTTLException(getTimes()[getTimes().length - 1], getTTLLowerBound(ttl)); } return firstAliveLoc; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/SeriesScanOptions.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/SeriesScanOptions.java index 38335a788f575..15a71e2364285 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/SeriesScanOptions.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/SeriesScanOptions.java @@ -22,8 +22,8 @@ import org.apache.iotdb.commons.path.AlignedFullPath; import org.apache.iotdb.commons.path.IFullPath; import org.apache.iotdb.commons.path.NonAlignedFullPath; -import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import org.apache.iotdb.db.queryengine.execution.operator.source.relational.TreeNonAlignedDeviceViewAggregationScanOperator; +import org.apache.iotdb.db.utils.CommonUtils; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.read.filter.factory.FilterFactory; @@ -156,12 +156,11 @@ public void setTTLForTableView(long ttlForTableView) { */ public static Filter updateFilterUsingTTL(Filter filter, long dataTTL) { if (dataTTL != Long.MAX_VALUE) { + long ttlLowerBound = CommonUtils.getTTLLowerBound(dataTTL); if (filter != null) { - filter = - FilterFactory.and( - filter, TimeFilterApi.gtEq(CommonDateTimeUtils.currentTime() - dataTTL)); + filter = FilterFactory.and(filter, TimeFilterApi.gtEq(ttlLowerBound)); } else { - filter = TimeFilterApi.gtEq(CommonDateTimeUtils.currentTime() - dataTTL); + filter = TimeFilterApi.gtEq(ttlLowerBound); } } return filter; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/GapFillStartAndEndTimeExtractVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/GapFillStartAndEndTimeExtractVisitor.java index bce19f447808f..0987e88fb460b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/GapFillStartAndEndTimeExtractVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/GapFillStartAndEndTimeExtractVisitor.java @@ -248,10 +248,16 @@ public long[] getTimeRange( } long[] result = new long[2]; if (leftOperator == GREATER_THAN) { + if (startTime == Long.MAX_VALUE) { + throw new SemanticException(CAN_NOT_INFER_TIME_RANGE); + } startTime++; } result[0] = dateBin(startTime, origin, monthDuration, nonMonthDuration, zoneId); if (rightOperator == LESS_THAN) { + if (endTime == Long.MIN_VALUE) { + throw new SemanticException(CAN_NOT_INFER_TIME_RANGE); + } endTime--; } result[1] = dateBin(endTime, origin, monthDuration, nonMonthDuration, zoneId); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index a19168b50f42f..64580c361fe78 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -3415,14 +3415,16 @@ private Long parseDateExpression( long time; time = parseDateTimeFormat(ctx.getChild(0).getText(), currentTime, zoneId); for (int i = 1; i < ctx.getChildCount(); i = i + 2) { - if ("+".equals(ctx.getChild(i).getText())) { - time += - DataNodeDateTimeUtils.convertDurationStrToLong( - time, ctx.getChild(i + 1).getText(), false); - } else { - time -= + try { + long duration = DataNodeDateTimeUtils.convertDurationStrToLong( time, ctx.getChild(i + 1).getText(), false); + time = + "+".equals(ctx.getChild(i).getText()) + ? Math.addExact(time, duration) + : Math.subtractExact(time, duration); + } catch (ArithmeticException e) { + throw new SemanticException("Date expression is out of range: " + ctx.getText()); } } return time; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java index 8ce0d534a4980..9d7d3dd033b40 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java @@ -252,7 +252,8 @@ public List getTimePartitionSlots() { long upperBoundOfTimePartition = TimePartitionUtils.getTimePartitionUpperBound(times[0]); TTimePartitionSlot timePartitionSlot = TimePartitionUtils.getTimePartitionSlot(times[0]); for (int i = 1; i < times.length; i++) { // times are sorted in session API. - if (times[i] >= upperBoundOfTimePartition) { + if (TimePartitionUtils.isAfterOrEqualToTimePartitionUpperBound( + times[i], timePartitionSlot.getStartTime(), upperBoundOfTimePartition)) { result.add(timePartitionSlot); // next init upperBoundOfTimePartition = TimePartitionUtils.getTimePartitionUpperBound(times[i]); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/IntermediateLayer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/IntermediateLayer.java index 13c73485ea801..a1282a8b0b292 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/IntermediateLayer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/IntermediateLayer.java @@ -86,6 +86,17 @@ protected abstract LayerRowWindowReader constructRowStateWindowReader( StateWindowAccessStrategy strategy, float memoryBudgetInMB) throws QueryProcessException, IOException; + protected static long saturatingAdd(long left, long right) { + long result = left + right; + if (right > 0 && result < left) { + return Long.MAX_VALUE; + } + if (right < 0 && result > left) { + return Long.MIN_VALUE; + } + return result; + } + @Override public String toString() { return expression.getExpressionString(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/MultiInputLayer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/MultiInputLayer.java index fc0bf8ba1ee0d..510548a0c1267 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/MultiInputLayer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/MultiInputLayer.java @@ -396,7 +396,8 @@ public YieldableState yield() throws Exception { return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; } - long nextWindowTimeEnd = Math.min(nextWindowTimeBegin + timeInterval, displayWindowEnd); + long nextWindowTimeEnd = + Math.min(saturatingAdd(nextWindowTimeBegin, timeInterval), displayWindowEnd); while (currentEndTime < nextWindowTimeEnd) { final YieldableState state = udfInputDataSet.yield(); if (state == YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA) { @@ -466,7 +467,7 @@ public YieldableState yield() throws Exception { nextIndexBegin, nextIndexEnd, nextWindowTimeBegin, - nextWindowTimeBegin + timeInterval - 1); + saturatingAdd(nextWindowTimeBegin, timeInterval - 1)); hasCached = !(nextIndexBegin == nextIndexEnd && nextIndexEnd == rowRecordList.size()); return hasCached ? YieldableState.YIELDABLE : YieldableState.NOT_YIELDABLE_NO_MORE_DATA; @@ -475,7 +476,7 @@ public YieldableState yield() throws Exception { @Override public void readyForNext() { hasCached = false; - nextWindowTimeBegin += slidingStep; + nextWindowTimeBegin = saturatingAdd(nextWindowTimeBegin, slidingStep); rowRecordList.setEvictionUpperBound(nextIndexBegin + 1); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/SingleInputMultiReferenceLayer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/SingleInputMultiReferenceLayer.java index e272a1b99b946..689992cc6e7ad 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/SingleInputMultiReferenceLayer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/SingleInputMultiReferenceLayer.java @@ -286,7 +286,8 @@ public YieldableState yield() throws Exception { return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; } - long nextWindowTimeEnd = Math.min(nextWindowTimeBegin + timeInterval, displayWindowEnd); + long nextWindowTimeEnd = + Math.min(saturatingAdd(nextWindowTimeBegin, timeInterval), displayWindowEnd); while (currentEndTime < nextWindowTimeEnd) { final YieldableState state = parentLayerReader.yield(); if (state == YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA) { @@ -357,7 +358,7 @@ public YieldableState yield() throws Exception { nextIndexBegin, nextIndexEnd, nextWindowTimeBegin, - nextWindowTimeBegin + timeInterval - 1); + saturatingAdd(nextWindowTimeBegin, timeInterval - 1)); hasCached = !(nextIndexBegin == nextIndexEnd && nextIndexEnd == tvList.size()); return hasCached ? YieldableState.YIELDABLE : YieldableState.NOT_YIELDABLE_NO_MORE_DATA; @@ -366,7 +367,7 @@ public YieldableState yield() throws Exception { @Override public void readyForNext() { hasCached = false; - nextWindowTimeBegin += slidingStep; + nextWindowTimeBegin = saturatingAdd(nextWindowTimeBegin, slidingStep); safetyPile.moveForwardTo(nextIndexBegin + 1); tvList.setEvictionUpperBound(safetyLine.getSafetyLine()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/SingleInputSingleReferenceLayer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/SingleInputSingleReferenceLayer.java index 928a2e68b53b1..0809f1e8138fb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/SingleInputSingleReferenceLayer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/SingleInputSingleReferenceLayer.java @@ -209,7 +209,8 @@ public YieldableState yield() throws Exception { return YieldableState.NOT_YIELDABLE_NO_MORE_DATA; } - long nextWindowTimeEnd = Math.min(nextWindowTimeBegin + timeInterval, displayWindowEnd); + long nextWindowTimeEnd = + Math.min(saturatingAdd(nextWindowTimeBegin, timeInterval), displayWindowEnd); while (currentEndTime < nextWindowTimeEnd) { final YieldableState state = parentLayerReader.yield(); if (state == YieldableState.NOT_YIELDABLE_WAITING_FOR_DATA) { @@ -280,7 +281,7 @@ public YieldableState yield() throws Exception { nextIndexBegin, nextIndexEnd, nextWindowTimeBegin, - nextWindowTimeBegin + timeInterval - 1); + saturatingAdd(nextWindowTimeBegin, timeInterval - 1)); hasCached = !(nextIndexBegin == nextIndexEnd && nextIndexEnd == tvList.size()); return hasCached ? YieldableState.YIELDABLE : YieldableState.NOT_YIELDABLE_NO_MORE_DATA; @@ -289,7 +290,7 @@ public YieldableState yield() throws Exception { @Override public void readyForNext() { hasCached = false; - nextWindowTimeBegin += slidingStep; + nextWindowTimeBegin = saturatingAdd(nextWindowTimeBegin, slidingStep); tvList.setEvictionUpperBound(nextIndexBegin + 1); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java index db51775281d43..17717780307c7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java @@ -178,6 +178,8 @@ public static StorageEngine getInstance() { } private static void initTimePartition() { + TimePartitionUtils.setTimePartitionOrigin( + CommonDescriptor.getInstance().getConfig().getTimePartitionOrigin()); TimePartitionUtils.setTimePartitionInterval( CommonDescriptor.getInstance().getConfig().getTimePartitionInterval()); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 8c719dfdaac35..58feaf2c545b0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -45,7 +45,6 @@ import org.apache.iotdb.commons.service.metric.PerformanceOverviewMetrics; import org.apache.iotdb.commons.service.metric.enums.Metric; import org.apache.iotdb.commons.service.metric.enums.Tag; -import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import org.apache.iotdb.commons.utils.RetryUtils; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.commons.utils.TimePartitionUtils; @@ -1186,8 +1185,7 @@ public void insert(InsertRowNode insertRowNode) throws WriteProcessException { // reject insertions that are out of ttl long ttl = getTTL(insertRowNode); if (!CommonUtils.isAlive(insertRowNode.getTime(), ttl)) { - throw new OutOfTTLException( - insertRowNode.getTime(), (CommonDateTimeUtils.currentTime() - ttl)); + throw new OutOfTTLException(insertRowNode.getTime(), CommonUtils.getTTLLowerBound(ttl)); } StorageEngine.blockInsertionIfReject(); long startTime = System.nanoTime(); @@ -4671,8 +4669,7 @@ public void insert(InsertRowsOfOneDeviceNode insertRowsOfOneDeviceNode) String.format( "Insertion time [%s] is less than ttl time bound [%s]", DateTimeUtils.convertLongToDate(insertRowNode.getTime()), - DateTimeUtils.convertLongToDate( - CommonDateTimeUtils.currentTime() - ttl)))); + DateTimeUtils.convertLongToDate(CommonUtils.getTTLLowerBound(ttl))))); continue; } // init map @@ -4787,8 +4784,7 @@ public void insert(InsertRowsNode insertRowsNode) String.format( "Insertion time [%s] is less than ttl time bound [%s]", DateTimeUtils.convertLongToDate(insertRowNode.getTime()), - DateTimeUtils.convertLongToDate( - CommonDateTimeUtils.currentTime() - ttl)))); + DateTimeUtils.convertLongToDate(CommonUtils.getTTLLowerBound(ttl))))); insertRowNode.setFailedMeasurementNumber(insertRowNode.getMeasurements().length); insertRowNode.setMeasurements(null); continue; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java index a639fba299cb9..d711cf9269125 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java @@ -26,7 +26,6 @@ import org.apache.iotdb.commons.path.PatternTreeMap; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; -import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeTTLCache; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.db.storageengine.dataregion.compaction.io.CompactionTsFileReader; @@ -35,6 +34,7 @@ import org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry; import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.utils.CommonUtils; import org.apache.iotdb.db.utils.EncryptDBUtils; import org.apache.iotdb.db.utils.ModificationUtils; import org.apache.iotdb.db.utils.datastructure.PatternTreeMapFactory; @@ -245,7 +245,7 @@ public Pair nextDevice() throws IllegalPathException, IOExce timeLowerBoundForCurrentDevice = ttlForCurrentDevice == Long.MAX_VALUE ? Long.MIN_VALUE - : CommonDateTimeUtils.currentTime() - ttlForCurrentDevice; + : CommonUtils.getTTLLowerBound(ttlForCurrentDevice); return currentDevice; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java index f2b37bda42ceb..8d8251ca3d1c9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java @@ -45,6 +45,7 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex; import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo; +import org.apache.iotdb.db.utils.CommonUtils; import org.apache.tsfile.exception.StopReadTsFileByInterruptException; import org.apache.tsfile.file.metadata.IDeviceID; @@ -360,7 +361,7 @@ public List selectCrossSpaceTask( boolean isInsertionTask) { // TODO: (xingtanzjr) need to confirm what this ttl is used for long startTime = System.currentTimeMillis(); - long ttlLowerBound = System.currentTimeMillis() - Long.MAX_VALUE; + long ttlLowerBound = CommonUtils.getTTLLowerBound(Long.MAX_VALUE); // we record the variable `candidate` here is used for selecting more than one // CrossCompactionTaskResources in this method. // Add read lock for candidate source files to avoid being deleted during the selection. diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java index 01a61eee5cca3..22ac44130bdf0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java @@ -309,10 +309,10 @@ public void initChunkMetaFromTVListsWithFakeStatistics() { (int) Math.min( MAX_NUMBER_OF_FAKE_PAGE, Math.max(1, rowNum / MAX_NUMBER_OF_POINTS_IN_FAKE_PAGE)); - long timeInterval = (chunkEndTime - chunkStartTime + 1) / pageNum; - for (int i = 0; i < pageNum; i++) { - long pageStartTime = chunkStartTime + i * timeInterval; - long pageEndTime = (i == pageNum - 1) ? chunkEndTime : (pageStartTime + timeInterval - 1); + for (long[] pageTimeRange : + MemChunkTimeRangeUtils.splitFakePageTimeRanges(chunkStartTime, chunkEndTime, pageNum)) { + long pageStartTime = pageTimeRange[0]; + long pageEndTime = pageTimeRange[1]; Statistics[] pageValueStatistics = new Statistics[dataTypes.size()]; for (int column = 0; column < dataTypes.size(); column++) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemChunkTimeRangeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemChunkTimeRangeUtils.java new file mode 100644 index 0000000000000..10ea5197630f8 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemChunkTimeRangeUtils.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion.memtable; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.List; + +final class MemChunkTimeRangeUtils { + + private MemChunkTimeRangeUtils() { + // Utility class. + } + + static List splitFakePageTimeRanges(long chunkStartTime, long chunkEndTime, int pageNum) { + BigInteger timeRange = + BigInteger.valueOf(chunkEndTime) + .subtract(BigInteger.valueOf(chunkStartTime)) + .add(BigInteger.ONE); + int effectivePageNum = + timeRange.compareTo(BigInteger.valueOf(pageNum)) < 0 ? timeRange.intValue() : pageNum; + BigInteger pageTimeInterval = timeRange.divide(BigInteger.valueOf(effectivePageNum)); + BigInteger chunkStartTimeAsBigInteger = BigInteger.valueOf(chunkStartTime); + + List pageTimeRanges = new ArrayList<>(effectivePageNum); + for (int i = 0; i < effectivePageNum; i++) { + BigInteger pageStartTime = + chunkStartTimeAsBigInteger.add(pageTimeInterval.multiply(BigInteger.valueOf(i))); + BigInteger pageEndTime = + i == effectivePageNum - 1 + ? BigInteger.valueOf(chunkEndTime) + : chunkStartTimeAsBigInteger + .add(pageTimeInterval.multiply(BigInteger.valueOf(i + 1))) + .subtract(BigInteger.ONE); + pageTimeRanges.add(new long[] {pageStartTime.longValue(), pageEndTime.longValue()}); + } + return pageTimeRanges; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java index 4c32be0e79828..c5990fb70f7a9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java @@ -249,10 +249,10 @@ public void initChunkMetaFromTVListsWithFakeStatistics() { (int) Math.min( MAX_NUMBER_OF_FAKE_PAGE, Math.max(1, rowNum / MAX_NUMBER_OF_POINTS_IN_FAKE_PAGE)); - long timeInterval = (chunkEndTime - chunkStartTime + 1) / pageNum; - for (int i = 0; i < pageNum; i++) { - long pageStartTime = chunkStartTime + i * timeInterval; - long pageEndTime = (i == pageNum - 1) ? chunkEndTime : (pageStartTime + timeInterval - 1); + for (long[] pageTimeRange : + MemChunkTimeRangeUtils.splitFakePageTimeRanges(chunkStartTime, chunkEndTime, pageNum)) { + long pageStartTime = pageTimeRange[0]; + long pageEndTime = pageTimeRange[1]; pageStatisticsList.add(generateFakeStatistics(dataType, pageStartTime, pageEndTime)); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java index c4e488f84863f..e9602015d65eb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java @@ -29,7 +29,6 @@ import org.apache.iotdb.commons.path.AlignedPath; import org.apache.iotdb.commons.path.IFullPath; import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; -import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; @@ -75,6 +74,7 @@ import org.apache.iotdb.db.storageengine.rescon.memory.MemTableManager; import org.apache.iotdb.db.storageengine.rescon.memory.PrimitiveArrayManager; import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo; +import org.apache.iotdb.db.utils.CommonUtils; import org.apache.iotdb.db.utils.EncryptDBUtils; import org.apache.iotdb.db.utils.MemUtils; import org.apache.iotdb.db.utils.ModificationUtils; @@ -2224,7 +2224,7 @@ private long getQueryTimeLowerBound(IDeviceID deviceID) { DataNodeTTLCache.getInstance() .getTTLForTable(this.dataRegionName, deviceID.getTableName()); } - return ttl != Long.MAX_VALUE ? CommonDateTimeUtils.currentTime() - ttl : Long.MIN_VALUE; + return ttl != Long.MAX_VALUE ? CommonUtils.getTTLLowerBound(ttl) : Long.MIN_VALUE; } public long getTimeRangeId() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java index 439286288d623..d396059654a94 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java @@ -25,7 +25,6 @@ import org.apache.iotdb.commons.path.IFullPath; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.pipe.datastructure.resource.PersistentResource; -import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; @@ -1055,13 +1054,6 @@ public boolean isSatisfied(IDeviceID deviceId, Filter timeFilter, boolean isSeq, return true; } - /** - * @return whether the given time falls in ttl - */ - private boolean isAlive(long time, long dataTTL) { - return dataTTL == Long.MAX_VALUE || (CommonDateTimeUtils.currentTime() - time) <= dataTTL; - } - /** * Check whether the given device may still alive or not. Return false if the device does not * exist or out of dated. @@ -1293,15 +1285,11 @@ public static int compareFileName(TsFileResource o1, TsFileResource o2) { public static int checkAndCompareFileName(String fileName1, String fileName2) throws IOException { TsFileNameGenerator.TsFileName tsFileName1 = TsFileNameGenerator.getTsFileName(fileName1); TsFileNameGenerator.TsFileName tsFileName2 = TsFileNameGenerator.getTsFileName(fileName2); - long timeDiff = tsFileName1.getTime() - tsFileName2.getTime(); - if (timeDiff != 0) { - return timeDiff < 0 ? -1 : 1; + int timeCompare = Long.compare(tsFileName1.getTime(), tsFileName2.getTime()); + if (timeCompare != 0) { + return timeCompare; } - long versionDiff = tsFileName1.getVersion() - tsFileName2.getVersion(); - if (versionDiff != 0) { - return versionDiff < 0 ? -1 : 1; - } - return 0; + return Long.compare(tsFileName1.getVersion(), tsFileName2.getVersion()); } /** @@ -1322,11 +1310,7 @@ public static int compareFileCreationOrderByDesc(TsFileResource o1, TsFileResour TsFileNameGenerator.getTsFileName(o1.getTsFile().getName()); TsFileNameGenerator.TsFileName n2 = TsFileNameGenerator.getTsFileName(o2.getTsFile().getName()); - long versionDiff = n2.getVersion() - n1.getVersion(); - if (versionDiff != 0) { - return versionDiff < 0 ? -1 : 1; - } - return 0; + return Long.compare(n2.getVersion(), n1.getVersion()); } catch (IOException e) { LOGGER.error(StorageEngineMessages.FILE_NAME_NOT_STANDARD, e); throw new RuntimeException(e.getMessage()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java index 37c8f1a964fa1..0b21d3ff9e44c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java @@ -414,7 +414,7 @@ public static TsFileResource getSettleCompactionTargetFileResources( } public static class TsFileName { - private static final String FILE_NAME_PATTERN = "(\\d+)-(\\d+)-(\\d+)-(\\d+).tsfile$"; + private static final String FILE_NAME_PATTERN = "(-?\\d+)-(-?\\d+)-(\\d+)-(\\d+)\\.tsfile$"; private static final Pattern FILE_NAME_MATCHER = Pattern.compile(TsFileName.FILE_NAME_PATTERN); private long time; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ArrayDeviceTimeIndex.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ArrayDeviceTimeIndex.java index a0ceb57de9a9a..b38bb8b8cc9e9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ArrayDeviceTimeIndex.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ArrayDeviceTimeIndex.java @@ -21,11 +21,11 @@ import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.exception.load.PartitionViolationException; import org.apache.iotdb.db.i18n.StorageEngineMessages; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.utils.CommonUtils; import com.google.common.util.concurrent.RateLimiter; import org.apache.tsfile.file.metadata.IDeviceID; @@ -438,7 +438,7 @@ public boolean definitelyNotContains(IDeviceID device) { @Override public boolean isDeviceAlive(IDeviceID device, long ttl) { return ttl == Long.MAX_VALUE - || endTimes[deviceToIndex.get(device)] >= CommonDateTimeUtils.currentTime() - ttl; + || endTimes[deviceToIndex.get(device)] >= CommonUtils.getTTLLowerBound(ttl); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java index e0761d9ef7c2d..3088fc1dff54c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java @@ -20,12 +20,12 @@ package org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex; import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import org.apache.iotdb.commons.utils.IOUtils; import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.exception.load.PartitionViolationException; import org.apache.iotdb.db.i18n.StorageEngineMessages; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.utils.CommonUtils; import com.google.common.util.concurrent.RateLimiter; import org.apache.tsfile.file.metadata.IDeviceID; @@ -278,7 +278,7 @@ public boolean definitelyNotContains(IDeviceID device) { @Override public boolean isDeviceAlive(IDeviceID device, long ttl) { - return ttl == Long.MAX_VALUE || endTime >= CommonDateTimeUtils.currentTime() - ttl; + return ttl == Long.MAX_VALUE || endTime >= CommonUtils.getTTLLowerBound(ttl); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/AlignedChunkData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/AlignedChunkData.java index d852ec3c9b85d..9d6897637ed8e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/AlignedChunkData.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/AlignedChunkData.java @@ -213,11 +213,7 @@ public void writeDecodePage(final long[] times, final Object[] values, final int pageNumbers.set(pageNumbers.size() - 1, pageNumbers.get(pageNumbers.size() - 1) + 1); satisfiedLengthQueue.offer(satisfiedLength); final long startTime = timePartitionSlot.getStartTime(); - // beware of overflow - long endTime = startTime + TimePartitionUtils.getTimePartitionInterval() - 1; - if (endTime <= startTime) { - endTime = Long.MAX_VALUE; - } + final long endTime = TimePartitionUtils.getTimePartitionEndTime(startTime); // serialize needDecode==true dataSize += ReadWriteIOUtils.write(true, stream); dataSize += ReadWriteIOUtils.write(satisfiedLength, stream); @@ -237,11 +233,7 @@ public void writeDecodeValuePage( throws IOException { pageNumbers.set(pageNumbers.size() - 1, pageNumbers.get(pageNumbers.size() - 1) + 1); final long startTime = timePartitionSlot.getStartTime(); - // beware of overflow - long endTime = startTime + TimePartitionUtils.getTimePartitionInterval() - 1; - if (endTime <= startTime) { - endTime = Long.MAX_VALUE; - } + final long endTime = TimePartitionUtils.getTimePartitionEndTime(startTime); final int satisfiedLength = satisfiedLengthQueue.poll(); // serialize needDecode==true dataSize += ReadWriteIOUtils.write(true, stream); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/BatchedAlignedValueChunkData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/BatchedAlignedValueChunkData.java index 6e8d3850b8e24..35cfb7da1b66c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/BatchedAlignedValueChunkData.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/BatchedAlignedValueChunkData.java @@ -64,11 +64,7 @@ public void writeDecodeValuePage(long[] times, TsPrimitiveType[] values, TSDataT throws IOException { pageNumbers.set(pageNumbers.size() - 1, pageNumbers.get(pageNumbers.size() - 1) + 1); final long startTime = timePartitionSlot.getStartTime(); - // beware of overflow - long endTime = startTime + TimePartitionUtils.getTimePartitionInterval() - 1; - if (endTime <= startTime) { - endTime = Long.MAX_VALUE; - } + final long endTime = TimePartitionUtils.getTimePartitionEndTime(startTime); final int satisfiedLength = satisfiedLengthQueue.poll(); // serialize needDecode==true dataSize += ReadWriteIOUtils.write(true, stream); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/NonAlignedChunkData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/NonAlignedChunkData.java index 7b3c2fd3f9523..d5f342cc5cc84 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/NonAlignedChunkData.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/NonAlignedChunkData.java @@ -181,11 +181,7 @@ public void writeDecodePage(final long[] times, final Object[] values, final int throws IOException { pageNumber += 1; final long startTime = timePartitionSlot.getStartTime(); - // beware of overflow - long endTime = startTime + TimePartitionUtils.getTimePartitionInterval() - 1; - if (endTime <= startTime) { - endTime = Long.MAX_VALUE; - } + final long endTime = TimePartitionUtils.getTimePartitionEndTime(startTime); dataSize += ReadWriteIOUtils.write(true, stream); dataSize += ReadWriteIOUtils.write(satisfiedLength, stream); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileSplitter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileSplitter.java index d43b828cced8f..5de16a822ef50 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileSplitter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileSplitter.java @@ -58,6 +58,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -71,7 +72,7 @@ public class TsFileSplitter { private final TsFileDataConsumer consumer; private Map offset2ChunkMetadata = new HashMap<>(); private List deletions = new ArrayList<>(); - private Map> pageIndex2ChunkData = new HashMap<>(); + private Map> pageIndex2ChunkData = new LinkedHashMap<>(); private Map pageIndex2Times = new HashMap<>(); private boolean isTimeChunkNeedDecode = true; private IDeviceID curDevice = null; @@ -169,7 +170,7 @@ private void processTimeChunkOrNonAlignedChunk(TsFileSequenceReader reader, byte == TsFileConstant.TIME_COLUMN_MASK); if (isAligned) { pageIndex2Times = new HashMap<>(); - pageIndex2ChunkData = new HashMap<>(); + pageIndex2ChunkData = new LinkedHashMap<>(); isTimeChunkNeedDecode = true; } @@ -260,13 +261,10 @@ private void decodeAndWriteTimeChunkOrNonAlignedChunk( int satisfiedLength = 0; long endTime = - timePartitionSlot.getStartTime() + TimePartitionUtils.getTimePartitionInterval(); - // beware of overflow - if (endTime <= timePartitionSlot.getStartTime()) { - endTime = Long.MAX_VALUE; - } + TimePartitionUtils.getTimePartitionUpperBound(timePartitionSlot.getStartTime()); for (int i = 0; i < times.length; i++) { - if (times[i] >= endTime) { + if (TimePartitionUtils.isAfterOrEqualToTimePartitionUpperBound( + times[i], timePartitionSlot.getStartTime(), endTime)) { chunkData.writeDecodePage(times, values, satisfiedLength); if (isAligned) { pageIndex2ChunkData @@ -279,10 +277,7 @@ private void decodeAndWriteTimeChunkOrNonAlignedChunk( timePartitionSlot = TimePartitionUtils.getTimePartitionSlot(times[i]); satisfiedLength = 0; endTime = - timePartitionSlot.getStartTime() + TimePartitionUtils.getTimePartitionInterval(); - if (endTime <= timePartitionSlot.getStartTime()) { - endTime = Long.MAX_VALUE; - } + TimePartitionUtils.getTimePartitionUpperBound(timePartitionSlot.getStartTime()); chunkData = ChunkData.createChunkData(isAligned, curDevice, header, timePartitionSlot); } satisfiedLength += 1; @@ -442,7 +437,7 @@ private void consumeAllAlignedChunkData( return; } - Map chunkDataMap = new HashMap<>(); + Map chunkDataMap = new LinkedHashMap<>(); for (Map.Entry> entry : pageIndex2ChunkData.entrySet()) { List alignedChunkDataList = entry.getValue(); for (int i = 0; i < alignedChunkDataList.size(); i++) { @@ -468,7 +463,7 @@ private void consumeAllAlignedChunkData( offset, chunkData)); } } - this.pageIndex2ChunkData = new HashMap<>(); + this.pageIndex2ChunkData = new LinkedHashMap<>(); } private void consumeChunkData(String measurement, long offset, ChunkData chunkData) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/validate/TsFileOverlapValidationAndRepairTool.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/validate/TsFileOverlapValidationAndRepairTool.java index bf46e95e07b62..29927c5f840df 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/validate/TsFileOverlapValidationAndRepairTool.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/validate/TsFileOverlapValidationAndRepairTool.java @@ -122,8 +122,17 @@ private static void moveSeqResourceToUnsequenceDir(TsFileResource resource) thro tsFileName.getInnerCompactionCnt(), 0); targetFile = new File(targetDir.getAbsolutePath() + File.separator + fileNameStr); + if (!targetFile.exists()) { + break; + } + if (tsFileName.getTime() == Long.MAX_VALUE) { + throw new IOException( + "Cannot repair " + + tsfile.getAbsolutePath() + + " because the target file already exists and the file timestamp is Long.MAX_VALUE"); + } tsFileName.setTime(tsFileName.getTime() + 1); - } while (targetFile.exists()); + } while (true); moveFile(tsfile, targetFile); moveFile( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java index a4656878bafc5..055500054dc83 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java @@ -363,7 +363,22 @@ public static Object[] deviceIdToObjArray(IDeviceID deviceID) { * @return whether the given time falls in ttl */ public static boolean isAlive(long time, long dataTTL) { - return dataTTL == Long.MAX_VALUE || (CommonDateTimeUtils.currentTime() - time) <= dataTTL; + return dataTTL == Long.MAX_VALUE || time >= getTTLLowerBound(dataTTL); + } + + public static long getTTLLowerBound(long dataTTL) { + if (dataTTL == Long.MAX_VALUE) { + return Long.MIN_VALUE; + } + + long currentTime = CommonDateTimeUtils.currentTime(); + if (dataTTL >= 0 && currentTime < Long.MIN_VALUE + dataTTL) { + return Long.MIN_VALUE; + } + if (dataTTL < 0 && currentTime > Long.MAX_VALUE + dataTTL) { + return Long.MAX_VALUE; + } + return currentTime - dataTTL; } public static Object createValueColumnOfDataType( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DataNodeDateTimeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DataNodeDateTimeUtils.java index fcee3bbf61d15..59c696d3aa7f1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DataNodeDateTimeUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DataNodeDateTimeUtils.java @@ -66,39 +66,39 @@ public static long convertDurationStrToLong( switch (durationUnit) { case y: case year: - res *= 365 * 86_400_000L; + res = Math.multiplyExact(value, 365 * 86_400_000L); break; case mo: case month: if (currentTime == -1) { - res *= 30 * 86_400_000L; + res = Math.multiplyExact(value, 30 * 86_400_000L); } else { Calendar calendar = Calendar.getInstance(); calendar.setTimeZone(SessionManager.getInstance().getSessionTimeZone()); calendar.setTimeInMillis(currentTime); - calendar.add(Calendar.MONTH, (int) (value)); - res = calendar.getTimeInMillis() - currentTime; + calendar.add(Calendar.MONTH, Math.toIntExact(value)); + res = Math.subtractExact(calendar.getTimeInMillis(), currentTime); } break; case w: case week: - res *= 7 * 86_400_000L; + res = Math.multiplyExact(value, 7 * 86_400_000L); break; case d: case day: - res *= 86_400_000L; + res = Math.multiplyExact(value, 86_400_000L); break; case h: case hour: - res *= 3_600_000L; + res = Math.multiplyExact(value, 3_600_000L); break; case m: case minute: - res *= 60_000L; + res = Math.multiplyExact(value, 60_000L); break; case s: case second: - res *= 1_000L; + res = Math.multiplyExact(value, 1_000L); break; default: break; @@ -112,7 +112,7 @@ public static long convertDurationStrToLong( || unit.equals(DateTimeUtils.DurationUnit.microsecond.toString())) { return value; } else { - return res * 1000; + return Math.multiplyExact(res, 1000); } } else if ("ns".equals(timestampPrecision) || "nanosecond".equals(timestampPrecision)) { if (unit.equals(DateTimeUtils.DurationUnit.ns.toString()) @@ -120,9 +120,9 @@ public static long convertDurationStrToLong( return value; } else if (unit.equals(DateTimeUtils.DurationUnit.us.toString()) || unit.equals(DateTimeUtils.DurationUnit.microsecond.toString())) { - return value * 1000; + return Math.multiplyExact(value, 1000); } else { - return res * 1000_000; + return Math.multiplyExact(res, 1000_000); } } else { if (unit.equals(DateTimeUtils.DurationUnit.ns.toString()) @@ -157,8 +157,7 @@ public static long convertDurationStrToLong( for (int i = 0; i < duration.length(); i++) { char ch = duration.charAt(i); if (Character.isDigit(ch)) { - temp *= 10; - temp += (ch - '0'); + temp = Math.addExact(Math.multiplyExact(temp, 10), ch - '0'); } else { String unit = String.valueOf(duration.charAt(i)); // This is to identify units with two letters. @@ -168,12 +167,17 @@ public static long convertDurationStrToLong( } unit = unit.toLowerCase(); if (convertYearToMonth && unit.equals("y")) { - temp *= 12; + temp = Math.multiplyExact(temp, 12); unit = "mo"; } - total += - convertDurationStrToLong( - currentTime == -1 ? -1 : currentTime + total, temp, unit, timestampPrecision); + total = + Math.addExact( + total, + convertDurationStrToLong( + currentTime == -1 ? -1 : Math.addExact(currentTime, total), + temp, + unit, + timestampPrecision)); temp = 0; } } @@ -229,8 +233,7 @@ public static TimeDuration constructTimeDuration(String duration) { for (; i < duration.length(); i++) { char ch = duration.charAt(i); if (Character.isDigit(ch)) { - temp *= 10; - temp += (ch - '0'); + temp = Math.addExact(Math.multiplyExact(temp, 10), ch - '0'); } else { StringBuilder unit = new StringBuilder(String.valueOf(duration.charAt(i))); i++; @@ -241,19 +244,22 @@ public static TimeDuration constructTimeDuration(String duration) { } i--; if ("y".contentEquals(unit) || "year".contentEquals(unit)) { - monthDuration += temp * 12; + monthDuration = Math.addExact(monthDuration, Math.multiplyExact(temp, 12)); temp = 0; continue; } if ("mo".contentEquals(unit) || "month".contentEquals(unit)) { - monthDuration += temp; + monthDuration = Math.addExact(monthDuration, temp); temp = 0; continue; } - nonMonthDuration += convertDurationStrToLong(-1, temp, unit.toString(), currTimePrecision); + nonMonthDuration = + Math.addExact( + nonMonthDuration, + convertDurationStrToLong(-1, temp, unit.toString(), currTimePrecision)); temp = 0; } } - return new TimeDuration((int) monthDuration, nonMonthDuration); + return new TimeDuration(Math.toIntExact(monthDuration), nonMonthDuration); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimeFilterForDeviceTTL.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimeFilterForDeviceTTL.java index 4df2cfbc459c7..43016d871ed05 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimeFilterForDeviceTTL.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimeFilterForDeviceTTL.java @@ -19,8 +19,6 @@ package org.apache.iotdb.db.utils; -import org.apache.iotdb.commons.utils.CommonDateTimeUtils; - import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.read.filter.basic.Filter; @@ -40,7 +38,7 @@ public TimeFilterForDeviceTTL(Filter timeFilter, Map ttlCached) public boolean satisfyStartEndTime(long startTime, long endTime, IDeviceID deviceID) { long ttl = getTTL(deviceID); if (ttl != Long.MAX_VALUE) { - long validStartTime = CommonDateTimeUtils.currentTime() - ttl; + long validStartTime = CommonUtils.getTTLLowerBound(ttl); if (validStartTime > endTime) { return false; } @@ -52,7 +50,7 @@ public boolean satisfyStartEndTime(long startTime, long endTime, IDeviceID devic public boolean satisfy(long time, IDeviceID deviceID) { long ttl = getTTL(deviceID); if (ttl != Long.MAX_VALUE) { - long validStartTime = CommonDateTimeUtils.currentTime() - ttl; + long validStartTime = CommonUtils.getTTLLowerBound(ttl); if (validStartTime > time) { return false; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/windowing/handler/SlidingTimeWindowEvaluationHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/windowing/handler/SlidingTimeWindowEvaluationHandler.java index 92c3044ff40fc..e7214925c4a77 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/windowing/handler/SlidingTimeWindowEvaluationHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/windowing/handler/SlidingTimeWindowEvaluationHandler.java @@ -25,54 +25,56 @@ import org.apache.iotdb.db.utils.windowing.runtime.WindowEvaluationTask; import org.apache.iotdb.db.utils.windowing.window.WindowImpl; +import java.math.BigInteger; import java.util.LinkedList; import java.util.Queue; public class SlidingTimeWindowEvaluationHandler extends SlidingWindowEvaluationHandler { - private final long timeInterval; - private final long slidingStep; + private final BigInteger timeInterval; + private final BigInteger slidingStep; private final Queue windowBeginIndexQueue; /** window: [begin, end). */ - private long currentWindowEndTime; + private BigInteger currentWindowEndTime; /** window: [begin, end). */ - private long nextWindowBeginTime; + private BigInteger nextWindowBeginTime; public SlidingTimeWindowEvaluationHandler( SlidingTimeWindowConfiguration configuration, Evaluator evaluator) throws WindowingException { super(configuration, evaluator); - timeInterval = configuration.getTimeInterval(); - slidingStep = configuration.getSlidingStep(); + timeInterval = BigInteger.valueOf(configuration.getTimeInterval()); + slidingStep = BigInteger.valueOf(configuration.getSlidingStep()); windowBeginIndexQueue = new LinkedList<>(); } @Override protected void createEvaluationTaskIfNecessary(long timestamp) { + BigInteger currentTimestamp = BigInteger.valueOf(timestamp); if (data.size() == 1) { windowBeginIndexQueue.add(0); - currentWindowEndTime = timestamp + timeInterval; - nextWindowBeginTime = timestamp + slidingStep; + currentWindowEndTime = currentTimestamp.add(timeInterval); + nextWindowBeginTime = currentTimestamp.add(slidingStep); return; } - while (nextWindowBeginTime <= timestamp) { + while (nextWindowBeginTime.compareTo(currentTimestamp) <= 0) { windowBeginIndexQueue.add(data.size() - 1); - nextWindowBeginTime += slidingStep; + nextWindowBeginTime = nextWindowBeginTime.add(slidingStep); } - while (currentWindowEndTime <= timestamp) { + while (currentWindowEndTime.compareTo(currentTimestamp) <= 0) { int windowBeginIndex = windowBeginIndexQueue.remove(); TASK_POOL_MANAGER.submit( new WindowEvaluationTask( evaluator, new WindowImpl(data, windowBeginIndex, data.size() - 1 - windowBeginIndex))); data.setEvictionUpperBound(windowBeginIndex); - currentWindowEndTime += slidingStep; + currentWindowEndTime = currentWindowEndTime.add(slidingStep); } } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/auth/LoginLockManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/auth/LoginLockManagerTest.java index 0a9e95886a82c..b94e0d1f3497a 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/auth/LoginLockManagerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/auth/LoginLockManagerTest.java @@ -34,6 +34,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import static org.junit.Assert.assertEquals; @@ -108,6 +109,24 @@ public void testAllConfigScenarios() { assertEquals(30, getField(customLockTime, "passwordLockTimeMinutes")); } + @Test + public void testLargeLockTimeDoesNotOverflowCutoff() { + long now = 1_000_000L; + assertEquals( + now - TimeUnit.MINUTES.toMillis(Integer.MAX_VALUE), + LoginLockManager.getLockWindowCutoffTime(now, Integer.MAX_VALUE)); + assertEquals(Long.MIN_VALUE, LoginLockManager.getLockWindowCutoffTime(Long.MIN_VALUE, 1)); + + LoginLockManager largeWindowManager = + new LoginLockManager(failedLoginAttempts, failedLoginAttemptsPerUser, Integer.MAX_VALUE); + for (int i = 0; i < failedLoginAttempts; i++) { + largeWindowManager.recordFailure(TEST_USER_ID, TEST_IP); + } + assertTrue( + "Large lock window should not overflow and discard recent failures", + largeWindowManager.checkLock(TEST_USER_ID, TEST_IP)); + } + private int getField(LoginLockManager manager, String fieldName) { try { java.lang.reflect.Field field = LoginLockManager.class.getDeclaredField(fieldName); @@ -285,7 +304,8 @@ public void testCleanExpiredLocks() { Deque timestamps = (Deque) tsField.get(lockInfo); timestamps.clear(); - timestamps.add(System.currentTimeMillis() - (passwordLockTimeMinutes * 60 * 1000L) - 5000); + timestamps.add( + System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(passwordLockTimeMinutes) - 5000); } catch (Exception e) { fail("Failed to modify failure timestamps via reflection: " + e.getMessage()); @@ -337,7 +357,8 @@ public void testFailuresOutsideWindowNotCounted() throws Exception { Deque timestamps = (Deque) tsField.get(lockInfo); timestamps.clear(); - timestamps.add(System.currentTimeMillis() - (passwordLockTimeMinutes * 60 * 1000L) - 5000); + timestamps.add( + System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(passwordLockTimeMinutes) - 5000); } catch (Exception e) { fail("Reflection modification failed: " + e.getMessage()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/processor/aggregate/window/processor/TumblingWindowingProcessorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/processor/aggregate/window/processor/TumblingWindowingProcessorTest.java new file mode 100644 index 0000000000000..a9c78e7d73cf8 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/processor/aggregate/window/processor/TumblingWindowingProcessorTest.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.processor.aggregate.window.processor; + +import org.apache.iotdb.commons.queryengine.utils.TimestampPrecisionUtils; +import org.apache.iotdb.db.pipe.processor.aggregate.window.datastructure.TimeSeriesWindow; +import org.apache.iotdb.db.pipe.processor.aggregate.window.datastructure.WindowOutput; +import org.apache.iotdb.db.pipe.processor.aggregate.window.datastructure.WindowState; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; + +import org.apache.tsfile.utils.Pair; +import org.junit.Assert; +import org.junit.Test; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.apache.iotdb.commons.pipe.config.constant.PipeProcessorConstant.PROCESSOR_SLIDING_BOUNDARY_TIME_KEY; +import static org.apache.iotdb.commons.pipe.config.constant.PipeProcessorConstant.PROCESSOR_SLIDING_SECONDS_KEY; + +public class TumblingWindowingProcessorTest { + + @Test + public void testMayAddWindowAlignsExtremeRangeWithoutOverflow() throws Exception { + final TumblingWindowingProcessor processor = createProcessor(Long.MIN_VALUE, 1); + final long interval = TimestampPrecisionUtils.convertToCurrPrecision(1, TimeUnit.SECONDS); + final List windows = new ArrayList<>(); + + Assert.assertEquals(1, processor.mayAddWindow(windows, Long.MAX_VALUE).size()); + Assert.assertEquals(1, windows.size()); + Assert.assertEquals( + alignWindowStart(Long.MAX_VALUE, Long.MIN_VALUE, interval), windows.get(0).getTimestamp()); + Assert.assertTrue(windows.get(0).getTimestamp() > Long.MIN_VALUE); + } + + @Test + public void testWindowEndOverflowDoesNotEmitEarly() throws Exception { + final TumblingWindowingProcessor processor = createProcessor(0, 1); + final long interval = TimestampPrecisionUtils.convertToCurrPrecision(1, TimeUnit.SECONDS); + final TimeSeriesWindow window = new TimeSeriesWindow(processor, null); + window.setTimestamp(Long.MAX_VALUE - interval + 1); + + final Pair result = + processor.updateAndMaySetWindowState(window, Long.MAX_VALUE); + + Assert.assertEquals(WindowState.COMPUTE, result.getLeft()); + Assert.assertNull(result.getRight()); + Assert.assertEquals(Long.MAX_VALUE, processor.forceOutput(window).getProgressTime()); + + final List windows = new ArrayList<>(); + windows.add(window); + Assert.assertTrue(processor.mayAddWindow(windows, Long.MAX_VALUE).isEmpty()); + } + + private static TumblingWindowingProcessor createProcessor( + final long slidingBoundaryTime, final long slidingSeconds) throws Exception { + final Map attributes = new HashMap<>(); + attributes.put(PROCESSOR_SLIDING_BOUNDARY_TIME_KEY, Long.toString(slidingBoundaryTime)); + attributes.put(PROCESSOR_SLIDING_SECONDS_KEY, Long.toString(slidingSeconds)); + final PipeParameters parameters = new PipeParameters(attributes); + final TumblingWindowingProcessor processor = new TumblingWindowingProcessor(); + processor.validate(new PipeParameterValidator(parameters)); + processor.customize(parameters, () -> null); + return processor; + } + + private static long alignWindowStart( + final long timestamp, final long baseTime, final long interval) { + final BigInteger base = BigInteger.valueOf(baseTime); + final BigInteger intervalValue = BigInteger.valueOf(interval); + return base.add( + BigInteger.valueOf(timestamp) + .subtract(base) + .divide(intervalValue) + .multiply(intervalValue)) + .longValueExact(); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/DataRegionWatermarkInjectorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/DataRegionWatermarkInjectorTest.java new file mode 100644 index 0000000000000..6141cbc7196ef --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/DataRegionWatermarkInjectorTest.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.source.dataregion; + +import org.junit.Assert; +import org.junit.Test; + +public class DataRegionWatermarkInjectorTest { + + @Test + public void testCalculateNextInjectionTime() { + Assert.assertEquals( + 90_000, DataRegionWatermarkInjector.calculateNextInjectionTime(60_001, 30_000)); + } + + @Test + public void testCalculateNextInjectionTimeSaturatesOnOverflow() { + Assert.assertEquals( + Long.MAX_VALUE, + DataRegionWatermarkInjector.calculateNextInjectionTime( + Long.MAX_VALUE, DataRegionWatermarkInjector.MIN_INJECTION_INTERVAL_IN_MS)); + Assert.assertEquals( + Long.MAX_VALUE, + DataRegionWatermarkInjector.calculateNextInjectionTime( + Long.MAX_VALUE - 1, DataRegionWatermarkInjector.MIN_INJECTION_INTERVAL_IN_MS)); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/aggregation/AccumulatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/aggregation/AccumulatorTest.java index 108f58224484b..8ac00f82ffbce 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/aggregation/AccumulatorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/aggregation/AccumulatorTest.java @@ -488,6 +488,25 @@ public void minTimeAccumulatorTest() { Assert.assertEquals(100, finalResult.build().getLong(0)); } + @Test + public void timeDurationAccumulatorSaturatesOverflow() { + TsBlockBuilder tsBlockBuilder = new TsBlockBuilder(Collections.singletonList(TSDataType.INT32)); + tsBlockBuilder.getTimeColumnBuilder().writeLong(Long.MIN_VALUE); + tsBlockBuilder.getColumnBuilder(0).writeInt(1); + tsBlockBuilder.declarePosition(); + tsBlockBuilder.getTimeColumnBuilder().writeLong(Long.MAX_VALUE); + tsBlockBuilder.getColumnBuilder(0).writeInt(2); + tsBlockBuilder.declarePosition(); + TsBlock tsBlock = tsBlockBuilder.build(); + + TimeDurationAccumulator accumulator = new TimeDurationAccumulator(); + accumulator.addInput(new Column[] {tsBlock.getTimeColumn(), tsBlock.getColumn(0)}, null); + + ColumnBuilder finalResult = new LongColumnBuilder(null, 1); + accumulator.outputFinal(finalResult); + Assert.assertEquals(Long.MAX_VALUE, finalResult.build().getLong(0)); + } + @Test public void maxValueAccumulatorTest() { Accumulator extremeAccumulator = diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/aggregation/TimeRangeIteratorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/aggregation/TimeRangeIteratorTest.java index 86d8ef8ad34b2..c8f71367c930c 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/aggregation/TimeRangeIteratorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/aggregation/TimeRangeIteratorTest.java @@ -66,6 +66,133 @@ public void testNotSplitTimeRange() { checkRes(descTimeRangeIterator, res); } + @Test + public void testNotSplitTimeRangeWithLongBoundaries() { + TimeDuration interval = new TimeDuration(0, 2); + TimeDuration slidingStep = new TimeDuration(0, 1); + + String[] maxLeftClosedRightOpenRes = { + "[ 9223372036854775805 : 9223372036854775806 ]", + "[ 9223372036854775806 : 9223372036854775806 ]" + }; + checkRes( + TimeRangeIteratorFactory.getTimeRangeIterator( + Long.MAX_VALUE - 2, + Long.MAX_VALUE, + interval, + slidingStep, + true, + true, + false, + ZoneId.systemDefault()), + maxLeftClosedRightOpenRes); + checkRes( + TimeRangeIteratorFactory.getTimeRangeIterator( + Long.MAX_VALUE - 2, + Long.MAX_VALUE, + interval, + slidingStep, + false, + true, + false, + ZoneId.systemDefault()), + maxLeftClosedRightOpenRes); + + String[] maxLeftOpenRightClosedRes = { + "[ 9223372036854775806 : 9223372036854775807 ]", + "[ 9223372036854775807 : 9223372036854775807 ]" + }; + checkRes( + TimeRangeIteratorFactory.getTimeRangeIterator( + Long.MAX_VALUE - 2, + Long.MAX_VALUE, + interval, + slidingStep, + true, + false, + false, + ZoneId.systemDefault()), + maxLeftOpenRightClosedRes); + checkRes( + TimeRangeIteratorFactory.getTimeRangeIterator( + Long.MAX_VALUE - 2, + Long.MAX_VALUE, + interval, + slidingStep, + false, + false, + false, + ZoneId.systemDefault()), + maxLeftOpenRightClosedRes); + + String[] minLeftClosedRightOpenRes = { + "[ -9223372036854775808 : -9223372036854775807 ]", + "[ -9223372036854775807 : -9223372036854775807 ]" + }; + checkRes( + TimeRangeIteratorFactory.getTimeRangeIterator( + Long.MIN_VALUE, + Long.MIN_VALUE + 2, + interval, + slidingStep, + true, + true, + false, + ZoneId.systemDefault()), + minLeftClosedRightOpenRes); + checkRes( + TimeRangeIteratorFactory.getTimeRangeIterator( + Long.MIN_VALUE, + Long.MIN_VALUE + 2, + interval, + slidingStep, + false, + true, + false, + ZoneId.systemDefault()), + minLeftClosedRightOpenRes); + + String[] maxSplitLeftClosedRightOpenRes = { + "[ 9223372036854775805 : 9223372036854775805 ]", + "[ 9223372036854775806 : 9223372036854775806 ]" + }; + checkRes( + TimeRangeIteratorFactory.getTimeRangeIterator( + Long.MAX_VALUE - 2, + Long.MAX_VALUE, + interval, + slidingStep, + true, + true, + true, + ZoneId.systemDefault()), + maxSplitLeftClosedRightOpenRes); + checkRes( + TimeRangeIteratorFactory.getTimeRangeIterator( + Long.MAX_VALUE - 2, + Long.MAX_VALUE, + interval, + slidingStep, + false, + true, + true, + ZoneId.systemDefault()), + maxSplitLeftClosedRightOpenRes); + + Assert.assertEquals( + Long.MAX_VALUE, + TimeRangeIteratorFactory.getTimeRangeIterator( + Long.MIN_VALUE, + Long.MAX_VALUE, + new TimeDuration(0, 1), + new TimeDuration(0, 1), + true, + true, + false, + ZoneId.systemDefault()) + .getTotalIntervalNum()); + } + @Test public void testSplitTimeRange() { String[] res4_1 = { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/ai/InferenceOperatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/ai/InferenceOperatorTest.java new file mode 100644 index 0000000000000..2ee0854c4b143 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/ai/InferenceOperatorTest.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process.ai; + +import org.apache.iotdb.db.exception.runtime.ModelInferenceProcessException; + +import org.junit.Assert; +import org.junit.Test; + +public class InferenceOperatorTest { + + @Test + public void testGeneratedTimeRejectsOverflow() { + Assert.assertEquals( + Long.MAX_VALUE, InferenceOperator.calculateGeneratedTime(Long.MAX_VALUE - 2, 1, 2)); + + Assert.assertThrows( + ModelInferenceProcessException.class, + () -> InferenceOperator.calculateGeneratedTime(Long.MAX_VALUE - 1, 1, 2)); + } + + @Test + public void testGeneratedTimeIntervalRejectsOverflow() { + Assert.assertEquals( + Long.MAX_VALUE, + InferenceOperator.calculateGeneratedTimeInterval(Long.MIN_VALUE, Long.MAX_VALUE, 2)); + + Assert.assertThrows( + ModelInferenceProcessException.class, + () -> InferenceOperator.calculateGeneratedTimeInterval(Long.MIN_VALUE, Long.MAX_VALUE, 1)); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/window/SessionWindowTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/window/SessionWindowTest.java index ea534a76ebc82..7393eeda7f630 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/window/SessionWindowTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/window/SessionWindowTest.java @@ -54,6 +54,14 @@ public void skipPointsOutOfCurWindowHandlesOverflowedTimeDistance() { Assert.assertEquals(Long.MAX_VALUE, skippedBlock.getTimeColumn().getLong(0)); } + @Test + public void containsRejectsOverflowedTimeRange() { + SessionWindow sessionWindow = new SessionWindow(Long.MAX_VALUE, true); + Column timeColumn = buildTimeColumn(Long.MIN_VALUE, Long.MAX_VALUE); + + Assert.assertFalse(sessionWindow.contains(timeColumn)); + } + private Column buildTimeColumn(long... timestamps) { return buildTsBlock(timestamps).getTimeColumn(); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/expression/predicate/ConvertPredicateToTimeFilterTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/expression/predicate/ConvertPredicateToTimeFilterTest.java index 76fd4303a0b74..9f50f7050175f 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/expression/predicate/ConvertPredicateToTimeFilterTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/expression/predicate/ConvertPredicateToTimeFilterTest.java @@ -105,6 +105,11 @@ public void testNormal() { parameter.getSlidingStep(), TimeZone.getTimeZone("+00:00"), TimestampPrecisionUtils.currPrecision)); + + GroupByTimeParameter rightClosedParameter = + new GroupByTimeParameter( + 1, Long.MAX_VALUE, new TimeDuration(0, 10), new TimeDuration(0, 10), false); + Assert.assertThrows(IllegalArgumentException.class, () -> groupByTime(rightClosedParameter)); } @Test diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeTest.java index 2cc9b8faf7e64..a7a0ecca628de 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeTest.java @@ -68,10 +68,12 @@ import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.DEVICE; import static org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.and; +import static org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.eq; import static org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.groupByTime; import static org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.gt; import static org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.gte; import static org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.longValue; +import static org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.or; import static org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.time; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; @@ -277,6 +279,17 @@ public void testAggregationQuery() { } } + @Test + public void testRightClosedGroupByTimePredicateWithLongMaxEndTime() { + String sql = "select count(s1) from root.sg.d1 " + "group by ((0, 9223372036854775807], 10ms);"; + + Analysis actualAnalysis = analyzeSQL(sql); + + assertEquals( + or(groupByTime(1, Long.MAX_VALUE, 10, 10), eq(time(), longValue(Long.MAX_VALUE))), + actualAnalysis.getGlobalTimePredicate()); + } + @Test public void testRawDataQueryAlignByDevice() { String sql = diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtilsTest.java index 5d0ccd74f4598..5e63746a77b8e 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtilsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtilsTest.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.queryengine.plan.analyze; +import org.apache.iotdb.commons.exception.SemanticException; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Identifier; @@ -33,6 +34,7 @@ import java.util.List; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; public class AnalyzeUtilsTest { @@ -52,4 +54,55 @@ public void testParseDeletePredicateWithRenamedTimeColumn() { assertEquals(Long.MIN_VALUE, entries.get(0).getStartTime()); assertEquals(100, entries.get(0).getEndTime()); } + + @Test + public void testParseDeleteTimePredicateWithBoundary() { + TsTable table = new TsTable("table1"); + table.addColumnSchema(new TimeColumnSchema("time", TSDataType.TIMESTAMP)); + + Expression expression = + new ComparisonExpression( + ComparisonExpression.Operator.GREATER_THAN, + new Identifier("time"), + new LongLiteral(String.valueOf(Long.MAX_VALUE - 1))); + List entries = AnalyzeUtils.parseExpressions2ModEntries(expression, table); + assertEquals(1, entries.size()); + assertEquals(Long.MAX_VALUE, entries.get(0).getStartTime()); + assertEquals(Long.MAX_VALUE, entries.get(0).getEndTime()); + + expression = + new ComparisonExpression( + ComparisonExpression.Operator.LESS_THAN, + new Identifier("time"), + new LongLiteral(String.valueOf(Long.MIN_VALUE + 1))); + entries = AnalyzeUtils.parseExpressions2ModEntries(expression, table); + assertEquals(1, entries.size()); + assertEquals(Long.MIN_VALUE, entries.get(0).getStartTime()); + assertEquals(Long.MIN_VALUE, entries.get(0).getEndTime()); + } + + @Test + public void testParseDeleteTimePredicateWithEmptyBoundary() { + TsTable table = new TsTable("table1"); + table.addColumnSchema(new TimeColumnSchema("time", TSDataType.TIMESTAMP)); + + assertThrows( + SemanticException.class, + () -> + AnalyzeUtils.parseExpressions2ModEntries( + new ComparisonExpression( + ComparisonExpression.Operator.GREATER_THAN, + new Identifier("time"), + new LongLiteral(String.valueOf(Long.MAX_VALUE))), + table)); + assertThrows( + SemanticException.class, + () -> + AnalyzeUtils.parseExpressions2ModEntries( + new ComparisonExpression( + ComparisonExpression.Operator.LESS_THAN, + new Identifier("time"), + new LongLiteral(String.valueOf(Long.MIN_VALUE))), + table)); + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/QueryTimePartitionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/QueryTimePartitionTest.java index 1e85a1461ee87..aa31e0ed3762d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/QueryTimePartitionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/QueryTimePartitionTest.java @@ -20,6 +20,7 @@ import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.common.QueryId; @@ -575,4 +576,30 @@ public void testGetTimePartitionSlotList() { assertFalse(res.right.left); assertFalse(res.right.right); } + + @Test + public void testEstimateTimePartitionSlotMemoryWithOverflow() { + assertEquals(Long.MAX_VALUE, AnalyzeVisitor.estimateTimePartitionSlotMemory(Long.MAX_VALUE)); + } + + @Test + public void testTimePartitionSlotListWithUpperOverflowPartition() { + MPPQueryContext context = new MPPQueryContext(new QueryId("test_query")); + long partitionStartTime = TimePartitionUtils.getTimePartitionSlot(Long.MAX_VALUE).startTime; + + Pair, Pair> res = + getTimePartitionSlotList( + TimeFilterApi.between(partitionStartTime - 1, Long.MAX_VALUE - 1), context); + List expected = + Arrays.asList( + TimePartitionUtils.getTimePartitionSlot(partitionStartTime - 1), + new TTimePartitionSlot(partitionStartTime)); + + assertEquals(expected.size(), res.left.size()); + for (int i = 0; i < expected.size(); i++) { + assertEquals(expected.get(i), res.left.get(i)); + } + assertFalse(res.right.left); + assertFalse(res.right.right); + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java index b98f34a2484d9..e759471c08469 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java @@ -56,6 +56,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.metadata.DatabaseSchemaStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.DeleteDatabaseStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.DeleteTimeSeriesStatement; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.GetRegionIdStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.template.BatchActivateTemplateStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.template.CreateSchemaTemplateStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.template.DropSchemaTemplateStatement; @@ -590,6 +591,65 @@ public void testDeleteData() throws IllegalPathException { assertEquals(100L, statement.getDeleteEndTime()); } + @Test + public void testDeleteDataWithTimeRangeBoundary() { + DeleteDataStatement statement = + (DeleteDataStatement) + StatementGenerator.createStatement( + "DELETE FROM root.sg.d1.s1 WHERE time > 9223372036854775806", + ZonedDateTime.now().getOffset()); + assertEquals(Long.MAX_VALUE, statement.getDeleteStartTime()); + assertEquals(Long.MAX_VALUE, statement.getDeleteEndTime()); + + statement = + (DeleteDataStatement) + StatementGenerator.createStatement( + "DELETE FROM root.sg.d1.s1 WHERE time < -9223372036854775807", + ZonedDateTime.now().getOffset()); + assertEquals(Long.MIN_VALUE, statement.getDeleteStartTime()); + assertEquals(Long.MIN_VALUE, statement.getDeleteEndTime()); + } + + @Test + public void testDeleteDataWithEmptyTimeRangeBoundary() { + Assert.assertThrows( + SemanticException.class, + () -> + StatementGenerator.createStatement( + "DELETE FROM root.sg.d1.s1 WHERE time > 9223372036854775807", + ZonedDateTime.now().getOffset())); + Assert.assertThrows( + SemanticException.class, + () -> + StatementGenerator.createStatement( + "DELETE FROM root.sg.d1.s1 WHERE time < -9223372036854775808", + ZonedDateTime.now().getOffset())); + } + + @Test + public void testGetRegionIdWithTimeRangeBoundary() { + GetRegionIdStatement statement = + (GetRegionIdStatement) + StatementGenerator.createStatement( + "SHOW DATA REGIONID WHERE DATABASE = root.sg AND time > 9223372036854775806", + ZonedDateTime.now().getOffset()); + assertEquals(Long.MAX_VALUE, statement.getStartTimeStamp()); + assertEquals(Long.MAX_VALUE, statement.getEndTimeStamp()); + + assertThrows( + SemanticException.class, + () -> + StatementGenerator.createStatement( + "SHOW DATA REGIONID WHERE DATABASE = root.sg AND time > 9223372036854775807", + ZonedDateTime.now().getOffset())); + assertThrows( + SemanticException.class, + () -> + StatementGenerator.createStatement( + "SHOW DATA REGIONID WHERE DATABASE = root.sg AND time < -9223372036854775808", + ZonedDateTime.now().getOffset())); + } + @Test public void testCreateSchemaTemplate() throws MetadataException, IOException, StatementExecutionException { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/WritePlanNodeSplitTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/WritePlanNodeSplitTest.java index 01857cb5f8aa9..f72eba7896168 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/WritePlanNodeSplitTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/WritePlanNodeSplitTest.java @@ -44,6 +44,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsOfOneDeviceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.Binary; @@ -55,6 +56,7 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -62,6 +64,7 @@ public class WritePlanNodeSplitTest { + long prevTimePartitionOrigin; long prevTimePartitionInterval; Map>>> @@ -77,9 +80,12 @@ public class WritePlanNodeSplitTest { @Before public void setUp() { + prevTimePartitionOrigin = CommonDescriptor.getInstance().getConfig().getTimePartitionOrigin(); prevTimePartitionInterval = CommonDescriptor.getInstance().getConfig().getTimePartitionInterval(); + CommonDescriptor.getInstance().getConfig().setTimePartitionOrigin(0); CommonDescriptor.getInstance().getConfig().setTimePartitionInterval(100); + TimePartitionUtils.setTimePartitionOrigin(0); TimePartitionUtils.setTimePartitionInterval(100); executorClassName = IoTDBDescriptor.getInstance().getConfig().getSeriesPartitionExecutorClass(); @@ -260,6 +266,47 @@ public void testSplitInsertTablet() throws IllegalPathException { } } + @Test + public void testInsertTabletDoesNotSplitAtLongMaxValueWithinLastPartition() + throws IllegalPathException { + final long lastPartitionStartTime = + TimePartitionUtils.getTimePartitionSlot(Long.MAX_VALUE).getStartTime(); + final List expectedTimePartitionSlots = + Collections.singletonList(new TTimePartitionSlot(lastPartitionStartTime)); + + InsertTabletNode insertTabletNode = new InsertTabletNode(new PlanNodeId("plan node boundary")); + insertTabletNode.setTargetPath(new PartialPath("root.sg1.d1")); + insertTabletNode.setTimes(new long[] {lastPartitionStartTime, Long.MAX_VALUE}); + + Assert.assertEquals(expectedTimePartitionSlots, insertTabletNode.getTimePartitionSlots()); + + InsertTabletStatement insertTabletStatement = new InsertTabletStatement(); + insertTabletStatement.setTimes(new long[] {lastPartitionStartTime, Long.MAX_VALUE}); + + Assert.assertEquals(expectedTimePartitionSlots, insertTabletStatement.getTimePartitionSlots()); + } + + @Test + public void testInsertTabletSplitsAtExactLongMaxValueUpperBound() throws IllegalPathException { + CommonDescriptor.getInstance().getConfig().setTimePartitionInterval(1); + TimePartitionUtils.setTimePartitionInterval(1); + final List expectedTimePartitionSlots = + Arrays.asList( + new TTimePartitionSlot(Long.MAX_VALUE - 1), new TTimePartitionSlot(Long.MAX_VALUE)); + + InsertTabletNode insertTabletNode = + new InsertTabletNode(new PlanNodeId("plan node exact boundary")); + insertTabletNode.setTargetPath(new PartialPath("root.sg1.d1")); + insertTabletNode.setTimes(new long[] {Long.MAX_VALUE - 1, Long.MAX_VALUE}); + + Assert.assertEquals(expectedTimePartitionSlots, insertTabletNode.getTimePartitionSlots()); + + InsertTabletStatement insertTabletStatement = new InsertTabletStatement(); + insertTabletStatement.setTimes(new long[] {Long.MAX_VALUE - 1, Long.MAX_VALUE}); + + Assert.assertEquals(expectedTimePartitionSlots, insertTabletStatement.getTimePartitionSlots()); + } + @Test public void testSplitRelationalInsertTablet() throws IllegalPathException { RelationalInsertTabletNode relationalInsertTabletNode = @@ -483,6 +530,8 @@ public void testInsertRowsOfOneDeviceNode() throws IllegalPathException { @After public void tearDown() { TimePartitionUtils.setTimePartitionInterval(prevTimePartitionInterval); + TimePartitionUtils.setTimePartitionOrigin(prevTimePartitionOrigin); CommonDescriptor.getInstance().getConfig().setTimePartitionInterval(prevTimePartitionInterval); + CommonDescriptor.getInstance().getConfig().setTimePartitionOrigin(prevTimePartitionOrigin); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionSelectorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionSelectorTest.java index 0a33a91799a46..d11110173da99 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionSelectorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionSelectorTest.java @@ -35,6 +35,7 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo; +import org.apache.iotdb.db.utils.CommonUtils; import org.apache.iotdb.db.utils.datastructure.FixedPriorityBlockingQueue; import org.apache.tsfile.exception.write.WriteProcessException; @@ -272,7 +273,7 @@ public void testSeqFileWithDeviceIndexBeenDeletedBeforeSelection() "", "", 0, null, new CompactionScheduleContext()); CrossSpaceCompactionCandidate candidate = new CrossSpaceCompactionCandidate( - seqResources, unseqResources, System.currentTimeMillis() - Long.MAX_VALUE); + seqResources, unseqResources, CommonUtils.getTTLLowerBound(Long.MAX_VALUE)); CrossCompactionTaskResource crossCompactionTaskResource = selector.selectOneTaskResources(candidate); @@ -343,7 +344,7 @@ public void testSeqFileWithDeviceIndexBeenDeletedDuringSelectionAndAfterCopyingL // copy candidate source file list and add read lock CrossSpaceCompactionCandidate candidate = new CrossSpaceCompactionCandidate( - seqResources, unseqResources, System.currentTimeMillis() - Long.MAX_VALUE); + seqResources, unseqResources, CommonUtils.getTTLLowerBound(Long.MAX_VALUE)); // the other thread holds write lock and delete file successfully after copying list cd1.countDown(); @@ -406,7 +407,7 @@ public void testSeqFileWithDeviceIndexBeenDeletedDuringSelectionAndBeforeSetting // copy candidate source file list and add read lock CrossSpaceCompactionCandidate candidate = new CrossSpaceCompactionCandidate( - seqResources, unseqResources, System.currentTimeMillis() - Long.MAX_VALUE); + seqResources, unseqResources, CommonUtils.getTTLLowerBound(Long.MAX_VALUE)); CrossCompactionTaskResource crossCompactionTaskResource = selector.selectOneTaskResources(candidate); @@ -512,7 +513,7 @@ public void testSeqFileWithDeviceIndexBeenDeletedDuringSelectionAndBeforeSetting // copy candidate source file list and add read lock CrossSpaceCompactionCandidate candidate = new CrossSpaceCompactionCandidate( - seqResources, unseqResources, System.currentTimeMillis() - Long.MAX_VALUE); + seqResources, unseqResources, CommonUtils.getTTLLowerBound(Long.MAX_VALUE)); CrossCompactionTaskResource crossCompactionTaskResource = selector.selectOneTaskResources(candidate); @@ -633,7 +634,7 @@ public void testSeqFileWithFileIndexBeenDeletedDuringSelectionAndAfterCopyingLis // copy candidate source file list and add read lock CrossSpaceCompactionCandidate candidate = new CrossSpaceCompactionCandidate( - seqResources, unseqResources, System.currentTimeMillis() - Long.MAX_VALUE); + seqResources, unseqResources, CommonUtils.getTTLLowerBound(Long.MAX_VALUE)); // the other thread holds write lock and delete file successfully after copying list cd1.countDown(); @@ -750,7 +751,7 @@ public void testSeqFileWithFileIndexBeenDeletedDuringSelectionAndBeforeSettingCa // copy candidate source file list and add read lock CrossSpaceCompactionCandidate candidate = new CrossSpaceCompactionCandidate( - seqResources, unseqResources, System.currentTimeMillis() - Long.MAX_VALUE); + seqResources, unseqResources, CommonUtils.getTTLLowerBound(Long.MAX_VALUE)); CrossCompactionTaskResource crossCompactionTaskResource = selector.selectOneTaskResources(candidate); @@ -860,7 +861,7 @@ public void testSeqFileWithFileIndexBeenDeletedDuringSelectionAndBeforeSettingCo // copy candidate source file list and add read lock CrossSpaceCompactionCandidate candidate = new CrossSpaceCompactionCandidate( - seqResources, unseqResources, System.currentTimeMillis() - Long.MAX_VALUE); + seqResources, unseqResources, CommonUtils.getTTLLowerBound(Long.MAX_VALUE)); CrossCompactionTaskResource crossCompactionTaskResource = selector.selectOneTaskResources(candidate); @@ -982,7 +983,7 @@ public void testSeqFileWithFileIndexBeenDeletedBeforeSelection() "", "", 0, null, new CompactionScheduleContext()); CrossSpaceCompactionCandidate candidate = new CrossSpaceCompactionCandidate( - seqResources, unseqResources, System.currentTimeMillis() - Long.MAX_VALUE); + seqResources, unseqResources, CommonUtils.getTTLLowerBound(Long.MAX_VALUE)); CrossCompactionTaskResource crossCompactionTaskResource = selector.selectOneTaskResources(candidate); @@ -1054,7 +1055,7 @@ public void testUnSeqFileWithDeviceIndexBeenDeletedBeforeSelection() "", "", 0, null, new CompactionScheduleContext()); CrossSpaceCompactionCandidate candidate = new CrossSpaceCompactionCandidate( - seqResources, unseqResources, System.currentTimeMillis() - Long.MAX_VALUE); + seqResources, unseqResources, CommonUtils.getTTLLowerBound(Long.MAX_VALUE)); CrossCompactionTaskResource crossCompactionTaskResource = selector.selectOneTaskResources(candidate); @@ -1125,7 +1126,7 @@ public void testUnSeqFileWithDeviceIndexBeenDeletedDuringSelectionAndAfterCopyin // copy candidate source file list and add read lock CrossSpaceCompactionCandidate candidate = new CrossSpaceCompactionCandidate( - seqResources, unseqResources, System.currentTimeMillis() - Long.MAX_VALUE); + seqResources, unseqResources, CommonUtils.getTTLLowerBound(Long.MAX_VALUE)); // the other thread holds write lock and delete file successfully after copying list cd1.countDown(); @@ -1189,7 +1190,7 @@ public void testUnSeqFileWithDeviceIndexBeenDeletedDuringSelectionAndBeforeSetti // copy candidate source file list and add read lock CrossSpaceCompactionCandidate candidate = new CrossSpaceCompactionCandidate( - seqResources, unseqResources, System.currentTimeMillis() - Long.MAX_VALUE); + seqResources, unseqResources, CommonUtils.getTTLLowerBound(Long.MAX_VALUE)); CrossCompactionTaskResource crossCompactionTaskResource = selector.selectOneTaskResources(candidate); @@ -1296,7 +1297,7 @@ public void testUnSeqFileWithDeviceIndexBeenDeletedDuringSelectionAndBeforeSetti // copy candidate source file list and add read lock CrossSpaceCompactionCandidate candidate = new CrossSpaceCompactionCandidate( - seqResources, unseqResources, System.currentTimeMillis() - Long.MAX_VALUE); + seqResources, unseqResources, CommonUtils.getTTLLowerBound(Long.MAX_VALUE)); CrossCompactionTaskResource crossCompactionTaskResource = selector.selectOneTaskResources(candidate); @@ -1418,7 +1419,7 @@ public void testUnSeqFileWithFileIndexBeenDeletedDuringSelectionAndAfterCopyingL // copy candidate source file list and add read lock CrossSpaceCompactionCandidate candidate = new CrossSpaceCompactionCandidate( - seqResources, unseqResources, System.currentTimeMillis() - Long.MAX_VALUE); + seqResources, unseqResources, CommonUtils.getTTLLowerBound(Long.MAX_VALUE)); // the other thread holds write lock and delete file successfully after copying list cd1.countDown(); @@ -1534,7 +1535,7 @@ public void testUnSeqFileWithFileIndexBeenDeletedDuringSelectionAndBeforeSetting // copy candidate source file list and add read lock CrossSpaceCompactionCandidate candidate = new CrossSpaceCompactionCandidate( - seqResources, unseqResources, System.currentTimeMillis() - Long.MAX_VALUE); + seqResources, unseqResources, CommonUtils.getTTLLowerBound(Long.MAX_VALUE)); CrossCompactionTaskResource crossCompactionTaskResource = selector.selectOneTaskResources(candidate); @@ -1644,7 +1645,7 @@ public void testUnSeqFileWithFileIndexBeenDeletedDuringSelectionAndBeforeSetting // copy candidate source file list and add read lock CrossSpaceCompactionCandidate candidate = new CrossSpaceCompactionCandidate( - seqResources, unseqResources, System.currentTimeMillis() - Long.MAX_VALUE); + seqResources, unseqResources, CommonUtils.getTTLLowerBound(Long.MAX_VALUE)); CrossCompactionTaskResource crossCompactionTaskResource = selector.selectOneTaskResources(candidate); @@ -1766,7 +1767,7 @@ public void testUnSeqFileWithFileIndexBeenDeletedBeforeSelection() "", "", 0, null, new CompactionScheduleContext()); CrossSpaceCompactionCandidate candidate = new CrossSpaceCompactionCandidate( - seqResources, unseqResources, System.currentTimeMillis() - Long.MAX_VALUE); + seqResources, unseqResources, CommonUtils.getTTLLowerBound(Long.MAX_VALUE)); CrossCompactionTaskResource crossCompactionTaskResource = selector.selectOneTaskResources(candidate); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithFastPerformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithFastPerformerTest.java index e327114ca5c99..d3ae001b21541 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithFastPerformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithFastPerformerTest.java @@ -43,6 +43,7 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceList; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator; +import org.apache.iotdb.db.utils.CommonUtils; import org.apache.iotdb.db.utils.EnvironmentUtils; import org.apache.tsfile.read.TimeValuePair; @@ -407,7 +408,7 @@ public void testOneSeqFileAndSixUnseqFile() throws Exception { seqTsFileResourceList.addAll(seqResources); TsFileResourceList unseqTsFileResourceList = new TsFileResourceList(); unseqTsFileResourceList.addAll(unseqResources); - long timeLowerBound = System.currentTimeMillis() - Long.MAX_VALUE; + long timeLowerBound = CommonUtils.getTTLLowerBound(Long.MAX_VALUE); CrossSpaceCompactionCandidate mergeResource = new CrossSpaceCompactionCandidate( seqTsFileResourceList, unseqTsFileResourceList, timeLowerBound); @@ -709,7 +710,7 @@ public void testFiveSeqFileAndOneUnseqFileWithSomeDeviceNotInSeqFiles() throws E seqTsFileResourceList.addAll(seqResources); TsFileResourceList unseqTsFileResourceList = new TsFileResourceList(); unseqTsFileResourceList.addAll(unseqResources); - long timeLowerBound = System.currentTimeMillis() - Long.MAX_VALUE; + long timeLowerBound = CommonUtils.getTTLLowerBound(Long.MAX_VALUE); CrossSpaceCompactionCandidate mergeResource = new CrossSpaceCompactionCandidate( seqTsFileResourceList, unseqTsFileResourceList, timeLowerBound); @@ -1009,7 +1010,7 @@ public void testFiveSeqFileAndOneUnseqFile() throws Exception { seqTsFileResourceList.addAll(seqResources); TsFileResourceList unseqTsFileResourceList = new TsFileResourceList(); unseqTsFileResourceList.addAll(unseqResources); - long timeLowerBound = System.currentTimeMillis() - Long.MAX_VALUE; + long timeLowerBound = CommonUtils.getTTLLowerBound(Long.MAX_VALUE); CrossSpaceCompactionCandidate mergeResource = new CrossSpaceCompactionCandidate( seqTsFileResourceList, unseqTsFileResourceList, timeLowerBound); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithReadPointPerformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithReadPointPerformerTest.java index 3c194a4e3d14d..9193c82322337 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithReadPointPerformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithReadPointPerformerTest.java @@ -43,6 +43,7 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceList; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator; +import org.apache.iotdb.db.utils.CommonUtils; import org.apache.iotdb.db.utils.EnvironmentUtils; import org.apache.tsfile.read.TimeValuePair; @@ -406,7 +407,7 @@ public void testOneSeqFileAndSixUnseqFile() throws Exception { seqTsFileResourceList.addAll(seqResources); TsFileResourceList unseqTsFileResourceList = new TsFileResourceList(); unseqTsFileResourceList.addAll(unseqResources); - long timeLowerBound = System.currentTimeMillis() - Long.MAX_VALUE; + long timeLowerBound = CommonUtils.getTTLLowerBound(Long.MAX_VALUE); CrossSpaceCompactionCandidate mergeResource = new CrossSpaceCompactionCandidate( seqTsFileResourceList, unseqTsFileResourceList, timeLowerBound); @@ -708,7 +709,7 @@ public void testFiveSeqFileAndOneUnseqFileWithSomeDeviceNotInSeqFiles() throws E seqTsFileResourceList.addAll(seqResources); TsFileResourceList unseqTsFileResourceList = new TsFileResourceList(); unseqTsFileResourceList.addAll(unseqResources); - long timeLowerBound = System.currentTimeMillis() - Long.MAX_VALUE; + long timeLowerBound = CommonUtils.getTTLLowerBound(Long.MAX_VALUE); CrossSpaceCompactionCandidate mergeResource = new CrossSpaceCompactionCandidate( seqTsFileResourceList, unseqTsFileResourceList, timeLowerBound); @@ -1008,7 +1009,7 @@ public void testFiveSeqFileAndOneUnseqFile() throws Exception { seqTsFileResourceList.addAll(seqResources); TsFileResourceList unseqTsFileResourceList = new TsFileResourceList(); unseqTsFileResourceList.addAll(unseqResources); - long timeLowerBound = System.currentTimeMillis() - Long.MAX_VALUE; + long timeLowerBound = CommonUtils.getTTLLowerBound(Long.MAX_VALUE); CrossSpaceCompactionCandidate mergeResource = new CrossSpaceCompactionCandidate( seqTsFileResourceList, unseqTsFileResourceList, timeLowerBound); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/RewriteCompactionFileSelectorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/RewriteCompactionFileSelectorTest.java index 3afaca0cf1b3a..2f0c95372190a 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/RewriteCompactionFileSelectorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/RewriteCompactionFileSelectorTest.java @@ -32,6 +32,7 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex; import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo; +import org.apache.iotdb.db.utils.CommonUtils; import org.apache.iotdb.db.utils.constant.TestConstant; import org.apache.tsfile.exception.write.WriteProcessException; @@ -237,7 +238,7 @@ public void testFileOpenSelectionFromCompaction() List newUnseqResources = new ArrayList<>(); newUnseqResources.add(largeUnseqTsFileResource); - long ttlLowerBound = System.currentTimeMillis() - Long.MAX_VALUE; + long ttlLowerBound = CommonUtils.getTTLLowerBound(Long.MAX_VALUE); CrossSpaceCompactionCandidate mergeResource = new CrossSpaceCompactionCandidate(seqResources, newUnseqResources, ttlLowerBound); assertEquals(5, mergeResource.getSeqFiles().size()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemChunkTimeRangeUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemChunkTimeRangeUtilsTest.java new file mode 100644 index 0000000000000..b01fe53be947c --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemChunkTimeRangeUtilsTest.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion.memtable; + +import org.junit.Test; + +import java.math.BigInteger; +import java.util.List; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class MemChunkTimeRangeUtilsTest { + + @Test + public void testSplitFakePageTimeRangesWithLongBoundaries() { + List pageTimeRanges = + MemChunkTimeRangeUtils.splitFakePageTimeRanges(Long.MIN_VALUE, Long.MAX_VALUE, 100); + + assertEquals(100, pageTimeRanges.size()); + assertEquals(Long.MIN_VALUE, pageTimeRanges.get(0)[0]); + assertEquals(Long.MAX_VALUE, pageTimeRanges.get(pageTimeRanges.size() - 1)[1]); + for (int i = 0; i < pageTimeRanges.size(); i++) { + long[] pageTimeRange = pageTimeRanges.get(i); + assertTrue(pageTimeRange[0] <= pageTimeRange[1]); + if (i > 0) { + assertEquals( + BigInteger.valueOf(pageTimeRanges.get(i - 1)[1]).add(BigInteger.ONE), + BigInteger.valueOf(pageTimeRange[0])); + } + } + } + + @Test + public void testSplitFakePageTimeRangesDoesNotCreateEmptyRanges() { + List pageTimeRanges = MemChunkTimeRangeUtils.splitFakePageTimeRanges(10, 10, 100); + + assertEquals(1, pageTimeRanges.size()); + assertArrayEquals(new long[] {10, 10}, pageTimeRanges.get(0)); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResourceTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResourceTest.java index d637450eee38f..50515e82aa793 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResourceTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResourceTest.java @@ -107,6 +107,36 @@ public void testSerializeAndDeserialize() throws IOException { Assert.assertEquals(tsFileResource, derTsFileResource); } + @Test + public void testCheckAndCompareFileNameWithLongBoundary() throws IOException { + String minTimeFile = TsFileNameGenerator.generateNewTsFileName(Long.MIN_VALUE, 0, 0, 0); + String maxTimeFile = TsFileNameGenerator.generateNewTsFileName(Long.MAX_VALUE, 0, 0, 0); + + Assert.assertTrue(TsFileResource.checkAndCompareFileName(minTimeFile, maxTimeFile) < 0); + Assert.assertTrue(TsFileResource.checkAndCompareFileName(maxTimeFile, minTimeFile) > 0); + + String minVersionFile = TsFileNameGenerator.generateNewTsFileName(0, Long.MIN_VALUE, 0, 0); + String maxVersionFile = TsFileNameGenerator.generateNewTsFileName(0, Long.MAX_VALUE, 0, 0); + + Assert.assertTrue(TsFileResource.checkAndCompareFileName(minVersionFile, maxVersionFile) < 0); + Assert.assertTrue(TsFileResource.checkAndCompareFileName(maxVersionFile, minVersionFile) > 0); + } + + @Test + public void testCompareFileCreationOrderByDescWithLongBoundaryVersion() { + TsFileResource minVersionResource = + new TsFileResource( + new File(TsFileNameGenerator.generateNewTsFileName(0, Long.MIN_VALUE, 0, 0))); + TsFileResource maxVersionResource = + new TsFileResource( + new File(TsFileNameGenerator.generateNewTsFileName(0, Long.MAX_VALUE, 0, 0))); + + Assert.assertTrue( + TsFileResource.compareFileCreationOrderByDesc(maxVersionResource, minVersionResource) < 0); + Assert.assertTrue( + TsFileResource.compareFileCreationOrderByDesc(minVersionResource, maxVersionResource) > 0); + } + @Test public void testDegradeAndFileTimeIndex() { Assert.assertEquals(ITimeIndex.ARRAY_DEVICE_TIME_INDEX_TYPE, tsFileResource.getTimeIndexType()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/TsFileSplitterTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/TsFileSplitterTest.java index 6610880567e90..58d68d8d675c2 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/TsFileSplitterTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/TsFileSplitterTest.java @@ -19,6 +19,10 @@ package org.apache.iotdb.db.storageengine.load.splitter; +import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; +import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.utils.TimePartitionUtils; + import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; @@ -53,7 +57,7 @@ public void testSplitTableTimeOnlyAlignedChunk() throws Exception { final IDeviceID deviceID = new StringArrayDeviceID("table1", "tagA"); try { - writeTableTsFileWithTimeOnlyChunk(sourceTsFile, deviceID); + writeTableTsFileWithTimeOnlyChunk(sourceTsFile, deviceID, 100, 101); final List emittedChunkDataList = new ArrayList<>(); final TsFileSplitter splitter = @@ -110,8 +114,156 @@ public void testSplitTableTimeOnlyAlignedChunk() throws Exception { } } - private void writeTableTsFileWithTimeOnlyChunk(final File tsFile, final IDeviceID deviceID) - throws Exception { + @Test + public void testSplitTableTimeOnlyAlignedChunkAtLongMaxPartitionEnd() throws Exception { + final File sourceTsFile = new File("split-table-time-only-long-max-source.tsfile"); + final File targetTsFile = new File("split-table-time-only-long-max-target.tsfile"); + final IDeviceID deviceID = new StringArrayDeviceID("table1", "tagA"); + final long lastPartitionStartTime = + TimePartitionUtils.getTimePartitionSlot(Long.MAX_VALUE).getStartTime(); + + try { + writeTableTsFileWithTimeOnlyChunk( + sourceTsFile, + deviceID, + lastPartitionStartTime - 1, + lastPartitionStartTime, + Long.MAX_VALUE); + + final List emittedChunkDataList = new ArrayList<>(); + final TsFileSplitter splitter = + new TsFileSplitter( + sourceTsFile, + tsFileData -> { + if (tsFileData instanceof ChunkData) { + emittedChunkDataList.add((ChunkData) tsFileData); + } + return true; + }); + splitter.splitTsFileByDataPartition(); + + if (targetTsFile.exists()) { + Assert.assertTrue(targetTsFile.delete()); + } + try (final TsFileIOWriter writer = new TsFileIOWriter(targetTsFile)) { + writer.setSchema(createSchema()); + writer.startChunkGroup(deviceID); + for (final ChunkData chunkData : emittedChunkDataList) { + writeSerializedChunkDataToWriter(chunkData, writer); + } + writer.endChunkGroup(); + writer.endFile(); + } + + Assert.assertEquals(2, emittedChunkDataList.size()); + Assert.assertEquals( + TimePartitionUtils.getTimePartitionSlot(lastPartitionStartTime - 1), + emittedChunkDataList.get(0).getTimePartitionSlot()); + Assert.assertEquals( + new TTimePartitionSlot(lastPartitionStartTime), + emittedChunkDataList.get(1).getTimePartitionSlot()); + try (final TsFileSequenceReader reader = + new TsFileSequenceReader(targetTsFile.getAbsolutePath())) { + final List chunkMetadataList = + reader.getAlignedChunkMetadata(deviceID, false); + Assert.assertEquals(2, chunkMetadataList.size()); + Assert.assertEquals( + 3, + chunkMetadataList.stream() + .mapToLong(metadata -> metadata.getTimeChunkMetadata().getStatistics().getCount()) + .sum()); + } + } finally { + if (sourceTsFile.exists()) { + Assert.assertTrue(sourceTsFile.delete()); + } + if (targetTsFile.exists()) { + Assert.assertTrue(targetTsFile.delete()); + } + } + } + + @Test + public void testSplitTableTimeOnlyAlignedChunkAtExactLongMaxUpperBound() throws Exception { + final File sourceTsFile = new File("split-table-time-only-exact-long-max-source.tsfile"); + final File targetTsFile = new File("split-table-time-only-exact-long-max-target.tsfile"); + final IDeviceID deviceID = new StringArrayDeviceID("table1", "tagA"); + final long previousTimePartitionOrigin = + CommonDescriptor.getInstance().getConfig().getTimePartitionOrigin(); + final long previousTimePartitionInterval = + CommonDescriptor.getInstance().getConfig().getTimePartitionInterval(); + + try { + CommonDescriptor.getInstance().getConfig().setTimePartitionOrigin(0); + CommonDescriptor.getInstance().getConfig().setTimePartitionInterval(1); + TimePartitionUtils.setTimePartitionOrigin(0); + TimePartitionUtils.setTimePartitionInterval(1); + + writeTableTsFileWithTimeOnlyChunk(sourceTsFile, deviceID, Long.MAX_VALUE - 1, Long.MAX_VALUE); + + final List emittedChunkDataList = new ArrayList<>(); + final TsFileSplitter splitter = + new TsFileSplitter( + sourceTsFile, + tsFileData -> { + if (tsFileData instanceof ChunkData) { + emittedChunkDataList.add((ChunkData) tsFileData); + } + return true; + }); + splitter.splitTsFileByDataPartition(); + + if (targetTsFile.exists()) { + Assert.assertTrue(targetTsFile.delete()); + } + try (final TsFileIOWriter writer = new TsFileIOWriter(targetTsFile)) { + writer.setSchema(createSchema()); + writer.startChunkGroup(deviceID); + for (final ChunkData chunkData : emittedChunkDataList) { + writeSerializedChunkDataToWriter(chunkData, writer); + } + writer.endChunkGroup(); + writer.endFile(); + } + + Assert.assertEquals(2, emittedChunkDataList.size()); + Assert.assertEquals( + new TTimePartitionSlot(Long.MAX_VALUE - 1), + emittedChunkDataList.get(0).getTimePartitionSlot()); + Assert.assertEquals( + new TTimePartitionSlot(Long.MAX_VALUE), + emittedChunkDataList.get(1).getTimePartitionSlot()); + try (final TsFileSequenceReader reader = + new TsFileSequenceReader(targetTsFile.getAbsolutePath())) { + final List chunkMetadataList = + reader.getAlignedChunkMetadata(deviceID, false); + Assert.assertEquals(2, chunkMetadataList.size()); + Assert.assertEquals( + 2, + chunkMetadataList.stream() + .mapToLong(metadata -> metadata.getTimeChunkMetadata().getStatistics().getCount()) + .sum()); + } + } finally { + TimePartitionUtils.setTimePartitionOrigin(previousTimePartitionOrigin); + TimePartitionUtils.setTimePartitionInterval(previousTimePartitionInterval); + CommonDescriptor.getInstance() + .getConfig() + .setTimePartitionOrigin(previousTimePartitionOrigin); + CommonDescriptor.getInstance() + .getConfig() + .setTimePartitionInterval(previousTimePartitionInterval); + if (sourceTsFile.exists()) { + Assert.assertTrue(sourceTsFile.delete()); + } + if (targetTsFile.exists()) { + Assert.assertTrue(targetTsFile.delete()); + } + } + } + + private void writeTableTsFileWithTimeOnlyChunk( + final File tsFile, final IDeviceID deviceID, final long... times) throws Exception { if (tsFile.exists()) { Assert.assertTrue(tsFile.delete()); } @@ -122,8 +274,9 @@ private void writeTableTsFileWithTimeOnlyChunk(final File tsFile, final IDeviceI final AlignedChunkWriterImpl chunkWriter = new AlignedChunkWriterImpl(Collections.emptyList()); - chunkWriter.write(100); - chunkWriter.write(101); + for (final long time : times) { + chunkWriter.write(time); + } chunkWriter.writeToFileWriter(writer); writer.endChunkGroup(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/CommonUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/CommonUtilsTest.java new file mode 100644 index 0000000000000..bd1b6c52ea710 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/CommonUtilsTest.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.utils; + +import org.junit.Assert; +import org.junit.Test; + +public class CommonUtilsTest { + + @Test + public void testIsAliveDoesNotOverflowForLongMinTimestamp() { + Assert.assertFalse(CommonUtils.isAlive(Long.MIN_VALUE, 1)); + Assert.assertTrue(CommonUtils.isAlive(Long.MAX_VALUE, 1)); + Assert.assertTrue(CommonUtils.isAlive(Long.MIN_VALUE, Long.MAX_VALUE)); + } + + @Test + public void testTTLLowerBoundDoesNotUnderflowWithHugeTTL() { + long ttl = Long.MAX_VALUE - 1; + long ttlLowerBound = CommonUtils.getTTLLowerBound(ttl); + + Assert.assertTrue(ttlLowerBound > Long.MIN_VALUE); + Assert.assertFalse(CommonUtils.isAlive(Long.MIN_VALUE, ttl)); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/DateTimeUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/DateTimeUtilsTest.java index 056aa7576b1e9..91d86d32d5b48 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/DateTimeUtilsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/DateTimeUtilsTest.java @@ -18,6 +18,7 @@ */ package org.apache.iotdb.db.utils; +import org.apache.iotdb.commons.exception.SemanticException; import org.apache.iotdb.commons.queryengine.utils.DateTimeUtils; import org.apache.iotdb.db.protocol.session.IClientSession; import org.apache.iotdb.db.protocol.session.InternalClientSession; @@ -116,6 +117,25 @@ public void convertDurationStrToLongTest1() { 7L, DataNodeDateTimeUtils.convertDurationStrToLongForTest(7000000, "ns", "ms")); } + @Test + public void convertDurationStrToLongOverflowTest() { + Assert.assertThrows( + ArithmeticException.class, + () -> DataNodeDateTimeUtils.convertDurationStrToLongForTest(Long.MAX_VALUE, "s", "ms")); + Assert.assertThrows( + ArithmeticException.class, + () -> DataNodeDateTimeUtils.convertDurationStrToLongForTest(Long.MAX_VALUE, "ms", "us")); + Assert.assertThrows( + ArithmeticException.class, + () -> DataNodeDateTimeUtils.convertDurationStrToLong("9223372036854775808ms")); + Assert.assertThrows( + ArithmeticException.class, + () -> DataNodeDateTimeUtils.convertDurationStrToLong(0, Long.MAX_VALUE, "mo", "ms")); + Assert.assertThrows( + ArithmeticException.class, + () -> DataNodeDateTimeUtils.convertDurationStrToLong(Long.MAX_VALUE, "1ms1s", "ms", false)); + } + /** Test time precision is us. */ @Test public void convertDurationStrToLongTest2() { @@ -373,4 +393,25 @@ public void testConstructTimeDuration() { timeDuration = DataNodeDateTimeUtils.constructTimeDuration("10000000000ms"); Assert.assertEquals(10000000000L, timeDuration.nonMonthDuration); } + + @Test + public void testConstructTimeDurationOverflow() { + Assert.assertThrows( + ArithmeticException.class, () -> DataNodeDateTimeUtils.constructTimeDuration("178956971y")); + Assert.assertThrows( + ArithmeticException.class, + () -> DataNodeDateTimeUtils.constructTimeDuration("9223372036854775808ms")); + Assert.assertThrows( + ArithmeticException.class, + () -> DataNodeDateTimeUtils.constructTimeDuration("9223372036854775807ms1ms")); + } + + @Test + public void testParseDateTimeExpressionOverflow() { + Assert.assertThrows( + SemanticException.class, + () -> + DataNodeDateTimeUtils.parseDateTimeExpressionToLong( + "1970-01-01T00:00:00.001 + 9223372036854775807ms", ZoneOffset.UTC)); + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/windowing/SlidingTimeWindowEvaluationHandlerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/windowing/SlidingTimeWindowEvaluationHandlerTest.java index 019aadde0f449..71110c8417b9b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/windowing/SlidingTimeWindowEvaluationHandlerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/windowing/SlidingTimeWindowEvaluationHandlerTest.java @@ -149,6 +149,21 @@ public void test19() throws WindowingException { doTest(1, 100, 101); } + @Test(timeout = 1000) + public void testNearLongMaxBoundaryDoesNotOverflow() throws WindowingException { + final AtomicInteger count = new AtomicInteger(0); + + SlidingTimeWindowEvaluationHandler handler = + new SlidingTimeWindowEvaluationHandler( + new SlidingTimeWindowConfiguration(TSDataType.INT32, 2, 1), + window -> count.incrementAndGet()); + + handler.collect(Long.MAX_VALUE - 1, 1); + handler.collect(Long.MAX_VALUE, 2); + + Assert.assertEquals(0, count.get()); + } + private void doTest(long timeInterval, long slidingStep, long totalTime) throws WindowingException { final AtomicInteger count = new AtomicInteger(0); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/SeriesPartitionTable.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/SeriesPartitionTable.java index b388121fcd0e9..2d5e36b338c42 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/SeriesPartitionTable.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/SeriesPartitionTable.java @@ -178,16 +178,22 @@ public List getTimeSlotList( TConsensusGroupId regionId, long startTime, long endTime) { if (regionId.getId() == -1) { return seriesPartitionMap.keySet().stream() - .filter(e -> e.getStartTime() >= startTime && e.getStartTime() < endTime) + .filter(e -> isTimeSlotInQueryRange(e, startTime, endTime)) .collect(Collectors.toList()); } else { return seriesPartitionMap.keySet().stream() - .filter(e -> e.getStartTime() >= startTime && e.getStartTime() < endTime) + .filter(e -> isTimeSlotInQueryRange(e, startTime, endTime)) .filter(e -> seriesPartitionMap.get(e).contains(regionId)) .collect(Collectors.toList()); } } + private static boolean isTimeSlotInQueryRange( + TTimePartitionSlot timePartitionSlot, long startTime, long endTime) { + final long slotStartTime = timePartitionSlot.getStartTime(); + return slotStartTime >= startTime && (endTime == Long.MAX_VALUE || slotStartTime < endTime); + } + /** * Create DataPartition within the specific SeriesPartitionSlot. * @@ -265,8 +271,7 @@ public List autoCleanPartitionTable( while (iterator.hasNext()) { Map.Entry> entry = iterator.next(); TTimePartitionSlot timePartitionSlot = entry.getKey(); - if (timePartitionSlot.getStartTime() + timePartitionInterval + TTL - <= currentTimeSlot.getStartTime()) { + if (isTimePartitionExpired(timePartitionSlot, timePartitionInterval, TTL, currentTimeSlot)) { removedTimePartitions.add(timePartitionSlot); iterator.remove(); } @@ -274,6 +279,24 @@ public List autoCleanPartitionTable( return removedTimePartitions; } + private static boolean isTimePartitionExpired( + TTimePartitionSlot timePartitionSlot, + long timePartitionInterval, + long TTL, + TTimePartitionSlot currentTimeSlot) { + long partitionEndTime = saturatingAdd(timePartitionSlot.getStartTime(), timePartitionInterval); + long expireTime = saturatingAdd(partitionEndTime, TTL); + return expireTime <= currentTimeSlot.getStartTime(); + } + + private static long saturatingAdd(long left, long right) { + long result = left + right; + if (((left ^ result) & (right ^ result)) < 0) { + return left < 0 ? Long.MIN_VALUE : Long.MAX_VALUE; + } + return result; + } + public void merge(SeriesPartitionTable sourceMap) { if (sourceMap == null) return; sourceMap.seriesPartitionMap.forEach( diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/relational/function/tvf/ForecastTableFunction.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/relational/function/tvf/ForecastTableFunction.java index 6cad7b09079b1..4c58e5855ce6a 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/relational/function/tvf/ForecastTableFunction.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/relational/function/tvf/ForecastTableFunction.java @@ -70,6 +70,9 @@ import static org.apache.iotdb.commons.queryengine.plan.relational.function.tvf.TableFunctionUtils.checkType; import static org.apache.iotdb.commons.queryengine.plan.relational.function.tvf.TableFunctionUtils.parseOptions; import static org.apache.iotdb.commons.queryengine.plan.relational.utils.ResultColumnAppender.createResultColumnAppender; +import static org.apache.iotdb.commons.queryengine.plan.udf.ForecastTimeUtils.calculateForecastInterval; +import static org.apache.iotdb.commons.queryengine.plan.udf.ForecastTimeUtils.calculateForecastOutputTime; +import static org.apache.iotdb.commons.queryengine.plan.udf.ForecastTimeUtils.calculateForecastStartTime; import static org.apache.iotdb.commons.udf.builtin.relational.tvf.WindowTVFUtils.findColumnIndex; public class ForecastTableFunction implements TableFunction { @@ -439,15 +442,10 @@ public void finish( String.format( QueryMessages.INPUT_END_TIME_LESS_THAN_START_TIME, inputStartTime, inputEndTime)); } - long interval = outputInterval; - if (outputInterval <= 0) { - interval = - inputRecords.size() == 1 - ? 0 - : (inputEndTime - inputStartTime) / (inputRecords.size() - 1); - } - long outputTime = - (outputStartTime == Long.MIN_VALUE) ? (inputEndTime + interval) : outputStartTime; + long interval = + calculateForecastInterval( + inputStartTime, inputEndTime, inputRecords.size(), outputInterval); + long outputTime = calculateForecastStartTime(inputEndTime, outputStartTime, interval); if (outputTime <= inputEndTime) { throw new SemanticException( String.format( @@ -457,7 +455,7 @@ public void finish( outputTime)); } for (int i = 0; i < outputLength; i++) { - properColumnBuilders.get(0).writeLong(outputTime + interval * i); + properColumnBuilders.get(0).writeLong(calculateForecastOutputTime(outputTime, interval, i)); } // predicated columns diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/udf/ForecastTimeUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/udf/ForecastTimeUtils.java new file mode 100644 index 0000000000000..b3e6f50636faf --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/udf/ForecastTimeUtils.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.queryengine.plan.udf; + +import org.apache.iotdb.commons.exception.SemanticException; + +import java.math.BigInteger; + +public final class ForecastTimeUtils { + + private static final BigInteger BIG_LONG_MAX = BigInteger.valueOf(Long.MAX_VALUE); + private static final String FORECAST_OUTPUT_TIME_OUT_OF_RANGE = + "Forecast output time is out of range."; + + private ForecastTimeUtils() { + // Utility class. + } + + public static long calculateForecastInterval( + long inputStartTime, long inputEndTime, int inputSize, long outputInterval) { + if (outputInterval > 0) { + return outputInterval; + } + if (inputSize <= 1) { + return 0; + } + + BigInteger interval = + BigInteger.valueOf(inputEndTime) + .subtract(BigInteger.valueOf(inputStartTime)) + .divide(BigInteger.valueOf(inputSize - 1L)); + if (interval.compareTo(BIG_LONG_MAX) > 0) { + throw new SemanticException(FORECAST_OUTPUT_TIME_OUT_OF_RANGE); + } + return interval.longValue(); + } + + public static long calculateForecastStartTime( + long inputEndTime, long outputStartTime, long interval) { + if (outputStartTime != Long.MIN_VALUE) { + return outputStartTime; + } + try { + return Math.addExact(inputEndTime, interval); + } catch (ArithmeticException e) { + throw new SemanticException(FORECAST_OUTPUT_TIME_OUT_OF_RANGE); + } + } + + public static long calculateForecastOutputTime(long outputStartTime, long interval, int index) { + try { + return Math.addExact(outputStartTime, Math.multiplyExact(interval, (long) index)); + } catch (ArithmeticException e) { + throw new SemanticException(FORECAST_OUTPUT_TIME_OUT_OF_RANGE); + } + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/udf/UDTFForecast.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/udf/UDTFForecast.java index 244fe6e1c5b60..de6bd3110072f 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/udf/UDTFForecast.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/udf/UDTFForecast.java @@ -47,6 +47,10 @@ import java.util.Set; import java.util.stream.Collectors; +import static org.apache.iotdb.commons.queryengine.plan.udf.ForecastTimeUtils.calculateForecastInterval; +import static org.apache.iotdb.commons.queryengine.plan.udf.ForecastTimeUtils.calculateForecastOutputTime; +import static org.apache.iotdb.commons.queryengine.plan.udf.ForecastTimeUtils.calculateForecastStartTime; + public class UDTFForecast implements UDTF { private static final TsBlockSerde serde = new TsBlockSerde(); private String model_id; @@ -225,15 +229,12 @@ public void terminate(PointCollector collector) throws Exception { String.format( QueryMessages.INPUT_END_TIME_LESS_THAN_START_TIME, inputStartTime, inputEndTime)); } - long interval = this.outputInterval; - if (outputInterval <= 0) { - interval = (inputEndTime - inputStartTime) / (inputRows.size() - 1); - } - long outputTime = - (this.outputStartTime == Long.MIN_VALUE) ? inputEndTime + interval : this.outputStartTime; + long interval = + calculateForecastInterval(inputStartTime, inputEndTime, inputRows.size(), outputInterval); + long outputTime = calculateForecastStartTime(inputEndTime, this.outputStartTime, interval); long[] outputTimes = new long[this.outputLength]; for (int i = 0; i < this.outputLength; i++) { - outputTimes[i] = outputTime + interval * i; + outputTimes[i] = calculateForecastOutputTime(outputTime, interval, i); } TsBlock forecastResult = forecast(); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketOutlierSample.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketOutlierSample.java index 85490ac534e47..c71a919ef366b 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketOutlierSample.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketOutlierSample.java @@ -185,7 +185,7 @@ public void outlierSampleInt(RowWindow rowWindow, PointCollector collector) thro double A = (double) row0y - row1y; double B = (double) row1x - row0x; - double C = (double) row0x * row1y - row1x * row0y; + double C = (double) row0x * row1y - (double) row1x * row0y; double denominator = Math.sqrt(A * A + B * B); for (int i = 1; i < windowSize - 1; i++) { @@ -215,7 +215,7 @@ public void outlierSampleLong(RowWindow rowWindow, PointCollector collector) double A = (double) row0y - row1y; double B = (double) row1x - row0x; - double C = (double) row0x * row1y - row1x * row0y; + double C = (double) row0x * row1y - (double) row1x * row0y; double denominator = Math.sqrt(A * A + B * B); for (int i = 1; i < windowSize - 1; i++) { @@ -245,7 +245,7 @@ public void outlierSampleFloat(RowWindow rowWindow, PointCollector collector) double A = (double) row0y - row1y; double B = (double) row1x - row0x; - double C = (double) row0x * row1y - row1x * row0y; + double C = (double) row0x * row1y - (double) row1x * row0y; double denominator = Math.sqrt(A * A + B * B); for (int i = 1; i < windowSize - 1; i++) { @@ -275,7 +275,7 @@ public void outlierSampleDouble(RowWindow rowWindow, PointCollector collector) double A = row0y - row1y; double B = (double) row1x - row0x; - double C = row0x * row1y - row1x * row0y; + double C = (double) row0x * row1y - (double) row1x * row0y; double denominator = Math.sqrt(A * A + B * B); for (int i = 1; i < windowSize - 1; i++) { @@ -301,8 +301,9 @@ public void outlierSampleInt(RowWindow rowWindow, PointCollector collector) thro PriorityQueue> pq = new PriorityQueue<>(number, Comparator.comparing(o -> -o.right)); - long lastTime, currentTime, nextTime, x1, x2; - int lastValue, currentValue, nextValue, y1, y2; + long lastTime, currentTime, nextTime; + int lastValue, currentValue, nextValue; + double x1, x2, y1, y2; double value; for (int i = 1; i < windowSize - 1; i++) { @@ -314,14 +315,12 @@ public void outlierSampleInt(RowWindow rowWindow, PointCollector collector) thro currentValue = rowWindow.getRow(i).getInt(0); nextValue = rowWindow.getRow(i + 1).getInt(0); - x1 = currentTime - lastTime; - x2 = nextTime - currentTime; - y1 = currentValue - lastValue; - y2 = nextValue - currentValue; + x1 = timeDifferenceAsDouble(currentTime, lastTime); + x2 = timeDifferenceAsDouble(nextTime, currentTime); + y1 = (double) currentValue - lastValue; + y2 = (double) nextValue - currentValue; - value = - (x1 * x2 + y1 * y2) - / (Math.sqrt((double) x1 * x1 + y1 * y1) * Math.sqrt((double) x2 * x2 + y2 * y2)); + value = (x1 * x2 + y1 * y2) / (Math.sqrt(x1 * x1 + y1 * y1) * Math.sqrt(x2 * x2 + y2 * y2)); addToMaxHeap(pq, i, value); } @@ -341,8 +340,9 @@ public void outlierSampleLong(RowWindow rowWindow, PointCollector collector) PriorityQueue> pq = new PriorityQueue<>(number, Comparator.comparing(o -> -o.right)); - long lastTime, currentTime, nextTime, x1, x2; - long lastValue, currentValue, nextValue, y1, y2; + long lastTime, currentTime, nextTime; + long lastValue, currentValue, nextValue; + double x1, x2, y1, y2; double value; for (int i = 1; i < windowSize - 1; i++) { @@ -354,14 +354,12 @@ public void outlierSampleLong(RowWindow rowWindow, PointCollector collector) currentValue = rowWindow.getRow(i).getLong(0); nextValue = rowWindow.getRow(i + 1).getLong(0); - x1 = currentTime - lastTime; - x2 = nextTime - currentTime; - y1 = currentValue - lastValue; - y2 = nextValue - currentValue; + x1 = timeDifferenceAsDouble(currentTime, lastTime); + x2 = timeDifferenceAsDouble(nextTime, currentTime); + y1 = (double) currentValue - (double) lastValue; + y2 = (double) nextValue - (double) currentValue; - value = - (x1 * x2 + y1 * y2) - / (Math.sqrt((double) x1 * x1 + y1 * y1) * Math.sqrt((double) x2 * x2 + y2 * y2)); + value = (x1 * x2 + y1 * y2) / (Math.sqrt(x1 * x1 + y1 * y1) * Math.sqrt(x2 * x2 + y2 * y2)); addToMaxHeap(pq, i, value); } @@ -381,8 +379,9 @@ public void outlierSampleFloat(RowWindow rowWindow, PointCollector collector) PriorityQueue> pq = new PriorityQueue<>(number, Comparator.comparing(o -> -o.right)); - long lastTime, currentTime, nextTime, x1, x2; - float lastValue, currentValue, nextValue, y1, y2; + long lastTime, currentTime, nextTime; + float lastValue, currentValue, nextValue; + double x1, x2, y1, y2; double value; for (int i = 1; i < windowSize - 1; i++) { @@ -394,10 +393,10 @@ public void outlierSampleFloat(RowWindow rowWindow, PointCollector collector) currentValue = rowWindow.getRow(i).getFloat(0); nextValue = rowWindow.getRow(i + 1).getFloat(0); - x1 = currentTime - lastTime; - x2 = nextTime - currentTime; - y1 = currentValue - lastValue; - y2 = nextValue - currentValue; + x1 = timeDifferenceAsDouble(currentTime, lastTime); + x2 = timeDifferenceAsDouble(nextTime, currentTime); + y1 = (double) currentValue - lastValue; + y2 = (double) nextValue - currentValue; value = (x1 * x2 + y1 * y2) / (Math.sqrt(x1 * x1 + y1 * y1) * Math.sqrt(x2 * x2 + y2 * y2)); @@ -419,8 +418,8 @@ public void outlierSampleDouble(RowWindow rowWindow, PointCollector collector) PriorityQueue> pq = new PriorityQueue<>(number, Comparator.comparing(o -> -o.right)); - long lastTime, currentTime, nextTime, x1, x2; - double lastValue, currentValue, nextValue, y1, y2; + long lastTime, currentTime, nextTime; + double lastValue, currentValue, nextValue, x1, x2, y1, y2; double value; for (int i = 1; i < windowSize - 1; i++) { @@ -432,8 +431,8 @@ public void outlierSampleDouble(RowWindow rowWindow, PointCollector collector) currentValue = rowWindow.getRow(i).getDouble(0); nextValue = rowWindow.getRow(i + 1).getDouble(0); - x1 = currentTime - lastTime; - x2 = nextTime - currentTime; + x1 = timeDifferenceAsDouble(currentTime, lastTime); + x2 = timeDifferenceAsDouble(nextTime, currentTime); y1 = currentValue - lastValue; y2 = nextValue - currentValue; @@ -458,8 +457,9 @@ public void outlierSampleInt(RowWindow rowWindow, PointCollector collector) thro PriorityQueue> pq = new PriorityQueue<>(number, Comparator.comparing(o -> o.right)); - long lastTime, currentTime, nextTime, x1, x2; - int lastValue, currentValue, nextValue, y1, y2; + long lastTime, currentTime, nextTime; + int lastValue, currentValue, nextValue; + double x1, x2, y1, y2; double value; for (int i = 1; i < windowSize - 1; i++) { @@ -471,10 +471,10 @@ public void outlierSampleInt(RowWindow rowWindow, PointCollector collector) thro currentValue = rowWindow.getRow(i).getInt(0); nextValue = rowWindow.getRow(i + 1).getInt(0); - x1 = Math.abs(currentTime - lastTime); - x2 = Math.abs(nextTime - currentTime); - y1 = Math.abs(currentValue - lastValue); - y2 = Math.abs(nextValue - currentValue); + x1 = timeDistanceAsDouble(currentTime, lastTime); + x2 = timeDistanceAsDouble(nextTime, currentTime); + y1 = Math.abs((double) currentValue - lastValue); + y2 = Math.abs((double) nextValue - currentValue); value = (double) x1 + y1 + x2 + y2; @@ -495,8 +495,9 @@ public void outlierSampleLong(RowWindow rowWindow, PointCollector collector) PriorityQueue> pq = new PriorityQueue<>(number, Comparator.comparing(o -> o.right)); - long lastTime, currentTime, nextTime, x1, x2; - long lastValue, currentValue, nextValue, y1, y2; + long lastTime, currentTime, nextTime; + long lastValue, currentValue, nextValue; + double x1, x2, y1, y2; double value; for (int i = 1; i < windowSize - 1; i++) { @@ -508,10 +509,10 @@ public void outlierSampleLong(RowWindow rowWindow, PointCollector collector) currentValue = rowWindow.getRow(i).getLong(0); nextValue = rowWindow.getRow(i + 1).getLong(0); - x1 = Math.abs(currentTime - lastTime); - x2 = Math.abs(nextTime - currentTime); - y1 = Math.abs(currentValue - lastValue); - y2 = Math.abs(nextValue - currentValue); + x1 = timeDistanceAsDouble(currentTime, lastTime); + x2 = timeDistanceAsDouble(nextTime, currentTime); + y1 = Math.abs((double) currentValue - (double) lastValue); + y2 = Math.abs((double) nextValue - (double) currentValue); value = (double) x1 + y1 + x2 + y2; @@ -532,8 +533,9 @@ public void outlierSampleFloat(RowWindow rowWindow, PointCollector collector) PriorityQueue> pq = new PriorityQueue<>(number, Comparator.comparing(o -> o.right)); - long lastTime, currentTime, nextTime, x1, x2; - float lastValue, currentValue, nextValue, y1, y2; + long lastTime, currentTime, nextTime; + float lastValue, currentValue, nextValue; + double x1, x2, y1, y2; double value; for (int i = 1; i < windowSize - 1; i++) { @@ -545,10 +547,10 @@ public void outlierSampleFloat(RowWindow rowWindow, PointCollector collector) currentValue = rowWindow.getRow(i).getFloat(0); nextValue = rowWindow.getRow(i + 1).getFloat(0); - x1 = Math.abs(currentTime - lastTime); - x2 = Math.abs(nextTime - currentTime); - y1 = Math.abs(currentValue - lastValue); - y2 = Math.abs(nextValue - currentValue); + x1 = timeDistanceAsDouble(currentTime, lastTime); + x2 = timeDistanceAsDouble(nextTime, currentTime); + y1 = Math.abs((double) currentValue - lastValue); + y2 = Math.abs((double) nextValue - currentValue); value = x1 + y1 + x2 + y2; @@ -569,7 +571,8 @@ public void outlierSampleDouble(RowWindow rowWindow, PointCollector collector) PriorityQueue> pq = new PriorityQueue<>(number, Comparator.comparing(o -> o.right)); - long lastTime, currentTime, nextTime, x1, x2; + long lastTime, currentTime, nextTime; + double x1, x2; double lastValue, currentValue, nextValue, y1, y2; double value; @@ -582,8 +585,8 @@ public void outlierSampleDouble(RowWindow rowWindow, PointCollector collector) currentValue = rowWindow.getRow(i).getDouble(0); nextValue = rowWindow.getRow(i + 1).getDouble(0); - x1 = Math.abs(currentTime - lastTime); - x2 = Math.abs(nextTime - currentTime); + x1 = timeDistanceAsDouble(currentTime, lastTime); + x2 = timeDistanceAsDouble(nextTime, currentTime); y1 = Math.abs(currentValue - lastValue); y2 = Math.abs(nextValue - currentValue); @@ -845,4 +848,12 @@ public boolean isWindowSizeTooSmallDouble( } return false; } + + private static double timeDifferenceAsDouble(long left, long right) { + return (double) left - (double) right; + } + + private static double timeDistanceAsDouble(long left, long right) { + return Math.abs(timeDifferenceAsDouble(left, right)); + } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFTimeDifference.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFTimeDifference.java index 6478eaf7fd74b..df5d8b12430d9 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFTimeDifference.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFTimeDifference.java @@ -49,7 +49,18 @@ public void transform(Row row, PointCollector collector) throws IOException { } long currentTime = row.getTime(); - collector.putLong(currentTime, currentTime - previousTime); + collector.putLong(currentTime, saturatingSubtract(currentTime, previousTime)); previousTime = currentTime; } + + private static long saturatingSubtract(long left, long right) { + long result = left - right; + if (left >= right && result < 0) { + return Long.MAX_VALUE; + } + if (left < right && result > 0) { + return Long.MIN_VALUE; + } + return result; + } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/CapacityTableFunction.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/CapacityTableFunction.java index a037500c00631..42afbfc724fd4 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/CapacityTableFunction.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/CapacityTableFunction.java @@ -134,17 +134,33 @@ public void process( // k * slide <= curIndex < k * slide + size, and k >= 0 // The first valid k: max(0, ceil((curIndex - size + 1) / slide)) // The last valid k: floor(curIndex / slide) - long firstWindow = Math.max(0, (curIndex - size + slide) / slide); + long firstWindow = curIndex < size ? 0 : ceilDiv(curIndex - size + 1, slide); long lastWindow = curIndex / slide; - for (long k = firstWindow; k <= lastWindow; k++) { - // Verify: k * slide <= curIndex < k * slide + size - long windowStart = k * slide; - if (windowStart <= curIndex && curIndex < windowStart + size) { - properColumnBuilders.get(0).writeLong(k); - passThroughIndexBuilder.writeLong(curIndex); + if (firstWindow <= lastWindow) { + for (long k = firstWindow; ; k++) { + // Verify: k * slide <= curIndex < k * slide + size + if (isIndexInWindow(curIndex, k, slide, size)) { + properColumnBuilders.get(0).writeLong(k); + passThroughIndexBuilder.writeLong(curIndex); + } + if (k == lastWindow) { + break; + } } } curIndex++; } + + private static long ceilDiv(long dividend, long divisor) { + return dividend == 0 ? 0 : (dividend - 1) / divisor + 1; + } + + private static boolean isIndexInWindow(long curIndex, long windowIndex, long slide, long size) { + if (windowIndex > Long.MAX_VALUE / slide) { + return false; + } + long windowStart = windowIndex * slide; + return windowStart <= curIndex && curIndex - windowStart < size; + } } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/CumulateTableFunction.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/CumulateTableFunction.java index e6627491f2b3c..0ee3449eaef7e 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/CumulateTableFunction.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/CumulateTableFunction.java @@ -38,12 +38,15 @@ import org.apache.tsfile.block.column.ColumnBuilder; +import java.math.BigInteger; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; import static org.apache.iotdb.commons.udf.builtin.relational.tvf.WindowTVFUtils.findColumnIndex; +import static org.apache.iotdb.commons.udf.builtin.relational.tvf.WindowTVFUtils.getWindowStart; +import static org.apache.iotdb.commons.udf.builtin.relational.tvf.WindowTVFUtils.saturateToLong; import static org.apache.iotdb.udf.api.relational.table.argument.ScalarArgumentChecker.POSITIVE_LONG_CHECKER; public class CumulateTableFunction implements TableFunction { @@ -163,12 +166,19 @@ public void process( // find the first windows long timeValue = input.getLong(0); if (timeValue >= origin) { - long windowStart = origin + (timeValue - origin) / size * size; - for (long steps = (timeValue - windowStart + step) / step * step; - steps <= size; - steps += step) { - properColumnBuilders.get(0).writeLong(windowStart); - properColumnBuilders.get(1).writeLong(windowStart + steps); + BigInteger windowStart = getWindowStart(timeValue, origin, size); + BigInteger stepValue = BigInteger.valueOf(step); + BigInteger sizeValue = BigInteger.valueOf(size); + for (BigInteger steps = + BigInteger.valueOf(timeValue) + .subtract(windowStart) + .add(stepValue) + .divide(stepValue) + .multiply(stepValue); + steps.compareTo(sizeValue) <= 0; + steps = steps.add(stepValue)) { + properColumnBuilders.get(0).writeLong(saturateToLong(windowStart)); + properColumnBuilders.get(1).writeLong(saturateToLong(windowStart.add(steps))); passThroughIndexBuilder.writeLong(curIndex); } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/HOPTableFunction.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/HOPTableFunction.java index a411bda4a7c41..7b4832cb1aa95 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/HOPTableFunction.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/HOPTableFunction.java @@ -37,12 +37,15 @@ import org.apache.tsfile.block.column.ColumnBuilder; +import java.math.BigInteger; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; import static org.apache.iotdb.commons.udf.builtin.relational.tvf.WindowTVFUtils.findColumnIndex; +import static org.apache.iotdb.commons.udf.builtin.relational.tvf.WindowTVFUtils.getWindowStart; +import static org.apache.iotdb.commons.udf.builtin.relational.tvf.WindowTVFUtils.saturateToLong; import static org.apache.iotdb.udf.api.relational.table.argument.ScalarArgumentChecker.POSITIVE_LONG_CHECKER; public class HOPTableFunction implements TableFunction { @@ -158,12 +161,16 @@ public void process( // n*slide + size long timeValue = input.getLong(0); if (timeValue >= origin) { - long window_start = origin + (timeValue - origin - size + slide) / slide * slide; - while (window_start <= timeValue && window_start + size > timeValue) { - properColumnBuilders.get(0).writeLong(window_start); - properColumnBuilders.get(1).writeLong(window_start + size); + BigInteger time = BigInteger.valueOf(timeValue); + BigInteger sizeValue = BigInteger.valueOf(size); + BigInteger slideValue = BigInteger.valueOf(slide); + BigInteger windowStart = + getWindowStart(timeValue, origin, slide, slideValue.subtract(sizeValue)); + while (windowStart.compareTo(time) <= 0 && windowStart.add(sizeValue).compareTo(time) > 0) { + properColumnBuilders.get(0).writeLong(saturateToLong(windowStart)); + properColumnBuilders.get(1).writeLong(saturateToLong(windowStart.add(sizeValue))); passThroughIndexBuilder.writeLong(curIndex); - window_start += slide; + windowStart = windowStart.add(slideValue); } } curIndex++; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/SessionTableFunction.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/SessionTableFunction.java index 19252a67df3dc..c42491a1366b5 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/SessionTableFunction.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/SessionTableFunction.java @@ -38,6 +38,7 @@ import org.apache.tsfile.block.column.ColumnBuilder; +import java.math.BigInteger; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -116,7 +117,8 @@ private static class SessionDataProcessor implements TableFunctionDataProcessor private long currentStartIndex = 0; private long curIndex = 0; private long windowStart = Long.MIN_VALUE; - private long windowEnd = Long.MIN_VALUE; + private long currentWindowEnd = Long.MIN_VALUE; + private BigInteger windowEnd = BigInteger.valueOf(Long.MIN_VALUE); public SessionDataProcessor(long gap) { this.gap = gap; @@ -128,12 +130,13 @@ public void process( List properColumnBuilders, ColumnBuilder passThroughIndexBuilder) { long timeValue = input.getLong(0); - if (timeValue > windowEnd) { + if (BigInteger.valueOf(timeValue).compareTo(windowEnd) > 0) { outputWindow(properColumnBuilders, passThroughIndexBuilder); currentStartIndex = curIndex; windowStart = timeValue; } - windowEnd = timeValue + gap; + currentWindowEnd = timeValue; + windowEnd = BigInteger.valueOf(timeValue).add(BigInteger.valueOf(gap)); curIndex++; } @@ -145,7 +148,6 @@ public void finish( private void outputWindow( List properColumnBuilders, ColumnBuilder passThroughIndexBuilder) { - long currentWindowEnd = windowEnd - gap; for (long i = currentStartIndex; i < curIndex; i++) { properColumnBuilders.get(0).writeLong(windowStart); properColumnBuilders.get(1).writeLong(currentWindowEnd); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/TumbleTableFunction.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/TumbleTableFunction.java index b94520c8c5cbd..d003360cb1371 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/TumbleTableFunction.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/TumbleTableFunction.java @@ -38,12 +38,15 @@ import org.apache.tsfile.block.column.ColumnBuilder; +import java.math.BigInteger; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; import static org.apache.iotdb.commons.udf.builtin.relational.tvf.WindowTVFUtils.findColumnIndex; +import static org.apache.iotdb.commons.udf.builtin.relational.tvf.WindowTVFUtils.getWindowStart; +import static org.apache.iotdb.commons.udf.builtin.relational.tvf.WindowTVFUtils.saturateToLong; import static org.apache.iotdb.udf.api.relational.table.argument.ScalarArgumentChecker.POSITIVE_LONG_CHECKER; public class TumbleTableFunction implements TableFunction { @@ -145,9 +148,11 @@ public void process( // find the proper window long timeValue = input.getLong(0); if (timeValue >= origin) { - long windowStart = origin + (timeValue - origin) / size * size; - properColumnBuilders.get(0).writeLong(windowStart); - properColumnBuilders.get(1).writeLong(windowStart + size); + BigInteger windowStart = getWindowStart(timeValue, origin, size); + properColumnBuilders.get(0).writeLong(saturateToLong(windowStart)); + properColumnBuilders + .get(1) + .writeLong(saturateToLong(windowStart.add(BigInteger.valueOf(size)))); passThroughIndexBuilder.writeLong(curIndex); } curIndex++; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/WindowTVFUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/WindowTVFUtils.java index f0f72858c005a..ba53ed9dc4d0c 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/WindowTVFUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/WindowTVFUtils.java @@ -25,10 +25,14 @@ import org.apache.iotdb.udf.api.relational.table.argument.TableArgument; import org.apache.iotdb.udf.api.type.Type; +import java.math.BigInteger; import java.util.Optional; import java.util.Set; public class WindowTVFUtils { + private static final BigInteger BIG_LONG_MIN = BigInteger.valueOf(Long.MIN_VALUE); + private static final BigInteger BIG_LONG_MAX = BigInteger.valueOf(Long.MAX_VALUE); + /** * Find the index of the column in the table argument. * @@ -54,4 +58,29 @@ public static int findColumnIndex( String.format( "Required column [%s] not found in the source table argument.", expectedFieldName)); } + + static BigInteger getWindowStart(long source, long origin, long interval) { + return getWindowStart(source, origin, interval, BigInteger.ZERO); + } + + static BigInteger getWindowStart( + long source, long origin, long interval, BigInteger distanceOffset) { + BigInteger intervalValue = BigInteger.valueOf(interval); + BigInteger stepCount = + BigInteger.valueOf(source) + .subtract(BigInteger.valueOf(origin)) + .add(distanceOffset) + .divide(intervalValue); + return BigInteger.valueOf(origin).add(stepCount.multiply(intervalValue)); + } + + static long saturateToLong(BigInteger value) { + if (value.compareTo(BIG_LONG_MAX) > 0) { + return Long.MAX_VALUE; + } + if (value.compareTo(BIG_LONG_MIN) < 0) { + return Long.MIN_VALUE; + } + return value.longValue(); + } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/TimePartitionUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/TimePartitionUtils.java index 250a347d1496b..c98909de41104 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/TimePartitionUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/TimePartitionUtils.java @@ -31,44 +31,46 @@ public class TimePartitionUtils { * Time partition origin for dividing database, the time unit is the same with IoTDB's * TimestampPrecision */ - private static long timePartitionOrigin = + private static volatile long timePartitionOrigin = CommonDescriptor.getInstance().getConfig().getTimePartitionOrigin(); /** Time range for dividing database, the time unit is the same with IoTDB's TimestampPrecision */ - private static long timePartitionInterval = + private static volatile long timePartitionInterval = CommonDescriptor.getInstance().getConfig().getTimePartitionInterval(); - private static final BigInteger bigTimePartitionOrigin = BigInteger.valueOf(timePartitionOrigin); - private static final BigInteger bigTimePartitionInterval = - BigInteger.valueOf(timePartitionInterval); - private static final boolean originMayCauseOverflow = (timePartitionOrigin != 0); - private static final long timePartitionLowerBoundWithoutOverflow; - private static final long timePartitionUpperBoundWithoutOverflow; + private static volatile boolean originMayCauseOverflow; + private static volatile long timePartitionLowerBoundWithoutOverflow; + private static volatile long timePartitionUpperBoundWithoutOverflow; static { + updateTimePartitionBound(); + } + + private static void updateTimePartitionBound() { long minPartition = getTimePartitionIdWithoutOverflow(Long.MIN_VALUE); long maxPartition = getTimePartitionIdWithoutOverflow(Long.MAX_VALUE); BigInteger minPartitionStartTime = BigInteger.valueOf(minPartition) - .multiply(bigTimePartitionInterval) - .add(bigTimePartitionOrigin); + .multiply(BigInteger.valueOf(timePartitionInterval)) + .add(BigInteger.valueOf(timePartitionOrigin)); BigInteger maxPartitionEndTime = BigInteger.valueOf(maxPartition) - .multiply(bigTimePartitionInterval) - .add(bigTimePartitionInterval) - .add(bigTimePartitionOrigin); + .multiply(BigInteger.valueOf(timePartitionInterval)) + .add(BigInteger.valueOf(timePartitionInterval)) + .add(BigInteger.valueOf(timePartitionOrigin)); if (minPartitionStartTime.compareTo(BigInteger.valueOf(Long.MIN_VALUE)) < 0) { timePartitionLowerBoundWithoutOverflow = - minPartitionStartTime.add(bigTimePartitionInterval).longValue(); + minPartitionStartTime.add(BigInteger.valueOf(timePartitionInterval)).longValue(); } else { timePartitionLowerBoundWithoutOverflow = minPartitionStartTime.longValue(); } if (maxPartitionEndTime.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0) { timePartitionUpperBoundWithoutOverflow = - maxPartitionEndTime.subtract(bigTimePartitionInterval).longValue(); + maxPartitionEndTime.subtract(BigInteger.valueOf(timePartitionInterval)).longValue(); } else { timePartitionUpperBoundWithoutOverflow = maxPartitionEndTime.longValue(); } + originMayCauseOverflow = (timePartitionOrigin != 0); } public static TTimePartitionSlot getTimePartitionSlot(long time) { @@ -85,11 +87,11 @@ public static long getTimePartitionLowerBound(long time) { if (time < timePartitionLowerBoundWithoutOverflow) { return Long.MIN_VALUE; } + if (time >= timePartitionUpperBoundWithoutOverflow) { + return timePartitionUpperBoundWithoutOverflow; + } if (originMayCauseOverflow) { - return BigInteger.valueOf(getTimePartitionIdWithoutOverflow(time)) - .multiply(bigTimePartitionInterval) - .add(bigTimePartitionOrigin) - .longValue(); + return getTimePartitionStartTime(getTimePartitionIdWithoutOverflow(time)); } else { return getTimePartitionId(time) * timePartitionInterval + timePartitionOrigin; } @@ -99,13 +101,41 @@ public static long getTimePartitionUpperBound(long time) { if (time >= timePartitionUpperBoundWithoutOverflow) { return Long.MAX_VALUE; } - long lowerBound = getTimePartitionLowerBound(time); - return lowerBound == Long.MIN_VALUE - ? timePartitionLowerBoundWithoutOverflow - : lowerBound + timePartitionInterval; + if (time < timePartitionLowerBoundWithoutOverflow) { + return timePartitionLowerBoundWithoutOverflow; + } + return getTimePartitionLowerBound(time) + timePartitionInterval; + } + + public static long getTimePartitionEndTime(long time) { + long upperBound = getTimePartitionUpperBound(time); + if (upperBound != Long.MAX_VALUE) { + return upperBound - 1; + } + return getTimePartitionLowerBound(time) == getTimePartitionLowerBound(Long.MAX_VALUE) + ? Long.MAX_VALUE + : Long.MAX_VALUE - 1; + } + + public static boolean isAfterOrEqualToTimePartitionUpperBound( + long time, long timePartitionStartTime, long timePartitionUpperBound) { + if (timePartitionUpperBound != Long.MAX_VALUE) { + return time >= timePartitionUpperBound; + } + return time == Long.MAX_VALUE && getTimePartitionLowerBound(time) != timePartitionStartTime; + } + + public static boolean isTimePartitionStartTime(long time) { + return getTimePartitionLowerBound(time) == time; } public static long getTimePartitionId(long time) { + if (originMayCauseOverflow) { + return getTimePartitionIdWithoutOverflow(time); + } + if (time >= timePartitionUpperBoundWithoutOverflow) { + return getTimePartitionIdWithoutOverflow(time); + } time -= timePartitionOrigin; return time > 0 || time % timePartitionInterval == 0 ? time / timePartitionInterval @@ -113,7 +143,8 @@ public static long getTimePartitionId(long time) { } public static long getTimePartitionIdWithoutOverflow(long time) { - BigInteger bigTime = BigInteger.valueOf(time).subtract(bigTimePartitionOrigin); + BigInteger bigTime = BigInteger.valueOf(time).subtract(BigInteger.valueOf(timePartitionOrigin)); + BigInteger bigTimePartitionInterval = BigInteger.valueOf(timePartitionInterval); BigInteger partitionId = bigTime.compareTo(BigInteger.ZERO) > 0 || bigTime.remainder(bigTimePartitionInterval).equals(BigInteger.ZERO) @@ -123,7 +154,7 @@ public static long getTimePartitionIdWithoutOverflow(long time) { } public static long getStartTimeByPartitionId(long partitionId) { - return (partitionId * timePartitionInterval) + timePartitionOrigin; + return getTimePartitionStartTime(partitionId); } public static boolean satisfyPartitionId(long startTime, long endTime, long partitionId) { @@ -142,40 +173,47 @@ public static boolean satisfyPartitionStartTime(Filter timeFilter, long partitio if (timeFilter == null) { return true; } - - long partitionEndTime = - partitionStartTime >= timePartitionLowerBoundWithoutOverflow - ? Long.MAX_VALUE - : (partitionStartTime + timePartitionInterval - 1); + long partitionEndTime = getTimePartitionEndTime(partitionStartTime); return timeFilter.satisfyStartEndTime(partitionStartTime, partitionEndTime); } public static boolean satisfyTimePartition(Filter timeFilter, long partitionId) { - long partitionStartTime; - if (originMayCauseOverflow) { - partitionStartTime = - BigInteger.valueOf(partitionId) - .multiply(bigTimePartitionInterval) - .add(bigTimePartitionOrigin) - .longValue(); - } else { - partitionStartTime = partitionId * timePartitionInterval + timePartitionOrigin; + return satisfyPartitionStartTime(timeFilter, getTimePartitionStartTime(partitionId)); + } + + private static long getTimePartitionStartTime(long partitionId) { + BigInteger partitionStartTime = + BigInteger.valueOf(partitionId) + .multiply(BigInteger.valueOf(timePartitionInterval)) + .add(BigInteger.valueOf(timePartitionOrigin)); + if (partitionStartTime.compareTo(BigInteger.valueOf(Long.MIN_VALUE)) < 0) { + return Long.MIN_VALUE; + } + if (partitionStartTime.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0) { + return Long.MAX_VALUE; } - return satisfyPartitionStartTime(timeFilter, partitionStartTime); + return partitionStartTime.longValue(); } public static void setTimePartitionInterval(long timePartitionInterval) { TimePartitionUtils.timePartitionInterval = timePartitionInterval; + updateTimePartitionBound(); + } + + public static void setTimePartitionOrigin(long timePartitionOrigin) { + TimePartitionUtils.timePartitionOrigin = timePartitionOrigin; + updateTimePartitionBound(); } public static long getEstimateTimePartitionSize(long startTime, long endTime) { - if (endTime > 0 && startTime < 0) { - return BigInteger.valueOf(endTime) - .subtract(BigInteger.valueOf(startTime)) - .divide(bigTimePartitionInterval) - .longValue() - + 1; + BigInteger estimateSize = + BigInteger.valueOf(endTime) + .subtract(BigInteger.valueOf(startTime)) + .divide(BigInteger.valueOf(timePartitionInterval)) + .add(BigInteger.ONE); + if (estimateSize.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0) { + return Long.MAX_VALUE; } - return (endTime - startTime) / timePartitionInterval + 1; + return estimateSize.longValue(); } } diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/partition/SeriesPartitionTableTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/partition/SeriesPartitionTableTest.java index ab63deb3c68f6..85cb63bd38c49 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/partition/SeriesPartitionTableTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/partition/SeriesPartitionTableTest.java @@ -108,4 +108,47 @@ public void snapshotSerDeTest() throws TException, IOException { table1.deserialize(inputStream, protocol); Assert.assertEquals(table0, table1); } + + @Test + public void autoCleanPartitionTableShouldNotExpireOnOverflow() { + TConsensusGroupId consensusGroupId = new TConsensusGroupId(TConsensusGroupType.DataRegion, 0); + + SeriesPartitionTable table = new SeriesPartitionTable(); + TTimePartitionSlot nearMaxSlot = new TTimePartitionSlot(Long.MAX_VALUE - 1); + table.putDataPartition(nearMaxSlot, consensusGroupId); + Assert.assertTrue( + table.autoCleanPartitionTable(0, new TTimePartitionSlot(Long.MAX_VALUE - 1)).isEmpty()); + Assert.assertTrue(table.getSeriesPartitionMap().containsKey(nearMaxSlot)); + + table = new SeriesPartitionTable(); + TTimePartitionSlot normalSlot = new TTimePartitionSlot(0); + table.putDataPartition(normalSlot, consensusGroupId); + Assert.assertTrue( + table + .autoCleanPartitionTable(Long.MAX_VALUE - 1, new TTimePartitionSlot(Long.MAX_VALUE - 1)) + .isEmpty()); + Assert.assertTrue(table.getSeriesPartitionMap().containsKey(normalSlot)); + } + + @Test + public void getTimeSlotListShouldIncludeLongMaxSlotWithUnboundedEndTime() { + final TConsensusGroupId consensusGroupId = + new TConsensusGroupId(TConsensusGroupType.DataRegion, 0); + final TConsensusGroupId allRegionId = new TConsensusGroupId(TConsensusGroupType.DataRegion, -1); + final TTimePartitionSlot previousSlot = new TTimePartitionSlot(Long.MAX_VALUE - 1); + final TTimePartitionSlot lastSlot = new TTimePartitionSlot(Long.MAX_VALUE); + + final SeriesPartitionTable table = new SeriesPartitionTable(); + table.putDataPartition(previousSlot, consensusGroupId); + table.putDataPartition(lastSlot, consensusGroupId); + + final List expected = new ArrayList<>(); + expected.add(previousSlot); + expected.add(lastSlot); + + Assert.assertEquals( + expected, table.getTimeSlotList(allRegionId, Long.MAX_VALUE - 1, Long.MAX_VALUE)); + Assert.assertEquals( + expected, table.getTimeSlotList(consensusGroupId, Long.MAX_VALUE - 1, Long.MAX_VALUE)); + } } diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/CapacityTableFunctionTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/CapacityTableFunctionTest.java index 4f40e73758621..da0eb4a333c33 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/CapacityTableFunctionTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/CapacityTableFunctionTest.java @@ -34,6 +34,7 @@ import org.mockito.ArgumentCaptor; import org.mockito.Mockito; +import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -94,16 +95,7 @@ public void testAnalyzeSizeNegative() throws UDFException { * through process(). Returns captured (windowIndex, passThroughIndex) pairs. */ private List runProcessor(long size, long slide, int rowCount) throws UDFException { - Map args = new HashMap<>(); - args.put("SIZE", new ScalarArgument(Type.INT64, size)); - args.put("SLIDE", new ScalarArgument(Type.INT64, slide == -1 ? -1L : slide)); - - TableFunctionAnalysis analysis = function.analyze(args); - TableFunctionHandle handle = analysis.getTableFunctionHandle(); - - TableFunctionProcessorProvider provider = function.getProcessorProvider(handle); - TableFunctionDataProcessor processor = provider.getDataProcessor(); - + TableFunctionDataProcessor processor = newProcessor(size, slide); Record mockRecord = Mockito.mock(Record.class); List results = new ArrayList<>(); @@ -128,6 +120,18 @@ private List runProcessor(long size, long slide, int rowCount) throws UD return results; } + private TableFunctionDataProcessor newProcessor(long size, long slide) throws UDFException { + Map args = new HashMap<>(); + args.put("SIZE", new ScalarArgument(Type.INT64, size)); + args.put("SLIDE", new ScalarArgument(Type.INT64, slide == -1 ? -1L : slide)); + + TableFunctionAnalysis analysis = function.analyze(args); + TableFunctionHandle handle = analysis.getTableFunctionHandle(); + + TableFunctionProcessorProvider provider = function.getProcessorProvider(handle); + return provider.getDataProcessor(); + } + @Test public void testSlideEqualsSize() throws UDFException { // SIZE=2, SLIDE=2 (non-overlapping), 5 rows @@ -184,6 +188,40 @@ public void testSingleRow() throws UDFException { assertResultsEqual(expected, results); } + @Test + public void testWindowEndAtLongMaxDoesNotOverflow() throws Exception { + TableFunctionDataProcessor processor = newProcessor(1, Long.MAX_VALUE); + Field curIndexField = processor.getClass().getDeclaredField("curIndex"); + curIndexField.setAccessible(true); + curIndexField.setLong(processor, Long.MAX_VALUE); + + ColumnBuilder properBuilder = Mockito.mock(ColumnBuilder.class); + ColumnBuilder passThroughBuilder = Mockito.mock(ColumnBuilder.class); + + processor.process( + Mockito.mock(Record.class), Collections.singletonList(properBuilder), passThroughBuilder); + + Mockito.verify(properBuilder).writeLong(1L); + Mockito.verify(passThroughBuilder).writeLong(Long.MAX_VALUE); + } + + @Test + public void testLastWindowIndexDoesNotLoopForever() throws Exception { + TableFunctionDataProcessor processor = newProcessor(1, 1); + Field curIndexField = processor.getClass().getDeclaredField("curIndex"); + curIndexField.setAccessible(true); + curIndexField.setLong(processor, Long.MAX_VALUE); + + ColumnBuilder properBuilder = Mockito.mock(ColumnBuilder.class); + ColumnBuilder passThroughBuilder = Mockito.mock(ColumnBuilder.class); + + processor.process( + Mockito.mock(Record.class), Collections.singletonList(properBuilder), passThroughBuilder); + + Mockito.verify(properBuilder).writeLong(Long.MAX_VALUE); + Mockito.verify(passThroughBuilder).writeLong(Long.MAX_VALUE); + } + @Test public void testGetArgumentsSpecifications() { assertEquals(3, function.getArgumentsSpecifications().size()); diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/ForecastTimeUtilsTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/ForecastTimeUtilsTest.java new file mode 100644 index 0000000000000..24a3b3676ee58 --- /dev/null +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/ForecastTimeUtilsTest.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.udf.builtin.relational.tvf; + +import org.apache.iotdb.commons.exception.SemanticException; +import org.apache.iotdb.commons.queryengine.plan.udf.ForecastTimeUtils; + +import org.junit.Assert; +import org.junit.Test; + +public class ForecastTimeUtilsTest { + + @Test + public void testForecastStartTimeRejectsOverflow() { + Assert.assertEquals( + Long.MAX_VALUE, + ForecastTimeUtils.calculateForecastStartTime(Long.MAX_VALUE - 1, Long.MIN_VALUE, 1)); + + Assert.assertThrows( + SemanticException.class, + () -> ForecastTimeUtils.calculateForecastStartTime(Long.MAX_VALUE, Long.MIN_VALUE, 1)); + } + + @Test + public void testForecastOutputTimeRejectsOverflow() { + Assert.assertEquals( + Long.MAX_VALUE, ForecastTimeUtils.calculateForecastOutputTime(Long.MAX_VALUE - 2, 1, 2)); + + Assert.assertThrows( + SemanticException.class, + () -> ForecastTimeUtils.calculateForecastOutputTime(Long.MAX_VALUE - 1, 1, 2)); + } + + @Test + public void testForecastIntervalUsesExactTimeRange() { + Assert.assertEquals( + Long.MAX_VALUE, + ForecastTimeUtils.calculateForecastInterval(Long.MIN_VALUE, Long.MAX_VALUE, 3, 0)); + + Assert.assertThrows( + SemanticException.class, + () -> ForecastTimeUtils.calculateForecastInterval(Long.MIN_VALUE, Long.MAX_VALUE, 2, 0)); + } +} diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/TimeWindowTableFunctionBoundaryTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/TimeWindowTableFunctionBoundaryTest.java new file mode 100644 index 0000000000000..c02f686217ef0 --- /dev/null +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/TimeWindowTableFunctionBoundaryTest.java @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.udf.builtin.relational.tvf; + +import org.apache.iotdb.udf.api.exception.UDFException; +import org.apache.iotdb.udf.api.relational.TableFunction; +import org.apache.iotdb.udf.api.relational.access.Record; +import org.apache.iotdb.udf.api.relational.table.argument.Argument; +import org.apache.iotdb.udf.api.relational.table.argument.ScalarArgument; +import org.apache.iotdb.udf.api.relational.table.argument.TableArgument; +import org.apache.iotdb.udf.api.relational.table.processor.TableFunctionDataProcessor; +import org.apache.iotdb.udf.api.type.Type; + +import org.apache.tsfile.block.column.ColumnBuilder; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.junit.Assert.assertEquals; + +public class TimeWindowTableFunctionBoundaryTest { + + @Test + public void testTumbleWindowEndSaturatesAtLongMax() throws UDFException { + Map arguments = baseTimeWindowArguments(); + arguments.put("SIZE", new ScalarArgument(Type.INT64, 2L)); + arguments.put("ORIGIN", new ScalarArgument(Type.TIMESTAMP, Long.MAX_VALUE - 1)); + + WindowOutput output = + runProcessor(newProcessor(new TumbleTableFunction(), arguments), 1, Long.MAX_VALUE); + + assertEquals(Collections.singletonList(Long.MAX_VALUE - 1), output.windowStarts); + assertEquals(Collections.singletonList(Long.MAX_VALUE), output.windowEnds); + assertEquals(Collections.singletonList(0L), output.passThroughIndexes); + } + + @Test + public void testHopWindowComparisonUsesExactEndTime() throws UDFException { + Map arguments = baseTimeWindowArguments(); + arguments.put("SIZE", new ScalarArgument(Type.INT64, 2L)); + arguments.put("SLIDE", new ScalarArgument(Type.INT64, 1L)); + arguments.put("ORIGIN", new ScalarArgument(Type.TIMESTAMP, Long.MAX_VALUE - 1)); + + WindowOutput output = + runProcessor(newProcessor(new HOPTableFunction(), arguments), 2, Long.MAX_VALUE); + + assertEquals(Arrays.asList(Long.MAX_VALUE - 1, Long.MAX_VALUE), output.windowStarts); + assertEquals(Arrays.asList(Long.MAX_VALUE, Long.MAX_VALUE), output.windowEnds); + assertEquals(Arrays.asList(0L, 0L), output.passThroughIndexes); + } + + @Test + public void testCumulateWindowEndSaturatesAtLongMax() throws UDFException { + Map arguments = baseTimeWindowArguments(); + arguments.put("SIZE", new ScalarArgument(Type.INT64, 4L)); + arguments.put("STEP", new ScalarArgument(Type.INT64, 2L)); + arguments.put("ORIGIN", new ScalarArgument(Type.TIMESTAMP, Long.MAX_VALUE - 3)); + + WindowOutput output = + runProcessor(newProcessor(new CumulateTableFunction(), arguments), 1, Long.MAX_VALUE); + + assertEquals(Collections.singletonList(Long.MAX_VALUE - 3), output.windowStarts); + assertEquals(Collections.singletonList(Long.MAX_VALUE), output.windowEnds); + assertEquals(Collections.singletonList(0L), output.passThroughIndexes); + } + + @Test + public void testSessionWindowDoesNotSplitWhenGapEndOverflows() throws UDFException { + Map arguments = baseTimeWindowArguments(); + arguments.put("GAP", new ScalarArgument(Type.INT64, 2L)); + + WindowOutput output = + runProcessor( + newProcessor(new SessionTableFunction(), arguments), + 2, + Long.MAX_VALUE - 1, + Long.MAX_VALUE); + + assertEquals(Arrays.asList(Long.MAX_VALUE - 1, Long.MAX_VALUE - 1), output.windowStarts); + assertEquals(Arrays.asList(Long.MAX_VALUE, Long.MAX_VALUE), output.windowEnds); + assertEquals(Arrays.asList(0L, 1L), output.passThroughIndexes); + } + + private static TableFunctionDataProcessor newProcessor( + TableFunction tableFunction, Map arguments) throws UDFException { + return tableFunction + .getProcessorProvider(tableFunction.analyze(arguments).getTableFunctionHandle()) + .getDataProcessor(); + } + + private static Map baseTimeWindowArguments() { + Map arguments = new HashMap<>(); + arguments.put( + "DATA", + new TableArgument( + Collections.singletonList(Optional.of("time")), + Collections.singletonList(Type.TIMESTAMP), + Collections.emptyList(), + Collections.emptyList(), + true)); + arguments.put("TIMECOL", new ScalarArgument(Type.STRING, "time")); + return arguments; + } + + private static WindowOutput runProcessor( + TableFunctionDataProcessor processor, int expectedOutputCount, long... times) { + ColumnBuilder startBuilder = Mockito.mock(ColumnBuilder.class); + ColumnBuilder endBuilder = Mockito.mock(ColumnBuilder.class); + ColumnBuilder passThroughIndexBuilder = Mockito.mock(ColumnBuilder.class); + List properColumnBuilders = Arrays.asList(startBuilder, endBuilder); + + for (long time : times) { + processor.process(recordWithTime(time), properColumnBuilders, passThroughIndexBuilder); + } + processor.finish(properColumnBuilders, passThroughIndexBuilder); + + return new WindowOutput( + captureWrites(startBuilder, expectedOutputCount), + captureWrites(endBuilder, expectedOutputCount), + captureWrites(passThroughIndexBuilder, expectedOutputCount)); + } + + private static Record recordWithTime(long time) { + Record record = Mockito.mock(Record.class); + Mockito.when(record.getLong(0)).thenReturn(time); + return record; + } + + private static List captureWrites(ColumnBuilder builder, int expectedOutputCount) { + ArgumentCaptor captor = ArgumentCaptor.forClass(Long.class); + Mockito.verify(builder, Mockito.times(expectedOutputCount)).writeLong(captor.capture()); + return captor.getAllValues(); + } + + private static class WindowOutput { + private final List windowStarts; + private final List windowEnds; + private final List passThroughIndexes; + + private WindowOutput( + List windowStarts, List windowEnds, List passThroughIndexes) { + this.windowStarts = windowStarts; + this.windowEnds = windowEnds; + this.passThroughIndexes = passThroughIndexes; + } + } +} diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/TimePartitionUtilsTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/TimePartitionUtilsTest.java index ea0eeda45d28f..ba31a7fbcb800 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/TimePartitionUtilsTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/TimePartitionUtilsTest.java @@ -22,6 +22,8 @@ import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.tsfile.read.filter.factory.TimeFilterApi; +import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -33,12 +35,31 @@ public class TimePartitionUtilsTest { private static final long TEST_TIME_PARTITION_ORIGIN = 1000L; private static final long TEST_TIME_PARTITION_INTERVAL = 3600000L; + private long previousTimePartitionOrigin; + private long previousTimePartitionInterval; + @Before public void setUp() { + previousTimePartitionOrigin = + CommonDescriptor.getInstance().getConfig().getTimePartitionOrigin(); + previousTimePartitionInterval = + CommonDescriptor.getInstance().getConfig().getTimePartitionInterval(); CommonDescriptor.getInstance().getConfig().setTimePartitionOrigin(TEST_TIME_PARTITION_ORIGIN); CommonDescriptor.getInstance() .getConfig() .setTimePartitionInterval(TEST_TIME_PARTITION_INTERVAL); + TimePartitionUtils.setTimePartitionOrigin(TEST_TIME_PARTITION_ORIGIN); + TimePartitionUtils.setTimePartitionInterval(TEST_TIME_PARTITION_INTERVAL); + } + + @After + public void tearDown() { + CommonDescriptor.getInstance().getConfig().setTimePartitionOrigin(previousTimePartitionOrigin); + CommonDescriptor.getInstance() + .getConfig() + .setTimePartitionInterval(previousTimePartitionInterval); + TimePartitionUtils.setTimePartitionOrigin(previousTimePartitionOrigin); + TimePartitionUtils.setTimePartitionInterval(previousTimePartitionInterval); } @Test @@ -105,4 +126,126 @@ public void testOverflow() { long upperBound = TimePartitionUtils.getTimePartitionUpperBound(testTime); assertEquals(Long.MAX_VALUE, upperBound); } + + @Test + public void testIsTimePartitionStartTimeWithOrigin() { + Assert.assertTrue(TimePartitionUtils.isTimePartitionStartTime(TEST_TIME_PARTITION_ORIGIN)); + Assert.assertFalse(TimePartitionUtils.isTimePartitionStartTime(TEST_TIME_PARTITION_ORIGIN + 1)); + Assert.assertTrue( + TimePartitionUtils.isTimePartitionStartTime( + TEST_TIME_PARTITION_ORIGIN + TEST_TIME_PARTITION_INTERVAL)); + } + + @Test + public void testSatisfyPartitionStartTimeWithNormalPartitionEnd() { + Assert.assertFalse( + TimePartitionUtils.satisfyPartitionStartTime( + TimeFilterApi.gtEq(TEST_TIME_PARTITION_ORIGIN + TEST_TIME_PARTITION_INTERVAL), + TEST_TIME_PARTITION_ORIGIN)); + Assert.assertFalse( + TimePartitionUtils.satisfyTimePartition( + TimeFilterApi.gtEq(TEST_TIME_PARTITION_ORIGIN + TEST_TIME_PARTITION_INTERVAL), 0)); + Assert.assertTrue( + TimePartitionUtils.satisfyPartitionStartTime( + TimeFilterApi.gtEq(TEST_TIME_PARTITION_ORIGIN + TEST_TIME_PARTITION_INTERVAL - 1), + TEST_TIME_PARTITION_ORIGIN)); + } + + @Test + public void testSatisfyPartitionStartTimeWithOverflowPartitionEnd() { + long partitionStartTime = TimePartitionUtils.getTimePartitionSlot(Long.MAX_VALUE).startTime; + + Assert.assertTrue( + TimePartitionUtils.satisfyPartitionStartTime( + TimeFilterApi.eq(Long.MAX_VALUE), partitionStartTime)); + } + + @Test + public void testPartitionEndTimeAndUpperBoundCheckWithOverflowPartitionEnd() { + long partitionStartTime = TimePartitionUtils.getTimePartitionSlot(Long.MAX_VALUE).startTime; + long upperBound = TimePartitionUtils.getTimePartitionUpperBound(partitionStartTime); + + assertEquals(Long.MAX_VALUE, TimePartitionUtils.getTimePartitionEndTime(partitionStartTime)); + assertEquals(Long.MAX_VALUE, upperBound); + Assert.assertFalse( + TimePartitionUtils.isAfterOrEqualToTimePartitionUpperBound( + Long.MAX_VALUE, partitionStartTime, upperBound)); + Assert.assertTrue( + TimePartitionUtils.isAfterOrEqualToTimePartitionUpperBound( + TEST_TIME_PARTITION_ORIGIN + TEST_TIME_PARTITION_INTERVAL, + TEST_TIME_PARTITION_ORIGIN, + TimePartitionUtils.getTimePartitionUpperBound(TEST_TIME_PARTITION_ORIGIN))); + } + + @Test + public void testExactLongMaxUpperBoundCheck() { + CommonDescriptor.getInstance().getConfig().setTimePartitionOrigin(0); + CommonDescriptor.getInstance().getConfig().setTimePartitionInterval(1); + TimePartitionUtils.setTimePartitionOrigin(0); + TimePartitionUtils.setTimePartitionInterval(1); + + assertEquals(Long.MAX_VALUE, TimePartitionUtils.getTimePartitionUpperBound(Long.MAX_VALUE - 1)); + assertEquals( + Long.MAX_VALUE - 1, TimePartitionUtils.getTimePartitionEndTime(Long.MAX_VALUE - 1)); + assertEquals(Long.MAX_VALUE, TimePartitionUtils.getTimePartitionEndTime(Long.MAX_VALUE)); + Assert.assertTrue( + TimePartitionUtils.isAfterOrEqualToTimePartitionUpperBound( + Long.MAX_VALUE, Long.MAX_VALUE - 1, Long.MAX_VALUE)); + Assert.assertFalse( + TimePartitionUtils.isAfterOrEqualToTimePartitionUpperBound( + Long.MAX_VALUE, Long.MAX_VALUE, Long.MAX_VALUE)); + } + + @Test + public void testSatisfyTimePartitionWithOverflowPartitionStart() { + long partitionId = TimePartitionUtils.getTimePartitionIdWithoutOverflow(Long.MIN_VALUE); + long nextPartitionStartTime = TimePartitionUtils.getTimePartitionUpperBound(Long.MIN_VALUE); + + Assert.assertTrue( + TimePartitionUtils.satisfyTimePartition(TimeFilterApi.eq(Long.MIN_VALUE), partitionId)); + Assert.assertFalse( + TimePartitionUtils.satisfyTimePartition( + TimeFilterApi.eq(nextPartitionStartTime), partitionId)); + } + + @Test + public void testExactLongMinPartitionStartUpperBound() { + CommonDescriptor.getInstance().getConfig().setTimePartitionOrigin(0); + CommonDescriptor.getInstance().getConfig().setTimePartitionInterval(1); + TimePartitionUtils.setTimePartitionOrigin(0); + TimePartitionUtils.setTimePartitionInterval(1); + + assertEquals(Long.MIN_VALUE, TimePartitionUtils.getTimePartitionLowerBound(Long.MIN_VALUE)); + assertEquals(Long.MIN_VALUE + 1, TimePartitionUtils.getTimePartitionUpperBound(Long.MIN_VALUE)); + assertEquals(Long.MIN_VALUE, TimePartitionUtils.getTimePartitionEndTime(Long.MIN_VALUE)); + Assert.assertTrue( + TimePartitionUtils.satisfyPartitionStartTime( + TimeFilterApi.eq(Long.MIN_VALUE), Long.MIN_VALUE)); + Assert.assertFalse( + TimePartitionUtils.satisfyPartitionStartTime( + TimeFilterApi.eq(Long.MIN_VALUE + 1), Long.MIN_VALUE)); + } + + @Test + public void testGetTimePartitionIdWithOverflowOrigin() { + assertEquals( + TimePartitionUtils.getTimePartitionIdWithoutOverflow(Long.MIN_VALUE), + TimePartitionUtils.getTimePartitionId(Long.MIN_VALUE)); + assertEquals( + TimePartitionUtils.getTimePartitionIdWithoutOverflow(Long.MAX_VALUE), + TimePartitionUtils.getTimePartitionId(Long.MAX_VALUE)); + } + + @Test + public void testGetEstimateTimePartitionSizeWithOverflow() { + long previousTimePartitionInterval = TimePartitionUtils.getTimePartitionInterval(); + try { + TimePartitionUtils.setTimePartitionInterval(1); + assertEquals( + Long.MAX_VALUE, + TimePartitionUtils.getEstimateTimePartitionSize(Long.MIN_VALUE, Long.MAX_VALUE)); + } finally { + TimePartitionUtils.setTimePartitionInterval(previousTimePartitionInterval); + } + } }