From 91dd4ebb994c2513132c9bb40a0a500654bbd89d Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 9 Jun 2026 15:54:32 +0800 Subject: [PATCH 1/6] Fix partial insert handling for null measurements --- .../parser/TabletInsertionEventParser.java | 117 +++++++++--- .../PipeConvertedInsertRowStatement.java | 3 + .../memory/InsertNodeMemoryEstimator.java | 15 +- .../sink/util/TabletStatementConverter.java | 2 +- .../planner/plan/node/write/InsertNode.java | 29 ++- .../plan/node/write/InsertRowNode.java | 30 ++- .../node/write/InsertRowsOfOneDeviceNode.java | 3 + .../plan/node/write/InsertTabletNode.java | 152 ++++++++++++---- .../write/RelationalInsertTabletNode.java | 10 +- .../dataregion/memtable/AbstractMemTable.java | 77 +++++--- .../dataregion/memtable/TsFileProcessor.java | 133 ++++++++------ .../memtable/WritableMemChunkGroup.java | 2 +- .../org/apache/iotdb/db/utils/MemUtils.java | 42 +++-- .../event/PipeTabletInsertionEventTest.java | 81 +++++++++ .../PipeConvertedInsertRowStatementTest.java | 55 ++++++ .../memory/InsertNodeMemoryEstimatorTest.java | 12 ++ .../util/TabletStatementConverterTest.java | 36 ++++ .../node/write/InsertRowNodeSerdeTest.java | 19 ++ .../InsertRowsOfOneDeviceNodeSerdeTest.java | 34 ++++ .../node/write/InsertTabletNodeSerdeTest.java | 63 +++++++ .../node/write/WritePlanNodeSplitTest.java | 41 +++++ .../InsertNodeIsMeasurementFailedTest.java | 135 ++++++++++++++ .../AbstractMemTablePartialInsertTest.java | 113 ++++++++++++ .../memtable/MemChunkDeserializeTest.java | 23 +++ .../memtable/TsFileProcessorTest.java | 172 ++++++++++++++++++ .../apache/iotdb/db/utils/MemUtilsTest.java | 78 ++++++++ 26 files changed, 1298 insertions(+), 179 deletions(-) create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertRowStatementTest.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventParser.java index 6903504129385..a01dfd5222b0b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventParser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventParser.java @@ -54,6 +54,7 @@ import java.util.Map; import java.util.Objects; import java.util.function.BiConsumer; +import java.util.function.IntPredicate; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -127,14 +128,26 @@ protected void parse(final InsertRowNode insertRowNode) { final List rowIndexList = generateRowIndexList(originTimestampColumn); this.timestampColumn = rowIndexList.stream().mapToLong(i -> originTimestampColumn[i]).toArray(); + final MeasurementSchema[] originMeasurementSchemaList = insertRowNode.getMeasurementSchemas(); + final String[] originColumnNameStringList = insertRowNode.getMeasurements(); + final TsTableColumnCategory[] originColumnCategories = insertRowNode.getColumnCategories(); + final TSDataType[] originValueDataTypes = insertRowNode.getDataTypes(); + final Object[] originValues = insertRowNode.getValues(); + generateColumnIndexMapper( - insertRowNode.getMeasurements(), originColumnIndex2FilteredColumnIndexMapperList); + originColumnNameStringList, originColumnIndex2FilteredColumnIndexMapperList); final int filteredColumnSize = - Arrays.stream(originColumnIndex2FilteredColumnIndexMapperList) - .filter(Objects::nonNull) - .toArray() - .length; + compactColumnIndexMapper( + originColumnIndex2FilteredColumnIndexMapperList, + i -> + isValidOriginColumn( + originColumnNameStringList, + originMeasurementSchemaList, + originValueDataTypes, + i) + && originValues != null + && i < originValues.length); this.measurementSchemaList = new MeasurementSchema[filteredColumnSize]; this.columnNameStringList = new String[filteredColumnSize]; @@ -143,19 +156,15 @@ protected void parse(final InsertRowNode insertRowNode) { this.valueColumns = new Object[filteredColumnSize]; this.nullValueColumnBitmaps = new BitMap[filteredColumnSize]; - final MeasurementSchema[] originMeasurementSchemaList = insertRowNode.getMeasurementSchemas(); - final String[] originColumnNameStringList = insertRowNode.getMeasurements(); - final TsTableColumnCategory[] originColumnCategories = insertRowNode.getColumnCategories(); - final TSDataType[] originValueDataTypes = insertRowNode.getDataTypes(); - final Object[] originValues = insertRowNode.getValues(); - for (int i = 0; i < originColumnIndex2FilteredColumnIndexMapperList.length; i++) { if (originColumnIndex2FilteredColumnIndexMapperList[i] != null) { final int filteredColumnIndex = originColumnIndex2FilteredColumnIndexMapperList[i]; this.measurementSchemaList[filteredColumnIndex] = originMeasurementSchemaList[i]; this.columnNameStringList[filteredColumnIndex] = originColumnNameStringList[i]; this.valueColumnTypes[filteredColumnIndex] = - originColumnCategories != null && originColumnCategories[i] != null + originColumnCategories != null + && i < originColumnCategories.length + && originColumnCategories[i] != null ? originColumnCategories[i].toTsFileColumnType() : ColumnCategory.FIELD; this.valueColumnDataTypes[filteredColumnIndex] = originValueDataTypes[i]; @@ -202,14 +211,29 @@ protected void parse(final InsertTabletNode insertTabletNode) throws IllegalPath final List rowIndexList = generateRowIndexList(originTimestampColumn); this.timestampColumn = rowIndexList.stream().mapToLong(i -> originTimestampColumn[i]).toArray(); + final MeasurementSchema[] originMeasurementSchemaList = + insertTabletNode.getMeasurementSchemas(); + final String[] originColumnNameStringList = insertTabletNode.getMeasurements(); + final TsTableColumnCategory[] originColumnCategories = insertTabletNode.getColumnCategories(); + final TSDataType[] originValueColumnDataTypes = insertTabletNode.getDataTypes(); + final Object[] originValueColumns = insertTabletNode.getColumns(); + final BitMap[] originBitMapList = insertTabletNode.getBitMaps(); + generateColumnIndexMapper( - insertTabletNode.getMeasurements(), originColumnIndex2FilteredColumnIndexMapperList); + originColumnNameStringList, originColumnIndex2FilteredColumnIndexMapperList); final int filteredColumnSize = - Arrays.stream(originColumnIndex2FilteredColumnIndexMapperList) - .filter(Objects::nonNull) - .toArray() - .length; + compactColumnIndexMapper( + originColumnIndex2FilteredColumnIndexMapperList, + i -> + isValidOriginColumn( + originColumnNameStringList, + originMeasurementSchemaList, + originValueColumnDataTypes, + i) + && originValueColumns != null + && i < originValueColumns.length + && originValueColumns[i] != null); this.measurementSchemaList = new MeasurementSchema[filteredColumnSize]; this.columnNameStringList = new String[filteredColumnSize]; @@ -218,21 +242,15 @@ protected void parse(final InsertTabletNode insertTabletNode) throws IllegalPath this.valueColumns = new Object[filteredColumnSize]; this.nullValueColumnBitmaps = new BitMap[filteredColumnSize]; - final MeasurementSchema[] originMeasurementSchemaList = - insertTabletNode.getMeasurementSchemas(); - final String[] originColumnNameStringList = insertTabletNode.getMeasurements(); - final TsTableColumnCategory[] originColumnCategories = insertTabletNode.getColumnCategories(); - final TSDataType[] originValueColumnDataTypes = insertTabletNode.getDataTypes(); - final Object[] originValueColumns = insertTabletNode.getColumns(); - final BitMap[] originBitMapList = insertTabletNode.getBitMaps(); - for (int i = 0; i < originColumnIndex2FilteredColumnIndexMapperList.length; i++) { if (originColumnIndex2FilteredColumnIndexMapperList[i] != null) { final int filteredColumnIndex = originColumnIndex2FilteredColumnIndexMapperList[i]; this.measurementSchemaList[filteredColumnIndex] = originMeasurementSchemaList[i]; this.columnNameStringList[filteredColumnIndex] = originColumnNameStringList[i]; this.valueColumnTypes[filteredColumnIndex] = - originColumnCategories != null && originColumnCategories[i] != null + originColumnCategories != null + && i < originColumnCategories.length + && originColumnCategories[i] != null ? originColumnCategories[i].toTsFileColumnType() : ColumnCategory.FIELD; this.valueColumnDataTypes[filteredColumnIndex] = originValueColumnDataTypes[i]; @@ -298,10 +316,9 @@ protected void parse(final Tablet tablet, final boolean isAligned) { originMeasurementList, originColumnIndex2FilteredColumnIndexMapperList); final int filteredColumnSize = - Arrays.stream(originColumnIndex2FilteredColumnIndexMapperList) - .filter(Objects::nonNull) - .toArray() - .length; + compactColumnIndexMapper( + originColumnIndex2FilteredColumnIndexMapperList, + i -> isValidOriginColumn(originMeasurementSchemaList, i)); this.measurementSchemaList = new MeasurementSchema[filteredColumnSize]; this.columnNameStringList = new String[filteredColumnSize]; @@ -373,6 +390,46 @@ protected abstract void generateColumnIndexMapper( final String[] originMeasurementList, final Integer[] originColumnIndex2FilteredColumnIndexMapperList); + private static int compactColumnIndexMapper( + final Integer[] originColumnIndex2FilteredColumnIndexMapperList, + final IntPredicate columnValidator) { + int filteredCount = 0; + for (int i = 0; i < originColumnIndex2FilteredColumnIndexMapperList.length; i++) { + if (originColumnIndex2FilteredColumnIndexMapperList[i] != null && columnValidator.test(i)) { + originColumnIndex2FilteredColumnIndexMapperList[i] = filteredCount++; + } else { + originColumnIndex2FilteredColumnIndexMapperList[i] = null; + } + } + return filteredCount; + } + + private static boolean isValidOriginColumn( + final String[] originColumnNameStringList, + final MeasurementSchema[] originMeasurementSchemaList, + final TSDataType[] originValueDataTypes, + final int index) { + return originColumnNameStringList != null + && index < originColumnNameStringList.length + && originColumnNameStringList[index] != null + && originMeasurementSchemaList != null + && index < originMeasurementSchemaList.length + && originMeasurementSchemaList[index] != null + && originMeasurementSchemaList[index].getType() != null + && originValueDataTypes != null + && index < originValueDataTypes.length + && originValueDataTypes[index] != null; + } + + private static boolean isValidOriginColumn( + final List originMeasurementSchemaList, final int index) { + return originMeasurementSchemaList != null + && index < originMeasurementSchemaList.size() + && originMeasurementSchemaList.get(index) != null + && originMeasurementSchemaList.get(index).getMeasurementName() != null + && originMeasurementSchemaList.get(index).getType() != null; + } + private List generateRowIndexList(final long[] originTimestampColumn) { final int rowCount = originTimestampColumn.length; if (Objects.isNull(sourceEvent) || !sourceEvent.shouldParseTime()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertRowStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertRowStatement.java index c3db7954b40be..742f5989553d0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertRowStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertRowStatement.java @@ -126,6 +126,9 @@ public void transferType(ZoneId zoneId) throws QueryProcessException { // parse string value to specific type dataTypes[i] = measurementSchemas[i].getType(); + if (values == null || i >= values.length || values[i] == null) { + continue; + } try { values[i] = ValueConverter.parse(values[i].toString(), dataTypes[i]); } catch (Exception e) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java index 677b404c217b8..54655704c1c53 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java @@ -548,6 +548,9 @@ private static long sizeOfBinary(final Binary binary) { public static long sizeOfColumns( final Object[] columns, final MeasurementSchema[] measurementSchemas) { + if (Objects.isNull(columns)) { + return 0L; + } // Directly calculate if measurementSchemas are absent if (Objects.isNull(measurementSchemas)) { return RamUsageEstimator.shallowSizeOf(columns) @@ -559,7 +562,10 @@ public static long sizeOfColumns( RamUsageEstimator.alignObjectSize( NUM_BYTES_ARRAY_HEADER + NUM_BYTES_OBJECT_REF * columns.length); for (int i = 0; i < columns.length; i++) { - if (measurementSchemas[i] == null || measurementSchemas[i].getType() == null) { + if (columns[i] == null + || i >= measurementSchemas.length + || measurementSchemas[i] == null + || measurementSchemas[i].getType() == null) { continue; } switch (measurementSchemas[i].getType()) { @@ -611,6 +617,9 @@ private static long getNumBytesUnknownObject(final Object obj) { public static long sizeOfValues( final Object[] values, final MeasurementSchema[] measurementSchemas) { + if (Objects.isNull(values)) { + return 0L; + } // Directly calculate if measurementSchemas are absent if (Objects.isNull(measurementSchemas)) { return RamUsageEstimator.shallowSizeOf(values) @@ -622,7 +631,9 @@ public static long sizeOfValues( RamUsageEstimator.alignObjectSize( NUM_BYTES_ARRAY_HEADER + NUM_BYTES_OBJECT_REF * values.length); for (int i = 0; i < values.length; i++) { - if (measurementSchemas[i] == null || measurementSchemas[i].getType() == null) { + if (i >= measurementSchemas.length + || measurementSchemas[i] == null + || measurementSchemas[i].getType() == null) { size += NUM_BYTES_OBJECT_HEADER; continue; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/TabletStatementConverter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/TabletStatementConverter.java index 773d40e99d1f7..d7be9f548f542 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/TabletStatementConverter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/TabletStatementConverter.java @@ -395,7 +395,7 @@ private static Pair readValuesFromBufferWithMemory( for (int i = 0; i < columns; i++) { final boolean isValueColumnsNotNull = BytesUtils.byteToBool(ReadWriteIOUtils.readByte(byteBuffer)); - if (isValueColumnsNotNull && types[i] == null) { + if (types[i] == null) { continue; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java index da5f2325acdbd..d9a202de22ba5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java @@ -174,6 +174,7 @@ public MeasurementSchema[] getMeasurementSchemas() { public void setMeasurementSchemas(MeasurementSchema[] measurementSchemas) { this.measurementSchemas = measurementSchemas; + measurementColumnCnt = -1; } public String[] getMeasurements() { @@ -189,13 +190,16 @@ public int measureColumnCnt() { } public boolean isValidMeasurement(int i) { - return measurementSchemas != null + return measurements != null + && measurements[i] != null + && measurementSchemas != null && measurementSchemas[i] != null && (columnCategories == null || columnCategories[i] == TsTableColumnCategory.FIELD); } public void setMeasurements(String[] measurements) { this.measurements = measurements; + measurementColumnCnt = -1; } public TSDataType[] getDataTypes() { @@ -327,8 +331,12 @@ public void markFailedMeasurement(int index) { } public boolean hasValidMeasurements() { - for (Object o : measurements) { - if (o != null) { + if (measurements == null) { + return false; + } + for (int i = 0; i < measurements.length; i++) { + if (measurements[i] != null + && (columnCategories == null || columnCategories[i] == TsTableColumnCategory.FIELD)) { return true; } } @@ -354,15 +362,16 @@ protected int getValidMeasurementNumber() { } public boolean isMeasurementFailed(int index) { - return measurements[index] == null; + return measurements == null || measurements[index] == null; + } + + protected boolean isWritableFieldMeasurement(int index) { + return !isMeasurementFailed(index) + && (columnCategories == null || columnCategories[index] == TsTableColumnCategory.FIELD); } public boolean allMeasurementFailed() { - if (measurements != null) { - return failedMeasurementNumber - >= measurements.length - (tagColumnIndices == null ? 0 : tagColumnIndices.size()); - } - return true; + return measurements == null || !hasValidMeasurements(); } // endregion @@ -418,6 +427,8 @@ public TsTableColumnCategory[] getColumnCategories() { public void setColumnCategories(TsTableColumnCategory[] columnCategories) { this.columnCategories = columnCategories; + measurementColumnCnt = -1; + tagColumnIndices = null; if (columnCategories != null) { tagColumnIndices = new ArrayList<>(); for (int i = 0; i < columnCategories.length; i++) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java index fcb96cf3f0795..9d4d826687d53 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java @@ -240,6 +240,7 @@ public void markFailedMeasurement(int index) { measurements[index] = null; dataTypes[index] = null; values[index] = null; + measurementColumnCnt = -1; } @Override @@ -268,7 +269,7 @@ void subSerialize(DataOutputStream stream) throws IOException { /** Serialize measurements and values, ignoring failed time series. */ void serializeMeasurementsAndValues(ByteBuffer buffer) { - ReadWriteIOUtils.write(measurements.length - getFailedMeasurementNumber(), buffer); + ReadWriteIOUtils.write(getValidMeasurementNumber(), buffer); serializeMeasurementsOrSchemas(buffer); putDataTypesAndValues(buffer); ReadWriteIOUtils.write((byte) (isNeedInferType ? 1 : 0), buffer); @@ -282,7 +283,7 @@ void serializeMeasurementsAndValues(ByteBuffer buffer) { * @throws IOException - If an I/O error occurs. */ void serializeMeasurementsAndValues(DataOutputStream stream) throws IOException { - ReadWriteIOUtils.write(measurements.length - getFailedMeasurementNumber(), stream); + ReadWriteIOUtils.write(getValidMeasurementNumber(), stream); serializeMeasurementsOrSchemas(stream); putDataTypesAndValues(stream); ReadWriteIOUtils.write((byte) (isNeedInferType ? 1 : 0), stream); @@ -637,7 +638,7 @@ protected void subSerialize(IWALByteBufferView buffer) { /** Serialize measurements and values, ignoring failed time series. */ private void serializeMeasurementsAndValues(IWALByteBufferView buffer) { - buffer.putInt(measurements.length - getFailedMeasurementNumber()); + buffer.putInt(getValidMeasurementNumber()); serializeMeasurementSchemasToWAL(buffer); putDataTypesAndValues(buffer); buffer.put((byte) (isAligned ? 1 : 0)); @@ -910,15 +911,26 @@ public R accept(IPlanVisitor visitor, C context) { } public TimeValuePair composeTimeValuePair(int columnIndex) { - if (columnIndex >= values.length - || Objects.isNull(dataTypes[columnIndex]) - || dataTypes[columnIndex] == TSDataType.OBJECT) { + if (!canComposeTimeValuePair(columnIndex)) { return null; } Object value = values[columnIndex]; - return Objects.nonNull(value) - ? new TimeValuePair(time, TsPrimitiveType.getByType(dataTypes[columnIndex], value)) - : null; + return new TimeValuePair(time, TsPrimitiveType.getByType(dataTypes[columnIndex], value)); + } + + private boolean canComposeTimeValuePair(final int columnIndex) { + return measurements != null + && columnIndex >= 0 + && columnIndex < measurements.length + && values != null + && columnIndex < values.length + && values[columnIndex] != null + && dataTypes != null + && columnIndex < dataTypes.length + && dataTypes[columnIndex] != null + && dataTypes[columnIndex] != TSDataType.OBJECT + && (columnCategories == null || columnIndex < columnCategories.length) + && isWritableFieldMeasurement(columnIndex); } public void updateLastCache(String databaseName) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsOfOneDeviceNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsOfOneDeviceNode.java index 457ab81dbca86..2acaeccffa698 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsOfOneDeviceNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsOfOneDeviceNode.java @@ -241,6 +241,9 @@ private void storeMeasurementsAndDataType() { String[] measurements = insertRowNode.getMeasurements(); TSDataType[] dataTypes = insertRowNode.getDataTypes(); for (int i = 0; i < measurements.length; i++) { + if (measurements[i] == null) { + continue; + } if (!measurementSet.contains(measurements[i])) { measurementList.add(measurements[i]); dataTypeList.add(dataTypes[i]); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java index 98d0eca98b0c3..9f1cb48708c35 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java @@ -332,7 +332,7 @@ protected List doSplit(Map> spli protected InsertTabletNode getEmptySplit(int count) { long[] subTimes = new long[count]; - Object[] values = initTabletValues(dataTypes.length, count, dataTypes); + Object[] values = initTabletValuesForSplit(dataTypes.length, count, dataTypes); BitMap[] newBitMaps = initBitmapsForSplit(dataTypes.length, count); return new InsertTabletNode( getPlanNodeId(), @@ -368,7 +368,7 @@ protected WritePlanNode generateOneSplit(Map.Entry rangeList) { size += Byte.BYTES; if (bitMaps != null) { for (int i = 0; i < bitMaps.length; i++) { - // ignore failed partial insert - if (measurements[i] == null) { + // ignore failed partial insert and null columns + if (!shouldSerializeMeasurement(i)) { continue; } @@ -864,10 +888,11 @@ private int subSerializeSizeByRange(List rangeList) { } // values size for (int i = 0; i < dataTypes.length; i++) { - if (columns[i] != null) { - for (int[] range : rangeList) { - size += getColumnSize(dataTypes[i], columns[i], range[0], range[1]); - } + if (!shouldSerializeMeasurement(i)) { + continue; + } + for (int[] range : rangeList) { + size += getColumnSize(dataTypes[i], columns[i], range[0], range[1]); } } // isAlign @@ -958,7 +983,7 @@ void subSerialize(IWALByteBufferView buffer, List rangeList, long encoded /** Serialize measurement schemas, ignoring failed time series */ protected void writeMeasurementSchemas(IWALByteBufferView buffer) { - buffer.putInt(measurements.length - getFailedMeasurementNumber()); + buffer.putInt(getValidMeasurementNumber()); serializeMeasurementSchemasToWAL(buffer); } @@ -976,8 +1001,8 @@ protected void writeBitMaps(IWALByteBufferView buffer, List rangeList, in buffer.put(BytesUtils.boolToByte(bitMaps != null)); if (bitMaps != null) { for (int i = 0; i < bitMaps.length; i++) { - // ignore failed partial insert - if (measurements[i] == null) { + // ignore failed partial insert and null columns + if (!shouldSerializeMeasurement(i)) { continue; } @@ -1001,8 +1026,8 @@ protected void writeBitMaps(IWALByteBufferView buffer, List rangeList, in /** Serialize values, ignoring failed time series */ protected void writeValues(IWALByteBufferView buffer, List rangeList) { for (int i = 0; i < columns.length; i++) { - // ignore failed partial insert - if (measurements[i] == null) { + // ignore failed partial insert and null columns + if (!shouldSerializeMeasurement(i)) { continue; } for (int[] startEnd : rangeList) { @@ -1011,6 +1036,51 @@ protected void writeValues(IWALByteBufferView buffer, List rangeList) { } } + @Override + protected int getValidMeasurementNumber() { + int validMeasurementNumber = 0; + for (int i = 0; measurements != null && i < measurements.length; i++) { + if (shouldSerializeMeasurement(i)) { + validMeasurementNumber++; + } + } + return validMeasurementNumber; + } + + @Override + protected int serializeMeasurementSchemasSize() { + int byteLen = 0; + for (int i = 0; measurements != null && i < measurements.length; i++) { + if (shouldSerializeMeasurement(i)) { + byteLen += WALWriteUtils.sizeToWrite(measurementSchemas[i]); + } + } + return byteLen; + } + + @Override + protected void serializeMeasurementSchemasToWAL(IWALByteBufferView buffer) { + for (int i = 0; measurements != null && i < measurements.length; i++) { + if (shouldSerializeMeasurement(i)) { + WALWriteUtils.write(measurementSchemas[i], buffer); + } + } + } + + protected boolean shouldSerializeMeasurement(int index) { + return measurements != null + && index < measurements.length + && measurements[index] != null + && (measurementSchemas == null + || index < measurementSchemas.length && measurementSchemas[index] != null) + && dataTypes != null + && index < dataTypes.length + && dataTypes[index] != null + && columns != null + && index < columns.length + && columns[index] != null; + } + private void serializeColumn( TSDataType dataType, Object column, IWALByteBufferView buffer, int start, int end) { switch (dataType) { @@ -1239,7 +1309,7 @@ public R accept(IPlanVisitor visitor, C context) { public TimeValuePair composeLastTimeValuePair( int measurementIndex, int startOffset, int endOffset) { - if (measurementIndex >= columns.length || Objects.isNull(dataTypes[measurementIndex])) { + if (!canComposeLastTimeValuePair(measurementIndex)) { return null; } @@ -1272,7 +1342,7 @@ protected TimeValuePair composeLastTimeValuePair( if (results == null) { return composeLastTimeValuePair(measurementIndex, startOffset, endOffset); } - if (measurementIndex >= columns.length || Objects.isNull(dataTypes[measurementIndex])) { + if (!canComposeLastTimeValuePair(measurementIndex)) { return null; } @@ -1293,6 +1363,20 @@ protected TimeValuePair composeLastTimeValuePair( return lastIdx < startOffset ? null : composeTimeValuePair(measurementIndex, lastIdx); } + private boolean canComposeLastTimeValuePair(final int measurementIndex) { + return measurements != null + && measurementIndex >= 0 + && measurementIndex < measurements.length + && columns != null + && measurementIndex < columns.length + && columns[measurementIndex] != null + && dataTypes != null + && measurementIndex < dataTypes.length + && dataTypes[measurementIndex] != null + && (columnCategories == null || measurementIndex < columnCategories.length) + && isWritableFieldMeasurement(measurementIndex); + } + private TimeValuePair composeTimeValuePair(final int measurementIndex, final int rowIndex) { TsPrimitiveType value; switch (dataTypes[measurementIndex]) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java index 839c3db5a7cf9..11694e6d5ff1d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java @@ -180,7 +180,7 @@ public R accept(IPlanVisitor visitor, C context) { @Override protected InsertTabletNode getEmptySplit(int count) { long[] subTimes = new long[count]; - Object[] values = initTabletValues(dataTypes.length, count, dataTypes); + Object[] values = initTabletValuesForSplit(dataTypes.length, count, dataTypes); BitMap[] newBitMaps = initBitmapsForSplit(dataTypes.length, count); RelationalInsertTabletNode split = new RelationalInsertTabletNode( @@ -250,7 +250,7 @@ public static RelationalInsertTabletNode deserialize(ByteBuffer byteBuffer) { protected void serializeAttributes(ByteBuffer byteBuffer) { super.serializeAttributes(byteBuffer); for (int i = 0; i < measurements.length; i++) { - if (measurements[i] != null) { + if (shouldSerializeMeasurement(i)) { columnCategories[i].serialize(byteBuffer); } } @@ -260,7 +260,7 @@ protected void serializeAttributes(ByteBuffer byteBuffer) { protected void serializeAttributes(DataOutputStream stream) throws IOException { super.serializeAttributes(stream); for (int i = 0; i < measurements.length; i++) { - if (measurements[i] != null) { + if (shouldSerializeMeasurement(i)) { columnCategories[i].serialize(stream); } } @@ -285,7 +285,7 @@ void subSerialize(IWALByteBufferView buffer, List rangeList) { void subSerialize(IWALByteBufferView buffer, List rangeList, long encodedSearchIndex) { super.subSerialize(buffer, rangeList, encodedSearchIndex); for (int i = 0; i < measurements.length; i++) { - if (measurements[i] != null) { + if (shouldSerializeMeasurement(i)) { buffer.put(columnCategories[i].getCategory()); } } @@ -450,7 +450,7 @@ private List generateOneSplitList( System.arraycopy(times, start, subNode.times, destLoc, length); for (int i = 0; i < subNode.columns.length; i++) { - if (dataTypes[i] != null) { + if (hasColumnForSplit(i)) { System.arraycopy(columns[i], start, subNode.columns[i], destLoc, length); } if (subNode.bitMaps != null diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java index 6ba5f6307cb82..866ef26d2d2f4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java @@ -176,14 +176,14 @@ private IWritableMemChunkGroup createMemChunkGroupIfNotExistAndGet( private IWritableMemChunkGroup createAlignedMemChunkGroupIfNotExistAndGet( IDeviceID deviceId, List schemaList) { + List filteredSchemaList = + schemaList.stream().filter(Objects::nonNull).collect(Collectors.toList()); IWritableMemChunkGroup memChunkGroup = memTableMap.computeIfAbsent( deviceId, k -> { - seriesNumber += schemaList.size(); - return new AlignedWritableMemChunkGroup( - schemaList.stream().filter(Objects::nonNull).collect(Collectors.toList()), - k.isTableModel()); + seriesNumber += filteredSchemaList.size(); + return new AlignedWritableMemChunkGroup(filteredSchemaList, k.isTableModel()); }); for (IMeasurementSchema schema : schemaList) { if (schema != null && !memChunkGroup.contains(schema.getMeasurementName())) { @@ -204,8 +204,8 @@ public int insert(InsertRowNode insertRowNode) { int nullPointsNumber = 0; for (int i = 0; i < insertRowNode.getMeasurements().length; i++) { // Use measurements[i] to ignore failed partial insert - if (measurements[i] == null || values[i] == null) { - if (values[i] == null) { + if (measurements[i] == null || values[i] == null || !isFieldMeasurement(insertRowNode, i)) { + if (isValidMeasurement(insertRowNode, i, true) && values[i] == null) { nullPointsNumber++; } schemaList.add(null); @@ -215,12 +215,11 @@ public int insert(InsertRowNode insertRowNode) { dataTypes.add(schema.getType()); } } - memSize += MemUtils.getRowRecordSize(dataTypes, values); + memSize += MemUtils.getRowRecordSize(dataTypes, values, insertRowNode.getColumnCategories()); write(insertRowNode.getDeviceID(), schemaList, insertRowNode.getTime(), values); int pointsInserted = - insertRowNode.getMeasurements().length - - insertRowNode.getFailedMeasurementNumber() + getValidMeasurementNumber(insertRowNode, true) - (IoTDBDescriptor.getInstance().getConfig().isIncludeNullValueInWriteThroughputMetric() ? 0 : nullPointsNumber); @@ -243,7 +242,7 @@ public int insertAlignedRow(InsertRowNode insertRowNode) { || values[i] == null || insertRowNode.getColumnCategories() != null && insertRowNode.getColumnCategories()[i] != TsTableColumnCategory.FIELD) { - if (measurements[i] != null && values[i] == null) { + if (isValidMeasurement(insertRowNode, i, true) && values[i] == null) { // do not include failed measurement to avoid a negative pointsInserted nullPointsNumber++; } @@ -261,8 +260,7 @@ public int insertAlignedRow(InsertRowNode insertRowNode) { MemUtils.getAlignedRowRecordSize(dataTypes, values, insertRowNode.getColumnCategories()); writeAlignedRow(insertRowNode.getDeviceID(), schemaList, insertRowNode.getTime(), values); int pointsInserted = - insertRowNode.getMeasurementColumnCnt() - - insertRowNode.getFailedMeasurementNumber() + getValidMeasurementNumber(insertRowNode, true) - (IoTDBDescriptor.getInstance().getConfig().isIncludeNullValueInWriteThroughputMetric() ? 0 : nullPointsNumber); @@ -274,12 +272,11 @@ public int insertAlignedRow(InsertRowNode insertRowNode) { public int insertTablet(InsertTabletNode insertTabletNode, int start, int end) throws WriteProcessException { try { - int nullPointsNumber = computeTabletNullPointsNumber(insertTabletNode, start, end); + int nullPointsNumber = computeTabletNullPointsNumber(insertTabletNode, start, end, true); writeTabletNode(insertTabletNode, start, end); memSize += MemUtils.getTabletSize(insertTabletNode, start, end); int pointsInserted = - ((insertTabletNode.getDataTypes().length - insertTabletNode.getFailedMeasurementNumber()) - * (end - start)) + (getValidMeasurementNumber(insertTabletNode, true) * (end - start)) - (IoTDBDescriptor.getInstance() .getConfig() .isIncludeNullValueInWriteThroughputMetric() @@ -297,14 +294,12 @@ public int insertAlignedTablet( InsertTabletNode insertTabletNode, int start, int end, TSStatus[] results) throws WriteProcessException { try { - int nullPointsNumber = computeTabletNullPointsNumber(insertTabletNode, start, end); + int nullPointsNumber = computeTabletNullPointsNumber(insertTabletNode, start, end, true); writeAlignedTablet(insertTabletNode, start, end, results); // TODO-Table: what is the relation between this and TsFileProcessor.checkMemCost memSize += MemUtils.getAlignedTabletSize(insertTabletNode, start, end, results); int pointsInserted = - ((insertTabletNode.getMeasurementColumnCnt() - - insertTabletNode.getFailedMeasurementNumber()) - * (end - start)) + (getValidMeasurementNumber(insertTabletNode, true) * (end - start)) - (IoTDBDescriptor.getInstance() .getConfig() .isIncludeNullValueInWriteThroughputMetric() @@ -318,12 +313,13 @@ public int insertAlignedTablet( } private static int computeTabletNullPointsNumber( - InsertTabletNode insertTabletNode, int start, int end) { + InsertTabletNode insertTabletNode, int start, int end, boolean countFieldOnly) { Object[] values = insertTabletNode.getBitMaps(); int nullPointsNumber = 0; if (values != null) { for (int i = 0; i < insertTabletNode.getMeasurements().length; i++) { - if (insertTabletNode.isMeasurementFailed(i)) { + if (!isValidMeasurement(insertTabletNode, i, countFieldOnly) + || insertTabletNode.getColumns()[i] == null) { // do not include failed measurement to avoid a negative pointsInserted continue; } @@ -341,6 +337,38 @@ private static int computeTabletNullPointsNumber( return nullPointsNumber; } + private static int getValidMeasurementNumber(InsertRowNode insertNode, boolean countFieldOnly) { + int count = 0; + for (int i = 0; i < insertNode.getMeasurements().length; i++) { + if (isValidMeasurement(insertNode, i, countFieldOnly)) { + count++; + } + } + return count; + } + + private static int getValidMeasurementNumber( + InsertTabletNode insertNode, boolean countFieldOnly) { + int count = 0; + for (int i = 0; i < insertNode.getMeasurements().length; i++) { + if (isValidMeasurement(insertNode, i, countFieldOnly) && insertNode.getColumns()[i] != null) { + count++; + } + } + return count; + } + + private static boolean isValidMeasurement( + InsertNode insertNode, int index, boolean countFieldOnly) { + return insertNode.getMeasurements()[index] != null + && (!countFieldOnly || isFieldMeasurement(insertNode, index)); + } + + private static boolean isFieldMeasurement(InsertNode insertNode, int index) { + return insertNode.getColumnCategories() == null + || insertNode.getColumnCategories()[index] == TsTableColumnCategory.FIELD; + } + @Override public void write( IDeviceID deviceId, @@ -366,7 +394,9 @@ public void writeAlignedRow( public void writeTabletNode(InsertTabletNode insertTabletNode, int start, int end) { List schemaList = new ArrayList<>(); for (int i = 0; i < insertTabletNode.getMeasurementSchemas().length; i++) { - if (insertTabletNode.getColumns()[i] == null) { + if (insertTabletNode.getMeasurements()[i] == null + || insertTabletNode.getColumns()[i] == null + || !isFieldMeasurement(insertTabletNode, i)) { schemaList.add(null); } else { schemaList.add(insertTabletNode.getMeasurementSchemas()[i]); @@ -388,7 +418,8 @@ public void writeAlignedTablet( InsertTabletNode insertTabletNode, int start, int end, TSStatus[] results) { List schemaList = new ArrayList<>(); for (int i = 0; i < insertTabletNode.getMeasurementSchemas().length; i++) { - if (insertTabletNode.getColumns()[i] == null + if (insertTabletNode.getMeasurements()[i] == null + || insertTabletNode.getColumns()[i] == null || (insertTabletNode.getColumnCategories() != null && insertTabletNode.getColumnCategories()[i] != TsTableColumnCategory.FIELD)) { schemaList.add(null); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java index c4e488f84863f..54adec65b97e3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java @@ -306,8 +306,11 @@ public void insert(InsertRowNode insertRowNode, long[] infoForMetrics) } else { memIncrements = checkMemCostAndAddToTspInfoForRow( - insertRowNode.getDeviceID(), insertRowNode.getMeasurements(), - insertRowNode.getDataTypes(), insertRowNode.getValues()); + insertRowNode.getDeviceID(), + insertRowNode.getMeasurements(), + insertRowNode.getDataTypes(), + insertRowNode.getValues(), + insertRowNode.getColumnCategories()); } // recordScheduleMemoryBlockCost infoForMetrics[1] += System.nanoTime() - memControlStartTime; @@ -523,6 +526,7 @@ private long[] checkMemCost( insertTabletNode.getMeasurements(), insertTabletNode.getDataTypes(), insertTabletNode.getColumns(), + insertTabletNode.getColumnCategories(), start, end); } @@ -676,7 +680,11 @@ public void insertTablet( @SuppressWarnings("squid:S3776") // High Cognitive Complexity private long[] checkMemCostAndAddToTspInfoForRow( - IDeviceID deviceId, String[] measurements, TSDataType[] dataTypes, Object[] values) + IDeviceID deviceId, + String[] measurements, + TSDataType[] dataTypes, + Object[] values, + TsTableColumnCategory[] columnCategories) throws WriteProcessException { // Memory of increased PrimitiveArray and TEXT values, e.g., add a long[128], add 128*8 long memTableIncrement = 0L; @@ -685,7 +693,7 @@ private long[] checkMemCostAndAddToTspInfoForRow( for (int i = 0; i < dataTypes.length; i++) { // Skip failed Measurements - if (dataTypes[i] == null || measurements[i] == null) { + if (!isWritableFieldMeasurement(measurements, dataTypes, values, columnCategories, i)) { continue; } IWritableMemChunk memChunk = workMemTable.getWritableMemChunk(deviceId, measurements[i]); @@ -725,7 +733,8 @@ private long[] checkMemCostAndAddToTspInfoForRows(List insertRowN String[] measurements = insertRowNode.getMeasurements(); for (int i = 0; i < dataTypes.length; i++) { // Skip failed Measurements - if (dataTypes[i] == null || measurements[i] == null) { + if (!isWritableFieldMeasurement( + measurements, dataTypes, values, insertRowNode.getColumnCategories(), i)) { continue; } IWritableMemChunk memChunk = workMemTable.getWritableMemChunk(deviceId, measurements[i]); @@ -779,21 +788,21 @@ private long[] checkAlignedMemCostAndAddToTspInfoForRow( IWritableMemChunk memChunk = workMemTable.getWritableMemChunk(deviceId, AlignedPath.VECTOR_PLACEHOLDER); if (memChunk == null) { + TSDataType[] writableFieldDataTypes = + getWritableFieldDataTypes(measurements, dataTypes, values, columnCategories); // For new device of this mem table // ChunkMetadataIncrement chunkMetadataIncrement += ChunkMetadata.calculateRamSize(AlignedPath.VECTOR_PLACEHOLDER, TSDataType.VECTOR) - * dataTypes.length; - memTableIncrement += AlignedTVList.alignedTvListArrayMemCost(dataTypes, columnCategories); + * writableFieldDataTypes.length; + memTableIncrement += AlignedTVList.alignedTvListArrayMemCost(writableFieldDataTypes, null); } else { // For existed device of this mem table AlignedWritableMemChunk alignedMemChunk = (AlignedWritableMemChunk) memChunk; List dataTypesInTVList = new ArrayList<>(); for (int i = 0; i < dataTypes.length; i++) { // Skip failed Measurements - if (dataTypes[i] == null - || measurements[i] == null - || (columnCategories != null && columnCategories[i] != TsTableColumnCategory.FIELD)) { + if (!isWritableFieldMeasurement(measurements, dataTypes, values, columnCategories, i)) { continue; } @@ -817,7 +826,8 @@ private long[] checkAlignedMemCostAndAddToTspInfoForRow( for (int i = 0; i < dataTypes.length; i++) { // TEXT data mem size - if (dataTypes[i] != null && dataTypes[i].isBinary() && values[i] != null) { + if (isWritableFieldMeasurement(measurements, dataTypes, values, columnCategories, i) + && dataTypes[i].isBinary()) { textDataIncrement += MemUtils.getBinarySize((Binary) values[i]); } } @@ -843,25 +853,26 @@ private long[] checkAlignedMemCostAndAddToTspInfoForRows(List ins IWritableMemChunk memChunk = workMemTable.getWritableMemChunk(deviceId, AlignedPath.VECTOR_PLACEHOLDER); if (memChunk == null && !increasingMemTableInfo.containsKey(deviceId)) { + TSDataType[] writableFieldDataTypes = + getWritableFieldDataTypes( + measurements, dataTypes, values, insertRowNode.getColumnCategories()); + Pair, Integer> addingPointNumInfo = + increasingMemTableInfo.computeIfAbsent(deviceId, k -> new Pair<>(new HashMap<>(), 0)); // For new device of this mem table // ChunkMetadataIncrement chunkMetadataIncrement += ChunkMetadata.calculateRamSize(AlignedPath.VECTOR_PLACEHOLDER, TSDataType.VECTOR) - * dataTypes.length; - memTableIncrement += AlignedTVList.alignedTvListArrayMemCost(dataTypes, null); + * writableFieldDataTypes.length; + memTableIncrement += AlignedTVList.alignedTvListArrayMemCost(writableFieldDataTypes, null); for (int i = 0; i < dataTypes.length; i++) { // Skip failed Measurements - if (dataTypes[i] == null - || measurements[i] == null - || (insertRowNode.getColumnCategories() != null - && insertRowNode.getColumnCategories()[i] != TsTableColumnCategory.FIELD)) { + if (!isWritableFieldMeasurement( + measurements, dataTypes, values, insertRowNode.getColumnCategories(), i)) { continue; } - increasingMemTableInfo - .computeIfAbsent(deviceId, k -> new Pair<>(new HashMap<>(), 1)) - .left - .put(measurements[i], dataTypes[i]); + addingPointNumInfo.left.put(measurements[i], dataTypes[i]); } + addingPointNumInfo.setRight(1); } else { // For existed device of this mem table @@ -872,10 +883,8 @@ private long[] checkAlignedMemCostAndAddToTspInfoForRows(List ins increasingMemTableInfo.computeIfAbsent(deviceId, k -> new Pair<>(new HashMap<>(), 0)); for (int i = 0; i < dataTypes.length; i++) { // Skip failed Measurements - if (dataTypes[i] == null - || measurements[i] == null - || (insertRowNode.getColumnCategories() != null - && insertRowNode.getColumnCategories()[i] != TsTableColumnCategory.FIELD)) { + if (!isWritableFieldMeasurement( + measurements, dataTypes, values, insertRowNode.getColumnCategories(), i)) { continue; } @@ -914,10 +923,8 @@ private long[] checkAlignedMemCostAndAddToTspInfoForRows(List ins for (int i = 0; i < dataTypes.length; i++) { // Skip failed Measurements - if (dataTypes[i] == null - || measurements[i] == null - || (insertRowNode.getColumnCategories() != null - && insertRowNode.getColumnCategories()[i] != TsTableColumnCategory.FIELD)) { + if (!isWritableFieldMeasurement( + measurements, dataTypes, values, insertRowNode.getColumnCategories(), i)) { continue; } // TEXT data mem size @@ -935,6 +942,7 @@ private long[] checkMemCostAndAddToTspInfoForTablet( String[] measurements, TSDataType[] dataTypes, Object[] columns, + TsTableColumnCategory[] columnCategories, int start, int end) throws WriteProcessException { @@ -945,7 +953,7 @@ private long[] checkMemCostAndAddToTspInfoForTablet( for (int i = 0; i < dataTypes.length; i++) { // Skip failed Measurements - if (dataTypes[i] == null || columns[i] == null || measurements[i] == null) { + if (!isWritableFieldMeasurement(measurements, dataTypes, columns, columnCategories, i)) { continue; } updateMemCost(dataTypes[i], measurements[i], deviceId, start, end, memIncrements, columns[i]); @@ -1053,16 +1061,8 @@ private void updateAlignedMemCost( } } - int measurementColumnNum = 0; - if (columnCategories == null) { - measurementColumnNum = dataTypes.length; - } else { - for (TsTableColumnCategory columnCategory : columnCategories) { - if (columnCategory == TsTableColumnCategory.FIELD) { - measurementColumnNum++; - } - } - } + TSDataType[] writableFieldDataTypes = + getWritableFieldDataTypes(measurementIds, dataTypes, columns, columnCategories); // memIncrements = [memTable, text, chunk metadata] respectively IWritableMemChunk memChunk = @@ -1071,7 +1071,7 @@ private void updateAlignedMemCost( // new devices introduce new ChunkMetadata // ChunkMetadata memory Increment memIncrements[2] += - measurementColumnNum + writableFieldDataTypes.length * ChunkMetadata.calculateRamSize(AlignedPath.VECTOR_PLACEHOLDER, TSDataType.VECTOR); // TVList memory @@ -1079,7 +1079,7 @@ private void updateAlignedMemCost( incomingPointNum / PrimitiveArrayManager.ARRAY_SIZE + (incomingPointNum % PrimitiveArrayManager.ARRAY_SIZE > 0 ? 1 : 0); memIncrements[0] += - numArraysToAdd * AlignedTVList.alignedTvListArrayMemCost(dataTypes, columnCategories); + numArraysToAdd * AlignedTVList.alignedTvListArrayMemCost(writableFieldDataTypes, null); } else { AlignedWritableMemChunk alignedMemChunk = (AlignedWritableMemChunk) memChunk; List dataTypesInTVList = new ArrayList<>(); @@ -1087,12 +1087,7 @@ private void updateAlignedMemCost( int newPointNum = currentPointNum + incomingPointNum; for (int i = 0; i < dataTypes.length; i++) { TSDataType dataType = dataTypes[i]; - String measurement = measurementIds[i]; - Object column = columns[i]; - if (dataType == null - || column == null - || measurement == null - || (columnCategories != null && columnCategories[i] != TsTableColumnCategory.FIELD)) { + if (!isWritableFieldMeasurement(measurementIds, dataTypes, columns, columnCategories, i)) { continue; } @@ -1125,12 +1120,7 @@ private void updateAlignedMemCost( // flexible-length data size for (int i = 0; i < dataTypes.length; i++) { TSDataType dataType = dataTypes[i]; - String measurement = measurementIds[i]; - Object column = columns[i]; - if (dataType == null - || column == null - || measurement == null - || (columnCategories != null && columnCategories[i] != TsTableColumnCategory.FIELD)) { + if (!isWritableFieldMeasurement(measurementIds, dataTypes, columns, columnCategories, i)) { continue; } @@ -1141,6 +1131,41 @@ private void updateAlignedMemCost( } } + private static TSDataType[] getWritableFieldDataTypes( + String[] measurementIds, + TSDataType[] dataTypes, + Object[] valuesOrColumns, + TsTableColumnCategory[] columnCategories) { + List writableFieldDataTypes = new ArrayList<>(); + for (int i = 0; i < dataTypes.length; i++) { + if (isWritableFieldMeasurement( + measurementIds, dataTypes, valuesOrColumns, columnCategories, i)) { + writableFieldDataTypes.add(dataTypes[i]); + } + } + return writableFieldDataTypes.toArray(new TSDataType[0]); + } + + private static boolean isWritableFieldMeasurement( + String[] measurementIds, + TSDataType[] dataTypes, + Object[] valuesOrColumns, + TsTableColumnCategory[] columnCategories, + int index) { + return isFieldMeasurement(measurementIds, dataTypes, columnCategories, index) + && valuesOrColumns[index] != null; + } + + private static boolean isFieldMeasurement( + String[] measurementIds, + TSDataType[] dataTypes, + TsTableColumnCategory[] columnCategories, + int index) { + return dataTypes[index] != null + && measurementIds[index] != null + && (columnCategories == null || columnCategories[index] == TsTableColumnCategory.FIELD); + } + private void updateMemoryInfo( long memTableIncrement, long chunkMetadataIncrement, long textDataIncrement) throws WriteProcessRejectException { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java index 89f28726c98d7..6f5c63824c491 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java @@ -179,7 +179,7 @@ public int serializedSize() { int size = 0; size += Integer.BYTES; for (Map.Entry entry : memChunkMap.entrySet()) { - size += ReadWriteIOUtils.sizeToWrite(entry.getKey()); + size += WALWriteUtils.sizeToWrite(entry.getKey()); size += entry.getValue().serializedSize(); } return size; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/MemUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/MemUtils.java index 3c4d24b5796d7..09b836db3ccf5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/MemUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/MemUtils.java @@ -67,14 +67,18 @@ public static long getRecordSize(TSDataType dataType, Object value, boolean addi * memory before insertion */ public static long getRowRecordSize(List dataTypes, Object[] value) { - int emptyRecordCount = 0; + return getRowRecordSize(dataTypes, value, null); + } + + public static long getRowRecordSize( + List dataTypes, Object[] value, TsTableColumnCategory[] columnCategories) { + int dataTypeIndex = 0; long memSize = 0L; for (int i = 0; i < value.length; i++) { - if (value[i] == null) { - emptyRecordCount++; + if (value[i] == null || !isFieldColumn(columnCategories, i)) { continue; } - memSize += getRecordSize(dataTypes.get(i - emptyRecordCount), value[i], false); + memSize += getRecordSize(dataTypes.get(dataTypeIndex++), value[i], false); } return memSize; } @@ -87,13 +91,15 @@ public static long getAlignedRowRecordSize( List dataTypes, Object[] value, TsTableColumnCategory[] columnCategories) { // time and index size long memSize = 8L + 4L; - for (int i = 0; i < dataTypes.size(); i++) { - if (value[i] == null - || dataTypes.get(i).isBinary() - || columnCategories != null && columnCategories[i] != TsTableColumnCategory.FIELD) { + int dataTypeIndex = 0; + for (int i = 0; i < value.length; i++) { + if (value[i] == null || !isFieldColumn(columnCategories, i)) { continue; } - memSize += dataTypes.get(i).getDataTypeSize(); + TSDataType dataType = dataTypes.get(dataTypeIndex++); + if (!dataType.isBinary()) { + memSize += dataType.getDataTypeSize(); + } } return memSize; } @@ -127,7 +133,7 @@ public static long getTabletSize(InsertTabletNode insertTabletNode, int start, i } long memSize = 0; for (int i = 0; i < insertTabletNode.getMeasurements().length; i++) { - if (insertTabletNode.getMeasurements()[i] == null) { + if (!isWritableTabletMeasurement(insertTabletNode, i, true)) { continue; } // Time column memSize @@ -144,7 +150,7 @@ public static long getAlignedTabletSize( } long memSize = 0; for (int i = 0; i < insertTabletNode.getMeasurements().length; i++) { - if (!insertTabletNode.isValidMeasurement(i)) { + if (!isWritableTabletMeasurement(insertTabletNode, i, true)) { continue; } if (results == null) { @@ -163,6 +169,20 @@ public static long getAlignedTabletSize( return memSize; } + private static boolean isWritableTabletMeasurement( + InsertTabletNode insertTabletNode, int index, boolean fieldOnly) { + return insertTabletNode.getMeasurements()[index] != null + && insertTabletNode.getColumns()[index] != null + && insertTabletNode.getDataTypes()[index] != null + && insertTabletNode.getMeasurementSchemas() != null + && insertTabletNode.getMeasurementSchemas()[index] != null + && (!fieldOnly || isFieldColumn(insertTabletNode.getColumnCategories(), index)); + } + + private static boolean isFieldColumn(TsTableColumnCategory[] columnCategories, int columnIndex) { + return columnCategories == null || columnCategories[columnIndex] == TsTableColumnCategory.FIELD; + } + /** Calculate how much memory will be used if the given record is written to sequence file. */ public static long getTsRecordMem(TSRecord tsRecord) { long memUsed = 8; // time diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTabletInsertionEventTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTabletInsertionEventTest.java index da3dee91caad9..6b1f9e3d3d3b1 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTabletInsertionEventTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTabletInsertionEventTest.java @@ -33,10 +33,12 @@ import org.apache.iotdb.db.auth.AuthorityChecker; import org.apache.iotdb.db.pipe.event.common.tablet.PipeInsertNodeTabletInsertionEvent; import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent; +import org.apache.iotdb.db.pipe.event.common.tablet.parser.TabletInsertionEventTablePatternParser; import org.apache.iotdb.db.pipe.event.common.tablet.parser.TabletInsertionEventTreePatternParser; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; import org.apache.iotdb.db.queryengine.plan.relational.security.AccessControl; import org.apache.tsfile.enums.TSDataType; @@ -350,6 +352,75 @@ nodeWithSparseColumn, new PrefixTreePattern(pattern)) Assert.assertTrue(tablet.isNull(1, 1)); } + @Test + public void convertToTabletSkipsFailedMeasurementsForCoveredTreePattern() throws Exception { + final InsertRowNode rowNode = + new InsertRowNode( + new PlanNodeId("plan node failed row"), + new PartialPath(deviceId), + false, + Arrays.copyOf(measurementIds, measurementIds.length), + Arrays.copyOf(dataTypes, dataTypes.length), + Arrays.copyOf(schemas, schemas.length), + times[0], + Arrays.copyOf(insertRowNode.getValues(), schemas.length), + false); + rowNode.markFailedMeasurement(1); + + final Tablet rowTablet = + new TabletInsertionEventTreePatternParser(rowNode, new PrefixTreePattern(pattern)) + .convertToTablet(); + assertTabletDoesNotContainMeasurement(rowTablet, "s2", schemas.length - 1); + + final InsertTabletNode tabletNode = + new InsertTabletNode( + new PlanNodeId("plan node failed tablet"), + new PartialPath(deviceId), + false, + Arrays.copyOf(measurementIds, measurementIds.length), + Arrays.copyOf(dataTypes, dataTypes.length), + Arrays.copyOf(schemas, schemas.length), + times, + null, + Arrays.copyOf(insertTabletNode.getColumns(), schemas.length), + times.length); + tabletNode.markFailedMeasurement(1); + + final Tablet tablet = + new TabletInsertionEventTreePatternParser(tabletNode, new PrefixTreePattern(pattern)) + .convertToTablet(); + assertTabletDoesNotContainMeasurement(tablet, "s2", schemas.length - 1); + } + + @Test + public void convertToTabletSkipsFailedMeasurementsForTablePattern() throws Exception { + final TsTableColumnCategory[] columnCategories = new TsTableColumnCategory[schemas.length]; + Arrays.fill(columnCategories, TsTableColumnCategory.FIELD); + columnCategories[0] = TsTableColumnCategory.TAG; + columnCategories[2] = TsTableColumnCategory.ATTRIBUTE; + + final RelationalInsertTabletNode tabletNode = + new RelationalInsertTabletNode( + new PlanNodeId("plan node failed relational tablet"), + new PartialPath("table1", false), + false, + Arrays.copyOf(measurementIds, measurementIds.length), + Arrays.copyOf(dataTypes, dataTypes.length), + times, + null, + Arrays.copyOf(insertTabletNode.getColumns(), schemas.length), + times.length, + columnCategories); + tabletNode.setMeasurementSchemas(Arrays.copyOf(schemas, schemas.length)); + tabletNode.markFailedMeasurement(1); + + final Tablet tablet = + new TabletInsertionEventTablePatternParser( + null, null, tabletNode, new TablePattern(true, null, null)) + .convertToTablet(); + assertTabletDoesNotContainMeasurement(tablet, "s2", schemas.length - 1); + } + @Test public void convertToTabletWithFilteredRowsForTest() throws Exception { TabletInsertionEventTreePatternParser container1 = @@ -538,4 +609,14 @@ public TSStatus checkSeriesPrivilege4Pipe( : new TSStatus(org.apache.iotdb.rpc.TSStatusCode.SUCCESS_STATUS.getStatusCode()); } } + + private static void assertTabletDoesNotContainMeasurement( + final Tablet tablet, final String measurement, final int expectedSchemaSize) { + Assert.assertEquals(expectedSchemaSize, tablet.getSchemas().size()); + for (int i = 0; i < tablet.getSchemas().size(); i++) { + Assert.assertNotNull(tablet.getSchemas().get(i)); + Assert.assertNotEquals(measurement, tablet.getSchemas().get(i).getMeasurementName()); + Assert.assertNotNull(tablet.getValues()[i]); + } + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertRowStatementTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertRowStatementTest.java new file mode 100644 index 0000000000000..8c563a84b1db4 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertRowStatementTest.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.receiver.transform.statement; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.Assert; +import org.junit.Test; + +import java.time.ZoneId; + +public class PipeConvertedInsertRowStatementTest { + + @Test + public void testTransferTypeKeepsNullValue() throws Exception { + final InsertRowStatement statement = new InsertRowStatement(); + statement.setDevicePath(new PartialPath("root.sg.d1")); + statement.setMeasurements(new String[] {"s0"}); + statement.setMeasurementSchemas( + new MeasurementSchema[] {new MeasurementSchema("s0", TSDataType.INT32)}); + statement.setDataTypes(new TSDataType[] {null}); + statement.setTime(1L); + statement.setValues(new Object[] {null}); + statement.setNeedInferType(true); + + final PipeConvertedInsertRowStatement convertedStatement = + new PipeConvertedInsertRowStatement(statement); + convertedStatement.transferType(ZoneId.systemDefault()); + + Assert.assertEquals(TSDataType.INT32, convertedStatement.getDataTypes()[0]); + Assert.assertEquals("s0", convertedStatement.getMeasurements()[0]); + Assert.assertNull(convertedStatement.getValues()[0]); + Assert.assertFalse(convertedStatement.isNeedInferType()); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimatorTest.java index cefc2377dc79e..06db08aaf57b0 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimatorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimatorTest.java @@ -134,6 +134,18 @@ public void testInsertMultiTabletsNodeLaterTabletSizeIsEstimated() throws Illega Assert.assertTrue(largerNodeSize > baselineSize); } + @Test + public void testInsertTabletNodeWithNullColumnIsEstimated() throws IllegalPathException { + InsertTabletNode tablet = createTextInsertTabletNode("tablet", "root.sg.d1", 2, 4, 8); + + long fullSize = InsertNodeMemoryEstimator.sizeOf(tablet); + tablet.getColumns()[1] = null; + long sizeWithNullColumn = InsertNodeMemoryEstimator.sizeOf(tablet); + + Assert.assertTrue(sizeWithNullColumn > 0); + Assert.assertTrue(sizeWithNullColumn < fullSize); + } + @Test public void testPlanNodeIdIsEstimated() throws IllegalPathException { InsertRowNode shortPlanNodeIdRow = diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/util/TabletStatementConverterTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/util/TabletStatementConverterTest.java index 410afc7613030..8eff4a591916e 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/util/TabletStatementConverterTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/util/TabletStatementConverterTest.java @@ -25,6 +25,7 @@ import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.DateUtils; import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -172,6 +173,41 @@ public void testRoundTripConversionTableModel() throws MetadataException { assertTabletsEqual(originalTablet, convertedTablet); } + @Test + public void testDeserializeStatementFromTabletFormatWithNullSchemaAndNullColumn() + throws IOException, MetadataException { + final PublicBAOS byteArrayOutputStream = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream); + + ReadWriteIOUtils.write("root.sg.device", outputStream); + ReadWriteIOUtils.write(1, outputStream); + + ReadWriteIOUtils.write(BytesUtils.boolToByte(true), outputStream); + ReadWriteIOUtils.write(1, outputStream); + ReadWriteIOUtils.write(BytesUtils.boolToByte(false), outputStream); + + ReadWriteIOUtils.write(BytesUtils.boolToByte(true), outputStream); + ReadWriteIOUtils.write(1L, outputStream); + + ReadWriteIOUtils.write(BytesUtils.boolToByte(false), outputStream); + + ReadWriteIOUtils.write(BytesUtils.boolToByte(true), outputStream); + ReadWriteIOUtils.write(BytesUtils.boolToByte(false), outputStream); + + ReadWriteIOUtils.write(true, outputStream); + + final ByteBuffer buffer = + ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); + + final InsertTabletStatement statement = + TabletStatementConverter.deserializeStatementFromTabletFormat(buffer); + + Assert.assertArrayEquals(new String[] {null}, statement.getMeasurements()); + Assert.assertArrayEquals(new TSDataType[] {null}, statement.getDataTypes()); + Assert.assertNull(statement.getColumns()[0]); + Assert.assertTrue(statement.isAligned()); + } + /** * Generate a Tablet for tree model with all data types and specified number of columns and rows. * diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertRowNodeSerdeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertRowNodeSerdeTest.java index f068173632981..a2d28762103c5 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertRowNodeSerdeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertRowNodeSerdeTest.java @@ -176,6 +176,25 @@ public void testRelationalSerializedSizeWithFailedMeasurement() { Assert.assertEquals(insertRowNode.serializedSize(), byteBuffer.position()); } + @Test + public void testDeserializeFromWALWithMarkedFailedMeasurementOnly() + throws IllegalPathException, IOException { + InsertRowNode insertRowNode = getInsertRowNodeWithMeasurementSchemas(); + insertRowNode.markFailedMeasurement(1); + + byte[] bytes = new byte[insertRowNode.serializedSize()]; + WALByteBufferForTest walBuffer = new WALByteBufferForTest(ByteBuffer.wrap(bytes)); + insertRowNode.serializeToWAL(walBuffer); + Assert.assertFalse(walBuffer.getBuffer().hasRemaining()); + + DataInputStream dataInputStream = new DataInputStream(new ByteArrayInputStream(bytes)); + Assert.assertEquals(PlanNodeType.INSERT_ROW.getNodeType(), dataInputStream.readShort()); + + InsertRowNode tmpNode = InsertRowNode.deserializeFromWAL(dataInputStream); + Assert.assertArrayEquals( + new String[] {"\u6e29\u5ea6", "s3", "s4", "s5"}, tmpNode.getMeasurements()); + } + private InsertRowNode getInsertRowNode() throws IllegalPathException { long time = 110L; TSDataType[] dataTypes = diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertRowsOfOneDeviceNodeSerdeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertRowsOfOneDeviceNodeSerdeTest.java index 2cffd65e41e21..35204133bb8ef 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertRowsOfOneDeviceNodeSerdeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertRowsOfOneDeviceNodeSerdeTest.java @@ -79,4 +79,38 @@ public void TestSerializeAndDeserialize() throws IllegalPathException { Assert.assertEquals(node, InsertRowsOfOneDeviceNode.deserialize(byteBuffer)); } + + @Test + public void testStoreMeasurementsSkipsFailedMeasurements() throws IllegalPathException { + PartialPath device = new PartialPath("root.sg.d"); + InsertRowsOfOneDeviceNode node = new InsertRowsOfOneDeviceNode(new PlanNodeId("plan node 1")); + + List insertRowNodeList = new ArrayList<>(); + InsertRowNode firstRow = + new InsertRowNode( + new PlanNodeId("plan node 1"), + device, + false, + new String[] {"s1", "failed"}, + new TSDataType[] {TSDataType.DOUBLE, TSDataType.FLOAT}, + 1000L, + new Object[] {1.0, 2f}, + false); + firstRow.markFailedMeasurement(1); + insertRowNodeList.add(firstRow); + insertRowNodeList.add( + new InsertRowNode( + new PlanNodeId("plan node 1"), + device, + false, + new String[] {"s2"}, + new TSDataType[] {TSDataType.INT64}, + 2000L, + new Object[] {300L}, + false)); + + node.setInsertRowNodeList(insertRowNodeList); + + Assert.assertArrayEquals(new String[] {"s1", "s2"}, node.getMeasurements()); + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertTabletNodeSerdeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertTabletNodeSerdeTest.java index 8cf0ce6e36e49..27a3b8d1ce5db 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertTabletNodeSerdeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertTabletNodeSerdeTest.java @@ -235,6 +235,69 @@ public void testRelationalSerializedSizeWithFailedMeasurement() { Assert.assertEquals(insertTabletNode.serializedSize(), byteBuffer.position()); } + @Test + public void testSerializedSizeWithClearedMeasurementAndRetainedColumn() + throws IllegalPathException { + InsertTabletNode insertTabletNode = getInsertTabletNodeWithSchema(); + insertTabletNode.getMeasurements()[1] = null; + insertTabletNode.getMeasurementSchemas()[1] = null; + insertTabletNode.getDataTypes()[1] = null; + + ByteBuffer byteBuffer = ByteBuffer.allocate(insertTabletNode.serializedSize()); + insertTabletNode.serializeToWAL(new WALByteBufferForTest(byteBuffer)); + + Assert.assertEquals(insertTabletNode.serializedSize(), byteBuffer.position()); + } + + @Test + public void testSerializedSizeWithRetainedMeasurementAndNullColumn() + throws IllegalPathException, IOException { + InsertTabletNode insertTabletNode = getInsertTabletNodeWithSchema(); + insertTabletNode.getColumns()[1] = null; + + byte[] bytes = new byte[insertTabletNode.serializedSize()]; + WALByteBufferForTest walBuffer = new WALByteBufferForTest(ByteBuffer.wrap(bytes)); + insertTabletNode.serializeToWAL(walBuffer); + Assert.assertFalse(walBuffer.getBuffer().hasRemaining()); + + DataInputStream dataInputStream = new DataInputStream(new ByteArrayInputStream(bytes)); + Assert.assertEquals(PlanNodeType.INSERT_TABLET.getNodeType(), dataInputStream.readShort()); + + InsertTabletNode tmpNode = InsertTabletNode.deserializeFromWAL(dataInputStream); + Assert.assertArrayEquals( + new String[] {"\u6e29\u5ea6", "s3", "s4", "s5"}, tmpNode.getMeasurements()); + } + + @Test + public void testRelationalSerializedSizeWithRetainedMeasurementAndNullColumn() { + RelationalInsertTabletNode insertTabletNode = getRelationalInsertTabletNodeWithSchema("table1"); + insertTabletNode.getColumns()[1] = null; + + ByteBuffer byteBuffer = ByteBuffer.allocate(insertTabletNode.serializedSize()); + insertTabletNode.serializeToWAL(new WALByteBufferForTest(byteBuffer)); + + Assert.assertEquals(insertTabletNode.serializedSize(), byteBuffer.position()); + } + + @Test + public void testDeserializeFromWALWithMarkedFailedMeasurementOnly() + throws IllegalPathException, IOException { + InsertTabletNode insertTabletNode = getInsertTabletNodeWithSchema(); + insertTabletNode.markFailedMeasurement(1); + + byte[] bytes = new byte[insertTabletNode.serializedSize()]; + WALByteBufferForTest walBuffer = new WALByteBufferForTest(ByteBuffer.wrap(bytes)); + insertTabletNode.serializeToWAL(walBuffer); + Assert.assertFalse(walBuffer.getBuffer().hasRemaining()); + + DataInputStream dataInputStream = new DataInputStream(new ByteArrayInputStream(bytes)); + Assert.assertEquals(PlanNodeType.INSERT_TABLET.getNodeType(), dataInputStream.readShort()); + + InsertTabletNode tmpNode = InsertTabletNode.deserializeFromWAL(dataInputStream); + Assert.assertArrayEquals( + new String[] {"\u6e29\u5ea6", "s3", "s4", "s5"}, tmpNode.getMeasurements()); + } + @Test public void testInitTabletValuesWithAllTypes() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/WritePlanNodeSplitTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/WritePlanNodeSplitTest.java index 01857cb5f8aa9..6d5185af4ba7b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/WritePlanNodeSplitTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/WritePlanNodeSplitTest.java @@ -260,6 +260,47 @@ public void testSplitInsertTablet() throws IllegalPathException { } } + @Test + public void testSplitInsertTabletSkipsClearedMeasurementWithRetainedColumn() + throws IllegalPathException { + InsertTabletNode insertTabletNode = new InsertTabletNode(new PlanNodeId("plan node 1")); + + insertTabletNode.setTargetPath(new PartialPath("root.sg1.d1")); + insertTabletNode.setMeasurements(new String[] {"s0", null}); + insertTabletNode.setTimes( + new long[] {-200, -101, 1, 60, 120, 180, 270, 290, 360, 375, 440, 470}); + insertTabletNode.setDataTypes(new TSDataType[] {TSDataType.INT32, TSDataType.INT32}); + insertTabletNode.setColumns( + new Object[] { + new int[] {-20, -10, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100}, + new int[] {-2, -1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10} + }); + insertTabletNode.setRowCount(insertTabletNode.getTimes().length); + final BitMap[] bitMaps = new BitMap[] {null, new BitMap(insertTabletNode.getRowCount())}; + bitMaps[1].mark(2); + insertTabletNode.setBitMaps(bitMaps); + + DataPartitionQueryParam dataPartitionQueryParam = new DataPartitionQueryParam(); + dataPartitionQueryParam.setDeviceID( + insertTabletNode.getTargetPath().getIDeviceIDAsFullDevice()); + dataPartitionQueryParam.setTimePartitionSlotList(insertTabletNode.getTimePartitionSlots()); + + DataPartition dataPartition = + getDataPartition(Collections.singletonList(dataPartitionQueryParam)); + Analysis analysis = new Analysis(); + analysis.setDataPartitionInfo(dataPartition); + + List insertTabletNodeList = insertTabletNode.splitByPartition(analysis); + + Assert.assertEquals(6, insertTabletNodeList.size()); + for (WritePlanNode insertNode : insertTabletNodeList) { + InsertTabletNode tabletNode = (InsertTabletNode) insertNode; + Assert.assertNotNull(tabletNode.getColumns()[0]); + Assert.assertNull(tabletNode.getColumns()[1]); + Assert.assertNull(tabletNode.getBitMaps()); + } + } + @Test public void testSplitRelationalInsertTablet() throws IllegalPathException { RelationalInsertTabletNode relationalInsertTabletNode = diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNodeIsMeasurementFailedTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNodeIsMeasurementFailedTest.java index e5ed818323bbd..8853348719bed 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNodeIsMeasurementFailedTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNodeIsMeasurementFailedTest.java @@ -22,12 +22,18 @@ import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Binary; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.Test; +import java.nio.charset.StandardCharsets; + import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; /** @@ -82,6 +88,46 @@ public void testInsertRowNode_markSameTwice_idempotent() throws IllegalPathExcep assertFalse(node.isMeasurementFailed(1)); } + @Test + public void testInsertRowNode_clearedMeasurementWithRetainedValue_isFailed() + throws IllegalPathException { + InsertRowNode node = buildInsertRowNode(new String[] {"s0", "s1"}); + node.getMeasurements()[0] = null; + + assertTrue(node.isMeasurementFailed(0)); + assertNull(node.composeTimeValuePair(0)); + } + + @Test + public void testInsertRowNode_retainedMeasurementWithNullValueDoesNotComposeLastCacheValue() + throws IllegalPathException { + InsertRowNode node = buildInsertRowNode(new String[] {"s0", "s1"}); + node.getValues()[0] = null; + + assertFalse(node.isMeasurementFailed(0)); + assertNull(node.composeTimeValuePair(0)); + } + + @Test + public void testInsertRowNode_nullMeasurements_nullSafe() throws IllegalPathException { + InsertRowNode node = buildInsertRowNode(new String[] {"s0"}); + node.setMeasurements(null); + + assertTrue(node.isMeasurementFailed(0)); + assertFalse(node.hasValidMeasurements()); + assertTrue(node.allMeasurementFailed()); + } + + @Test + public void testRelationalInsertRowNode_nonFieldColumnsDoNotComposeLastCacheValue() + throws IllegalPathException { + RelationalInsertRowNode node = buildRelationalInsertRowNode(); + + assertNull(node.composeTimeValuePair(0)); + assertNull(node.composeTimeValuePair(1)); + assertNotNull(node.composeTimeValuePair(2)); + } + // ----------------------------------------------------------------------- // InsertTabletNode // ----------------------------------------------------------------------- @@ -115,6 +161,36 @@ public void testInsertTabletNode_markLastFailed_lastReturnsTrue() throws Illegal assertTrue(node.isMeasurementFailed(1)); } + @Test + public void testInsertTabletNode_clearedMeasurementWithRetainedColumn_isFailed() + throws IllegalPathException { + InsertTabletNode node = buildInsertTabletNode(new String[] {"s0", "s1"}); + node.getMeasurements()[0] = null; + + assertTrue(node.isMeasurementFailed(0)); + assertNull(node.composeLastTimeValuePair(0)); + } + + @Test + public void testInsertTabletNode_retainedMeasurementWithNullColumnDoesNotComposeLastCacheValue() + throws IllegalPathException { + InsertTabletNode node = buildInsertTabletNode(new String[] {"s0", "s1"}); + node.getColumns()[0] = null; + + assertFalse(node.isMeasurementFailed(0)); + assertNull(node.composeLastTimeValuePair(0)); + } + + @Test + public void testRelationalInsertTabletNode_nonFieldColumnsDoNotComposeLastCacheValue() + throws IllegalPathException { + RelationalInsertTabletNode node = buildRelationalInsertTabletNode(); + + assertNull(node.composeLastTimeValuePair(0)); + assertNull(node.composeLastTimeValuePair(1)); + assertNotNull(node.composeLastTimeValuePair(2)); + } + // ----------------------------------------------------------------------- // Helpers // ----------------------------------------------------------------------- @@ -144,6 +220,35 @@ private static InsertRowNode buildInsertRowNode(String[] measurementNames) return node; } + private static RelationalInsertRowNode buildRelationalInsertRowNode() + throws IllegalPathException { + String[] measurements = {"tag0", "attr0", "field0"}; + TSDataType[] dataTypes = {TSDataType.STRING, TSDataType.STRING, TSDataType.INT32}; + MeasurementSchema[] schemas = { + new MeasurementSchema("tag0", TSDataType.STRING), + new MeasurementSchema("attr0", TSDataType.STRING), + new MeasurementSchema("field0", TSDataType.INT32) + }; + Object[] values = { + new Binary("tag".getBytes(StandardCharsets.UTF_8)), + new Binary("attr".getBytes(StandardCharsets.UTF_8)), + 1 + }; + return new RelationalInsertRowNode( + new PlanNodeId("test"), + new PartialPath("table1", false), + true, + measurements, + dataTypes, + schemas, + 1L, + values, + false, + new TsTableColumnCategory[] { + TsTableColumnCategory.TAG, TsTableColumnCategory.ATTRIBUTE, TsTableColumnCategory.FIELD + }); + } + private static InsertTabletNode buildInsertTabletNode(String[] measurementNames) throws IllegalPathException { int n = measurementNames.length; @@ -171,4 +276,34 @@ private static InsertTabletNode buildInsertTabletNode(String[] measurementNames) rowCount); return node; } + + private static RelationalInsertTabletNode buildRelationalInsertTabletNode() + throws IllegalPathException { + String[] measurements = {"tag0", "attr0", "field0"}; + TSDataType[] dataTypes = {TSDataType.STRING, TSDataType.STRING, TSDataType.INT32}; + MeasurementSchema[] schemas = { + new MeasurementSchema("tag0", TSDataType.STRING), + new MeasurementSchema("attr0", TSDataType.STRING), + new MeasurementSchema("field0", TSDataType.INT32) + }; + Object[] columns = { + new Binary[] {new Binary("tag".getBytes(StandardCharsets.UTF_8))}, + new Binary[] {new Binary("attr".getBytes(StandardCharsets.UTF_8))}, + new int[] {1} + }; + return new RelationalInsertTabletNode( + new PlanNodeId("test"), + new PartialPath("table1", false), + true, + measurements, + dataTypes, + schemas, + new long[] {1L}, + null, + columns, + 1, + new TsTableColumnCategory[] { + TsTableColumnCategory.TAG, TsTableColumnCategory.ATTRIBUTE, TsTableColumnCategory.FIELD + }); + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTablePartialInsertTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTablePartialInsertTest.java index 2f4a85d5840a2..bdad14c20e33f 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTablePartialInsertTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTablePartialInsertTest.java @@ -22,6 +22,7 @@ import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.WriteProcessException; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; @@ -112,6 +113,20 @@ public void testInsertAlignedRow_oneFailedMeasurement_pointsInsertedNotNegative( assertEquals(1, memTable.getTotalPointsNum()); } + @Test + public void testInsertAlignedRow_markedFailedMeasurementOnly_pointsInsertedMatchesWrittenPoints() + throws IllegalPathException { + InsertRowNode node = + buildAlignedInsertRowNode( + new String[] {"s0", "s1"}, new Object[] {1, 2}, -1 /* no failure */); + node.markFailedMeasurement(0); + + int points = memTable.insertAlignedRow(node); + + assertEquals(1, points); + assertEquals(1, memTable.getTotalPointsNum()); + } + /** All measurements fail → insertAlignedRow returns 0 early (schemaList is empty). */ @Test public void testInsertAlignedRow_allMeasurementsFailed_pointsInsertedIsZero() @@ -132,6 +147,67 @@ public void testInsertAlignedRow_allMeasurementsFailed_pointsInsertedIsZero() assertEquals(0, memTable.getTotalPointsNum()); } + @Test + public void testInsertAlignedRow_tableNonFieldAndFailedMeasurementsNotCountedAsSeries() + throws IllegalPathException { + InsertRowNode node = + buildAlignedInsertRowNode( + new String[] {"tag1", "attr1", "s0", "s1"}, + new Object[] {1, 2, 3, 4}, + -1 /* no failure */); + node.setColumnCategories( + new TsTableColumnCategory[] { + TsTableColumnCategory.TAG, + TsTableColumnCategory.ATTRIBUTE, + TsTableColumnCategory.FIELD, + TsTableColumnCategory.FIELD + }); + node.markFailedMeasurement(3); + + int points = memTable.insertAlignedRow(node); + + assertEquals(1, points); + assertEquals(1, memTable.getTotalPointsNum()); + assertEquals(1, memTable.getSeriesNumber()); + } + + @Test + public void testInsertRow_tableNonFieldAndFailedMeasurementsNotCountedAsSeries() + throws IllegalPathException { + InsertRowNode node = + new InsertRowNode( + new PlanNodeId("test"), + new PartialPath("root.sg.d1"), + false, + new String[] {"tag1", "attr1", "s0", "s1"}, + new TSDataType[] { + TSDataType.INT32, TSDataType.INT32, TSDataType.INT32, TSDataType.INT32 + }, + new MeasurementSchema[] { + new MeasurementSchema("tag1", TSDataType.INT32), + new MeasurementSchema("attr1", TSDataType.INT32), + new MeasurementSchema("s0", TSDataType.INT32), + new MeasurementSchema("s1", TSDataType.INT32) + }, + 1L, + new Object[] {1, 2, 3, 4}, + false); + node.setColumnCategories( + new TsTableColumnCategory[] { + TsTableColumnCategory.TAG, + TsTableColumnCategory.ATTRIBUTE, + TsTableColumnCategory.FIELD, + TsTableColumnCategory.FIELD + }); + node.markFailedMeasurement(3); + + int points = memTable.insert(node); + + assertEquals(1, points); + assertEquals(1, memTable.getTotalPointsNum()); + assertEquals(1, memTable.getSeriesNumber()); + } + // ========================================================================= // insertTablet – failed measurement must be skipped in null-point counting // ========================================================================= @@ -179,6 +255,43 @@ public void testInsertTablet_oneFailedMeasurement_withBitmap_pointsInsertedNotNe assertEquals(3, memTable.getTotalPointsNum()); } + @Test + public void testInsertTablet_markedFailedMeasurementOnly_pointsInsertedMatchesWrittenPoints() + throws IllegalPathException, WriteProcessException { + int rowCount = 3; + InsertTabletNode node = + buildInsertTabletNode(new String[] {"s0", "s1"}, rowCount, null, -1 /* no failure */); + node.markFailedMeasurement(0); + + int points = memTable.insertTablet(node, 0, rowCount); + + assertEquals(3, points); + assertEquals(3, memTable.getTotalPointsNum()); + } + + @Test + public void testInsertTablet_tableNonFieldAndFailedMeasurementsNotCountedAsSeries() + throws IllegalPathException, WriteProcessException { + int rowCount = 3; + InsertTabletNode node = + buildInsertTabletNode( + new String[] {"tag1", "attr1", "s0", "s1"}, rowCount, null, -1 /* no failure */); + node.setColumnCategories( + new TsTableColumnCategory[] { + TsTableColumnCategory.TAG, + TsTableColumnCategory.ATTRIBUTE, + TsTableColumnCategory.FIELD, + TsTableColumnCategory.FIELD + }); + node.markFailedMeasurement(3); + + int points = memTable.insertTablet(node, 0, rowCount); + + assertEquals(rowCount, points); + assertEquals(rowCount, memTable.getTotalPointsNum()); + assertEquals(1, memTable.getSeriesNumber()); + } + /** All measurements fail → pointsInserted == 0. formula: (2-2)*3 - 0 = 0 */ @Test public void testInsertTablet_allMeasurementsFailed_pointsInsertedIsZero() diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemChunkDeserializeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemChunkDeserializeTest.java index 8e03776231eae..5595390752a39 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemChunkDeserializeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemChunkDeserializeTest.java @@ -283,6 +283,29 @@ public void testAlignedSeries() throws IOException, QueryProcessException, Metad } } + @Test + public void testNonAlignedMemChunkGroupSerializedSizeWithNonAsciiMeasurement() + throws IOException { + String measurement = "\u6e29\u5ea6"; + WritableMemChunk series = + new WritableMemChunk( + new MeasurementSchema(measurement, TSDataType.INT32, TSEncoding.PLAIN)); + series.writeNonAlignedPoint(1, 1); + + WritableMemChunkGroup group = new WritableMemChunkGroup(); + group.getMemChunkMap().put(measurement, series); + + WALByteBufferForTest walBuffer = + new WALByteBufferForTest(ByteBuffer.allocate(group.serializedSize())); + group.serializeToWAL(walBuffer); + Assert.assertEquals(group.serializedSize(), walBuffer.getBuffer().position()); + + DataInputStream inputStream = + new DataInputStream(new ByteArrayInputStream(walBuffer.getBuffer().array())); + WritableMemChunkGroup deserialized = WritableMemChunkGroup.deserialize(inputStream); + Assert.assertTrue(deserialized.getMemChunkMap().containsKey(measurement)); + } + private WritableMemChunk createWritableMemChunkFromBytes(WritableMemChunk series) throws IOException { int serializedSize = series.serializedSize(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessorTest.java index 3cc50496fb64f..2609cb1acb5e6 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessorTest.java @@ -26,6 +26,7 @@ import org.apache.iotdb.commons.path.NonAlignedFullPath; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.DataRegionException; @@ -36,6 +37,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowNode; import org.apache.iotdb.db.storageengine.dataregion.DataRegionInfo; import org.apache.iotdb.db.storageengine.dataregion.DataRegionTest; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; @@ -56,6 +58,7 @@ import org.apache.tsfile.read.expression.QueryExpression; import org.apache.tsfile.read.query.dataset.QueryDataSet; import org.apache.tsfile.read.reader.IPointReader; +import org.apache.tsfile.utils.Binary; import org.apache.tsfile.write.record.TSRecord; import org.apache.tsfile.write.record.datapoint.DataPoint; import org.apache.tsfile.write.schema.MeasurementSchema; @@ -68,6 +71,7 @@ import java.io.File; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -898,6 +902,174 @@ record = new TSRecord("root.vehicle.d2", 102); Assert.assertEquals(memTable1.memSize(), memTable2.memSize()); } + @Test + public void testAlignedRamCostIgnoresRelationalNonFieldAndNullFieldColumns() + throws IllegalPathException, WriteProcessException, IOException { + TsFileProcessor relationalProcessor = + new TsFileProcessor( + storageGroup, + SystemFileFactory.INSTANCE.getFile(filePath), + sgInfo, + this::closeTsFileProcessor, + (tsFileProcessor, updateMap, systemFlushTime) -> {}, + true); + TsFileProcessorInfo relationalInfo = new TsFileProcessorInfo(sgInfo); + relationalProcessor.setTsFileProcessorInfo(relationalInfo); + this.sgInfo.initTsFileProcessorInfo(relationalProcessor); + SystemInfo.getInstance().reportStorageGroupStatus(sgInfo, relationalProcessor); + + RelationalInsertRowNode relationalNode = + new RelationalInsertRowNode( + new PlanNodeId("relational"), + new PartialPath("table1", false), + true, + new String[] {"tag1", "attr1", "s1", "s2"}, + new TSDataType[] {TSDataType.TEXT, TSDataType.TEXT, TSDataType.INT32, TSDataType.INT64}, + new MeasurementSchema[] { + new MeasurementSchema("tag1", TSDataType.TEXT), + new MeasurementSchema("attr1", TSDataType.TEXT), + new MeasurementSchema("s1", TSDataType.INT32), + new MeasurementSchema("s2", TSDataType.INT64) + }, + 1L, + new Object[] { + new Binary("tag-value".getBytes(StandardCharsets.UTF_8)), + new Binary("attr-value".getBytes(StandardCharsets.UTF_8)), + 1, + null + }, + false, + new TsTableColumnCategory[] { + TsTableColumnCategory.TAG, + TsTableColumnCategory.ATTRIBUTE, + TsTableColumnCategory.FIELD, + TsTableColumnCategory.FIELD + }); + relationalProcessor.insert(relationalNode, new long[5]); + + TsFileProcessor fieldOnlyProcessor = + new TsFileProcessor( + storageGroup, + SystemFileFactory.INSTANCE.getFile(filePath), + sgInfo, + this::closeTsFileProcessor, + (tsFileProcessor, updateMap, systemFlushTime) -> {}, + true); + TsFileProcessorInfo fieldOnlyInfo = new TsFileProcessorInfo(sgInfo); + fieldOnlyProcessor.setTsFileProcessorInfo(fieldOnlyInfo); + this.sgInfo.initTsFileProcessorInfo(fieldOnlyProcessor); + SystemInfo.getInstance().reportStorageGroupStatus(sgInfo, fieldOnlyProcessor); + + InsertRowNode fieldOnlyNode = + new InsertRowNode( + new PlanNodeId("field-only"), + new PartialPath(deviceId), + true, + new String[] {"s1", "s2"}, + new TSDataType[] {TSDataType.INT32, TSDataType.INT64}, + new MeasurementSchema[] { + new MeasurementSchema("s1", TSDataType.INT32), + new MeasurementSchema("s2", TSDataType.INT64) + }, + 1L, + new Object[] {1, null}, + false); + fieldOnlyProcessor.insert(fieldOnlyNode, new long[5]); + + IMemTable relationalMemTable = relationalProcessor.getWorkMemTable(); + IMemTable fieldOnlyMemTable = fieldOnlyProcessor.getWorkMemTable(); + Assert.assertEquals( + fieldOnlyMemTable.getTVListsRamCost(), relationalMemTable.getTVListsRamCost()); + Assert.assertEquals(fieldOnlyInfo.getMemCost(), relationalInfo.getMemCost()); + Assert.assertEquals(fieldOnlyMemTable.memSize(), relationalMemTable.memSize()); + Assert.assertEquals(1, relationalMemTable.getTotalPointsNum()); + Assert.assertEquals(1, relationalMemTable.getSeriesNumber()); + } + + @Test + public void testNonAlignedRamCostIgnoresRelationalNonFieldAndNullFieldColumns() + throws IllegalPathException, WriteProcessException, IOException { + TsFileProcessor relationalProcessor = + new TsFileProcessor( + storageGroup, + SystemFileFactory.INSTANCE.getFile(filePath), + sgInfo, + this::closeTsFileProcessor, + (tsFileProcessor, updateMap, systemFlushTime) -> {}, + true); + TsFileProcessorInfo relationalInfo = new TsFileProcessorInfo(sgInfo); + relationalProcessor.setTsFileProcessorInfo(relationalInfo); + this.sgInfo.initTsFileProcessorInfo(relationalProcessor); + SystemInfo.getInstance().reportStorageGroupStatus(sgInfo, relationalProcessor); + + RelationalInsertRowNode relationalNode = + new RelationalInsertRowNode( + new PlanNodeId("relational"), + new PartialPath("table1", false), + false, + new String[] {"tag1", "attr1", "s1", "s2"}, + new TSDataType[] {TSDataType.TEXT, TSDataType.TEXT, TSDataType.INT32, TSDataType.INT64}, + new MeasurementSchema[] { + new MeasurementSchema("tag1", TSDataType.TEXT), + new MeasurementSchema("attr1", TSDataType.TEXT), + new MeasurementSchema("s1", TSDataType.INT32), + new MeasurementSchema("s2", TSDataType.INT64) + }, + 1L, + new Object[] { + new Binary("tag-value".getBytes(StandardCharsets.UTF_8)), + new Binary("attr-value".getBytes(StandardCharsets.UTF_8)), + 1, + null + }, + false, + new TsTableColumnCategory[] { + TsTableColumnCategory.TAG, + TsTableColumnCategory.ATTRIBUTE, + TsTableColumnCategory.FIELD, + TsTableColumnCategory.FIELD + }); + relationalProcessor.insert(relationalNode, new long[5]); + + TsFileProcessor fieldOnlyProcessor = + new TsFileProcessor( + storageGroup, + SystemFileFactory.INSTANCE.getFile(filePath), + sgInfo, + this::closeTsFileProcessor, + (tsFileProcessor, updateMap, systemFlushTime) -> {}, + true); + TsFileProcessorInfo fieldOnlyInfo = new TsFileProcessorInfo(sgInfo); + fieldOnlyProcessor.setTsFileProcessorInfo(fieldOnlyInfo); + this.sgInfo.initTsFileProcessorInfo(fieldOnlyProcessor); + SystemInfo.getInstance().reportStorageGroupStatus(sgInfo, fieldOnlyProcessor); + + InsertRowNode fieldOnlyNode = + new InsertRowNode( + new PlanNodeId("field-only"), + new PartialPath(deviceId), + false, + new String[] {"s1", "s2"}, + new TSDataType[] {TSDataType.INT32, TSDataType.INT64}, + new MeasurementSchema[] { + new MeasurementSchema("s1", TSDataType.INT32), + new MeasurementSchema("s2", TSDataType.INT64) + }, + 1L, + new Object[] {1, null}, + false); + fieldOnlyProcessor.insert(fieldOnlyNode, new long[5]); + + IMemTable relationalMemTable = relationalProcessor.getWorkMemTable(); + IMemTable fieldOnlyMemTable = fieldOnlyProcessor.getWorkMemTable(); + Assert.assertEquals( + fieldOnlyMemTable.getTVListsRamCost(), relationalMemTable.getTVListsRamCost()); + Assert.assertEquals(fieldOnlyInfo.getMemCost(), relationalInfo.getMemCost()); + Assert.assertEquals(fieldOnlyMemTable.memSize(), relationalMemTable.memSize()); + Assert.assertEquals(1, relationalMemTable.getTotalPointsNum()); + Assert.assertEquals(1, relationalMemTable.getSeriesNumber()); + } + @Test public void testRamCostInsertSameDataBy2Ways() throws MetadataException, WriteProcessException, IOException { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/MemUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/MemUtilsTest.java index 7af2a9775fb0f..1c1c1fca1220a 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/MemUtilsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/MemUtilsTest.java @@ -21,6 +21,7 @@ import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; import org.apache.tsfile.common.conf.TSFileConfig; @@ -91,6 +92,46 @@ public void getRecordSizeWithInsertAlignedRowNodeTest() { Assert.assertEquals(sizeSum, MemUtils.getAlignedRowRecordSize(dataTypes, row, null)); } + @Test + public void getAlignedRowRecordSizeWithSkippedSlotsTest() { + Object[] row = {null, 1, 2L}; + List dataTypes = new ArrayList<>(); + dataTypes.add(TSDataType.INT32); + + int sizeSum = 8 + 4; + sizeSum += TSDataType.INT32.getDataTypeSize(); + + Assert.assertEquals( + sizeSum, + MemUtils.getAlignedRowRecordSize( + dataTypes, + row, + new TsTableColumnCategory[] { + TsTableColumnCategory.FIELD, TsTableColumnCategory.FIELD, TsTableColumnCategory.TAG + })); + } + + @Test + public void getRowRecordSizeWithSkippedSlotsTest() { + Object[] row = {1, 2, 3L, null}; + List dataTypes = new ArrayList<>(); + dataTypes.add(TSDataType.INT64); + + int sizeSum = 8 + TSDataType.INT64.getDataTypeSize(); + + Assert.assertEquals( + sizeSum, + MemUtils.getRowRecordSize( + dataTypes, + row, + new TsTableColumnCategory[] { + TsTableColumnCategory.TAG, + TsTableColumnCategory.ATTRIBUTE, + TsTableColumnCategory.FIELD, + TsTableColumnCategory.FIELD + })); + } + @Test public void getRecordSizeWithInsertTableNodeTest() throws IllegalPathException { PartialPath device = new PartialPath("root.sg.d1"); @@ -125,6 +166,14 @@ public void getRecordSizeWithInsertTableNodeTest() throws IllegalPathException { null, columns, 1); + insertNode.setMeasurementSchemas( + new MeasurementSchema[] { + new MeasurementSchema("s1", TSDataType.INT32), + new MeasurementSchema("s2", TSDataType.INT64), + new MeasurementSchema("s3", TSDataType.FLOAT), + new MeasurementSchema("s4", TSDataType.DOUBLE), + new MeasurementSchema("s5", TSDataType.TEXT) + }); Assert.assertEquals(sizeSum, MemUtils.getTabletSize(insertNode, 0, 1)); } @@ -175,6 +224,35 @@ public void getRecordSizeWithInsertAlignedTableNodeTest() throws IllegalPathExce Assert.assertEquals(sizeSum, MemUtils.getAlignedTabletSize(insertNode, 0, 1, null)); } + @Test + public void getAlignedTabletSizeWithMarkedFailedMeasurementTest() throws IllegalPathException { + PartialPath device = new PartialPath("root.sg.d1"); + String[] measurements = {"s1", "s2"}; + Object[] columns = {new int[] {1, 2}, new long[] {3, 4}}; + TSDataType[] dataTypes = {TSDataType.INT32, TSDataType.INT64}; + InsertTabletNode insertNode = + new InsertTabletNode( + new PlanNodeId(""), + device, + true, + measurements, + dataTypes, + new long[] {1, 2}, + null, + columns, + 2); + insertNode.setMeasurementSchemas( + new MeasurementSchema[] { + new MeasurementSchema("s1", TSDataType.INT32), + new MeasurementSchema("s2", TSDataType.INT64) + }); + insertNode.markFailedMeasurement(0); + + long sizeSum = 2L * TSDataType.INT64.getDataTypeSize(); + sizeSum += 2L * (8L + 4L); + Assert.assertEquals(sizeSum, MemUtils.getAlignedTabletSize(insertNode, 0, 2, null)); + } + /** This method tests MemUtils.getStringMem() and MemUtils.getDataPointMem() */ @Test public void getMemSizeTest() { From cfe622fff760d156ab82833b2c05d99802384f18 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 9 Jun 2026 18:04:43 +0800 Subject: [PATCH 2/6] Fix partial insert handling for missing columns --- .../parser/TabletInsertionEventParser.java | 10 +- .../PipeConvertedInsertRowStatement.java | 6 + .../analyze/schema/NormalSchemaFetcher.java | 4 +- .../planner/plan/node/write/InsertNode.java | 25 ++- .../plan/node/write/InsertRowNode.java | 177 +++++++++++---- .../node/write/InsertRowsOfOneDeviceNode.java | 4 +- .../node/write/RelationalInsertRowNode.java | 38 +++- .../node/write/RelationalInsertRowsNode.java | 33 ++- .../write/RelationalInsertTabletNode.java | 54 ++++- .../fetcher/cache/TreeDeviceNormalSchema.java | 4 +- .../cache/TreeDeviceSchemaCacheManager.java | 9 +- .../plan/relational/sql/ast/InsertRow.java | 5 +- .../plan/relational/sql/ast/InsertRows.java | 14 +- .../plan/relational/sql/ast/InsertTablet.java | 13 +- .../sql/ast/WrappedInsertStatement.java | 22 +- .../statement/crud/InsertBaseStatement.java | 49 +++- .../statement/crud/InsertRowStatement.java | 65 +++++- .../crud/InsertRowsOfOneDeviceStatement.java | 12 + .../statement/crud/InsertTabletStatement.java | 105 +++++++-- .../storageengine/dataregion/DataRegion.java | 3 +- .../dataregion/memtable/AbstractMemTable.java | 120 +++++++--- .../dataregion/memtable/TsFileProcessor.java | 35 +-- .../ConsensusLogToTabletConverter.java | 70 ++++-- .../trigger/executor/TriggerFireVisitor.java | 53 ++++- .../org/apache/iotdb/db/utils/MemUtils.java | 33 ++- .../util/TabletStatementConverterTest.java | 28 +++ .../node/write/InsertRowNodeSerdeTest.java | 108 +++++++++ .../node/write/InsertRowsNodeSerdeTest.java | 35 +++ .../node/write/InsertTabletNodeSerdeTest.java | 28 +++ .../InsertStatementPartialInsertTest.java | 209 ++++++++++++++++++ .../ConsensusLogToTabletConverterTest.java | 58 +++++ .../apache/iotdb/db/utils/MemUtilsTest.java | 49 ++++ 32 files changed, 1259 insertions(+), 219 deletions(-) create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertStatementPartialInsertTest.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventParser.java index a01dfd5222b0b..fa99fa73a6ca3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventParser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventParser.java @@ -116,7 +116,9 @@ public void markAsNeedToReport() { //////////////////////////// parse //////////////////////////// protected void parse(final InsertRowNode insertRowNode) { - final int originColumnSize = insertRowNode.getMeasurements().length; + final String[] originColumnNameStringList = insertRowNode.getMeasurements(); + final int originColumnSize = + originColumnNameStringList == null ? 0 : originColumnNameStringList.length; final Integer[] originColumnIndex2FilteredColumnIndexMapperList = new Integer[originColumnSize]; // The full path is always cached when device path is deserialized @@ -129,7 +131,6 @@ protected void parse(final InsertRowNode insertRowNode) { this.timestampColumn = rowIndexList.stream().mapToLong(i -> originTimestampColumn[i]).toArray(); final MeasurementSchema[] originMeasurementSchemaList = insertRowNode.getMeasurementSchemas(); - final String[] originColumnNameStringList = insertRowNode.getMeasurements(); final TsTableColumnCategory[] originColumnCategories = insertRowNode.getColumnCategories(); final TSDataType[] originValueDataTypes = insertRowNode.getDataTypes(); final Object[] originValues = insertRowNode.getValues(); @@ -199,7 +200,9 @@ protected void parse(final InsertRowNode insertRowNode) { } protected void parse(final InsertTabletNode insertTabletNode) throws IllegalPathException { - final int originColumnSize = insertTabletNode.getMeasurements().length; + final String[] originColumnNameStringList = insertTabletNode.getMeasurements(); + final int originColumnSize = + originColumnNameStringList == null ? 0 : originColumnNameStringList.length; final Integer[] originColumnIndex2FilteredColumnIndexMapperList = new Integer[originColumnSize]; // The full path is always cached when device path is deserialized @@ -213,7 +216,6 @@ protected void parse(final InsertTabletNode insertTabletNode) throws IllegalPath final MeasurementSchema[] originMeasurementSchemaList = insertTabletNode.getMeasurementSchemas(); - final String[] originColumnNameStringList = insertTabletNode.getMeasurements(); final TsTableColumnCategory[] originColumnCategories = insertTabletNode.getColumnCategories(); final TSDataType[] originValueColumnDataTypes = insertTabletNode.getDataTypes(); final Object[] originValueColumns = insertTabletNode.getColumns(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertRowStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertRowStatement.java index 742f5989553d0..5f70750864b9e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertRowStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertRowStatement.java @@ -107,7 +107,13 @@ protected boolean checkAndCastDataType(int columnIndex, TSDataType dataType) { @Override public void transferType(ZoneId zoneId) throws QueryProcessException { + if (measurementSchemas == null) { + return; + } for (int i = 0; i < measurementSchemas.length; i++) { + if (!isColumnPresent(i) || dataTypes == null || i >= dataTypes.length) { + continue; + } // null when time series doesn't exist if (measurementSchemas[i] == null) { if (!IoTDBDescriptor.getInstance().getConfig().isEnablePartialInsert()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/NormalSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/NormalSchemaFetcher.java index d7d5275ef5567..04206a7f2083f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/NormalSchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/NormalSchemaFetcher.java @@ -358,7 +358,9 @@ void processNormalTimeSeries( schemaComputationWithAutoCreationList.stream() .map( o -> { - TSDataType[] dataTypes = new TSDataType[o.getMeasurements().length]; + final String[] measurements = o.getMeasurements(); + TSDataType[] dataTypes = + new TSDataType[measurements == null ? 0 : measurements.length]; for (int i = 0, length = dataTypes.length; i < length; i++) { dataTypes[i] = o.getDataType(i); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java index d9a202de22ba5..d71963f94028a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java @@ -191,10 +191,14 @@ public int measureColumnCnt() { public boolean isValidMeasurement(int i) { return measurements != null + && i >= 0 + && i < measurements.length && measurements[i] != null && measurementSchemas != null + && i < measurementSchemas.length && measurementSchemas[i] != null - && (columnCategories == null || columnCategories[i] == TsTableColumnCategory.FIELD); + && (columnCategories == null + || i < columnCategories.length && columnCategories[i] == TsTableColumnCategory.FIELD); } public void setMeasurements(String[] measurements) { @@ -336,7 +340,9 @@ public boolean hasValidMeasurements() { } for (int i = 0; i < measurements.length; i++) { if (measurements[i] != null - && (columnCategories == null || columnCategories[i] == TsTableColumnCategory.FIELD)) { + && (columnCategories == null + || i < columnCategories.length + && columnCategories[i] == TsTableColumnCategory.FIELD)) { return true; } } @@ -353,8 +359,8 @@ public int getFailedMeasurementNumber() { protected int getValidMeasurementNumber() { int validMeasurementNumber = 0; - for (String measurement : measurements) { - if (measurement != null) { + for (int i = 0; measurements != null && i < measurements.length; i++) { + if (measurements[i] != null) { validMeasurementNumber++; } } @@ -362,7 +368,10 @@ protected int getValidMeasurementNumber() { } public boolean isMeasurementFailed(int index) { - return measurements == null || measurements[index] == null; + return measurements == null + || index < 0 + || index >= measurements.length + || measurements[index] == null; } protected boolean isWritableFieldMeasurement(int index) { @@ -432,7 +441,7 @@ public void setColumnCategories(TsTableColumnCategory[] columnCategories) { if (columnCategories != null) { tagColumnIndices = new ArrayList<>(); for (int i = 0; i < columnCategories.length; i++) { - if (columnCategories[i].equals(TsTableColumnCategory.TAG)) { + if (columnCategories[i] == TsTableColumnCategory.TAG) { tagColumnIndices.add(i); } } @@ -453,7 +462,9 @@ public String[] getRawMeasurements() { MeasurementSchema[] measurementSchemas = getMeasurementSchemas(); String[] rawMeasurements = new String[measurements.length]; for (int i = 0; i < measurements.length; i++) { - if (measurementSchemas[i] != null) { + if (measurementSchemas != null + && i < measurementSchemas.length + && measurementSchemas[i] != null) { // get raw measurement rather than alias rawMeasurements[i] = measurementSchemas[i].getMeasurementName(); } else { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java index 9d4d826687d53..f27fa7bdbc3d5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java @@ -184,9 +184,13 @@ public List getOutputColumnNames() { @Override public TSDataType[] getDataTypes() { if (isNeedInferType) { - TSDataType[] predictedDataTypes = new TSDataType[dataTypes.length]; - for (int i = 0; i < dataTypes.length; i++) { - predictedDataTypes[i] = TypeInferenceUtils.getPredictedDataType(values[i], true); + TSDataType[] predictedDataTypes = + new TSDataType + [dataTypes == null ? (values == null ? 0 : values.length) : dataTypes.length]; + for (int i = 0; i < predictedDataTypes.length; i++) { + predictedDataTypes[i] = + TypeInferenceUtils.getPredictedDataType( + values != null && i < values.length ? values[i] : null, true); } return predictedDataTypes; } @@ -197,9 +201,10 @@ public TSDataType[] getDataTypes() { @Override public TSDataType getDataType(int index) { if (isNeedInferType) { - return TypeInferenceUtils.getPredictedDataType(values[index], true); + return TypeInferenceUtils.getPredictedDataType( + values != null && index >= 0 && index < values.length ? values[index] : null, true); } else { - return dataTypes[index]; + return getDataTypeIfPresent(index); } } @@ -234,15 +239,87 @@ public List getTimePartitionSlots() { @Override public void markFailedMeasurement(int index) { - if (measurements[index] == null) { + if (measurements == null + || index < 0 + || index >= measurements.length + || measurements[index] == null) { return; } measurements[index] = null; - dataTypes[index] = null; - values[index] = null; + if (dataTypes != null && index < dataTypes.length) { + dataTypes[index] = null; + } + if (values != null && index < values.length) { + values[index] = null; + } measurementColumnCnt = -1; } + @Override + protected int getValidMeasurementNumber() { + int validMeasurementNumber = 0; + for (int i = 0; measurements != null && i < measurements.length; i++) { + if (shouldSerializeMeasurement(i)) { + validMeasurementNumber++; + } + } + return validMeasurementNumber; + } + + protected int getValidMeasurementNumberForWAL() { + int validMeasurementNumber = 0; + for (int i = 0; measurements != null && i < measurements.length; i++) { + if (shouldSerializeMeasurementToWAL(i)) { + validMeasurementNumber++; + } + } + return validMeasurementNumber; + } + + @Override + protected int serializeMeasurementSchemasSize() { + int byteLen = 0; + for (int i = 0; measurements != null && i < measurements.length; i++) { + if (shouldSerializeMeasurementToWAL(i)) { + byteLen += WALWriteUtils.sizeToWrite(measurementSchemas[i]); + } + } + return byteLen; + } + + @Override + protected void serializeMeasurementSchemasToWAL(IWALByteBufferView buffer) { + for (int i = 0; measurements != null && i < measurements.length; i++) { + if (shouldSerializeMeasurementToWAL(i)) { + WALWriteUtils.write(measurementSchemas[i], buffer); + } + } + } + + protected boolean shouldSerializeMeasurement(final int index) { + return measurements != null + && index >= 0 + && index < measurements.length + && measurements[index] != null + && values != null + && index < values.length + && (measurementSchemas == null + || index < measurementSchemas.length && measurementSchemas[index] != null) + && (values[index] == null || isNeedInferType || getDataTypeIfPresent(index) != null); + } + + protected boolean shouldSerializeMeasurementToWAL(final int index) { + return shouldSerializeMeasurement(index) + && measurementSchemas != null + && index < measurementSchemas.length + && measurementSchemas[index] != null + && (values[index] == null || !isNeedInferType && getDataTypeIfPresent(index) != null); + } + + private TSDataType getDataTypeIfPresent(final int index) { + return dataTypes != null && index >= 0 && index < dataTypes.length ? dataTypes[index] : null; + } + @Override protected void serializeAttributes(ByteBuffer byteBuffer) { getType().serialize(byteBuffer); @@ -293,9 +370,9 @@ void serializeMeasurementsAndValues(DataOutputStream stream) throws IOException /** Serialize measurements or measurement schemas, ignoring failed time series. */ private void serializeMeasurementsOrSchemas(ByteBuffer buffer) { ReadWriteIOUtils.write((byte) (measurementSchemas != null ? 1 : 0), buffer); - for (int i = 0; i < measurements.length; i++) { - // ignore failed partial insert - if (measurements[i] == null) { + for (int i = 0; measurements != null && i < measurements.length; i++) { + // ignore failed partial insert and incomplete columns + if (!shouldSerializeMeasurement(i)) { continue; } // serialize measurement schemas when exist @@ -315,9 +392,9 @@ private void serializeMeasurementsOrSchemas(ByteBuffer buffer) { */ private void serializeMeasurementsOrSchemas(DataOutputStream stream) throws IOException { ReadWriteIOUtils.write((byte) (measurementSchemas != null ? 1 : 0), stream); - for (int i = 0; i < measurements.length; i++) { - // ignore failed partial insert - if (measurements[i] == null) { + for (int i = 0; measurements != null && i < measurements.length; i++) { + // ignore failed partial insert and incomplete columns + if (!shouldSerializeMeasurement(i)) { continue; } // serialize measurement schemas when exist @@ -336,17 +413,18 @@ private void serializeMeasurementsOrSchemas(DataOutputStream stream) throws IOEx * @throws UnSupportedDataTypeException - If meets unsupported data type. */ private void putDataTypesAndValues(ByteBuffer buffer) { - for (int i = 0; i < values.length; i++) { - // ignore failed partial insert - if (measurements[i] == null) { + for (int i = 0; values != null && i < values.length; i++) { + // ignore failed partial insert and incomplete columns + if (!shouldSerializeMeasurement(i)) { continue; } + final TSDataType dataType = getDataTypeIfPresent(i); // serialize null value if (values[i] == null) { ReadWriteIOUtils.write( - dataTypes[i] == null ? TYPE_NULL_WITHOUT_TYPE : TYPE_NULL_WITH_TYPE, buffer); - if (dataTypes[i] != null) { - ReadWriteIOUtils.write(dataTypes[i], buffer); + dataType == null ? TYPE_NULL_WITHOUT_TYPE : TYPE_NULL_WITH_TYPE, buffer); + if (dataType != null) { + ReadWriteIOUtils.write(dataType, buffer); } continue; } @@ -356,8 +434,8 @@ private void putDataTypesAndValues(ByteBuffer buffer) { ReadWriteIOUtils.write(TYPE_RAW_STRING, buffer); ReadWriteIOUtils.write(values[i].toString(), buffer); } else { - ReadWriteIOUtils.write(dataTypes[i], buffer); - switch (dataTypes[i]) { + ReadWriteIOUtils.write(dataType, buffer); + switch (dataType) { case BOOLEAN: ReadWriteIOUtils.write((Boolean) values[i], buffer); break; @@ -382,7 +460,7 @@ private void putDataTypesAndValues(ByteBuffer buffer) { ReadWriteIOUtils.write((Binary) values[i], buffer); break; default: - throw new UnSupportedDataTypeException(UNSUPPORTED_DATA_TYPE + dataTypes[i]); + throw new UnSupportedDataTypeException(UNSUPPORTED_DATA_TYPE + dataType); } } } @@ -396,17 +474,18 @@ private void putDataTypesAndValues(ByteBuffer buffer) { * @throws UnSupportedDataTypeException - If meets unsupported data type. */ private void putDataTypesAndValues(DataOutputStream stream) throws IOException { - for (int i = 0; i < values.length; i++) { - // ignore failed partial insert - if (measurements[i] == null) { + for (int i = 0; values != null && i < values.length; i++) { + // ignore failed partial insert and incomplete columns + if (!shouldSerializeMeasurement(i)) { continue; } + final TSDataType dataType = getDataTypeIfPresent(i); // serialize null value if (values[i] == null) { ReadWriteIOUtils.write( - dataTypes[i] == null ? TYPE_NULL_WITHOUT_TYPE : TYPE_NULL_WITH_TYPE, stream); - if (dataTypes[i] != null) { - ReadWriteIOUtils.write(dataTypes[i], stream); + dataType == null ? TYPE_NULL_WITHOUT_TYPE : TYPE_NULL_WITH_TYPE, stream); + if (dataType != null) { + ReadWriteIOUtils.write(dataType, stream); } continue; } @@ -416,8 +495,8 @@ private void putDataTypesAndValues(DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(TYPE_RAW_STRING, stream); ReadWriteIOUtils.write(values[i].toString(), stream); } else { - ReadWriteIOUtils.write(dataTypes[i], stream); - switch (dataTypes[i]) { + ReadWriteIOUtils.write(dataType, stream); + switch (dataType) { case BOOLEAN: ReadWriteIOUtils.write((Boolean) values[i], stream); break; @@ -442,7 +521,7 @@ private void putDataTypesAndValues(DataOutputStream stream) throws IOException { ReadWriteIOUtils.write((Binary) values[i], stream); break; default: - throw new UnSupportedDataTypeException(UNSUPPORTED_DATA_TYPE + dataTypes[i]); + throw new UnSupportedDataTypeException(UNSUPPORTED_DATA_TYPE + dataType); } } } @@ -568,21 +647,22 @@ private int serializeMeasurementsAndValuesSize() { size += serializeMeasurementSchemasSize(); // putValues - for (int i = 0; i < values.length; i++) { - // ignore failed partial insert - if (measurements[i] == null) { + for (int i = 0; values != null && i < values.length; i++) { + // ignore failed partial insert and incomplete columns + if (!shouldSerializeMeasurementToWAL(i)) { continue; } + final TSDataType dataType = getDataTypeIfPresent(i); // serialize null value if (values[i] == null) { size += Byte.BYTES; - if (dataTypes[i] != null) { + if (dataType != null) { size += Byte.BYTES; } continue; } size += Byte.BYTES; - switch (dataTypes[i]) { + switch (dataType) { case BOOLEAN: size += Byte.BYTES; break; @@ -607,7 +687,7 @@ private int serializeMeasurementsAndValuesSize() { size += ReadWriteIOUtils.sizeToWrite((Binary) values[i]); break; default: - throw new UnSupportedDataTypeException(UNSUPPORTED_DATA_TYPE + dataTypes[i]); + throw new UnSupportedDataTypeException(UNSUPPORTED_DATA_TYPE + dataType); } } @@ -638,7 +718,7 @@ protected void subSerialize(IWALByteBufferView buffer) { /** Serialize measurements and values, ignoring failed time series. */ private void serializeMeasurementsAndValues(IWALByteBufferView buffer) { - buffer.putInt(getValidMeasurementNumber()); + buffer.putInt(getValidMeasurementNumberForWAL()); serializeMeasurementSchemasToWAL(buffer); putDataTypesAndValues(buffer); buffer.put((byte) (isAligned ? 1 : 0)); @@ -651,22 +731,23 @@ private void serializeMeasurementsAndValues(IWALByteBufferView buffer) { * @throws UnSupportedDataTypeException - If meets unsupported data type. */ private void putDataTypesAndValues(IWALByteBufferView buffer) { - for (int i = 0; i < values.length; i++) { - // ignore failed partial insert - if (measurements[i] == null) { + for (int i = 0; values != null && i < values.length; i++) { + // ignore failed partial insert and incomplete columns + if (!shouldSerializeMeasurementToWAL(i)) { continue; } + final TSDataType dataType = getDataTypeIfPresent(i); // serialize null value if (values[i] == null) { WALWriteUtils.write( - dataTypes[i] == null ? TYPE_NULL_WITHOUT_TYPE : TYPE_NULL_WITH_TYPE, buffer); - if (dataTypes[i] != null) { - WALWriteUtils.write(dataTypes[i], buffer); + dataType == null ? TYPE_NULL_WITHOUT_TYPE : TYPE_NULL_WITH_TYPE, buffer); + if (dataType != null) { + WALWriteUtils.write(dataType, buffer); } continue; } - WALWriteUtils.write(dataTypes[i], buffer); - switch (dataTypes[i]) { + WALWriteUtils.write(dataType, buffer); + switch (dataType) { case BOOLEAN: WALWriteUtils.write((Boolean) values[i], buffer); break; @@ -691,7 +772,7 @@ private void putDataTypesAndValues(IWALByteBufferView buffer) { WALWriteUtils.write((Binary) values[i], buffer); break; default: - throw new UnSupportedDataTypeException(UNSUPPORTED_DATA_TYPE + dataTypes[i]); + throw new UnSupportedDataTypeException(UNSUPPORTED_DATA_TYPE + dataType); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsOfOneDeviceNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsOfOneDeviceNode.java index 2acaeccffa698..ccc4ca810d848 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsOfOneDeviceNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsOfOneDeviceNode.java @@ -240,8 +240,8 @@ private void storeMeasurementsAndDataType() { for (InsertRowNode insertRowNode : insertRowNodeList) { String[] measurements = insertRowNode.getMeasurements(); TSDataType[] dataTypes = insertRowNode.getDataTypes(); - for (int i = 0; i < measurements.length; i++) { - if (measurements[i] == null) { + for (int i = 0; measurements != null && i < measurements.length; i++) { + if (measurements[i] == null || dataTypes == null || i >= dataTypes.length) { continue; } if (!measurementSet.contains(measurements[i])) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowNode.java index 93838ffaf8b49..8340ed062901d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowNode.java @@ -95,8 +95,11 @@ public IDeviceID getDeviceID() { deviceIdSegments[0] = this.getTableName(); for (int i = 0; i < tagColumnIndices.size(); i++) { final Integer columnIndex = tagColumnIndices.get(i); - deviceIdSegments[i + 1] = - getValues()[columnIndex] != null ? getValues()[columnIndex].toString() : null; + final Object value = + getValues() != null && columnIndex < getValues().length + ? getValues()[columnIndex] + : null; + deviceIdSegments[i + 1] = value != null ? value.toString() : null; } deviceID = Factory.DEFAULT_FACTORY.create(deviceIdSegments); } @@ -178,11 +181,28 @@ protected static RelationalInsertRowNode subDeserializeFromWAL(ByteBuffer buffer return insertNode; } + @Override + protected boolean shouldSerializeMeasurement(final int index) { + return super.shouldSerializeMeasurement(index) && hasColumnCategory(index); + } + + @Override + protected boolean shouldSerializeMeasurementToWAL(final int index) { + return super.shouldSerializeMeasurementToWAL(index) && hasColumnCategory(index); + } + + private boolean hasColumnCategory(final int index) { + return columnCategories != null + && index >= 0 + && index < columnCategories.length + && columnCategories[index] != null; + } + @Override void subSerialize(ByteBuffer buffer) { super.subSerialize(buffer); - for (int i = 0; i < measurements.length; i++) { - if (measurements[i] != null) { + for (int i = 0; measurements != null && i < measurements.length; i++) { + if (shouldSerializeMeasurement(i)) { columnCategories[i].serialize(buffer); } } @@ -191,8 +211,8 @@ void subSerialize(ByteBuffer buffer) { @Override void subSerialize(DataOutputStream stream) throws IOException { super.subSerialize(stream); - for (int i = 0; i < measurements.length; i++) { - if (measurements[i] != null) { + for (int i = 0; measurements != null && i < measurements.length; i++) { + if (shouldSerializeMeasurement(i)) { columnCategories[i].serialize(stream); } } @@ -201,8 +221,8 @@ void subSerialize(DataOutputStream stream) throws IOException { @Override protected void subSerialize(IWALByteBufferView buffer) { super.subSerialize(buffer); - for (int i = 0; i < measurements.length; i++) { - if (measurements[i] != null) { + for (int i = 0; measurements != null && i < measurements.length; i++) { + if (shouldSerializeMeasurementToWAL(i)) { buffer.put(columnCategories[i].getCategory()); } } @@ -219,7 +239,7 @@ public void subDeserialize(ByteBuffer buffer) { @Override protected int subSerializeSize() { - return super.subSerializeSize() + getValidMeasurementNumber() * Byte.BYTES; + return super.subSerializeSize() + getValidMeasurementNumberForWAL() * Byte.BYTES; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowsNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowsNode.java index 7d7e1e035159a..56ae8cbd51dae 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowsNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowsNode.java @@ -24,6 +24,7 @@ import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.IPlanVisitor; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeType; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.exception.DataTypeInconsistentException; import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; @@ -62,12 +63,14 @@ public IDeviceID getDeviceID(int rowIdx) { deviceIDs = new IDeviceID[getInsertRowNodeList().size()]; } if (deviceIDs[rowIdx] == null) { - String[] deviceIdSegments = new String[tagColumnIndices.size() + 1]; - deviceIdSegments[0] = this.getTableName(); - for (int i = 0; i < tagColumnIndices.size(); i++) { - final Integer columnIndex = tagColumnIndices.get(i); - deviceIdSegments[i + 1] = - ((Object[]) getInsertRowNodeList().get(i).getValues()[columnIndex])[rowIdx].toString(); + final InsertRowNode insertRowNode = getInsertRowNodeList().get(rowIdx); + final List currentTagColumnIndices = getTagColumnIndices(insertRowNode); + String[] deviceIdSegments = new String[currentTagColumnIndices.size() + 1]; + final String tableName = insertRowNode.getTableName(); + deviceIdSegments[0] = tableName != null ? tableName : this.getTableName(); + for (int i = 0; i < currentTagColumnIndices.size(); i++) { + final Object idSegment = getValue(insertRowNode, currentTagColumnIndices.get(i)); + deviceIdSegments[i + 1] = idSegment != null ? idSegment.toString() : null; } deviceIDs[rowIdx] = Factory.DEFAULT_FACTORY.create(deviceIdSegments); } @@ -75,6 +78,24 @@ public IDeviceID getDeviceID(int rowIdx) { return deviceIDs[rowIdx]; } + private List getTagColumnIndices(final InsertRowNode insertRowNode) { + final TsTableColumnCategory[] columnCategories = insertRowNode.getColumnCategories(); + final List currentTagColumnIndices = new ArrayList<>(); + for (int i = 0; columnCategories != null && i < columnCategories.length; i++) { + if (columnCategories[i] == TsTableColumnCategory.TAG) { + currentTagColumnIndices.add(i); + } + } + return currentTagColumnIndices; + } + + private Object getValue(final InsertRowNode insertRowNode, final int columnIndex) { + final Object[] values = insertRowNode.getValues(); + return values != null && columnIndex >= 0 && columnIndex < values.length + ? values[columnIndex] + : null; + } + @Override public R accept(IPlanVisitor visitor, C context) { return ((PlanVisitor) visitor).visitRelationalInsertRows(this, context); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java index 11694e6d5ff1d..a951d1a513c05 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java @@ -118,7 +118,9 @@ public void setSingleDevice() { public List getObjectColumns() { List objectColumns = new ArrayList<>(); - for (int i = 0; i < columns.length; i++) { + for (int i = 0; + columns != null && dataTypes != null && i < columns.length && i < dataTypes.length; + i++) { if (dataTypes[i] == TSDataType.OBJECT) { objectColumns.add((Binary[]) columns[i]); } @@ -137,9 +139,8 @@ public IDeviceID getDeviceID(int rowIdx) { deviceIdSegments[0] = this.getTableName(); for (int i = 0; i < tagColumnIndices.size(); i++) { final Integer columnIndex = tagColumnIndices.get(i); - Object idSeg = ((Object[]) columns[columnIndex])[0]; - boolean isNull = - bitMaps != null && bitMaps[columnIndex] != null && bitMaps[columnIndex].isMarked(0); + Object idSeg = getColumnValue(columnIndex, 0); + boolean isNull = isNullValue(columnIndex, 0); deviceIdSegments[i + 1] = !isNull && idSeg != null ? idSeg.toString() : null; } deviceIDs[0] = Factory.DEFAULT_FACTORY.create(deviceIdSegments); @@ -154,11 +155,8 @@ public IDeviceID getDeviceID(int rowIdx) { deviceIdSegments[0] = this.getTableName(); for (int i = 0; i < tagColumnIndices.size(); i++) { final Integer columnIndex = tagColumnIndices.get(i); - Object idSeg = ((Object[]) columns[columnIndex])[rowIdx]; - boolean isNull = - bitMaps != null - && bitMaps[columnIndex] != null - && bitMaps[columnIndex].isMarked(rowIdx); + Object idSeg = getColumnValue(columnIndex, rowIdx); + boolean isNull = isNullValue(columnIndex, rowIdx); deviceIdSegments[i + 1] = !isNull && idSeg != null ? idSeg.toString() : null; } IDeviceID currentDeviceId = Factory.DEFAULT_FACTORY.create(deviceIdSegments); @@ -177,6 +175,38 @@ public R accept(IPlanVisitor visitor, C context) { return ((PlanVisitor) visitor).visitRelationalInsertTablet(this, context); } + @Override + protected boolean shouldSerializeMeasurement(final int index) { + return super.shouldSerializeMeasurement(index) && hasColumnCategory(index); + } + + private boolean hasColumnCategory(final int index) { + return columnCategories != null + && index >= 0 + && index < columnCategories.length + && columnCategories[index] != null; + } + + private Object getColumnValue(final int columnIndex, final int rowIndex) { + if (columns == null + || columnIndex < 0 + || columnIndex >= columns.length + || columns[columnIndex] == null + || !(columns[columnIndex] instanceof Object[])) { + return null; + } + final Object[] values = (Object[]) columns[columnIndex]; + return rowIndex >= 0 && rowIndex < values.length ? values[rowIndex] : null; + } + + private boolean isNullValue(final int columnIndex, final int rowIndex) { + return bitMaps != null + && columnIndex >= 0 + && columnIndex < bitMaps.length + && bitMaps[columnIndex] != null + && bitMaps[columnIndex].isMarked(rowIndex); + } + @Override protected InsertTabletNode getEmptySplit(int count) { long[] subTimes = new long[count]; @@ -249,7 +279,7 @@ public static RelationalInsertTabletNode deserialize(ByteBuffer byteBuffer) { @Override protected void serializeAttributes(ByteBuffer byteBuffer) { super.serializeAttributes(byteBuffer); - for (int i = 0; i < measurements.length; i++) { + for (int i = 0; measurements != null && i < measurements.length; i++) { if (shouldSerializeMeasurement(i)) { columnCategories[i].serialize(byteBuffer); } @@ -259,7 +289,7 @@ protected void serializeAttributes(ByteBuffer byteBuffer) { @Override protected void serializeAttributes(DataOutputStream stream) throws IOException { super.serializeAttributes(stream); - for (int i = 0; i < measurements.length; i++) { + for (int i = 0; measurements != null && i < measurements.length; i++) { if (shouldSerializeMeasurement(i)) { columnCategories[i].serialize(stream); } @@ -284,7 +314,7 @@ void subSerialize(IWALByteBufferView buffer, List rangeList) { @Override void subSerialize(IWALByteBufferView buffer, List rangeList, long encodedSearchIndex) { super.subSerialize(buffer, rangeList, encodedSearchIndex); - for (int i = 0; i < measurements.length; i++) { + for (int i = 0; measurements != null && i < measurements.length; i++) { if (shouldSerializeMeasurement(i)) { buffer.put(columnCategories[i].getCategory()); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TreeDeviceNormalSchema.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TreeDeviceNormalSchema.java index da37202c47043..a030946034a08 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TreeDeviceNormalSchema.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TreeDeviceNormalSchema.java @@ -67,7 +67,9 @@ public int update(final String[] measurements, final IMeasurementSchema[] schema for (int i = 0; i < length; ++i) { // Skip this to avoid instance creation/gc for writing performance - if (measurements[i] == null || measurementMap.containsKey(measurements[i])) { + if (measurements[i] == null + || schemas[i] == null + || measurementMap.containsKey(measurements[i])) { continue; } diff += putEntry(measurements[i], schemas[i], null); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TreeDeviceSchemaCacheManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TreeDeviceSchemaCacheManager.java index 93d75aeff2d8c..c2588606cea69 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TreeDeviceSchemaCacheManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TreeDeviceSchemaCacheManager.java @@ -169,17 +169,18 @@ public ClusterSchemaTree getMatchedNormalSchema(final PartialPath fullPath) { public List computeWithoutTemplate(final ISchemaComputation schemaComputation) { final List indexOfMissingMeasurements = new ArrayList<>(); final String[] measurements = schemaComputation.getMeasurements(); + if (measurements == null) { + return indexOfMissingMeasurements; + } final IDeviceSchema schema = tableDeviceSchemaCache.getDeviceSchema(schemaComputation.getDevicePath().getNodes()); if (!(schema instanceof TreeDeviceNormalSchema)) { - return IntStream.range(0, schemaComputation.getMeasurements().length) - .boxed() - .collect(Collectors.toList()); + return IntStream.range(0, measurements.length).boxed().collect(Collectors.toList()); } final TreeDeviceNormalSchema treeSchema = (TreeDeviceNormalSchema) schema; - for (int i = 0; i < schemaComputation.getMeasurements().length; i++) { + for (int i = 0; i < measurements.length; i++) { final SchemaCacheEntry value = treeSchema.getSchemaCacheEntry(measurements[i]); if (value == null) { indexOfMissingMeasurements.add(i); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertRow.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertRow.java index b5440ba1d8c4c..6acb68adad5aa 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertRow.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertRow.java @@ -80,10 +80,13 @@ public List getAttributeColumnNameList() { public List getAttributeValueList() { final InsertRowStatement insertRowStatement = getInnerTreeStatement(); final List attrColumnIndices = insertRowStatement.getAttrColumnIndices(); + final Object[] values = insertRowStatement.getValues(); Object[] attrValues = new Object[attrColumnIndices.size()]; for (int j = 0; j < attrColumnIndices.size(); j++) { final int columnIndex = attrColumnIndices.get(j); - attrValues[j] = insertRowStatement.getValues()[columnIndex]; + if (values != null && columnIndex >= 0 && columnIndex < values.length) { + attrValues[j] = values[columnIndex]; + } } return Collections.singletonList(attrValues); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertRows.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertRows.java index 0e53064e95282..4eb49df718360 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertRows.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertRows.java @@ -140,9 +140,17 @@ public List getAttributeColumnNameList() { @Override public List getAttributeValueList() { List attributeValueList = new ArrayList<>(); - for (int i = 0; i < insertRowStatement.getColumnCategories().length; i++) { - if (insertRowStatement.getColumnCategories()[i] == TsTableColumnCategory.ATTRIBUTE) { - attributeValueList.add(insertRowStatement.getValues()[i]); + final TsTableColumnCategory[] columnCategories = insertRowStatement.getColumnCategories(); + final String[] measurements = insertRowStatement.getMeasurements(); + final Object[] values = insertRowStatement.getValues(); + for (int i = 0; columnCategories != null && i < columnCategories.length; i++) { + if (columnCategories[i] == TsTableColumnCategory.ATTRIBUTE + && measurements != null + && i < measurements.length + && measurements[i] != null + && values != null + && i < values.length) { + attributeValueList.add(values[i]); } } return Collections.singletonList(attributeValueList.toArray()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertTablet.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertTablet.java index 5d25b01ecf348..639e35187a955 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertTablet.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertTablet.java @@ -103,9 +103,16 @@ public List getAttributeValueList() { deviceID2AttributeValues.get(insertTabletStatement.getTableDeviceID(rowIndex)); for (int attrColNum = 0; attrColNum < attrColumnIndices.size(); attrColNum++) { final int columnIndex = attrColumnIndices.get(attrColNum); - if (!insertTabletStatement.isNull(rowIndex, columnIndex)) { - attrValues[attrColNum] = - ((Object[]) insertTabletStatement.getColumns()[columnIndex])[rowIndex]; + final Object[] columns = insertTabletStatement.getColumns(); + if (!insertTabletStatement.isNull(rowIndex, columnIndex) + && columns != null + && columnIndex >= 0 + && columnIndex < columns.length + && columns[columnIndex] instanceof Object[]) { + final Object[] columnValues = (Object[]) columns[columnIndex]; + if (rowIndex < columnValues.length) { + attrValues[attrColNum] = columnValues[rowIndex]; + } } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WrappedInsertStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WrappedInsertStatement.java index f3151fbfe10fe..a476018a745ea 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WrappedInsertStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WrappedInsertStatement.java @@ -87,10 +87,11 @@ public InsertNodeMeasurementInfo getInsertNodeMeasurementInfo() { protected TableSchema toTableSchema(InsertBaseStatement insertBaseStatement) { String tableName = insertBaseStatement.getDevicePath().getFullPath(); + final String[] measurements = insertBaseStatement.getMeasurements(); List columnSchemas = - new ArrayList<>(insertBaseStatement.getMeasurements().length); - for (int i = 0; i < insertBaseStatement.getMeasurements().length; i++) { - if (insertBaseStatement.getMeasurements()[i] != null) { + new ArrayList<>(measurements == null ? 0 : measurements.length); + for (int i = 0; measurements != null && i < measurements.length; i++) { + if (insertBaseStatement.isColumnPresent(i)) { TSDataType dataType = insertBaseStatement.getDataType(i); if (dataType == null) { dataType = @@ -99,7 +100,7 @@ protected TableSchema toTableSchema(InsertBaseStatement insertBaseStatement) { } columnSchemas.add( new ColumnSchema( - insertBaseStatement.getMeasurements()[i], + measurements[i], dataType != null ? TypeFactory.getType(dataType) : null, false, insertBaseStatement.getColumnCategory(i))); @@ -113,12 +114,20 @@ protected TableSchema toTableSchema(InsertBaseStatement insertBaseStatement) { protected InsertNodeMeasurementInfo toInsertNodeMeasurementInfo( InsertBaseStatement insertBaseStatement) { String tableName = insertBaseStatement.getDevicePath().getFullPath().toLowerCase(); + final String[] measurements = insertBaseStatement.getMeasurements(); + final String[] validMeasurements = + measurements == null ? null : new String[measurements.length]; + for (int i = 0; measurements != null && i < measurements.length; i++) { + if (insertBaseStatement.isColumnPresent(i)) { + validMeasurements[i] = measurements[i]; + } + } // Use lazy initialization with measurements and dataTypes return new InsertNodeMeasurementInfo( tableName, insertBaseStatement.getColumnCategories(), - insertBaseStatement.getMeasurements(), + validMeasurements, insertBaseStatement.getDataTypes(), index -> TypeInferenceUtils.getPredictedDataType( @@ -243,6 +252,9 @@ void adjustTagColumns( if (baseStatement == null || existingTagColumnIndexMap.isEmpty()) { return; } + if (baseStatement.getMeasurements() == null) { + return; + } // Phase 1: Analyze and determine action in one pass final int oldLength = baseStatement.getMeasurements().length; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertBaseStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertBaseStatement.java index ddefab4d2d416..49d82ebf6a1f9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertBaseStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertBaseStatement.java @@ -166,7 +166,7 @@ public TSDataType[] getDataTypes() { } public TSDataType getDataType(int i) { - if (dataTypes == null) { + if (dataTypes == null || i < 0 || i >= dataTypes.length) { return null; } return dataTypes[i]; @@ -300,20 +300,33 @@ public void removeAllFailedMeasurementMarks() { } public boolean hasValidMeasurements() { - for (Object o : measurements) { - if (o != null) { + if (measurements == null) { + return false; + } + for (int i = 0; i < measurements.length; i++) { + if (isColumnPresent(i) + && (columnCategories == null + || i < columnCategories.length + && columnCategories[i] == TsTableColumnCategory.FIELD)) { return true; } } return false; } + public boolean isColumnPresent(final int index) { + return measurements != null + && index >= 0 + && index < measurements.length + && measurements[index] != null; + } + public TsTableColumnCategory[] getColumnCategories() { return columnCategories; } public TsTableColumnCategory getColumnCategory(int i) { - if (columnCategories == null) { + if (columnCategories == null || i < 0 || i >= columnCategories.length) { return null; } return columnCategories[i]; @@ -335,7 +348,7 @@ public List getTagColumnIndices() { if (tagColumnIndices == null && columnCategories != null) { tagColumnIndices = new ArrayList<>(); for (int i = 0; i < columnCategories.length; i++) { - if (columnCategories[i].equals(TsTableColumnCategory.TAG)) { + if (isColumnPresent(i) && columnCategories[i] == TsTableColumnCategory.TAG) { tagColumnIndices.add(i); } } @@ -347,7 +360,7 @@ public List getAttrColumnIndices() { if (attrColumnIndices == null && columnCategories != null) { attrColumnIndices = new ArrayList<>(); for (int i = 0; i < columnCategories.length; i++) { - if (columnCategories[i].equals(TsTableColumnCategory.ATTRIBUTE)) { + if (isColumnPresent(i) && columnCategories[i] == TsTableColumnCategory.ATTRIBUTE) { attrColumnIndices.add(i); } } @@ -434,7 +447,7 @@ public void removeAttributeColumns() { List columnsToKeep = new ArrayList<>(); for (int i = 0; i < columnCategories.length; i++) { - if (!columnCategories[i].equals(TsTableColumnCategory.ATTRIBUTE)) { + if (columnCategories[i] != TsTableColumnCategory.ATTRIBUTE) { columnsToKeep.add(i); } } @@ -452,13 +465,24 @@ public void removeAttributeColumns() { if (measurementSchemas != null) { measurementSchemas = - columnsToKeep.stream().map(i -> measurementSchemas[i]).toArray(MeasurementSchema[]::new); + columnsToKeep.stream() + .filter(i -> i < measurementSchemas.length) + .map(i -> measurementSchemas[i]) + .toArray(MeasurementSchema[]::new); } if (measurements != null) { - measurements = columnsToKeep.stream().map(i -> measurements[i]).toArray(String[]::new); + measurements = + columnsToKeep.stream() + .filter(i -> i < measurements.length) + .map(i -> measurements[i]) + .toArray(String[]::new); } if (dataTypes != null) { - dataTypes = columnsToKeep.stream().map(i -> dataTypes[i]).toArray(TSDataType[]::new); + dataTypes = + columnsToKeep.stream() + .filter(i -> i < dataTypes.length) + .map(i -> dataTypes[i]) + .toArray(TSDataType[]::new); } if (columnCategories != null) { columnCategories = @@ -796,8 +820,9 @@ public void toLowerCaseForDevicePath() { @TableModel public List getAttributeColumnNameList() { final List attributeColumnNameList = new ArrayList<>(); - for (int i = 0; i < getColumnCategories().length; i++) { - if (getColumnCategories()[i] == TsTableColumnCategory.ATTRIBUTE) { + final TsTableColumnCategory[] columnCategories = getColumnCategories(); + for (int i = 0; columnCategories != null && i < columnCategories.length; i++) { + if (isColumnPresent(i) && columnCategories[i] == TsTableColumnCategory.ATTRIBUTE) { attributeColumnNameList.add(getMeasurements()[i]); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java index d0ec0b19ee081..1f494b08deeab 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java @@ -26,6 +26,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.queryengine.plan.relational.metadata.ColumnSchema; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Statement; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.commons.schema.view.LogicalViewSchema; import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.conf.IoTDBDescriptor; @@ -99,7 +100,13 @@ public InsertRowStatement() { @Override public List getPaths() { List ret = new ArrayList<>(); + if (measurements == null) { + return ret; + } for (String m : measurements) { + if (m == null) { + continue; + } PartialPath fullPath = devicePath.concatAsMeasurementPath(m); ret.add(fullPath); } @@ -120,6 +127,13 @@ public Object[] getValues() { public void setValues(Object[] values) { this.values = values; + deviceID = null; + } + + @Override + public void setColumnCategories(TsTableColumnCategory[] columnCategories) { + super.setColumnCategories(columnCategories); + deviceID = null; } public boolean isNeedInferType() { @@ -193,9 +207,17 @@ public long getMinTime() { @Override public Object getFirstValueOfIndex(int index) { + if (values == null || index < 0 || index >= values.length) { + return null; + } return values[index]; } + @Override + public boolean isColumnPresent(final int index) { + return super.isColumnPresent(index) && values != null && index < values.length; + } + @Override protected boolean checkAndCastDataType(int columnIndex, TSDataType dataType) { if (dataType.isCompatible(dataTypes[columnIndex])) { @@ -213,8 +235,18 @@ protected boolean checkAndCastDataType(int columnIndex, TSDataType dataType) { */ @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning public void transferType(ZoneId zoneId) throws QueryProcessException { + if (measurementSchemas == null) { + return; + } for (int i = 0; i < measurementSchemas.length; i++) { + if (!isColumnPresent(i) + || values == null + || i >= values.length + || dataTypes == null + || i >= dataTypes.length) { + continue; + } // null when time series doesn't exist if (measurementSchemas[i] == null) { if (!IoTDBDescriptor.getInstance().getConfig().isEnablePartialInsert()) { @@ -266,7 +298,10 @@ public void transferType(ZoneId zoneId) throws QueryProcessException { @Override public void markFailedMeasurement(int index, Exception cause) { - if (measurements[index] == null) { + if (measurements == null + || index < 0 + || index >= measurements.length + || measurements[index] == null) { return; } @@ -276,12 +311,19 @@ public void markFailedMeasurement(int index, Exception cause) { InsertBaseStatement.FailedMeasurementInfo failedMeasurementInfo = new InsertBaseStatement.FailedMeasurementInfo( - measurements[index], dataTypes[index], values[index], cause); + measurements[index], + dataTypes != null && index < dataTypes.length ? dataTypes[index] : null, + values != null && index < values.length ? values[index] : null, + cause); failedMeasurementIndex2Info.putIfAbsent(index, failedMeasurementInfo); measurements[index] = null; - dataTypes[index] = null; - values[index] = null; + if (dataTypes != null && index < dataTypes.length) { + dataTypes[index] = null; + } + if (values != null && index < values.length) { + values[index] = null; + } } @Override @@ -291,15 +333,15 @@ public void removeAllFailedMeasurementMarks() { } failedMeasurementIndex2Info.forEach( (index, info) -> { - if (measurements != null) { + if (measurements != null && index < measurements.length) { measurements[index] = info.getMeasurement(); } - if (dataTypes != null) { + if (dataTypes != null && index < dataTypes.length) { dataTypes[index] = info.getDataType(); } - if (values != null) { + if (values != null && index < values.length) { values[index] = info.getValue(); } }); @@ -506,8 +548,11 @@ public IDeviceID getTableDeviceID() { deviceIdSegments[0] = this.getTableName(); for (int i = 0; i < getTagColumnIndices().size(); i++) { final Integer columnIndex = getTagColumnIndices().get(i); - deviceIdSegments[i + 1] = - values[columnIndex] != null ? values[columnIndex].toString() : null; + final Object idSegment = + values != null && columnIndex >= 0 && columnIndex < values.length + ? values[columnIndex] + : null; + deviceIdSegments[i + 1] = idSegment != null ? idSegment.toString() : null; } deviceID = Factory.DEFAULT_FACTORY.create(deviceIdSegments); } @@ -596,7 +641,7 @@ protected long calculateBytesUsed() { @Override protected void subRemoveAttributeColumns(List columnsToKeep) { if (values != null) { - values = columnsToKeep.stream().map(i -> values[i]).toArray(); + values = columnsToKeep.stream().filter(i -> i < values.length).map(i -> values[i]).toArray(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowsOfOneDeviceStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowsOfOneDeviceStatement.java index e4052e505607c..bfae0797a4aa5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowsOfOneDeviceStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowsOfOneDeviceStatement.java @@ -78,7 +78,13 @@ public void setInsertRowStatementList(List insertRowStatemen List measurementList = new ArrayList<>(); for (InsertRowStatement insertRowStatement : insertRowStatementList) { String[] measurements = insertRowStatement.getMeasurements(); + if (measurements == null) { + continue; + } for (String measurement : measurements) { + if (measurement == null) { + continue; + } if (!measurementSet.contains(measurement)) { measurementList.add(measurement); measurementSet.add(measurement); @@ -105,7 +111,13 @@ public R accept(StatementVisitor visitor, C context) { @Override public List getPaths() { List ret = new ArrayList<>(); + if (measurements == null) { + return ret; + } for (String m : measurements) { + if (m == null) { + continue; + } PartialPath fullPath = devicePath.concatAsMeasurementPath(m); ret.add(fullPath); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java index 8ce0d534a4980..a329e01ad45ce 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java @@ -220,6 +220,7 @@ public Object[] getColumns() { public void setColumns(Object[] columns) { this.columns = columns; + deviceIDs = null; } public BitMap[] getBitMaps() { @@ -228,6 +229,13 @@ public BitMap[] getBitMaps() { public void setBitMaps(BitMap[] bitMaps) { this.nullBitMaps = bitMaps; + deviceIDs = null; + } + + @Override + public void setColumnCategories(TsTableColumnCategory[] columnCategories) { + super.setColumnCategories(columnCategories); + deviceIDs = null; } public long[] getTimes() { @@ -275,7 +283,13 @@ public R accept(StatementVisitor visitor, C context) { @Override public List getPaths() { List ret = new ArrayList<>(); + if (measurements == null) { + return ret; + } for (String m : measurements) { + if (m == null) { + continue; + } PartialPath fullPath = devicePath.concatAsMeasurementPath(m); ret.add(fullPath); } @@ -304,7 +318,10 @@ protected boolean checkAndCastDataType(int columnIndex, TSDataType dataType) { @Override public void markFailedMeasurement(int index, Exception cause) { - if (measurements[index] == null) { + if (measurements == null + || index < 0 + || index >= measurements.length + || measurements[index] == null) { return; } @@ -314,12 +331,19 @@ public void markFailedMeasurement(int index, Exception cause) { InsertBaseStatement.FailedMeasurementInfo failedMeasurementInfo = new InsertBaseStatement.FailedMeasurementInfo( - measurements[index], dataTypes[index], columns[index], cause); + measurements[index], + dataTypes != null && index < dataTypes.length ? dataTypes[index] : null, + columns != null && index < columns.length ? columns[index] : null, + cause); failedMeasurementIndex2Info.putIfAbsent(index, failedMeasurementInfo); measurements[index] = null; - dataTypes[index] = null; - columns[index] = null; + if (dataTypes != null && index < dataTypes.length) { + dataTypes[index] = null; + } + if (columns != null && index < columns.length) { + columns[index] = null; + } } @Override @@ -329,9 +353,15 @@ public void removeAllFailedMeasurementMarks() { } failedMeasurementIndex2Info.forEach( (index, info) -> { - measurements[index] = info.getMeasurement(); - dataTypes[index] = info.getDataType(); - columns[index] = info.getValue(); + if (measurements != null && index < measurements.length) { + measurements[index] = info.getMeasurement(); + } + if (dataTypes != null && index < dataTypes.length) { + dataTypes[index] = info.getDataType(); + } + if (columns != null && index < columns.length) { + columns[index] = info.getValue(); + } }); failedMeasurementIndex2Info.clear(); } @@ -420,6 +450,15 @@ public long getMinTime() { @Override public Object getFirstValueOfIndex(int index) { + if (dataTypes == null + || columns == null + || index < 0 + || index >= dataTypes.length + || index >= columns.length + || dataTypes[index] == null + || columns[index] == null) { + return null; + } Object value; switch (dataTypes[index]) { case INT32: @@ -457,6 +496,14 @@ public Object getFirstValueOfIndex(int index) { return value; } + @Override + public boolean isColumnPresent(final int index) { + return super.isColumnPresent(index) + && columns != null + && index < columns.length + && columns[index] != null; + } + @Override public TSDataType getDataType(int index) { return dataTypes != null ? dataTypes[index] : null; @@ -562,9 +609,9 @@ public IDeviceID getTableDeviceID(int rowIdx) { deviceIdSegments[0] = this.getTableName(); for (int i = 0; i < getTagColumnIndices().size(); i++) { final Integer columnIndex = getTagColumnIndices().get(i); - boolean isNull = isNull(rowIdx, i); - deviceIdSegments[i + 1] = - isNull ? null : ((Object[]) columns[columnIndex])[rowIdx].toString(); + final Object idSegment = + isNull(rowIdx, columnIndex) ? null : getColumnValue(rowIdx, columnIndex); + deviceIdSegments[i + 1] = idSegment != null ? idSegment.toString() : null; } deviceIDs[rowIdx] = Factory.DEFAULT_FACTORY.create(deviceIdSegments); } @@ -701,19 +748,38 @@ protected long calculateBytesUsed() { } public boolean isNull(int row, int col) { - if (nullBitMaps == null || nullBitMaps[col] == null) { - return false; + return nullBitMaps != null + && row >= 0 + && col >= 0 + && col < nullBitMaps.length + && nullBitMaps[col] != null + && nullBitMaps[col].isMarked(row); + } + + private Object getColumnValue(final int rowIdx, final int columnIndex) { + if (columns == null + || columnIndex < 0 + || columnIndex >= columns.length + || columns[columnIndex] == null + || !(columns[columnIndex] instanceof Object[])) { + return null; } - return nullBitMaps[col].isMarked(row); + final Object[] values = (Object[]) columns[columnIndex]; + return rowIdx >= 0 && rowIdx < values.length ? values[rowIdx] : null; } @Override protected void subRemoveAttributeColumns(List columnsToKeep) { if (columns != null) { - columns = columnsToKeep.stream().map(i -> columns[i]).toArray(); + columns = + columnsToKeep.stream().filter(i -> i < columns.length).map(i -> columns[i]).toArray(); } if (nullBitMaps != null) { - nullBitMaps = columnsToKeep.stream().map(i -> nullBitMaps[i]).toArray(BitMap[]::new); + nullBitMaps = + columnsToKeep.stream() + .filter(i -> i < nullBitMaps.length) + .map(i -> nullBitMaps[i]) + .toArray(BitMap[]::new); } } @@ -743,10 +809,14 @@ public Tablet convertToTablet() throws MetadataException { final List schemas = new ArrayList<>(originalSchemaSize); final int[] validColumnIndices = new int[originalSchemaSize]; int validColumnCount = 0; - if (dataTypes != null) { + final Object[] statementColumns = this.getColumns(); + if (dataTypes != null && statementColumns != null) { final int dataTypeSize = Math.min(originalSchemaSize, dataTypes.length); for (int i = 0; i < dataTypeSize; i++) { - if (measurements[i] != null && dataTypes[i] != null) { + if (measurements[i] != null + && dataTypes[i] != null + && i < statementColumns.length + && statementColumns[i] != null) { final MeasurementSchema measurementSchema = measurementSchemas != null && i < measurementSchemas.length ? measurementSchemas[i] @@ -803,7 +873,6 @@ public Tablet convertToTablet() throws MetadataException { // Get values - convert Statement columns to Tablet format, only for valid columns // All arrays are copied to rowSize length - final Object[] statementColumns = this.getColumns(); final Object[] tabletValues = new Object[schemaSize]; if (statementColumns != null && statementColumns.length > 0) { for (int i = 0; i < schemaSize; i++) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index bbafc1c4cd915..12113e030b07d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -4787,7 +4787,8 @@ public void insert(InsertRowsNode insertRowsNode) DateTimeUtils.convertLongToDate(insertRowNode.getTime()), DateTimeUtils.convertLongToDate( CommonDateTimeUtils.currentTime() - ttl)))); - insertRowNode.setFailedMeasurementNumber(insertRowNode.getMeasurements().length); + insertRowNode.setFailedMeasurementNumber( + insertRowNode.getMeasurements() == null ? 0 : insertRowNode.getMeasurements().length); insertRowNode.setMeasurements(null); continue; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java index 866ef26d2d2f4..33064fe294975 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java @@ -201,11 +201,14 @@ public int insert(InsertRowNode insertRowNode) { List schemaList = new ArrayList<>(); List dataTypes = new ArrayList<>(); + Object[] writableValues = new Object[measurements == null ? 0 : measurements.length]; int nullPointsNumber = 0; - for (int i = 0; i < insertRowNode.getMeasurements().length; i++) { + for (int i = 0; measurements != null && i < measurements.length; i++) { + final boolean hasValue = values != null && i < values.length; + final Object value = hasValue ? values[i] : null; // Use measurements[i] to ignore failed partial insert - if (measurements[i] == null || values[i] == null || !isFieldMeasurement(insertRowNode, i)) { - if (isValidMeasurement(insertRowNode, i, true) && values[i] == null) { + if (!isValidMeasurement(insertRowNode, i, true) || !hasValue || value == null) { + if (hasValue && isValidMeasurement(insertRowNode, i, true) && value == null) { nullPointsNumber++; } schemaList.add(null); @@ -213,10 +216,12 @@ public int insert(InsertRowNode insertRowNode) { IMeasurementSchema schema = insertRowNode.getMeasurementSchemas()[i]; schemaList.add(schema); dataTypes.add(schema.getType()); + writableValues[i] = value; } } - memSize += MemUtils.getRowRecordSize(dataTypes, values, insertRowNode.getColumnCategories()); - write(insertRowNode.getDeviceID(), schemaList, insertRowNode.getTime(), values); + memSize += + MemUtils.getRowRecordSize(dataTypes, writableValues, insertRowNode.getColumnCategories()); + write(insertRowNode.getDeviceID(), schemaList, insertRowNode.getTime(), writableValues); int pointsInserted = getValidMeasurementNumber(insertRowNode, true) @@ -235,14 +240,14 @@ public int insertAlignedRow(InsertRowNode insertRowNode) { Object[] values = insertRowNode.getValues(); List schemaList = new ArrayList<>(); List dataTypes = new ArrayList<>(); + Object[] writableValues = new Object[measurements == null ? 0 : measurements.length]; int nullPointsNumber = 0; - for (int i = 0; i < insertRowNode.getMeasurements().length; i++) { + for (int i = 0; measurements != null && i < measurements.length; i++) { + final boolean hasValue = values != null && i < values.length; + final Object value = hasValue ? values[i] : null; // Use measurements[i] to ignore failed partial insert - if (measurements[i] == null - || values[i] == null - || insertRowNode.getColumnCategories() != null - && insertRowNode.getColumnCategories()[i] != TsTableColumnCategory.FIELD) { - if (isValidMeasurement(insertRowNode, i, true) && values[i] == null) { + if (!isValidMeasurement(insertRowNode, i, true) || !hasValue || value == null) { + if (hasValue && isValidMeasurement(insertRowNode, i, true) && value == null) { // do not include failed measurement to avoid a negative pointsInserted nullPointsNumber++; } @@ -252,13 +257,16 @@ public int insertAlignedRow(InsertRowNode insertRowNode) { IMeasurementSchema schema = insertRowNode.getMeasurementSchemas()[i]; schemaList.add(schema); dataTypes.add(schema.getType()); + writableValues[i] = value; } if (schemaList.isEmpty()) { return 0; } memSize += - MemUtils.getAlignedRowRecordSize(dataTypes, values, insertRowNode.getColumnCategories()); - writeAlignedRow(insertRowNode.getDeviceID(), schemaList, insertRowNode.getTime(), values); + MemUtils.getAlignedRowRecordSize( + dataTypes, writableValues, insertRowNode.getColumnCategories()); + writeAlignedRow( + insertRowNode.getDeviceID(), schemaList, insertRowNode.getTime(), writableValues); int pointsInserted = getValidMeasurementNumber(insertRowNode, true) - (IoTDBDescriptor.getInstance().getConfig().isIncludeNullValueInWriteThroughputMetric() @@ -317,14 +325,19 @@ private static int computeTabletNullPointsNumber( Object[] values = insertTabletNode.getBitMaps(); int nullPointsNumber = 0; if (values != null) { - for (int i = 0; i < insertTabletNode.getMeasurements().length; i++) { + for (int i = 0; + insertTabletNode.getMeasurements() != null + && i < insertTabletNode.getMeasurements().length; + i++) { if (!isValidMeasurement(insertTabletNode, i, countFieldOnly) + || insertTabletNode.getColumns() == null + || i >= insertTabletNode.getColumns().length || insertTabletNode.getColumns()[i] == null) { // do not include failed measurement to avoid a negative pointsInserted continue; } - BitMap bitMap = (BitMap) values[i]; + BitMap bitMap = i < values.length ? (BitMap) values[i] : null; if (bitMap != null && !bitMap.isAllUnmarked()) { for (int j = start; j < end; j++) { if (bitMap.isMarked(j)) { @@ -339,8 +352,12 @@ private static int computeTabletNullPointsNumber( private static int getValidMeasurementNumber(InsertRowNode insertNode, boolean countFieldOnly) { int count = 0; - for (int i = 0; i < insertNode.getMeasurements().length; i++) { - if (isValidMeasurement(insertNode, i, countFieldOnly)) { + for (int i = 0; + insertNode.getMeasurements() != null && i < insertNode.getMeasurements().length; + i++) { + if (insertNode.getValues() != null + && i < insertNode.getValues().length + && isValidMeasurement(insertNode, i, countFieldOnly)) { count++; } } @@ -350,8 +367,13 @@ private static int getValidMeasurementNumber(InsertRowNode insertNode, boolean c private static int getValidMeasurementNumber( InsertTabletNode insertNode, boolean countFieldOnly) { int count = 0; - for (int i = 0; i < insertNode.getMeasurements().length; i++) { - if (isValidMeasurement(insertNode, i, countFieldOnly) && insertNode.getColumns()[i] != null) { + for (int i = 0; + insertNode.getMeasurements() != null && i < insertNode.getMeasurements().length; + i++) { + if (isValidMeasurement(insertNode, i, countFieldOnly) + && insertNode.getColumns() != null + && i < insertNode.getColumns().length + && insertNode.getColumns()[i] != null) { count++; } } @@ -360,13 +382,20 @@ private static int getValidMeasurementNumber( private static boolean isValidMeasurement( InsertNode insertNode, int index, boolean countFieldOnly) { - return insertNode.getMeasurements()[index] != null + return insertNode.getMeasurements() != null + && index >= 0 + && index < insertNode.getMeasurements().length + && insertNode.getMeasurements()[index] != null + && insertNode.getMeasurementSchemas() != null + && index < insertNode.getMeasurementSchemas().length + && insertNode.getMeasurementSchemas()[index] != null && (!countFieldOnly || isFieldMeasurement(insertNode, index)); } private static boolean isFieldMeasurement(InsertNode insertNode, int index) { return insertNode.getColumnCategories() == null - || insertNode.getColumnCategories()[index] == TsTableColumnCategory.FIELD; + || index < insertNode.getColumnCategories().length + && insertNode.getColumnCategories()[index] == TsTableColumnCategory.FIELD; } @Override @@ -393,21 +422,33 @@ public void writeAlignedRow( public void writeTabletNode(InsertTabletNode insertTabletNode, int start, int end) { List schemaList = new ArrayList<>(); - for (int i = 0; i < insertTabletNode.getMeasurementSchemas().length; i++) { - if (insertTabletNode.getMeasurements()[i] == null - || insertTabletNode.getColumns()[i] == null - || !isFieldMeasurement(insertTabletNode, i)) { + final int measurementSize = + insertTabletNode.getMeasurements() == null ? 0 : insertTabletNode.getMeasurements().length; + final Object[] writableColumns = new Object[measurementSize]; + final BitMap[] writableBitMaps = + insertTabletNode.getBitMaps() == null ? null : new BitMap[measurementSize]; + for (int i = 0; i < measurementSize; i++) { + if (!isValidMeasurement(insertTabletNode, i, true) + || insertTabletNode.getColumns() == null + || i >= insertTabletNode.getColumns().length + || insertTabletNode.getColumns()[i] == null) { schemaList.add(null); } else { schemaList.add(insertTabletNode.getMeasurementSchemas()[i]); + writableColumns[i] = insertTabletNode.getColumns()[i]; + if (writableBitMaps != null + && insertTabletNode.getBitMaps() != null + && i < insertTabletNode.getBitMaps().length) { + writableBitMaps[i] = insertTabletNode.getBitMaps()[i]; + } } } IWritableMemChunkGroup memChunkGroup = createMemChunkGroupIfNotExistAndGet(insertTabletNode.getDeviceID(), schemaList); memChunkGroup.writeTablet( insertTabletNode.getTimes(), - insertTabletNode.getColumns(), - insertTabletNode.getBitMaps(), + writableColumns, + writableBitMaps, schemaList, start, end, @@ -417,14 +458,25 @@ public void writeTabletNode(InsertTabletNode insertTabletNode, int start, int en public void writeAlignedTablet( InsertTabletNode insertTabletNode, int start, int end, TSStatus[] results) { List schemaList = new ArrayList<>(); - for (int i = 0; i < insertTabletNode.getMeasurementSchemas().length; i++) { - if (insertTabletNode.getMeasurements()[i] == null - || insertTabletNode.getColumns()[i] == null - || (insertTabletNode.getColumnCategories() != null - && insertTabletNode.getColumnCategories()[i] != TsTableColumnCategory.FIELD)) { + final int measurementSize = + insertTabletNode.getMeasurements() == null ? 0 : insertTabletNode.getMeasurements().length; + final Object[] writableColumns = new Object[measurementSize]; + final BitMap[] writableBitMaps = + insertTabletNode.getBitMaps() == null ? null : new BitMap[measurementSize]; + for (int i = 0; i < measurementSize; i++) { + if (!isValidMeasurement(insertTabletNode, i, true) + || insertTabletNode.getColumns() == null + || i >= insertTabletNode.getColumns().length + || insertTabletNode.getColumns()[i] == null) { schemaList.add(null); } else { schemaList.add(insertTabletNode.getMeasurementSchemas()[i]); + writableColumns[i] = insertTabletNode.getColumns()[i]; + if (writableBitMaps != null + && insertTabletNode.getBitMaps() != null + && i < insertTabletNode.getBitMaps().length) { + writableBitMaps[i] = insertTabletNode.getBitMaps()[i]; + } } } if (schemaList.isEmpty()) { @@ -440,8 +492,8 @@ public void writeAlignedTablet( createAlignedMemChunkGroupIfNotExistAndGet(deviceID, schemaList); memChunkGroup.writeTablet( insertTabletNode.getTimes(), - insertTabletNode.getColumns(), - insertTabletNode.getBitMaps(), + writableColumns, + writableBitMaps, schemaList, splitStart, splitEnd, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java index 54adec65b97e3..2572cbec5020d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java @@ -691,7 +691,7 @@ private long[] checkMemCostAndAddToTspInfoForRow( long textDataIncrement = 0L; long chunkMetadataIncrement = 0L; - for (int i = 0; i < dataTypes.length; i++) { + for (int i = 0; dataTypes != null && i < dataTypes.length; i++) { // Skip failed Measurements if (!isWritableFieldMeasurement(measurements, dataTypes, values, columnCategories, i)) { continue; @@ -731,7 +731,7 @@ private long[] checkMemCostAndAddToTspInfoForRows(List insertRowN TSDataType[] dataTypes = insertRowNode.getDataTypes(); Object[] values = insertRowNode.getValues(); String[] measurements = insertRowNode.getMeasurements(); - for (int i = 0; i < dataTypes.length; i++) { + for (int i = 0; dataTypes != null && i < dataTypes.length; i++) { // Skip failed Measurements if (!isWritableFieldMeasurement( measurements, dataTypes, values, insertRowNode.getColumnCategories(), i)) { @@ -800,7 +800,7 @@ private long[] checkAlignedMemCostAndAddToTspInfoForRow( // For existed device of this mem table AlignedWritableMemChunk alignedMemChunk = (AlignedWritableMemChunk) memChunk; List dataTypesInTVList = new ArrayList<>(); - for (int i = 0; i < dataTypes.length; i++) { + for (int i = 0; dataTypes != null && i < dataTypes.length; i++) { // Skip failed Measurements if (!isWritableFieldMeasurement(measurements, dataTypes, values, columnCategories, i)) { continue; @@ -824,7 +824,7 @@ private long[] checkAlignedMemCostAndAddToTspInfoForRow( } } - for (int i = 0; i < dataTypes.length; i++) { + for (int i = 0; dataTypes != null && i < dataTypes.length; i++) { // TEXT data mem size if (isWritableFieldMeasurement(measurements, dataTypes, values, columnCategories, i) && dataTypes[i].isBinary()) { @@ -864,7 +864,7 @@ private long[] checkAlignedMemCostAndAddToTspInfoForRows(List ins ChunkMetadata.calculateRamSize(AlignedPath.VECTOR_PLACEHOLDER, TSDataType.VECTOR) * writableFieldDataTypes.length; memTableIncrement += AlignedTVList.alignedTvListArrayMemCost(writableFieldDataTypes, null); - for (int i = 0; i < dataTypes.length; i++) { + for (int i = 0; dataTypes != null && i < dataTypes.length; i++) { // Skip failed Measurements if (!isWritableFieldMeasurement( measurements, dataTypes, values, insertRowNode.getColumnCategories(), i)) { @@ -881,7 +881,7 @@ private long[] checkAlignedMemCostAndAddToTspInfoForRows(List ins List dataTypesInTVList = new ArrayList<>(); Pair, Integer> addingPointNumInfo = increasingMemTableInfo.computeIfAbsent(deviceId, k -> new Pair<>(new HashMap<>(), 0)); - for (int i = 0; i < dataTypes.length; i++) { + for (int i = 0; dataTypes != null && i < dataTypes.length; i++) { // Skip failed Measurements if (!isWritableFieldMeasurement( measurements, dataTypes, values, insertRowNode.getColumnCategories(), i)) { @@ -921,7 +921,7 @@ private long[] checkAlignedMemCostAndAddToTspInfoForRows(List ins addingPointNumInfo.setRight(addingPointNum + 1); } - for (int i = 0; i < dataTypes.length; i++) { + for (int i = 0; dataTypes != null && i < dataTypes.length; i++) { // Skip failed Measurements if (!isWritableFieldMeasurement( measurements, dataTypes, values, insertRowNode.getColumnCategories(), i)) { @@ -951,7 +951,7 @@ private long[] checkMemCostAndAddToTspInfoForTablet( } long[] memIncrements = new long[3]; // memTable, text, chunk metadata - for (int i = 0; i < dataTypes.length; i++) { + for (int i = 0; dataTypes != null && i < dataTypes.length; i++) { // Skip failed Measurements if (!isWritableFieldMeasurement(measurements, dataTypes, columns, columnCategories, i)) { continue; @@ -1085,7 +1085,7 @@ private void updateAlignedMemCost( List dataTypesInTVList = new ArrayList<>(); int currentPointNum = alignedMemChunk.alignedListSize(); int newPointNum = currentPointNum + incomingPointNum; - for (int i = 0; i < dataTypes.length; i++) { + for (int i = 0; dataTypes != null && i < dataTypes.length; i++) { TSDataType dataType = dataTypes[i]; if (!isWritableFieldMeasurement(measurementIds, dataTypes, columns, columnCategories, i)) { continue; @@ -1118,7 +1118,7 @@ private void updateAlignedMemCost( } // flexible-length data size - for (int i = 0; i < dataTypes.length; i++) { + for (int i = 0; dataTypes != null && i < dataTypes.length; i++) { TSDataType dataType = dataTypes[i]; if (!isWritableFieldMeasurement(measurementIds, dataTypes, columns, columnCategories, i)) { continue; @@ -1137,7 +1137,7 @@ private static TSDataType[] getWritableFieldDataTypes( Object[] valuesOrColumns, TsTableColumnCategory[] columnCategories) { List writableFieldDataTypes = new ArrayList<>(); - for (int i = 0; i < dataTypes.length; i++) { + for (int i = 0; dataTypes != null && i < dataTypes.length; i++) { if (isWritableFieldMeasurement( measurementIds, dataTypes, valuesOrColumns, columnCategories, i)) { writableFieldDataTypes.add(dataTypes[i]); @@ -1153,6 +1153,8 @@ private static boolean isWritableFieldMeasurement( TsTableColumnCategory[] columnCategories, int index) { return isFieldMeasurement(measurementIds, dataTypes, columnCategories, index) + && valuesOrColumns != null + && index < valuesOrColumns.length && valuesOrColumns[index] != null; } @@ -1161,9 +1163,16 @@ private static boolean isFieldMeasurement( TSDataType[] dataTypes, TsTableColumnCategory[] columnCategories, int index) { - return dataTypes[index] != null + return measurementIds != null + && dataTypes != null + && index >= 0 + && index < measurementIds.length + && index < dataTypes.length + && dataTypes[index] != null && measurementIds[index] != null - && (columnCategories == null || columnCategories[index] == TsTableColumnCategory.FIELD); + && (columnCategories == null + || index < columnCategories.length + && columnCategories[index] == TsTableColumnCategory.FIELD); } private void updateMemoryInfo( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusLogToTabletConverter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusLogToTabletConverter.java index ad247680cb818..300c3c792e940 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusLogToTabletConverter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusLogToTabletConverter.java @@ -230,7 +230,8 @@ private List convertInsertRowNode(final InsertRowNode node) { final String[] measurements = node.getMeasurements(); final TSDataType[] dataTypes = node.getDataTypes(); final Object[] values = node.getValues(); - final List matchedColumnIndices = getMatchedTreeColumnIndices(deviceId, measurements); + final List matchedColumnIndices = + getMatchedTreeColumnIndices(deviceId, measurements, dataTypes, values, false); if (matchedColumnIndices.isEmpty()) { return Collections.emptyList(); @@ -283,7 +284,8 @@ private List convertInsertTabletNode(final InsertTabletNode node) { final int rowCount = node.getRowCount(); // Column filtering - final List matchedColumnIndices = getMatchedTreeColumnIndices(deviceId, measurements); + final List matchedColumnIndices = + getMatchedTreeColumnIndices(deviceId, measurements, dataTypes, columns, true); if (matchedColumnIndices.isEmpty()) { return Collections.emptyList(); } @@ -378,7 +380,9 @@ private MatchedTreeRow matchTreeInsertRowNode(final InsertRowNode node) { final String[] measurements = node.getMeasurements(); final TSDataType[] dataTypes = node.getDataTypes(); - final List matchedColumnIndices = getMatchedTreeColumnIndices(deviceId, measurements); + final Object[] values = node.getValues(); + final List matchedColumnIndices = + getMatchedTreeColumnIndices(deviceId, measurements, dataTypes, values, false); return matchedColumnIndices.isEmpty() ? null : new MatchedTreeRow( @@ -455,7 +459,8 @@ private List convertRelationalInsertRowNode(final RelationalInsertRowNod final TSDataType[] dataTypes = node.getDataTypes(); final Object[] values = node.getValues(); final List matchedColumnIndices = - getMatchedTableColumnIndices(measurements, node.getColumnCategories()); + getMatchedTableColumnIndices( + measurements, dataTypes, values, node.getColumnCategories(), false); if (matchedColumnIndices.isEmpty()) { return Collections.emptyList(); } @@ -512,7 +517,8 @@ private List convertRelationalInsertTabletNode(final RelationalInsertTab final BitMap[] bitMaps = node.getBitMaps(); final int rowCount = node.getRowCount(); final List matchedColumnIndices = - getMatchedTableColumnIndices(measurements, node.getColumnCategories()); + getMatchedTableColumnIndices( + measurements, dataTypes, columns, node.getColumnCategories(), true); if (matchedColumnIndices.isEmpty()) { return Collections.emptyList(); } @@ -568,12 +574,19 @@ private List convertRelationalInsertRowsNode(final RelationalInsertRowsN * column indices are returned. */ private List getMatchedTreeColumnIndices( - final IDeviceID deviceId, final String[] measurements) { + final IDeviceID deviceId, + final String[] measurements, + final TSDataType[] dataTypes, + final Object[] valuesOrColumns, + final boolean requireNonNullValue) { + if (measurements == null) { + return Collections.emptyList(); + } if (treePattern == null || treePattern.isRoot() || treePattern.coversDevice(deviceId)) { // All columns match final List allIndices = new ArrayList<>(measurements.length); for (int i = 0; i < measurements.length; i++) { - if (measurements[i] != null) { + if (isValidColumn(measurements, dataTypes, valuesOrColumns, i, requireNonNullValue)) { allIndices.add(i); } } @@ -582,7 +595,8 @@ private List getMatchedTreeColumnIndices( final List matchedIndices = new ArrayList<>(); for (int i = 0; i < measurements.length; i++) { - if (measurements[i] != null && treePattern.matchesMeasurement(deviceId, measurements[i])) { + if (isValidColumn(measurements, dataTypes, valuesOrColumns, i, requireNonNullValue) + && treePattern.matchesMeasurement(deviceId, measurements[i])) { matchedIndices.add(i); } } @@ -595,11 +609,18 @@ private List getMatchedTreeColumnIndices( * If no table column pattern is specified, all non-null columns are returned. */ private List getMatchedTableColumnIndices( - final String[] measurements, final TsTableColumnCategory[] columnCategories) { + final String[] measurements, + final TSDataType[] dataTypes, + final Object[] valuesOrColumns, + final TsTableColumnCategory[] columnCategories, + final boolean requireNonNullValue) { + if (measurements == null) { + return Collections.emptyList(); + } final boolean[] selectedColumns = new boolean[measurements.length]; boolean hasMatchedColumn = false; for (int i = 0; i < measurements.length; i++) { - if (measurements[i] == null) { + if (!isValidColumn(measurements, dataTypes, valuesOrColumns, i, requireNonNullValue)) { continue; } if (tableColumnPattern == null || tableColumnPattern.matcher(measurements[i]).matches()) { @@ -613,7 +634,8 @@ private List getMatchedTableColumnIndices( } for (int i = 0; i < measurements.length; i++) { - if (measurements[i] != null && isTagColumn(columnCategories, i)) { + if (isValidColumn(measurements, dataTypes, valuesOrColumns, i, requireNonNullValue) + && isTagColumn(columnCategories, i)) { selectedColumns[i] = true; } } @@ -629,16 +651,38 @@ private List getMatchedTableColumnIndices( private boolean isTagColumn( final TsTableColumnCategory[] columnCategories, final int columnIndex) { - return columnCategories != null && columnCategories[columnIndex] == TsTableColumnCategory.TAG; + return columnCategories != null + && columnIndex < columnCategories.length + && columnCategories[columnIndex] == TsTableColumnCategory.TAG; } private ColumnCategory toTsFileColumnCategory( final TsTableColumnCategory[] columnCategories, final int columnIndex) { - return columnCategories != null && columnCategories[columnIndex] != null + return columnCategories != null + && columnIndex < columnCategories.length + && columnCategories[columnIndex] != null ? columnCategories[columnIndex].toTsFileColumnType() : ColumnCategory.FIELD; } + private boolean isValidColumn( + final String[] measurements, + final TSDataType[] dataTypes, + final Object[] valuesOrColumns, + final int index, + final boolean requireNonNullValue) { + return measurements != null + && index >= 0 + && index < measurements.length + && measurements[index] != null + && dataTypes != null + && index < dataTypes.length + && dataTypes[index] != null + && valuesOrColumns != null + && index < valuesOrColumns.length + && (!requireNonNullValue || valuesOrColumns[index] != null); + } + /** * Adds a single value to the tablet at the specified position. * diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/trigger/executor/TriggerFireVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/trigger/executor/TriggerFireVisitor.java index 6fc369d89e34c..fab71cf8014b6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/trigger/executor/TriggerFireVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/trigger/executor/TriggerFireVisitor.java @@ -26,6 +26,7 @@ import org.apache.iotdb.commons.consensus.ConfigRegionId; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.commons.trigger.TriggerInformation; import org.apache.iotdb.commons.trigger.TriggerTable; import org.apache.iotdb.commons.trigger.exception.TriggerExecutionException; @@ -272,17 +273,20 @@ private Map constructMeasurementToSchemaIndexMap( // The index of measurement and schema is the same now. // However, in case one day the order changes, we need to construct an index map. Map indexMap = new HashMap<>(); + if (measurements == null || schemas == null) { + return indexMap; + } for (int i = 0, n = measurements.length; i < n; i++) { - if (measurements[i] == null) { + if (measurements[i] == null || i >= schemas.length) { continue; } // It is the same now - if (schemas[i] != null && schemas[i].getMeasurementName().equals(measurements[i])) { + if (schemas[i] != null && measurements[i].equals(schemas[i].getMeasurementName())) { indexMap.put(measurements[i], i); continue; } for (int j = 0, m = schemas.length; j < m; j++) { - if (schemas[j] != null && schemas[j].getMeasurementName().equals(measurements[i])) { + if (schemas[j] != null && measurements[i].equals(schemas[j].getMeasurementName())) { indexMap.put(measurements[i], j); break; } @@ -299,9 +303,13 @@ private Map> constructTriggerNameToMeasurementListMap( return Collections.emptyMap(); } List measurements = new ArrayList<>(); - for (String measurement : node.getMeasurements()) { - if (measurement != null) { - measurements.add(measurement); + final String[] nodeMeasurements = node.getMeasurements(); + if (nodeMeasurements == null) { + return Collections.emptyMap(); + } + for (int i = 0; i < nodeMeasurements.length; i++) { + if (isTriggerableMeasurement(node, i)) { + measurements.add(nodeMeasurements[i]); } } @@ -334,6 +342,39 @@ private Map> constructTriggerNameToMeasurementListMap( return triggerNameToPaths; } + private boolean isTriggerableMeasurement(final InsertNode node, final int index) { + final String[] measurements = node.getMeasurements(); + final MeasurementSchema[] schemas = node.getMeasurementSchemas(); + if (measurements == null + || index < 0 + || index >= measurements.length + || measurements[index] == null + || schemas == null + || index >= schemas.length + || schemas[index] == null + || schemas[index].getMeasurementName() == null + || schemas[index].getType() == null + || !isFieldMeasurement(node, index)) { + return false; + } + if (node instanceof InsertTabletNode) { + final Object[] columns = ((InsertTabletNode) node).getColumns(); + return columns != null && index < columns.length && columns[index] != null; + } + if (node instanceof InsertRowNode) { + final Object[] values = ((InsertRowNode) node).getValues(); + return values != null && index < values.length; + } + return true; + } + + private boolean isFieldMeasurement(final InsertNode node, final int index) { + final TsTableColumnCategory[] columnCategories = node.getColumnCategories(); + return columnCategories == null + || index < columnCategories.length + && columnCategories[index] == TsTableColumnCategory.FIELD; + } + private TriggerFireResult fire(String triggerName, Tablet tablet, TriggerEvent event) { TriggerFireResult result = TriggerFireResult.SUCCESS; for (int i = 0; i < FIRE_RETRY_NUM; i++) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/MemUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/MemUtils.java index 09b836db3ccf5..3d60d664d2460 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/MemUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/MemUtils.java @@ -72,9 +72,12 @@ public static long getRowRecordSize(List dataTypes, Object[] value) public static long getRowRecordSize( List dataTypes, Object[] value, TsTableColumnCategory[] columnCategories) { + if (dataTypes == null) { + return 0L; + } int dataTypeIndex = 0; long memSize = 0L; - for (int i = 0; i < value.length; i++) { + for (int i = 0; value != null && i < value.length && dataTypeIndex < dataTypes.size(); i++) { if (value[i] == null || !isFieldColumn(columnCategories, i)) { continue; } @@ -89,10 +92,13 @@ public static long getRowRecordSize( */ public static long getAlignedRowRecordSize( List dataTypes, Object[] value, TsTableColumnCategory[] columnCategories) { + if (dataTypes == null) { + return 8L + 4L; + } // time and index size long memSize = 8L + 4L; int dataTypeIndex = 0; - for (int i = 0; i < value.length; i++) { + for (int i = 0; value != null && i < value.length && dataTypeIndex < dataTypes.size(); i++) { if (value[i] == null || !isFieldColumn(columnCategories, i)) { continue; } @@ -132,7 +138,9 @@ public static long getTabletSize(InsertTabletNode insertTabletNode, int start, i return 0L; } long memSize = 0; - for (int i = 0; i < insertTabletNode.getMeasurements().length; i++) { + for (int i = 0; + insertTabletNode.getMeasurements() != null && i < insertTabletNode.getMeasurements().length; + i++) { if (!isWritableTabletMeasurement(insertTabletNode, i, true)) { continue; } @@ -149,7 +157,9 @@ public static long getAlignedTabletSize( return 0L; } long memSize = 0; - for (int i = 0; i < insertTabletNode.getMeasurements().length; i++) { + for (int i = 0; + insertTabletNode.getMeasurements() != null && i < insertTabletNode.getMeasurements().length; + i++) { if (!isWritableTabletMeasurement(insertTabletNode, i, true)) { continue; } @@ -171,16 +181,27 @@ public static long getAlignedTabletSize( private static boolean isWritableTabletMeasurement( InsertTabletNode insertTabletNode, int index, boolean fieldOnly) { - return insertTabletNode.getMeasurements()[index] != null + return insertTabletNode.getMeasurements() != null + && index >= 0 + && index < insertTabletNode.getMeasurements().length + && insertTabletNode.getMeasurements()[index] != null + && insertTabletNode.getColumns() != null + && index < insertTabletNode.getColumns().length && insertTabletNode.getColumns()[index] != null + && insertTabletNode.getDataTypes() != null + && index < insertTabletNode.getDataTypes().length && insertTabletNode.getDataTypes()[index] != null && insertTabletNode.getMeasurementSchemas() != null + && index < insertTabletNode.getMeasurementSchemas().length && insertTabletNode.getMeasurementSchemas()[index] != null && (!fieldOnly || isFieldColumn(insertTabletNode.getColumnCategories(), index)); } private static boolean isFieldColumn(TsTableColumnCategory[] columnCategories, int columnIndex) { - return columnCategories == null || columnCategories[columnIndex] == TsTableColumnCategory.FIELD; + return columnCategories == null + || columnIndex >= 0 + && columnIndex < columnCategories.length + && columnCategories[columnIndex] == TsTableColumnCategory.FIELD; } /** Calculate how much memory will be used if the given record is written to sequence file. */ diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/util/TabletStatementConverterTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/util/TabletStatementConverterTest.java index 8eff4a591916e..0a954ef4b5e97 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/util/TabletStatementConverterTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/util/TabletStatementConverterTest.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.pipe.sink.util; import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; import org.apache.tsfile.enums.ColumnCategory; @@ -208,6 +209,33 @@ public void testDeserializeStatementFromTabletFormatWithNullSchemaAndNullColumn( Assert.assertTrue(statement.isAligned()); } + @Test + public void testConvertStatementToTabletSkipsNullColumn() throws Exception { + final InsertTabletStatement statement = new InsertTabletStatement(); + statement.setDevicePath(new PartialPath("root.sg.device")); + statement.setTimes(new long[] {1L, 2L}); + statement.setRowCount(2); + statement.setMeasurements(new String[] {"s1", "s2", "s3"}); + statement.setDataTypes( + new TSDataType[] {TSDataType.INT32, TSDataType.INT64, TSDataType.DOUBLE}); + statement.setMeasurementSchemas( + new MeasurementSchema[] { + new MeasurementSchema("s1", TSDataType.INT32), + new MeasurementSchema("s2", TSDataType.INT64), + new MeasurementSchema("s3", TSDataType.DOUBLE) + }); + statement.setColumns(new Object[] {new int[] {1, 2}, null, new double[] {1.0, 2.0}}); + + final Tablet convertedTablet = statement.convertToTablet(); + + Assert.assertEquals(2, convertedTablet.getSchemas().size()); + Assert.assertEquals("s1", convertedTablet.getSchemas().get(0).getMeasurementName()); + Assert.assertEquals("s3", convertedTablet.getSchemas().get(1).getMeasurementName()); + Assert.assertArrayEquals(new int[] {1, 2}, (int[]) convertedTablet.getValues()[0]); + Assert.assertArrayEquals( + new double[] {1.0, 2.0}, (double[]) convertedTablet.getValues()[1], 0.0); + } + /** * Generate a Tablet for tree model with all data types and specified number of columns and rows. * diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertRowNodeSerdeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertRowNodeSerdeTest.java index a2d28762103c5..5a5f2d9d35d8c 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertRowNodeSerdeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertRowNodeSerdeTest.java @@ -176,6 +176,114 @@ public void testRelationalSerializedSizeWithFailedMeasurement() { Assert.assertEquals(insertRowNode.serializedSize(), byteBuffer.position()); } + @Test + public void testSerializeSkipsRetainedMeasurementWithMissingValue() throws IllegalPathException { + InsertRowNode insertRowNode = + new InsertRowNode( + new PlanNodeId("plannode missing value"), + new PartialPath("root.sg.d1"), + false, + new String[] {"s1", "s2", "s3"}, + new TSDataType[] {TSDataType.INT32, TSDataType.INT32, TSDataType.INT32}, + 1L, + new Object[] {1, 2}, + false); + + ByteBuffer byteBuffer = ByteBuffer.allocate(10000); + insertRowNode.serialize(byteBuffer); + byteBuffer.flip(); + + Assert.assertEquals(PlanNodeType.INSERT_ROW.getNodeType(), byteBuffer.getShort()); + + InsertRowNode tmpNode = InsertRowNode.deserialize(byteBuffer); + Assert.assertArrayEquals(new String[] {"s1", "s2"}, tmpNode.getMeasurements()); + } + + @Test + public void testSerializeKeepsNullRowValueWithoutType() throws IllegalPathException { + InsertRowNode insertRowNode = + new InsertRowNode( + new PlanNodeId("plannode null value"), + new PartialPath("root.sg.d1"), + false, + new String[] {"s1", "s2"}, + new TSDataType[] {TSDataType.INT32, null}, + 1L, + new Object[] {1, null}, + false); + + ByteBuffer byteBuffer = ByteBuffer.allocate(10000); + insertRowNode.serialize(byteBuffer); + byteBuffer.flip(); + + Assert.assertEquals(PlanNodeType.INSERT_ROW.getNodeType(), byteBuffer.getShort()); + + InsertRowNode tmpNode = InsertRowNode.deserialize(byteBuffer); + Assert.assertArrayEquals(new String[] {"s1", "s2"}, tmpNode.getMeasurements()); + Assert.assertArrayEquals(new TSDataType[] {TSDataType.INT32, null}, tmpNode.getDataTypes()); + Assert.assertArrayEquals(new Object[] {1, null}, tmpNode.getValues()); + } + + @Test + public void testDeserializeFromWALSkipsRetainedMeasurementWithNullSchema() + throws IllegalPathException, IOException { + InsertRowNode insertRowNode = getInsertRowNodeWithMeasurementSchemas(); + insertRowNode.getMeasurementSchemas()[1] = null; + + byte[] bytes = new byte[insertRowNode.serializedSize()]; + WALByteBufferForTest walBuffer = new WALByteBufferForTest(ByteBuffer.wrap(bytes)); + insertRowNode.serializeToWAL(walBuffer); + Assert.assertFalse(walBuffer.getBuffer().hasRemaining()); + + DataInputStream dataInputStream = new DataInputStream(new ByteArrayInputStream(bytes)); + Assert.assertEquals(PlanNodeType.INSERT_ROW.getNodeType(), dataInputStream.readShort()); + + InsertRowNode tmpNode = InsertRowNode.deserializeFromWAL(dataInputStream); + Assert.assertArrayEquals( + new String[] {"\u6e29\u5ea6", "s3", "s4", "s5"}, tmpNode.getMeasurements()); + } + + @Test + public void testDeserializeFromWALSkipsRetainedMeasurementWithMissingValue() + throws IllegalPathException, IOException { + InsertRowNode insertRowNode = getInsertRowNodeWithMeasurementSchemas(); + insertRowNode.setValues(new Object[] {5.0, 6.0f}); + + byte[] bytes = new byte[insertRowNode.serializedSize()]; + WALByteBufferForTest walBuffer = new WALByteBufferForTest(ByteBuffer.wrap(bytes)); + insertRowNode.serializeToWAL(walBuffer); + Assert.assertFalse(walBuffer.getBuffer().hasRemaining()); + + DataInputStream dataInputStream = new DataInputStream(new ByteArrayInputStream(bytes)); + Assert.assertEquals(PlanNodeType.INSERT_ROW.getNodeType(), dataInputStream.readShort()); + + InsertRowNode tmpNode = InsertRowNode.deserializeFromWAL(dataInputStream); + Assert.assertArrayEquals( + new String[] {"\u6e29\u5ea6", "\u6e7f\u5ea6"}, tmpNode.getMeasurements()); + } + + @Test + public void testRelationalDeserializeFromWALSkipsRetainedMeasurementWithNullCategory() + throws IOException { + RelationalInsertRowNode insertRowNode = getRelationalInsertRowNodeWithMeasurementSchemas(); + insertRowNode.getColumnCategories()[1] = null; + + byte[] bytes = new byte[insertRowNode.serializedSize()]; + WALByteBufferForTest walBuffer = new WALByteBufferForTest(ByteBuffer.wrap(bytes)); + insertRowNode.serializeToWAL(walBuffer); + Assert.assertFalse(walBuffer.getBuffer().hasRemaining()); + + DataInputStream dataInputStream = new DataInputStream(new ByteArrayInputStream(bytes)); + Assert.assertEquals( + PlanNodeType.RELATIONAL_INSERT_ROW.getNodeType(), dataInputStream.readShort()); + + RelationalInsertRowNode tmpNode = RelationalInsertRowNode.deserializeFromWAL(dataInputStream); + Assert.assertArrayEquals(new String[] {"id", "value"}, tmpNode.getMeasurements()); + Assert.assertArrayEquals( + new TsTableColumnCategory[] {TsTableColumnCategory.TAG, TsTableColumnCategory.FIELD}, + tmpNode.getColumnCategories()); + } + @Test public void testDeserializeFromWALWithMarkedFailedMeasurementOnly() throws IllegalPathException, IOException { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertRowsNodeSerdeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertRowsNodeSerdeTest.java index 034d4cb57de40..907af18ef9adb 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertRowsNodeSerdeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertRowsNodeSerdeTest.java @@ -31,6 +31,7 @@ import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALByteBufferForTest; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.IDeviceID.Factory; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.Assert; @@ -270,6 +271,40 @@ public void TestSerializeAndDeserializeRelational() throws IllegalPathException } } + @Test + public void testRelationalRowsGetDeviceIDSkipsMissingTagValue() throws IllegalPathException { + RelationalInsertRowsNode node = new RelationalInsertRowsNode(new PlanNodeId("plan node 1")); + node.addOneInsertRowNode( + new RelationalInsertRowNode( + new PlanNodeId("plan node 1"), + new PartialPath("table1", false), + false, + new String[] {"id", "value"}, + new TSDataType[] {TSDataType.STRING, TSDataType.DOUBLE}, + 1000L, + new Object[] {"id1", 1.0}, + false, + new TsTableColumnCategory[] {TsTableColumnCategory.TAG, TsTableColumnCategory.FIELD}), + 0); + node.addOneInsertRowNode( + new RelationalInsertRowNode( + new PlanNodeId("plan node 1"), + new PartialPath("table1", false), + false, + new String[] {"id", "value"}, + new TSDataType[] {TSDataType.STRING, TSDataType.DOUBLE}, + 2000L, + new Object[] {2.0}, + false, + new TsTableColumnCategory[] {TsTableColumnCategory.FIELD, TsTableColumnCategory.TAG}), + 1); + + Assert.assertEquals( + Factory.DEFAULT_FACTORY.create(new String[] {"table1", "id1"}), node.getDeviceID(0)); + Assert.assertEquals( + Factory.DEFAULT_FACTORY.create(new String[] {"table1", null}), node.getDeviceID(1)); + } + @Test public void testSerializeAndDeserializeForWALRelational() throws IOException { for (String tableName : new String[] {"table1", "ta`ble1", "root.table1"}) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertTabletNodeSerdeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertTabletNodeSerdeTest.java index 27a3b8d1ce5db..4cd4c1859077d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertTabletNodeSerdeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertTabletNodeSerdeTest.java @@ -279,6 +279,34 @@ public void testRelationalSerializedSizeWithRetainedMeasurementAndNullColumn() { Assert.assertEquals(insertTabletNode.serializedSize(), byteBuffer.position()); } + @Test + public void testRelationalDeserializeFromWALSkipsRetainedMeasurementWithNullCategory() + throws IOException { + RelationalInsertTabletNode insertTabletNode = getRelationalInsertTabletNodeWithSchema("table1"); + insertTabletNode.getColumnCategories()[1] = null; + + byte[] bytes = new byte[insertTabletNode.serializedSize()]; + WALByteBufferForTest walBuffer = new WALByteBufferForTest(ByteBuffer.wrap(bytes)); + insertTabletNode.serializeToWAL(walBuffer); + Assert.assertFalse(walBuffer.getBuffer().hasRemaining()); + + DataInputStream dataInputStream = new DataInputStream(new ByteArrayInputStream(bytes)); + Assert.assertEquals( + PlanNodeType.RELATIONAL_INSERT_TABLET.getNodeType(), dataInputStream.readShort()); + + RelationalInsertTabletNode tmpNode = + RelationalInsertTabletNode.deserializeFromWAL(dataInputStream); + Assert.assertArrayEquals(new String[] {"s1", "s3", "s4", "s5"}, tmpNode.getMeasurements()); + Assert.assertArrayEquals( + new TsTableColumnCategory[] { + TsTableColumnCategory.TAG, + TsTableColumnCategory.ATTRIBUTE, + TsTableColumnCategory.TAG, + TsTableColumnCategory.FIELD + }, + tmpNode.getColumnCategories()); + } + @Test public void testDeserializeFromWALWithMarkedFailedMeasurementOnly() throws IllegalPathException, IOException { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertStatementPartialInsertTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertStatementPartialInsertTest.java new file mode 100644 index 0000000000000..943c640e83360 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertStatementPartialInsertTest.java @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.statement.crud; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BitMap; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.Assert; +import org.junit.Test; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; + +public class InsertStatementPartialInsertTest { + + @Test + public void testInsertRowStatementGetPathsSkipsFailedMeasurements() throws Exception { + final InsertRowStatement statement = createInsertRowStatement(); + statement.markFailedMeasurement(0, new RuntimeException("failed")); + + Assert.assertEquals(1, statement.getPaths().size()); + Assert.assertEquals("root.sg.d1.s2", statement.getPaths().get(0).getFullPath()); + } + + @Test + public void testInsertTabletStatementGetPathsSkipsFailedMeasurements() throws Exception { + final InsertTabletStatement statement = createInsertTabletStatement(); + statement.markFailedMeasurement(1, new RuntimeException("failed")); + + Assert.assertEquals(1, statement.getPaths().size()); + Assert.assertEquals("root.sg.d1.s1", statement.getPaths().get(0).getFullPath()); + } + + @Test + public void testInsertRowsOfOneDeviceStatementSkipsFailedMeasurements() throws Exception { + final InsertRowStatement statement = createInsertRowStatement(); + statement.markFailedMeasurement(0, new RuntimeException("failed")); + final InsertRowsOfOneDeviceStatement rowsOfOneDeviceStatement = + new InsertRowsOfOneDeviceStatement(); + rowsOfOneDeviceStatement.setInsertRowStatementList(Arrays.asList(statement)); + + Assert.assertArrayEquals(new String[] {"s2"}, rowsOfOneDeviceStatement.getMeasurements()); + Assert.assertEquals(1, rowsOfOneDeviceStatement.getPaths().size()); + Assert.assertEquals("root.sg.d1.s2", rowsOfOneDeviceStatement.getPaths().get(0).getFullPath()); + } + + @Test + public void testInsertRowStatementMarkFailedMeasurementHandlesMissingValue() throws Exception { + final InsertRowStatement statement = createInsertRowStatement(); + statement.setValues(new Object[] {1}); + + statement.markFailedMeasurement(1, new RuntimeException("failed")); + + Assert.assertNull(statement.getMeasurements()[1]); + Assert.assertNull(statement.getDataTypes()[1]); + + statement.removeAllFailedMeasurementMarks(); + + Assert.assertEquals("s2", statement.getMeasurements()[1]); + Assert.assertEquals(TSDataType.INT64, statement.getDataTypes()[1]); + } + + @Test + public void testInsertTabletStatementMarkFailedMeasurementHandlesMissingColumn() + throws Exception { + final InsertTabletStatement statement = createInsertTabletStatement(); + statement.setColumns(new Object[] {new int[] {1, 2}}); + + statement.markFailedMeasurement(1, new RuntimeException("failed")); + + Assert.assertNull(statement.getMeasurements()[1]); + Assert.assertNull(statement.getDataTypes()[1]); + + statement.removeAllFailedMeasurementMarks(); + + Assert.assertEquals("s2", statement.getMeasurements()[1]); + Assert.assertEquals(TSDataType.INT64, statement.getDataTypes()[1]); + } + + @Test + public void testHasValidMeasurementsIgnoresNonFieldColumns() throws Exception { + final InsertRowStatement statement = createInsertRowStatement(); + statement.setColumnCategories( + new TsTableColumnCategory[] {TsTableColumnCategory.TAG, TsTableColumnCategory.ATTRIBUTE}); + + Assert.assertFalse(statement.hasValidMeasurements()); + + statement.setColumnCategories( + new TsTableColumnCategory[] {TsTableColumnCategory.TAG, TsTableColumnCategory.FIELD}); + + Assert.assertTrue(statement.hasValidMeasurements()); + } + + @Test + public void testInsertTabletStatementGetFirstValueOfIndexReturnsNullForNullColumn() + throws Exception { + final InsertTabletStatement statement = createInsertTabletStatement(); + statement.getColumns()[0] = null; + + Assert.assertNull(statement.getFirstValueOfIndex(0)); + } + + @Test + public void testInsertTabletStatementTagAndAttributeIndicesSkipNullColumn() throws Exception { + final InsertTabletStatement statement = createInsertTabletStatement(); + statement.setColumnCategories( + new TsTableColumnCategory[] {TsTableColumnCategory.TAG, TsTableColumnCategory.ATTRIBUTE}); + statement.getColumns()[0] = null; + + Assert.assertTrue(statement.getTagColumnIndices().isEmpty()); + Assert.assertEquals(Arrays.asList(1), statement.getAttrColumnIndices()); + Assert.assertEquals(Arrays.asList("s2"), statement.getAttributeColumnNameList()); + } + + @Test + public void testInsertRowStatementTableDeviceIDHandlesMissingTagValue() throws Exception { + final InsertRowStatement statement = createInsertRowStatement(); + statement.setColumnCategories( + new TsTableColumnCategory[] {TsTableColumnCategory.FIELD, TsTableColumnCategory.TAG}); + statement.getTagColumnIndices(); + statement.setValues(new Object[] {1}); + + Assert.assertArrayEquals( + new Object[] {"root.sg.d1"}, statement.getTableDeviceID().getSegments()); + } + + @Test + public void testInsertTabletStatementTableDeviceIDUsesTagColumnBitmap() throws Exception { + final InsertTabletStatement statement = createInsertTabletStatement(); + statement.setDataTypes(new TSDataType[] {TSDataType.INT32, TSDataType.STRING}); + statement.setColumnCategories( + new TsTableColumnCategory[] {TsTableColumnCategory.FIELD, TsTableColumnCategory.TAG}); + statement.setColumns( + new Object[] { + new int[] {1, 2}, + new Binary[] { + new Binary("tag1", StandardCharsets.UTF_8), new Binary("tag2", StandardCharsets.UTF_8) + } + }); + statement.setBitMaps(new BitMap[] {new BitMap(2, new byte[] {1}), new BitMap(2)}); + + Assert.assertArrayEquals( + new Object[] {"root.sg.d1", "tag1"}, statement.getTableDeviceID(0).getSegments()); + } + + @Test + public void testInsertTabletStatementTableDeviceIDHandlesMissingTagColumn() throws Exception { + final InsertTabletStatement statement = createInsertTabletStatement(); + statement.setColumnCategories( + new TsTableColumnCategory[] {TsTableColumnCategory.FIELD, TsTableColumnCategory.TAG}); + statement.getTagColumnIndices(); + statement.setColumns(new Object[] {new int[] {1, 2}}); + + Assert.assertArrayEquals( + new Object[] {"root.sg.d1"}, statement.getTableDeviceID(0).getSegments()); + } + + private static InsertRowStatement createInsertRowStatement() throws Exception { + final InsertRowStatement statement = new InsertRowStatement(); + statement.setDevicePath(new PartialPath("root.sg.d1")); + statement.setMeasurements(new String[] {"s1", "s2"}); + statement.setDataTypes(new TSDataType[] {TSDataType.INT32, TSDataType.INT64}); + statement.setMeasurementSchemas( + new MeasurementSchema[] { + new MeasurementSchema("s1", TSDataType.INT32), + new MeasurementSchema("s2", TSDataType.INT64) + }); + statement.setValues(new Object[] {1, 2L}); + statement.setTime(1L); + return statement; + } + + private static InsertTabletStatement createInsertTabletStatement() throws Exception { + final InsertTabletStatement statement = new InsertTabletStatement(); + statement.setDevicePath(new PartialPath("root.sg.d1")); + statement.setMeasurements(new String[] {"s1", "s2"}); + statement.setDataTypes(new TSDataType[] {TSDataType.INT32, TSDataType.INT64}); + statement.setMeasurementSchemas( + new MeasurementSchema[] { + new MeasurementSchema("s1", TSDataType.INT32), + new MeasurementSchema("s2", TSDataType.INT64) + }); + statement.setColumns(new Object[] {new int[] {1, 2}, new long[] {3L, 4L}}); + statement.setTimes(new long[] {1L, 2L}); + statement.setRowCount(2); + return statement; + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusLogToTabletConverterTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusLogToTabletConverterTest.java index 4ae6268ebce8e..5cc56bb016222 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusLogToTabletConverterTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusLogToTabletConverterTest.java @@ -27,6 +27,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsOfOneDeviceNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; import org.apache.iotdb.db.queryengine.plan.statement.StatementTestUtils; @@ -34,6 +35,7 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.Assert; import org.junit.Test; @@ -104,6 +106,33 @@ public void testConvertRelationalInsertTabletNodeWithSingleMatchedColumn() { Assert.assertSame(node.getColumns()[2], tablet.getValues()[1]); } + @Test + public void testConvertRelationalInsertTabletNodeSkipsNullMatchedFieldColumn() { + final ConsensusLogToTabletConverter converter = createConverter("m1"); + + final RelationalInsertTabletNode node = StatementTestUtils.genInsertTabletNode(3, 10); + node.getColumns()[2] = null; + + Assert.assertTrue(converter.convert(node).isEmpty()); + } + + @Test + public void testConvertRelationalInsertTabletNodeSkipsNullTagColumn() { + final ConsensusLogToTabletConverter converter = createConverter("m1"); + + final RelationalInsertTabletNode node = StatementTestUtils.genInsertTabletNode(3, 10); + node.getColumns()[0] = null; + final List tablets = converter.convert(node); + + Assert.assertEquals(1, tablets.size()); + final Tablet tablet = tablets.get(0); + Assert.assertEquals(1, tablet.getSchemas().size()); + Assert.assertEquals("m1", tablet.getSchemas().get(0).getMeasurementName()); + Assert.assertEquals(ColumnCategory.FIELD, tablet.getColumnTypes().get(0)); + Assert.assertArrayEquals( + new double[] {10.0, 11.0, 12.0}, (double[]) tablet.getValues()[0], 0.0); + } + @Test public void testConvertRelationalInsertRowNodeKeepsTagColumnsForMatchedField() { final ConsensusLogToTabletConverter converter = createConverter("m1"); @@ -177,6 +206,35 @@ public void testConvertInsertRowsNodeKeepsOrderWhenGroupingTreeRows() Assert.assertEquals(1, tablets.get(2).getRowSize()); } + @Test + public void testConvertTreeInsertTabletNodeSkipsNullColumn() throws IllegalPathException { + final ConsensusLogToTabletConverter converter = createTreeConverter(); + final InsertTabletNode node = + new InsertTabletNode( + new PlanNodeId("tablet"), + new PartialPath("root.sg.d1"), + false, + new String[] {"s1", "s2"}, + new TSDataType[] {TSDataType.INT32, TSDataType.DOUBLE}, + new MeasurementSchema[] { + new MeasurementSchema("s1", TSDataType.INT32), + new MeasurementSchema("s2", TSDataType.DOUBLE) + }, + new long[] {1L, 2L}, + null, + new Object[] {new int[] {1, 2}, null}, + 2); + + final List tablets = converter.convert(node); + + Assert.assertEquals(1, tablets.size()); + final Tablet tablet = tablets.get(0); + Assert.assertEquals("root.sg.d1", tablet.getDeviceId()); + Assert.assertEquals(1, tablet.getSchemas().size()); + Assert.assertEquals("s1", tablet.getSchemas().get(0).getMeasurementName()); + Assert.assertArrayEquals(new int[] {1, 2}, (int[]) tablet.getValues()[0]); + } + private static ConsensusLogToTabletConverter createConverter(final String columnPattern) { return new ConsensusLogToTabletConverter( null, diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/MemUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/MemUtilsTest.java index 1c1c1fca1220a..9bdbba74c09f8 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/MemUtilsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/MemUtilsTest.java @@ -132,6 +132,34 @@ public void getRowRecordSizeWithSkippedSlotsTest() { })); } + @Test + public void getRowRecordSizeWithShortColumnCategoriesTest() { + Object[] row = {1, 2L}; + List dataTypes = new ArrayList<>(); + dataTypes.add(TSDataType.INT32); + + int sizeSum = 8 + TSDataType.INT32.getDataTypeSize(); + + Assert.assertEquals( + sizeSum, + MemUtils.getRowRecordSize( + dataTypes, row, new TsTableColumnCategory[] {TsTableColumnCategory.FIELD})); + } + + @Test + public void getAlignedRowRecordSizeWithShortColumnCategoriesTest() { + Object[] row = {1, 2L}; + List dataTypes = new ArrayList<>(); + dataTypes.add(TSDataType.INT32); + + int sizeSum = 8 + 4 + TSDataType.INT32.getDataTypeSize(); + + Assert.assertEquals( + sizeSum, + MemUtils.getAlignedRowRecordSize( + dataTypes, row, new TsTableColumnCategory[] {TsTableColumnCategory.FIELD})); + } + @Test public void getRecordSizeWithInsertTableNodeTest() throws IllegalPathException { PartialPath device = new PartialPath("root.sg.d1"); @@ -253,6 +281,27 @@ public void getAlignedTabletSizeWithMarkedFailedMeasurementTest() throws Illegal Assert.assertEquals(sizeSum, MemUtils.getAlignedTabletSize(insertNode, 0, 2, null)); } + @Test + public void getTabletSizeWithShortValueArraysTest() throws IllegalPathException { + PartialPath device = new PartialPath("root.sg.d1"); + InsertTabletNode insertNode = + new InsertTabletNode( + new PlanNodeId(""), + device, + false, + new String[] {"s1", "s2"}, + new TSDataType[] {TSDataType.INT32}, + new long[] {1}, + null, + new Object[] {new int[] {1}}, + 1); + insertNode.setMeasurementSchemas( + new MeasurementSchema[] {new MeasurementSchema("s1", TSDataType.INT32)}); + + long sizeSum = 8 + TSDataType.INT32.getDataTypeSize(); + Assert.assertEquals(sizeSum, MemUtils.getTabletSize(insertNode, 0, 1)); + } + /** This method tests MemUtils.getStringMem() and MemUtils.getDataPointMem() */ @Test public void getMemSizeTest() { From 41dd69199cfaeff654782a5f449ea2f204aa8ff8 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 10 Jun 2026 09:36:05 +0800 Subject: [PATCH 3/6] Fix table insert schema validation for partial insert --- .../IoTDBPipeTypeConversionISessionIT.java | 2 +- .../planner/plan/node/write/InsertNode.java | 26 +++++- .../plan/node/write/InsertRowNode.java | 11 +++ .../plan/node/write/InsertTabletNode.java | 80 +++++++++++-------- .../dataregion/memtable/AbstractMemTable.java | 71 +++------------- .../org/apache/iotdb/db/utils/MemUtils.java | 2 + .../table/InsertNodeMeasurementInfo.java | 7 ++ .../table/InsertNodeMeasurementInfoTest.java | 61 ++++++++++++++ 8 files changed, 162 insertions(+), 98 deletions(-) create mode 100644 iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/schema/table/InsertNodeMeasurementInfoTest.java diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionISessionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionISessionIT.java index 54f4fe2c51fe0..c99a119d61ea3 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionISessionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionISessionIT.java @@ -324,7 +324,7 @@ private void prepareTypeConversionTest( generateMeasurementSchemas(); // Generate createTimeSeries in sender and receiver - String uuid = "bcdedit"; + String uuid = "bcdedit" + Long.toHexString(System.nanoTime()); for (Pair pair : measurementSchemas) { createTimeSeries(uuid, pair.left.getMeasurementName(), pair.left.getType().name(), senderEnv); createTimeSeries( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java index d71963f94028a..17af08644d920 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java @@ -190,6 +190,10 @@ public int measureColumnCnt() { } public boolean isValidMeasurement(int i) { + return isValidMeasurement(i, true); + } + + public boolean isValidMeasurement(int i, boolean countFieldOnly) { return measurements != null && i >= 0 && i < measurements.length @@ -197,8 +201,7 @@ public boolean isValidMeasurement(int i) { && measurementSchemas != null && i < measurementSchemas.length && measurementSchemas[i] != null - && (columnCategories == null - || i < columnCategories.length && columnCategories[i] == TsTableColumnCategory.FIELD); + && (!countFieldOnly || isFieldMeasurement(i)); } public void setMeasurements(String[] measurements) { @@ -367,6 +370,16 @@ protected int getValidMeasurementNumber() { return validMeasurementNumber; } + public int getValidMeasurementNumber(boolean countFieldOnly) { + int validMeasurementNumber = 0; + for (int i = 0; measurements != null && i < measurements.length; i++) { + if (isValidMeasurement(i, countFieldOnly)) { + validMeasurementNumber++; + } + } + return validMeasurementNumber; + } + public boolean isMeasurementFailed(int index) { return measurements == null || index < 0 @@ -375,8 +388,13 @@ public boolean isMeasurementFailed(int index) { } protected boolean isWritableFieldMeasurement(int index) { - return !isMeasurementFailed(index) - && (columnCategories == null || columnCategories[index] == TsTableColumnCategory.FIELD); + return !isMeasurementFailed(index) && isFieldMeasurement(index); + } + + public boolean isFieldMeasurement(int index) { + return columnCategories == null + || index < columnCategories.length + && columnCategories[index] == TsTableColumnCategory.FIELD; } public boolean allMeasurementFailed() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java index f27fa7bdbc3d5..73463864c0660 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java @@ -266,6 +266,17 @@ protected int getValidMeasurementNumber() { return validMeasurementNumber; } + @Override + public int getValidMeasurementNumber(boolean countFieldOnly) { + int validMeasurementNumber = 0; + for (int i = 0; measurements != null && i < measurements.length; i++) { + if (values != null && i < values.length && isValidMeasurement(i, countFieldOnly)) { + validMeasurementNumber++; + } + } + return validMeasurementNumber; + } + protected int getValidMeasurementNumberForWAL() { int validMeasurementNumber = 0; for (int i = 0; measurements != null && i < measurements.length; i++) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java index 9f1cb48708c35..a15d06370e41d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java @@ -405,47 +405,47 @@ public List getTimePartitionSlots() { } protected Object[] initTabletValues(int columnSize, int rowSize, TSDataType[] dataTypes) { + return initTabletValues(columnSize, rowSize, dataTypes, false); + } + + private Object[] initTabletValues( + int columnSize, int rowSize, TSDataType[] dataTypes, boolean forSplit) { Object[] values = new Object[columnSize]; for (int i = 0; i < values.length; i++) { - if (dataTypes[i] != null) { - switch (dataTypes[i]) { - case TEXT: - case BLOB: - case STRING: - case OBJECT: - values[i] = new Binary[rowSize]; - break; - case FLOAT: - values[i] = new float[rowSize]; - break; - case INT32: - case DATE: - values[i] = new int[rowSize]; - break; - case TIMESTAMP: - case INT64: - values[i] = new long[rowSize]; - break; - case DOUBLE: - values[i] = new double[rowSize]; - break; - case BOOLEAN: - values[i] = new boolean[rowSize]; - break; - } + if (dataTypes[i] == null || forSplit && !hasColumnForSplit(i)) { + continue; + } + switch (dataTypes[i]) { + case TEXT: + case BLOB: + case STRING: + case OBJECT: + values[i] = new Binary[rowSize]; + break; + case FLOAT: + values[i] = new float[rowSize]; + break; + case INT32: + case DATE: + values[i] = new int[rowSize]; + break; + case TIMESTAMP: + case INT64: + values[i] = new long[rowSize]; + break; + case DOUBLE: + values[i] = new double[rowSize]; + break; + case BOOLEAN: + values[i] = new boolean[rowSize]; + break; } } return values; } protected Object[] initTabletValuesForSplit(int columnSize, int rowSize, TSDataType[] dataTypes) { - Object[] values = initTabletValues(columnSize, rowSize, dataTypes); - for (int i = 0; i < values.length; i++) { - if (!hasColumnForSplit(i)) { - values[i] = null; - } - } - return values; + return initTabletValues(columnSize, rowSize, dataTypes, true); } protected BitMap[] initBitmaps(int columnSize, int rowSize) { @@ -1047,6 +1047,20 @@ protected int getValidMeasurementNumber() { return validMeasurementNumber; } + @Override + public int getValidMeasurementNumber(boolean countFieldOnly) { + int validMeasurementNumber = 0; + for (int i = 0; measurements != null && i < measurements.length; i++) { + if (isValidMeasurement(i, countFieldOnly) + && columns != null + && i < columns.length + && columns[i] != null) { + validMeasurementNumber++; + } + } + return validMeasurementNumber; + } + @Override protected int serializeMeasurementSchemasSize() { int byteLen = 0; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java index 33064fe294975..eea69f98a3f08 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java @@ -26,7 +26,6 @@ import org.apache.iotdb.commons.path.IFullPath; import org.apache.iotdb.commons.path.NonAlignedFullPath; import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.DataTypeInconsistentException; import org.apache.iotdb.db.exception.WriteProcessException; @@ -207,8 +206,8 @@ public int insert(InsertRowNode insertRowNode) { final boolean hasValue = values != null && i < values.length; final Object value = hasValue ? values[i] : null; // Use measurements[i] to ignore failed partial insert - if (!isValidMeasurement(insertRowNode, i, true) || !hasValue || value == null) { - if (hasValue && isValidMeasurement(insertRowNode, i, true) && value == null) { + if (!insertRowNode.isValidMeasurement(i, true) || !hasValue || value == null) { + if (hasValue && insertRowNode.isValidMeasurement(i, true) && value == null) { nullPointsNumber++; } schemaList.add(null); @@ -224,7 +223,7 @@ public int insert(InsertRowNode insertRowNode) { write(insertRowNode.getDeviceID(), schemaList, insertRowNode.getTime(), writableValues); int pointsInserted = - getValidMeasurementNumber(insertRowNode, true) + insertRowNode.getValidMeasurementNumber(true) - (IoTDBDescriptor.getInstance().getConfig().isIncludeNullValueInWriteThroughputMetric() ? 0 : nullPointsNumber); @@ -246,8 +245,8 @@ public int insertAlignedRow(InsertRowNode insertRowNode) { final boolean hasValue = values != null && i < values.length; final Object value = hasValue ? values[i] : null; // Use measurements[i] to ignore failed partial insert - if (!isValidMeasurement(insertRowNode, i, true) || !hasValue || value == null) { - if (hasValue && isValidMeasurement(insertRowNode, i, true) && value == null) { + if (!insertRowNode.isValidMeasurement(i, true) || !hasValue || value == null) { + if (hasValue && insertRowNode.isValidMeasurement(i, true) && value == null) { // do not include failed measurement to avoid a negative pointsInserted nullPointsNumber++; } @@ -268,7 +267,7 @@ public int insertAlignedRow(InsertRowNode insertRowNode) { writeAlignedRow( insertRowNode.getDeviceID(), schemaList, insertRowNode.getTime(), writableValues); int pointsInserted = - getValidMeasurementNumber(insertRowNode, true) + insertRowNode.getValidMeasurementNumber(true) - (IoTDBDescriptor.getInstance().getConfig().isIncludeNullValueInWriteThroughputMetric() ? 0 : nullPointsNumber); @@ -284,7 +283,7 @@ public int insertTablet(InsertTabletNode insertTabletNode, int start, int end) writeTabletNode(insertTabletNode, start, end); memSize += MemUtils.getTabletSize(insertTabletNode, start, end); int pointsInserted = - (getValidMeasurementNumber(insertTabletNode, true) * (end - start)) + (insertTabletNode.getValidMeasurementNumber(true) * (end - start)) - (IoTDBDescriptor.getInstance() .getConfig() .isIncludeNullValueInWriteThroughputMetric() @@ -307,7 +306,7 @@ public int insertAlignedTablet( // TODO-Table: what is the relation between this and TsFileProcessor.checkMemCost memSize += MemUtils.getAlignedTabletSize(insertTabletNode, start, end, results); int pointsInserted = - (getValidMeasurementNumber(insertTabletNode, true) * (end - start)) + (insertTabletNode.getValidMeasurementNumber(true) * (end - start)) - (IoTDBDescriptor.getInstance() .getConfig() .isIncludeNullValueInWriteThroughputMetric() @@ -329,7 +328,7 @@ private static int computeTabletNullPointsNumber( insertTabletNode.getMeasurements() != null && i < insertTabletNode.getMeasurements().length; i++) { - if (!isValidMeasurement(insertTabletNode, i, countFieldOnly) + if (!insertTabletNode.isValidMeasurement(i, countFieldOnly) || insertTabletNode.getColumns() == null || i >= insertTabletNode.getColumns().length || insertTabletNode.getColumns()[i] == null) { @@ -350,54 +349,6 @@ private static int computeTabletNullPointsNumber( return nullPointsNumber; } - private static int getValidMeasurementNumber(InsertRowNode insertNode, boolean countFieldOnly) { - int count = 0; - for (int i = 0; - insertNode.getMeasurements() != null && i < insertNode.getMeasurements().length; - i++) { - if (insertNode.getValues() != null - && i < insertNode.getValues().length - && isValidMeasurement(insertNode, i, countFieldOnly)) { - count++; - } - } - return count; - } - - private static int getValidMeasurementNumber( - InsertTabletNode insertNode, boolean countFieldOnly) { - int count = 0; - for (int i = 0; - insertNode.getMeasurements() != null && i < insertNode.getMeasurements().length; - i++) { - if (isValidMeasurement(insertNode, i, countFieldOnly) - && insertNode.getColumns() != null - && i < insertNode.getColumns().length - && insertNode.getColumns()[i] != null) { - count++; - } - } - return count; - } - - private static boolean isValidMeasurement( - InsertNode insertNode, int index, boolean countFieldOnly) { - return insertNode.getMeasurements() != null - && index >= 0 - && index < insertNode.getMeasurements().length - && insertNode.getMeasurements()[index] != null - && insertNode.getMeasurementSchemas() != null - && index < insertNode.getMeasurementSchemas().length - && insertNode.getMeasurementSchemas()[index] != null - && (!countFieldOnly || isFieldMeasurement(insertNode, index)); - } - - private static boolean isFieldMeasurement(InsertNode insertNode, int index) { - return insertNode.getColumnCategories() == null - || index < insertNode.getColumnCategories().length - && insertNode.getColumnCategories()[index] == TsTableColumnCategory.FIELD; - } - @Override public void write( IDeviceID deviceId, @@ -428,7 +379,7 @@ public void writeTabletNode(InsertTabletNode insertTabletNode, int start, int en final BitMap[] writableBitMaps = insertTabletNode.getBitMaps() == null ? null : new BitMap[measurementSize]; for (int i = 0; i < measurementSize; i++) { - if (!isValidMeasurement(insertTabletNode, i, true) + if (!insertTabletNode.isValidMeasurement(i, true) || insertTabletNode.getColumns() == null || i >= insertTabletNode.getColumns().length || insertTabletNode.getColumns()[i] == null) { @@ -464,7 +415,7 @@ public void writeAlignedTablet( final BitMap[] writableBitMaps = insertTabletNode.getBitMaps() == null ? null : new BitMap[measurementSize]; for (int i = 0; i < measurementSize; i++) { - if (!isValidMeasurement(insertTabletNode, i, true) + if (!insertTabletNode.isValidMeasurement(i, true) || insertTabletNode.getColumns() == null || i >= insertTabletNode.getColumns().length || insertTabletNode.getColumns()[i] == null) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/MemUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/MemUtils.java index 3d60d664d2460..b724c4d3c6f98 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/MemUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/MemUtils.java @@ -72,6 +72,7 @@ public static long getRowRecordSize(List dataTypes, Object[] value) public static long getRowRecordSize( List dataTypes, Object[] value, TsTableColumnCategory[] columnCategories) { + // dataTypes contains only non-null FIELD types, in the same order as non-null FIELD values. if (dataTypes == null) { return 0L; } @@ -92,6 +93,7 @@ public static long getRowRecordSize( */ public static long getAlignedRowRecordSize( List dataTypes, Object[] value, TsTableColumnCategory[] columnCategories) { + // dataTypes contains only non-null FIELD types, in the same order as non-null FIELD values. if (dataTypes == null) { return 8L + 4L; } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InsertNodeMeasurementInfo.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InsertNodeMeasurementInfo.java index eb08a20f283a9..b5e5cb58ce33c 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InsertNodeMeasurementInfo.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InsertNodeMeasurementInfo.java @@ -170,6 +170,13 @@ public TSDataType getTypeForFirstValue(int index) { *

The delegate is optional. If absent, this method is a no-op. */ public void toLowerCase() { + if (measurements != null) { + for (int i = 0; i < measurements.length; i++) { + if (measurements[i] != null) { + measurements[i] = measurements[i].toLowerCase(); + } + } + } if (toLowerCaseAction != null) { toLowerCaseAction.run(); } diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/schema/table/InsertNodeMeasurementInfoTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/schema/table/InsertNodeMeasurementInfoTest.java new file mode 100644 index 0000000000000..3767611b564d6 --- /dev/null +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/schema/table/InsertNodeMeasurementInfoTest.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.schema.table; + +import org.junit.Test; + +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertTrue; + +public class InsertNodeMeasurementInfoTest { + + @Test + public void testToLowerCaseUpdatesLocalMeasurementsAndDelegate() { + final String[] measurementInfoMeasurements = new String[] {"TAG1", null, "S1"}; + final String[] statementMeasurements = new String[] {"TAG1", "ATTR1", "S1"}; + final AtomicBoolean delegateCalled = new AtomicBoolean(false); + + final InsertNodeMeasurementInfo measurementInfo = + new InsertNodeMeasurementInfo( + "TABLE1", + null, + measurementInfoMeasurements, + null, + index -> null, + () -> { + delegateCalled.set(true); + for (int i = 0; i < statementMeasurements.length; i++) { + statementMeasurements[i] = statementMeasurements[i].toLowerCase(); + } + }, + null, + null, + null, + null); + + measurementInfo.toLowerCase(); + + assertArrayEquals(new String[] {"tag1", null, "s1"}, measurementInfoMeasurements); + assertArrayEquals(new String[] {"tag1", "attr1", "s1"}, statementMeasurements); + assertTrue(delegateCalled.get()); + } +} From d62a996d7b6f09c873a97b0dd7ea52b616e0d054 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 10 Jun 2026 11:03:10 +0800 Subject: [PATCH 4/6] Fix partial insert null column handling --- .../planner/plan/node/write/InsertNode.java | 2 +- .../plan/node/write/InsertTabletNode.java | 151 +++++++++++----- .../node/write/RelationalInsertRowNode.java | 25 ++- .../node/write/RelationalInsertRowsNode.java | 14 +- .../write/RelationalInsertTabletNode.java | 51 ++++-- .../statement/crud/InsertBaseStatement.java | 161 ++++++++++++------ .../statement/crud/InsertRowStatement.java | 63 +++++-- .../statement/crud/InsertTabletStatement.java | 56 ++++-- .../node/write/InsertTabletNodeSerdeTest.java | 39 +++++ .../InsertNodeIsMeasurementFailedTest.java | 120 +++++++++++++ .../InsertStatementPartialInsertTest.java | 111 ++++++++++++ .../table/InsertNodeMeasurementInfo.java | 2 +- .../table/InsertNodeMeasurementInfoTest.java | 22 +++ 13 files changed, 681 insertions(+), 136 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java index 17af08644d920..0855dbeb63a04 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java @@ -228,7 +228,7 @@ public int getMeasurementColumnCnt() { } public TSDataType getDataType(int index) { - return dataTypes[index]; + return dataTypes == null || index < 0 || index >= dataTypes.length ? null : dataTypes[index]; } public void setDataTypes(TSDataType[] dataTypes) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java index a15d06370e41d..becd347cfee8b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java @@ -332,8 +332,9 @@ protected List doSplit(Map> spli protected InsertTabletNode getEmptySplit(int count) { long[] subTimes = new long[count]; - Object[] values = initTabletValuesForSplit(dataTypes.length, count, dataTypes); - BitMap[] newBitMaps = initBitmapsForSplit(dataTypes.length, count); + final int columnSize = getColumnArrayLength(); + Object[] values = initTabletValuesForSplit(columnSize, count, dataTypes); + BitMap[] newBitMaps = initBitmapsForSplit(columnSize, count); return new InsertTabletNode( getPlanNodeId(), targetPath, @@ -371,11 +372,9 @@ protected WritePlanNode generateOneSplit(Map.Entry= dataTypes.length + || dataTypes[i] == null + || forSplit && !hasColumnForSplit(i)) { continue; } switch (dataTypes[i]) { @@ -475,6 +477,29 @@ protected BitMap[] initBitmapsForSplit(int columnSize, int rowSize) { return hasBitMap ? splitBitMaps : null; } + protected int getColumnArrayLength() { + int length = 0; + if (measurements != null) { + length = Math.max(length, measurements.length); + } + if (measurementSchemas != null) { + length = Math.max(length, measurementSchemas.length); + } + if (dataTypes != null) { + length = Math.max(length, dataTypes.length); + } + if (columns != null) { + length = Math.max(length, columns.length); + } + if (bitMaps != null) { + length = Math.max(length, bitMaps.length); + } + if (columnCategories != null) { + length = Math.max(length, columnCategories.length); + } + return length; + } + protected boolean hasColumnForSplit(int index) { return dataTypes != null && index < dataTypes.length @@ -489,12 +514,19 @@ protected boolean hasColumnForSplit(int index) { @Override public void markFailedMeasurement(int index) { - if (measurements[index] == null) { + if (measurements == null + || index < 0 + || index >= measurements.length + || measurements[index] == null) { return; } measurements[index] = null; - dataTypes[index] = null; - columns[index] = null; + if (dataTypes != null && index < dataTypes.length) { + dataTypes[index] = null; + } + if (columns != null && index < columns.length) { + columns[index] = null; + } measurementColumnCnt = -1; } @@ -540,7 +572,7 @@ private void writeMeasurementsOrSchemas(ByteBuffer buffer) { ReadWriteIOUtils.write(getValidMeasurementNumber(), buffer); ReadWriteIOUtils.write((byte) (measurementSchemas != null ? 1 : 0), buffer); - for (int i = 0; i < measurements.length; i++) { + for (int i = 0; measurements != null && i < measurements.length; i++) { // ignore failed partial insert and null columns if (!shouldSerializeMeasurement(i)) { continue; @@ -559,7 +591,7 @@ private void writeMeasurementsOrSchemas(DataOutputStream stream) throws IOExcept ReadWriteIOUtils.write(getValidMeasurementNumber(), stream); ReadWriteIOUtils.write((byte) (measurementSchemas != null ? 1 : 0), stream); - for (int i = 0; i < measurements.length; i++) { + for (int i = 0; measurements != null && i < measurements.length; i++) { // ignore failed partial insert and null columns if (!shouldSerializeMeasurement(i)) { continue; @@ -575,7 +607,7 @@ private void writeMeasurementsOrSchemas(DataOutputStream stream) throws IOExcept /** Serialize data types, ignoring failed time series */ private void writeDataTypes(ByteBuffer buffer) { - for (int i = 0; i < dataTypes.length; i++) { + for (int i = 0; dataTypes != null && i < dataTypes.length; i++) { // ignore failed partial insert and null columns if (!shouldSerializeMeasurement(i)) { continue; @@ -586,7 +618,7 @@ private void writeDataTypes(ByteBuffer buffer) { /** Serialize data types, ignoring failed time series */ private void writeDataTypes(DataOutputStream stream) throws IOException { - for (int i = 0; i < dataTypes.length; i++) { + for (int i = 0; dataTypes != null && i < dataTypes.length; i++) { // ignore failed partial insert and null columns if (!shouldSerializeMeasurement(i)) { continue; @@ -613,17 +645,18 @@ private void writeTimes(final DataOutputStream stream) throws IOException { private void writeBitMaps(ByteBuffer buffer) { ReadWriteIOUtils.write(BytesUtils.boolToByte(bitMaps != null), buffer); if (bitMaps != null) { - for (int i = 0; i < bitMaps.length; i++) { + for (int i = 0; measurements != null && i < measurements.length; i++) { // ignore failed partial insert and null columns if (!shouldSerializeMeasurement(i)) { continue; } - if (bitMaps[i] == null) { + final BitMap bitMap = getBitMapIfPresent(i); + if (bitMap == null) { ReadWriteIOUtils.write(BytesUtils.boolToByte(false), buffer); } else { ReadWriteIOUtils.write(BytesUtils.boolToByte(true), buffer); - buffer.put(bitMaps[i].getByteArray(), 0, BitMap.getSizeOfBytes(rowCount)); + buffer.put(bitMap.getByteArray(), 0, BitMap.getSizeOfBytes(rowCount)); } } } @@ -633,17 +666,18 @@ private void writeBitMaps(ByteBuffer buffer) { private void writeBitMaps(DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(BytesUtils.boolToByte(bitMaps != null), stream); if (bitMaps != null) { - for (int i = 0; i < bitMaps.length; i++) { + for (int i = 0; measurements != null && i < measurements.length; i++) { // ignore failed partial insert and null columns if (!shouldSerializeMeasurement(i)) { continue; } - if (bitMaps[i] == null) { + final BitMap bitMap = getBitMapIfPresent(i); + if (bitMap == null) { ReadWriteIOUtils.write(BytesUtils.boolToByte(false), stream); } else { ReadWriteIOUtils.write(BytesUtils.boolToByte(true), stream); - stream.write(bitMaps[i].getByteArray(), 0, BitMap.getSizeOfBytes(rowCount)); + stream.write(bitMap.getByteArray(), 0, BitMap.getSizeOfBytes(rowCount)); } } } @@ -651,7 +685,7 @@ private void writeBitMaps(DataOutputStream stream) throws IOException { /** Serialize values, ignoring failed time series */ private void writeValues(ByteBuffer buffer) { - for (int i = 0; i < columns.length; i++) { + for (int i = 0; columns != null && i < columns.length; i++) { // ignore failed partial insert and null columns if (!shouldSerializeMeasurement(i)) { continue; @@ -662,7 +696,7 @@ private void writeValues(ByteBuffer buffer) { /** Serialize values, ignoring failed time series */ private void writeValues(DataOutputStream stream) throws IOException { - for (int i = 0; i < columns.length; i++) { + for (int i = 0; columns != null && i < columns.length; i++) { // ignore failed partial insert and null columns if (!shouldSerializeMeasurement(i)) { continue; @@ -867,19 +901,20 @@ private int subSerializeSizeByRange(List rangeList) { // bitmaps size size += Byte.BYTES; if (bitMaps != null) { - for (int i = 0; i < bitMaps.length; i++) { + for (int i = 0; measurements != null && i < measurements.length; i++) { // ignore failed partial insert and null columns - if (!shouldSerializeMeasurement(i)) { + if (!shouldSerializeMeasurementToWAL(i)) { continue; } size += Byte.BYTES; - if (bitMaps[i] != null) { + final BitMap bitMap = getBitMapIfPresent(i); + if (bitMap != null) { BitMap partBitMap = new BitMap(rowNumInRange); int copiedLength = 0; for (int[] range : rangeList) { int len = range[1] - range[0]; - BitMap.copyOfRange(bitMaps[i], range[0], partBitMap, copiedLength, len); + BitMap.copyOfRange(bitMap, range[0], partBitMap, copiedLength, len); copiedLength += len; } size += partBitMap.getByteArray().length; @@ -887,8 +922,8 @@ private int subSerializeSizeByRange(List rangeList) { } } // values size - for (int i = 0; i < dataTypes.length; i++) { - if (!shouldSerializeMeasurement(i)) { + for (int i = 0; measurements != null && i < measurements.length; i++) { + if (!shouldSerializeMeasurementToWAL(i)) { continue; } for (int[] range : rangeList) { @@ -983,7 +1018,7 @@ void subSerialize(IWALByteBufferView buffer, List rangeList, long encoded /** Serialize measurement schemas, ignoring failed time series */ protected void writeMeasurementSchemas(IWALByteBufferView buffer) { - buffer.putInt(getValidMeasurementNumber()); + buffer.putInt(getValidMeasurementNumberForWAL()); serializeMeasurementSchemasToWAL(buffer); } @@ -1000,13 +1035,14 @@ protected void writeTimes(IWALByteBufferView buffer, List rangeList, int protected void writeBitMaps(IWALByteBufferView buffer, List rangeList, int rowNumInRange) { buffer.put(BytesUtils.boolToByte(bitMaps != null)); if (bitMaps != null) { - for (int i = 0; i < bitMaps.length; i++) { + for (int i = 0; measurements != null && i < measurements.length; i++) { // ignore failed partial insert and null columns - if (!shouldSerializeMeasurement(i)) { + if (!shouldSerializeMeasurementToWAL(i)) { continue; } - if (bitMaps[i] == null) { + final BitMap bitMap = getBitMapIfPresent(i); + if (bitMap == null) { buffer.put(BytesUtils.boolToByte(false)); } else { buffer.put(BytesUtils.boolToByte(true)); @@ -1014,7 +1050,7 @@ protected void writeBitMaps(IWALByteBufferView buffer, List rangeList, in int copiedLength = 0; for (int[] startEnd : rangeList) { int len = startEnd[1] - startEnd[0]; - BitMap.copyOfRange(bitMaps[i], startEnd[0], partBitMap, copiedLength, len); + BitMap.copyOfRange(bitMap, startEnd[0], partBitMap, copiedLength, len); copiedLength += len; } buffer.put(partBitMap.getByteArray()); @@ -1025,9 +1061,9 @@ protected void writeBitMaps(IWALByteBufferView buffer, List rangeList, in /** Serialize values, ignoring failed time series */ protected void writeValues(IWALByteBufferView buffer, List rangeList) { - for (int i = 0; i < columns.length; i++) { + for (int i = 0; measurements != null && i < measurements.length; i++) { // ignore failed partial insert and null columns - if (!shouldSerializeMeasurement(i)) { + if (!shouldSerializeMeasurementToWAL(i)) { continue; } for (int[] startEnd : rangeList) { @@ -1047,6 +1083,16 @@ protected int getValidMeasurementNumber() { return validMeasurementNumber; } + protected int getValidMeasurementNumberForWAL() { + int validMeasurementNumber = 0; + for (int i = 0; measurements != null && i < measurements.length; i++) { + if (shouldSerializeMeasurementToWAL(i)) { + validMeasurementNumber++; + } + } + return validMeasurementNumber; + } + @Override public int getValidMeasurementNumber(boolean countFieldOnly) { int validMeasurementNumber = 0; @@ -1065,7 +1111,7 @@ public int getValidMeasurementNumber(boolean countFieldOnly) { protected int serializeMeasurementSchemasSize() { int byteLen = 0; for (int i = 0; measurements != null && i < measurements.length; i++) { - if (shouldSerializeMeasurement(i)) { + if (shouldSerializeMeasurementToWAL(i)) { byteLen += WALWriteUtils.sizeToWrite(measurementSchemas[i]); } } @@ -1075,7 +1121,7 @@ protected int serializeMeasurementSchemasSize() { @Override protected void serializeMeasurementSchemasToWAL(IWALByteBufferView buffer) { for (int i = 0; measurements != null && i < measurements.length; i++) { - if (shouldSerializeMeasurement(i)) { + if (shouldSerializeMeasurementToWAL(i)) { WALWriteUtils.write(measurementSchemas[i], buffer); } } @@ -1095,6 +1141,13 @@ protected boolean shouldSerializeMeasurement(int index) { && columns[index] != null; } + protected boolean shouldSerializeMeasurementToWAL(int index) { + return shouldSerializeMeasurement(index) + && measurementSchemas != null + && index < measurementSchemas.length + && measurementSchemas[index] != null; + } + private void serializeColumn( TSDataType dataType, Object column, IWALByteBufferView buffer, int start, int end) { switch (dataType) { @@ -1267,8 +1320,9 @@ private boolean equals(Object[] columns) { } for (int i = 0; i < columns.length; i++) { - if (dataTypes[i] != null) { - switch (dataTypes[i]) { + final TSDataType dataType = getDataType(i); + if (dataType != null) { + switch (dataType) { case INT32: case DATE: if (!Arrays.equals((int[]) this.columns[i], (int[]) columns[i])) { @@ -1305,10 +1359,9 @@ private boolean equals(Object[] columns) { } break; default: - throw new UnSupportedDataTypeException( - String.format(DATATYPE_UNSUPPORTED, dataTypes[i])); + throw new UnSupportedDataTypeException(String.format(DATATYPE_UNSUPPORTED, dataType)); } - } else if (!columns[i].equals(columns)) { + } else if (!Objects.equals(this.columns[i], columns[i])) { return false; } } @@ -1329,8 +1382,8 @@ public TimeValuePair composeLastTimeValuePair( // get non-null value int lastIdx = Math.min(endOffset - 1, rowCount - 1); - if (bitMaps != null && bitMaps[measurementIndex] != null) { - BitMap bitMap = bitMaps[measurementIndex]; + final BitMap bitMap = getBitMapIfPresent(measurementIndex); + if (bitMap != null) { while (lastIdx >= startOffset) { if (!bitMap.isMarked(lastIdx)) { break; @@ -1360,7 +1413,7 @@ protected TimeValuePair composeLastTimeValuePair( return null; } - final BitMap bitMap = bitMaps == null ? null : bitMaps[measurementIndex]; + final BitMap bitMap = getBitMapIfPresent(measurementIndex); int lastIdx = Math.min(endOffset - 1, rowCount - 1); while (lastIdx >= startOffset) { if (results[lastIdx] != null @@ -1391,6 +1444,12 @@ private boolean canComposeLastTimeValuePair(final int measurementIndex) { && isWritableFieldMeasurement(measurementIndex); } + private BitMap getBitMapIfPresent(final int measurementIndex) { + return bitMaps != null && measurementIndex >= 0 && measurementIndex < bitMaps.length + ? bitMaps[measurementIndex] + : null; + } + private TimeValuePair composeTimeValuePair(final int measurementIndex, final int rowIndex) { TsPrimitiveType value; switch (dataTypes[measurementIndex]) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowNode.java index 8340ed062901d..d6ae162c6d04f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowNode.java @@ -43,6 +43,8 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; public class RelationalInsertRowNode extends InsertRowNode { @@ -91,10 +93,11 @@ public RelationalInsertRowNode( @Override public IDeviceID getDeviceID() { if (deviceID == null) { - String[] deviceIdSegments = new String[tagColumnIndices.size() + 1]; + final List presentTagColumnIndices = getPresentTagColumnIndices(); + String[] deviceIdSegments = new String[presentTagColumnIndices.size() + 1]; deviceIdSegments[0] = this.getTableName(); - for (int i = 0; i < tagColumnIndices.size(); i++) { - final Integer columnIndex = tagColumnIndices.get(i); + for (int i = 0; i < presentTagColumnIndices.size(); i++) { + final Integer columnIndex = presentTagColumnIndices.get(i); final Object value = getValues() != null && columnIndex < getValues().length ? getValues()[columnIndex] @@ -107,6 +110,22 @@ public IDeviceID getDeviceID() { return deviceID; } + private List getPresentTagColumnIndices() { + final List presentTagColumnIndices = new ArrayList<>(); + final Object[] values = getValues(); + for (int i = 0; columnCategories != null && i < columnCategories.length; i++) { + if (columnCategories[i] == TsTableColumnCategory.TAG + && measurements != null + && i < measurements.length + && measurements[i] != null + && values != null + && i < values.length) { + presentTagColumnIndices.add(i); + } + } + return presentTagColumnIndices; + } + @Override public R accept(IPlanVisitor visitor, C context) { return ((PlanVisitor) visitor).visitRelationalInsertRow(this, context); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowsNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowsNode.java index 56ae8cbd51dae..c76e7f327dd27 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowsNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowsNode.java @@ -82,13 +82,25 @@ private List getTagColumnIndices(final InsertRowNode insertRowNode) { final TsTableColumnCategory[] columnCategories = insertRowNode.getColumnCategories(); final List currentTagColumnIndices = new ArrayList<>(); for (int i = 0; columnCategories != null && i < columnCategories.length; i++) { - if (columnCategories[i] == TsTableColumnCategory.TAG) { + if (columnCategories[i] == TsTableColumnCategory.TAG + && isTagColumnPresent(insertRowNode, i)) { currentTagColumnIndices.add(i); } } return currentTagColumnIndices; } + private boolean isTagColumnPresent(final InsertRowNode insertRowNode, final int columnIndex) { + final String[] measurements = insertRowNode.getMeasurements(); + final Object[] values = insertRowNode.getValues(); + return measurements != null + && columnIndex >= 0 + && columnIndex < measurements.length + && measurements[columnIndex] != null + && values != null + && columnIndex < values.length; + } + private Object getValue(final InsertRowNode insertRowNode, final int columnIndex) { final Object[] values = insertRowNode.getValues(); return values != null && columnIndex >= 0 && columnIndex < values.length diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java index a951d1a513c05..b842cfc5db718 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java @@ -135,10 +135,11 @@ public IDeviceID getDeviceID(int rowIdx) { deviceIDs = new IDeviceID[1]; } if (deviceIDs[0] == null) { - String[] deviceIdSegments = new String[tagColumnIndices.size() + 1]; + final List presentTagColumnIndices = getPresentTagColumnIndices(); + String[] deviceIdSegments = new String[presentTagColumnIndices.size() + 1]; deviceIdSegments[0] = this.getTableName(); - for (int i = 0; i < tagColumnIndices.size(); i++) { - final Integer columnIndex = tagColumnIndices.get(i); + for (int i = 0; i < presentTagColumnIndices.size(); i++) { + final Integer columnIndex = presentTagColumnIndices.get(i); Object idSeg = getColumnValue(columnIndex, 0); boolean isNull = isNullValue(columnIndex, 0); deviceIdSegments[i + 1] = !isNull && idSeg != null ? idSeg.toString() : null; @@ -151,10 +152,11 @@ public IDeviceID getDeviceID(int rowIdx) { deviceIDs = new IDeviceID[rowCount]; } if (deviceIDs[rowIdx] == null) { - String[] deviceIdSegments = new String[tagColumnIndices.size() + 1]; + final List presentTagColumnIndices = getPresentTagColumnIndices(); + String[] deviceIdSegments = new String[presentTagColumnIndices.size() + 1]; deviceIdSegments[0] = this.getTableName(); - for (int i = 0; i < tagColumnIndices.size(); i++) { - final Integer columnIndex = tagColumnIndices.get(i); + for (int i = 0; i < presentTagColumnIndices.size(); i++) { + final Integer columnIndex = presentTagColumnIndices.get(i); Object idSeg = getColumnValue(columnIndex, rowIdx); boolean isNull = isNullValue(columnIndex, rowIdx); deviceIdSegments[i + 1] = !isNull && idSeg != null ? idSeg.toString() : null; @@ -170,6 +172,26 @@ public IDeviceID getDeviceID(int rowIdx) { return deviceIDs[rowIdx]; } + private List getPresentTagColumnIndices() { + final List presentTagColumnIndices = new ArrayList<>(); + for (int i = 0; columnCategories != null && i < columnCategories.length; i++) { + if (columnCategories[i] == TsTableColumnCategory.TAG && isTagColumnPresent(i)) { + presentTagColumnIndices.add(i); + } + } + return presentTagColumnIndices; + } + + private boolean isTagColumnPresent(final int columnIndex) { + return measurements != null + && columnIndex >= 0 + && columnIndex < measurements.length + && measurements[columnIndex] != null + && columns != null + && columnIndex < columns.length + && columns[columnIndex] != null; + } + @Override public R accept(IPlanVisitor visitor, C context) { return ((PlanVisitor) visitor).visitRelationalInsertTablet(this, context); @@ -180,6 +202,11 @@ protected boolean shouldSerializeMeasurement(final int index) { return super.shouldSerializeMeasurement(index) && hasColumnCategory(index); } + @Override + protected boolean shouldSerializeMeasurementToWAL(final int index) { + return super.shouldSerializeMeasurementToWAL(index) && hasColumnCategory(index); + } + private boolean hasColumnCategory(final int index) { return columnCategories != null && index >= 0 @@ -210,8 +237,9 @@ private boolean isNullValue(final int columnIndex, final int rowIndex) { @Override protected InsertTabletNode getEmptySplit(int count) { long[] subTimes = new long[count]; - Object[] values = initTabletValuesForSplit(dataTypes.length, count, dataTypes); - BitMap[] newBitMaps = initBitmapsForSplit(dataTypes.length, count); + final int columnSize = getColumnArrayLength(); + Object[] values = initTabletValuesForSplit(columnSize, count, dataTypes); + BitMap[] newBitMaps = initBitmapsForSplit(columnSize, count); RelationalInsertTabletNode split = new RelationalInsertTabletNode( getPlanNodeId(), @@ -315,7 +343,7 @@ void subSerialize(IWALByteBufferView buffer, List rangeList) { void subSerialize(IWALByteBufferView buffer, List rangeList, long encodedSearchIndex) { super.subSerialize(buffer, rangeList, encodedSearchIndex); for (int i = 0; measurements != null && i < measurements.length; i++) { - if (shouldSerializeMeasurement(i)) { + if (shouldSerializeMeasurementToWAL(i)) { buffer.put(columnCategories[i].getCategory()); } } @@ -359,12 +387,12 @@ public static RelationalInsertTabletNode deserializeFromWAL(ByteBuffer buffer) { @Override int subSerializeSize(int start, int end) { - return super.subSerializeSize(start, end) + getValidMeasurementNumber() * Byte.BYTES; + return super.subSerializeSize(start, end) + getValidMeasurementNumberForWAL() * Byte.BYTES; } @Override int subSerializeSize(List rangeList) { - return super.subSerializeSize(rangeList) + getValidMeasurementNumber() * Byte.BYTES; + return super.subSerializeSize(rangeList) + getValidMeasurementNumberForWAL() * Byte.BYTES; } @Override @@ -485,6 +513,7 @@ private List generateOneSplitList( } if (subNode.bitMaps != null && subNode.bitMaps[i] != null + && this.bitMaps != null && i < this.bitMaps.length && this.bitMaps[i] != null) { BitMap.copyOfRange(this.bitMaps[i], start, subNode.bitMaps[i], destLoc, length); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertBaseStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertBaseStatement.java index 49d82ebf6a1f9..cdcbc7b836cbe 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertBaseStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertBaseStatement.java @@ -147,8 +147,11 @@ public void setMeasurementSchemas(MeasurementSchema[] measurementSchemas) { } public void setMeasurementSchema(MeasurementSchema measurementSchema, int i) { - if (measurementSchemas == null) { - measurementSchemas = new MeasurementSchema[measurements.length]; + if (measurementSchemas == null || i >= measurementSchemas.length) { + measurementSchemas = + measurementSchemas == null + ? new MeasurementSchema[getRequiredColumnArrayLength(i)] + : Arrays.copyOf(measurementSchemas, getRequiredColumnArrayLength(i)); } measurementSchemas[i] = measurementSchema; } @@ -177,8 +180,11 @@ public void setDataTypes(TSDataType[] dataTypes) { } public void setDataType(TSDataType dataType, int i) { - if (dataTypes == null) { - dataTypes = new TSDataType[measurements.length]; + if (dataTypes == null || i >= dataTypes.length) { + dataTypes = + dataTypes == null + ? new TSDataType[getRequiredColumnArrayLength(i)] + : Arrays.copyOf(dataTypes, getRequiredColumnArrayLength(i)); } this.dataTypes[i] = dataType; } @@ -208,35 +214,46 @@ public void updateAfterSchemaValidation(MPPQueryContext context) throws QueryPro /** Check whether data types are matched with measurement schemas */ public void selfCheckDataTypes(int index) throws DataTypeMismatchException, PathNotExistException { + final MeasurementSchema measurementSchema = + measurementSchemas != null && index >= 0 && index < measurementSchemas.length + ? measurementSchemas[index] + : null; + final TSDataType dataType = getDataType(index); + final String measurement = + measurements != null && index >= 0 && index < measurements.length + ? measurements[index] + : null; + final String fullPath = + measurement == null + ? devicePath.getFullPath() + : devicePath.concatNode(measurement).getFullPath(); if (IoTDBDescriptor.getInstance().getConfig().isEnablePartialInsert()) { // if enable partial insert, mark failed measurements with exception - if (measurementSchemas[index] == null) { - markFailedMeasurement( - index, - new PathNotExistException(devicePath.concatNode(measurements[index]).getFullPath())); - } else if ((dataTypes[index] != measurementSchemas[index].getType() - && !checkAndCastDataType(index, measurementSchemas[index].getType()))) { + if (measurementSchema == null) { + markFailedMeasurement(index, new PathNotExistException(fullPath)); + } else if ((dataType != measurementSchema.getType() + && !checkAndCastDataType(index, measurementSchema.getType()))) { markFailedMeasurement( index, new DataTypeMismatchException( devicePath.getFullPath(), - measurements[index], - dataTypes[index], - measurementSchemas[index].getType(), + measurement, + dataType, + measurementSchema.getType(), getMinTime(), getFirstValueOfIndex(index))); } } else { // if not enable partial insert, throw the exception directly - if (measurementSchemas[index] == null) { - throw new PathNotExistException(devicePath.concatNode(measurements[index]).getFullPath()); - } else if ((dataTypes[index] != measurementSchemas[index].getType() - && !checkAndCastDataType(index, measurementSchemas[index].getType()))) { + if (measurementSchema == null) { + throw new PathNotExistException(fullPath); + } else if ((dataType != measurementSchema.getType() + && !checkAndCastDataType(index, measurementSchema.getType()))) { throw new DataTypeMismatchException( devicePath.getFullPath(), - measurements[index], - dataTypes[index], - measurementSchemas[index].getType(), + measurement, + dataType, + measurementSchema.getType(), getMinTime(), getFirstValueOfIndex(index)); } @@ -334,14 +351,20 @@ public TsTableColumnCategory getColumnCategory(int i) { public void setColumnCategories(TsTableColumnCategory[] columnCategories) { this.columnCategories = columnCategories; + this.tagColumnIndices = null; + this.attrColumnIndices = null; } public void setColumnCategory(TsTableColumnCategory columnCategory, int i) { - if (columnCategories == null) { - columnCategories = new TsTableColumnCategory[measurements.length]; + if (columnCategories == null || i >= columnCategories.length) { + columnCategories = + columnCategories == null + ? new TsTableColumnCategory[getRequiredColumnArrayLength(i)] + : Arrays.copyOf(columnCategories, getRequiredColumnArrayLength(i)); } this.columnCategories[i] = columnCategory; this.tagColumnIndices = null; + this.attrColumnIndices = null; } public List getTagColumnIndices() { @@ -460,6 +483,7 @@ public void removeAttributeColumns() { if (failedMeasurementIndex2Info != null) { failedMeasurementIndex2Info = failedMeasurementIndex2Info.entrySet().stream() + .filter(e -> columnsToKeep.contains(e.getKey())) .collect(Collectors.toMap(e -> columnsToKeep.indexOf(e.getKey()), Entry::getValue)); } @@ -566,6 +590,9 @@ protected Map>> getMapFromDeviceToMeasur Map>> mapFromDeviceToMeasurementAndIndex = new HashMap<>(); for (int i = 0; i < this.measurements.length; i++) { + if (!isColumnPresent(i)) { + continue; + } PartialPath targetDevicePath; String measurementName; if (isLogicalView[i]) { @@ -621,57 +648,53 @@ protected static void validateMapFromDeviceToMeasurement( } public void insertColumn(final int pos, final ColumnSchema columnSchema) { - if (pos < 0 || pos > measurements.length) { + final int oldLength = measurements.length; + if (pos < 0 || pos > oldLength) { throw new ArrayIndexOutOfBoundsException(pos); } + final int newLength = oldLength + 1; if (measurementSchemas != null) { - final MeasurementSchema[] tmp = new MeasurementSchema[measurementSchemas.length + 1]; - System.arraycopy(measurementSchemas, 0, tmp, 0, pos); + final MeasurementSchema[] tmp = new MeasurementSchema[newLength]; + copyWithInsertedSlot(measurementSchemas, tmp, pos); tmp[pos] = new MeasurementSchema( columnSchema.getName(), InternalTypeManager.getTSDataType(columnSchema.getType())); - System.arraycopy(measurementSchemas, pos, tmp, pos + 1, measurementSchemas.length - pos); measurementSchemas = tmp; } - String[] tmpMeasurements = new String[measurements.length + 1]; - System.arraycopy(measurements, 0, tmpMeasurements, 0, pos); + String[] tmpMeasurements = new String[newLength]; + copyWithInsertedSlot(measurements, tmpMeasurements, pos); tmpMeasurements[pos] = columnSchema.getName(); - System.arraycopy(measurements, pos, tmpMeasurements, pos + 1, measurements.length - pos); measurements = tmpMeasurements; if (dataTypes == null) { // sql insertion - dataTypes = new TSDataType[measurements.length + 1]; - dataTypes[pos] = InternalTypeManager.getTSDataType(columnSchema.getType()); + dataTypes = new TSDataType[newLength]; } else { - final TSDataType[] tmpTypes = new TSDataType[dataTypes.length + 1]; - System.arraycopy(dataTypes, 0, tmpTypes, 0, pos); - tmpTypes[pos] = InternalTypeManager.getTSDataType(columnSchema.getType()); - System.arraycopy(dataTypes, pos, tmpTypes, pos + 1, dataTypes.length - pos); + final TSDataType[] tmpTypes = new TSDataType[newLength]; + copyWithInsertedSlot(dataTypes, tmpTypes, pos); dataTypes = tmpTypes; } + dataTypes[pos] = InternalTypeManager.getTSDataType(columnSchema.getType()); if (columnCategories == null) { - columnCategories = new TsTableColumnCategory[measurements.length + 1]; - columnCategories[pos] = columnSchema.getColumnCategory(); + columnCategories = new TsTableColumnCategory[newLength]; } else { - final TsTableColumnCategory[] tmpCategories = - new TsTableColumnCategory[columnCategories.length + 1]; - System.arraycopy(columnCategories, 0, tmpCategories, 0, pos); - tmpCategories[pos] = columnSchema.getColumnCategory(); - System.arraycopy( - columnCategories, pos, tmpCategories, pos + 1, columnCategories.length - pos); + final TsTableColumnCategory[] tmpCategories = new TsTableColumnCategory[newLength]; + copyWithInsertedSlot(columnCategories, tmpCategories, pos); columnCategories = tmpCategories; - tagColumnIndices = null; } + columnCategories[pos] = columnSchema.getColumnCategory(); + tagColumnIndices = null; + attrColumnIndices = null; } public void swapColumn(int src, int target) { if (src < 0 || src >= measurements.length || target < 0 || target >= measurements.length) { throw new ArrayIndexOutOfBoundsException(src + "/" + target); } + ensureBaseArraysLength(measurements.length); if (measurementSchemas != null) { CommonUtils.swapArray(measurementSchemas, src, target); } @@ -690,6 +713,7 @@ public void swapColumn(int src, int target) { CommonUtils.swapArray(inputLocations, src, target); } tagColumnIndices = null; + attrColumnIndices = null; } /** @@ -734,19 +758,19 @@ public void rebuildArraysAfterExpansion( // typeConvertors and inputLocations remain null for missing columns } else { // Copy from old array - if (oldMeasurementSchemas != null) { + if (oldMeasurementSchemas != null && oldIdx < oldMeasurementSchemas.length) { newMeasurementSchemas[newIdx] = oldMeasurementSchemas[oldIdx]; } - if (oldDataTypes != null) { + if (oldDataTypes != null && oldIdx < oldDataTypes.length) { newDataTypes[newIdx] = oldDataTypes[oldIdx]; } - if (oldColumnCategories != null) { + if (oldColumnCategories != null && oldIdx < oldColumnCategories.length) { newColumnCategories[newIdx] = oldColumnCategories[oldIdx]; } - if (oldTypeConvertors != null) { + if (oldTypeConvertors != null && oldIdx < oldTypeConvertors.length) { newTypeConvertors[newIdx] = oldTypeConvertors[oldIdx]; } - if (oldInputLocations != null) { + if (oldInputLocations != null && oldIdx < oldInputLocations.length) { newInputLocations[newIdx] = oldInputLocations[oldIdx]; } } @@ -764,6 +788,45 @@ public void rebuildArraysAfterExpansion( attrColumnIndices = null; } + private int getRequiredColumnArrayLength(final int index) { + return Math.max(measurements == null ? 0 : measurements.length, index + 1); + } + + private void ensureBaseArraysLength(final int length) { + if (measurementSchemas != null && measurementSchemas.length < length) { + measurementSchemas = Arrays.copyOf(measurementSchemas, length); + } + if (dataTypes != null && dataTypes.length < length) { + dataTypes = Arrays.copyOf(dataTypes, length); + } + if (columnCategories != null && columnCategories.length < length) { + columnCategories = Arrays.copyOf(columnCategories, length); + } + if (typeConvertors != null && typeConvertors.length < length) { + typeConvertors = Arrays.copyOf(typeConvertors, length); + } + if (inputLocations != null && inputLocations.length < length) { + inputLocations = Arrays.copyOf(inputLocations, length); + } + } + + protected static void copyWithInsertedSlot( + final T[] source, final T[] target, final int pos) { + if (source == null) { + return; + } + final int prefixLength = Math.min(pos, source.length); + if (prefixLength > 0) { + System.arraycopy(source, 0, target, 0, prefixLength); + } + if (pos < source.length) { + final int suffixLength = Math.min(source.length - pos, target.length - pos - 1); + if (suffixLength > 0) { + System.arraycopy(source, pos, target, pos + 1, suffixLength); + } + } + } + public boolean isWriteToTable() { return writeToTable; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java index 1f494b08deeab..fc12391eba1b5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java @@ -220,6 +220,14 @@ public boolean isColumnPresent(final int index) { @Override protected boolean checkAndCastDataType(int columnIndex, TSDataType dataType) { + if (dataTypes == null + || values == null + || columnIndex < 0 + || columnIndex >= dataTypes.length + || columnIndex >= values.length + || dataTypes[columnIndex] == null) { + return false; + } if (dataType.isCompatible(dataTypes[columnIndex])) { values[columnIndex] = dataType.castFromSingleValue(dataTypes[columnIndex], values[columnIndex]); @@ -389,12 +397,19 @@ public List getSplitList() { TSDataType[] dataTypes = new TSDataType[pairList.size()]; for (int i = 0; i < pairList.size(); i++) { int realIndex = pairList.get(i).right; - copiedValues[i] = this.values[realIndex]; + copiedValues[i] = + this.values != null && realIndex < this.values.length ? this.values[realIndex] : null; measurements[i] = Objects.nonNull(this.measurements[realIndex]) ? pairList.get(i).left : null; - measurementSchemas[i] = this.measurementSchemas[realIndex]; - dataTypes[i] = this.dataTypes[realIndex]; - if (this.measurementIsAligned != null) { + measurementSchemas[i] = + this.measurementSchemas != null && realIndex < this.measurementSchemas.length + ? this.measurementSchemas[realIndex] + : null; + dataTypes[i] = + this.dataTypes != null && realIndex < this.dataTypes.length + ? this.dataTypes[realIndex] + : null; + if (this.measurementIsAligned != null && realIndex < this.measurementIsAligned.length) { statement.setAligned(this.measurementIsAligned[realIndex]); } } @@ -441,14 +456,22 @@ public void updateAfterSchemaValidation(MPPQueryContext context) throws QueryPro @Override public TSDataType getDataType(int index) { - if (isNeedInferType && (dataTypes == null || dataTypes[index] == null)) { + if (index < 0 || measurements == null || index >= measurements.length) { + return null; + } + if (isNeedInferType + && (dataTypes == null || index >= dataTypes.length || dataTypes[index] == null)) { if (dataTypes == null) { dataTypes = new TSDataType[measurements.length]; + } else if (index >= dataTypes.length) { + dataTypes = Arrays.copyOf(dataTypes, measurements.length); } - dataTypes[index] = TypeInferenceUtils.getPredictedDataType(values[index], true); + dataTypes[index] = + TypeInferenceUtils.getPredictedDataType( + values != null && index < values.length ? values[index] : null, true); return dataTypes[index]; } else { - return dataTypes != null ? dataTypes[index] : null; + return dataTypes != null && index < dataTypes.length ? dataTypes[index] : null; } } @@ -471,6 +494,8 @@ public void validateDeviceSchema(boolean isAligned) { public void validateMeasurementSchema(int index, IMeasurementSchemaInfo measurementSchemaInfo) { if (measurementSchemas == null) { measurementSchemas = new MeasurementSchema[measurements.length]; + } else if (index >= measurementSchemas.length) { + measurementSchemas = Arrays.copyOf(measurementSchemas, measurements.length); } if (measurementSchemaInfo == null) { measurementSchemas[index] = null; @@ -505,6 +530,9 @@ public void validateMeasurementSchema( if (this.measurementIsAligned == null) { this.measurementIsAligned = new boolean[this.measurements.length]; Arrays.fill(this.measurementIsAligned, this.isAligned); + } else if (index >= this.measurementIsAligned.length) { + this.measurementIsAligned = + Arrays.copyOf(this.measurementIsAligned, this.measurements.length); } this.measurementIsAligned[index] = isAligned; } @@ -575,17 +603,23 @@ public Statement toRelationalStatement(MPPQueryContext context) { @Override public void insertColumn(int pos, ColumnSchema columnSchema) { super.insertColumn(pos, columnSchema); - Object[] tmpValues = new Object[values.length + 1]; - System.arraycopy(values, 0, tmpValues, 0, pos); - System.arraycopy(values, pos, tmpValues, pos + 1, values.length - pos); + Object[] tmpValues = new Object[measurements.length]; + copyWithInsertedSlot(values, tmpValues, pos); values = tmpValues; + deviceID = null; } @TableModel @Override public void swapColumn(int src, int target) { super.swapColumn(src, target); + if (values == null) { + values = new Object[measurements.length]; + } else if (values.length < measurements.length) { + values = Arrays.copyOf(values, measurements.length); + } CommonUtils.swapArray(values, src, target); + deviceID = null; } @TableModel @@ -619,15 +653,19 @@ public void rebuildArraysAfterExpansion( } } else { // Copy from old array - values[newIdx] = oldValues[oldIdx]; + if (oldValues != null && oldIdx < oldValues.length) { + values[newIdx] = oldValues[oldIdx]; + } if (newMeasurementIsAligned != null && oldMeasurementIsAligned != null) { - newMeasurementIsAligned[newIdx] = oldMeasurementIsAligned[oldIdx]; + newMeasurementIsAligned[newIdx] = + oldIdx < oldMeasurementIsAligned.length && oldMeasurementIsAligned[oldIdx]; } } } // Replace old array with new array measurementIsAligned = newMeasurementIsAligned; + deviceID = null; } @Override @@ -643,6 +681,7 @@ protected void subRemoveAttributeColumns(List columnsToKeep) { if (values != null) { values = columnsToKeep.stream().filter(i -> i < values.length).map(i -> values[i]).toArray(); } + deviceID = null; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java index a329e01ad45ce..28f1043e86d15 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java @@ -308,6 +308,15 @@ public List getSchemaValidationList() { @Override protected boolean checkAndCastDataType(int columnIndex, TSDataType dataType) { + if (dataTypes == null + || columns == null + || columnIndex < 0 + || columnIndex >= dataTypes.length + || columnIndex >= columns.length + || dataTypes[columnIndex] == null + || columns[columnIndex] == null) { + return false; + } if (dataType.isCompatible(dataTypes[columnIndex])) { columns[columnIndex] = dataType.castFromArray(dataTypes[columnIndex], columns[columnIndex]); dataTypes[columnIndex] = dataType; @@ -404,15 +413,24 @@ public List getSplitList() { TSDataType[] dataTypes = new TSDataType[pairList.size()]; for (int i = 0; i < pairList.size(); i++) { int realIndex = pairList.get(i).right; - copiedColumns[i] = this.columns[realIndex]; + copiedColumns[i] = + this.columns != null && realIndex < this.columns.length + ? this.columns[realIndex] + : null; measurements[i] = Objects.nonNull(this.measurements[realIndex]) ? pairList.get(i).left : null; - measurementSchemas[i] = this.measurementSchemas[realIndex]; - dataTypes[i] = this.dataTypes[realIndex]; - if (this.nullBitMaps != null) { + measurementSchemas[i] = + this.measurementSchemas != null && realIndex < this.measurementSchemas.length + ? this.measurementSchemas[realIndex] + : null; + dataTypes[i] = + this.dataTypes != null && realIndex < this.dataTypes.length + ? this.dataTypes[realIndex] + : null; + if (this.nullBitMaps != null && realIndex < this.nullBitMaps.length) { copiedBitMaps[i] = this.nullBitMaps[realIndex]; } - if (this.measurementIsAligned != null) { + if (this.measurementIsAligned != null && realIndex < this.measurementIsAligned.length) { statement.setAligned(this.measurementIsAligned[realIndex]); } } @@ -506,7 +524,7 @@ public boolean isColumnPresent(final int index) { @Override public TSDataType getDataType(int index) { - return dataTypes != null ? dataTypes[index] : null; + return dataTypes != null && index >= 0 && index < dataTypes.length ? dataTypes[index] : null; } @Override @@ -528,6 +546,8 @@ public void validateDeviceSchema(boolean isAligned) { public void validateMeasurementSchema(int index, IMeasurementSchemaInfo measurementSchemaInfo) { if (measurementSchemas == null) { measurementSchemas = new MeasurementSchema[measurements.length]; + } else if (index >= measurementSchemas.length) { + measurementSchemas = Arrays.copyOf(measurementSchemas, measurements.length); } if (measurementSchemaInfo == null) { measurementSchemas[index] = null; @@ -559,6 +579,9 @@ public void validateMeasurementSchema( if (this.measurementIsAligned == null) { this.measurementIsAligned = new boolean[this.measurements.length]; Arrays.fill(this.measurementIsAligned, this.isAligned); + } else if (index >= this.measurementIsAligned.length) { + this.measurementIsAligned = + Arrays.copyOf(this.measurementIsAligned, this.measurements.length); } this.measurementIsAligned[index] = isAligned; } @@ -652,14 +675,13 @@ public void insertColumn(int pos, ColumnSchema columnSchema) { nullBitMaps = tmpBitmaps; } - Object[] tmpColumns = new Object[columns.length + 1]; - System.arraycopy(columns, 0, tmpColumns, 0, pos); + Object[] tmpColumns = new Object[measurements.length]; + copyWithInsertedSlot(columns, tmpColumns, pos); tmpColumns[pos] = CommonUtils.createValueColumnOfDataType( InternalTypeManager.getTSDataType(columnSchema.getType()), columnSchema.getColumnCategory(), rowCount); - System.arraycopy(columns, pos, tmpColumns, pos + 1, columns.length - pos); columns = tmpColumns; deviceIDs = null; @@ -669,8 +691,16 @@ public void insertColumn(int pos, ColumnSchema columnSchema) { public void swapColumn(int src, int target) { super.swapColumn(src, target); if (nullBitMaps != null) { + if (nullBitMaps.length < measurements.length) { + nullBitMaps = Arrays.copyOf(nullBitMaps, measurements.length); + } CommonUtils.swapArray(nullBitMaps, src, target); } + if (columns == null) { + columns = new Object[measurements.length]; + } else if (columns.length < measurements.length) { + columns = Arrays.copyOf(columns, measurements.length); + } CommonUtils.swapArray(columns, src, target); deviceIDs = null; } @@ -714,14 +744,15 @@ public void rebuildArraysAfterExpansion( } } else { // Copy from old array - if (oldNullBitMaps != null) { + if (oldNullBitMaps != null && oldIdx < oldNullBitMaps.length) { newNullBitMaps[newIdx] = oldNullBitMaps[oldIdx]; } - if (newColumns != null && oldColumns != null) { + if (newColumns != null && oldColumns != null && oldIdx < oldColumns.length) { newColumns[newIdx] = oldColumns[oldIdx]; } if (newMeasurementIsAligned != null && oldMeasurementIsAligned != null) { - newMeasurementIsAligned[newIdx] = oldMeasurementIsAligned[oldIdx]; + newMeasurementIsAligned[newIdx] = + oldIdx < oldMeasurementIsAligned.length && oldMeasurementIsAligned[oldIdx]; } } } @@ -781,6 +812,7 @@ protected void subRemoveAttributeColumns(List columnsToKeep) { .map(i -> nullBitMaps[i]) .toArray(BitMap[]::new); } + deviceIDs = null; } /** diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertTabletNodeSerdeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertTabletNodeSerdeTest.java index 4cd4c1859077d..a2a407a66d451 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertTabletNodeSerdeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertTabletNodeSerdeTest.java @@ -30,6 +30,7 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.Assert; import org.junit.Test; @@ -268,6 +269,44 @@ public void testSerializedSizeWithRetainedMeasurementAndNullColumn() new String[] {"\u6e29\u5ea6", "s3", "s4", "s5"}, tmpNode.getMeasurements()); } + @Test + public void testSerializeToWALWithoutMeasurementSchemas() throws Exception { + InsertTabletNode insertTabletNode = getInsertTabletNode(); + + byte[] bytes = new byte[insertTabletNode.serializedSize()]; + WALByteBufferForTest walBuffer = new WALByteBufferForTest(ByteBuffer.wrap(bytes)); + insertTabletNode.serializeToWAL(walBuffer); + Assert.assertFalse(walBuffer.getBuffer().hasRemaining()); + + DataInputStream dataInputStream = new DataInputStream(new ByteArrayInputStream(bytes)); + Assert.assertEquals(PlanNodeType.INSERT_TABLET.getNodeType(), dataInputStream.readShort()); + + InsertTabletNode tmpNode = InsertTabletNode.deserializeFromWAL(dataInputStream); + Assert.assertArrayEquals(new String[0], tmpNode.getMeasurements()); + } + + @Test + public void testSerializeToWALWithShortBitMaps() throws Exception { + InsertTabletNode insertTabletNode = getInsertTabletNodeWithSchema(); + BitMap bitMap = new BitMap(insertTabletNode.getRowCount()); + bitMap.mark(0); + insertTabletNode.setBitMaps(new BitMap[] {bitMap}); + + byte[] bytes = new byte[insertTabletNode.serializedSize()]; + WALByteBufferForTest walBuffer = new WALByteBufferForTest(ByteBuffer.wrap(bytes)); + insertTabletNode.serializeToWAL(walBuffer); + Assert.assertFalse(walBuffer.getBuffer().hasRemaining()); + + DataInputStream dataInputStream = new DataInputStream(new ByteArrayInputStream(bytes)); + Assert.assertEquals(PlanNodeType.INSERT_TABLET.getNodeType(), dataInputStream.readShort()); + + InsertTabletNode tmpNode = InsertTabletNode.deserializeFromWAL(dataInputStream); + Assert.assertArrayEquals( + new String[] {"\u6e29\u5ea6", "\u6e7f\u5ea6", "s3", "s4", "s5"}, tmpNode.getMeasurements()); + Assert.assertNotNull(tmpNode.getBitMaps()); + Assert.assertTrue(tmpNode.getBitMaps()[0].isMarked(0)); + } + @Test public void testRelationalSerializedSizeWithRetainedMeasurementAndNullColumn() { RelationalInsertTabletNode insertTabletNode = getRelationalInsertTabletNodeWithSchema("table1"); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNodeIsMeasurementFailedTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNodeIsMeasurementFailedTest.java index 8853348719bed..082156e134efc 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNodeIsMeasurementFailedTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNodeIsMeasurementFailedTest.java @@ -26,11 +26,13 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.Test; import java.nio.charset.StandardCharsets; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; @@ -128,6 +130,25 @@ public void testRelationalInsertRowNode_nonFieldColumnsDoNotComposeLastCacheValu assertNotNull(node.composeTimeValuePair(2)); } + @Test + public void testRelationalInsertRowNode_deviceIDSkipsMissingTagValue() + throws IllegalPathException { + RelationalInsertRowNode node = buildRelationalInsertRowNodeWithTwoTags(new Object[] {"tag0"}); + + assertArrayEquals(new Object[] {"table1", "tag0"}, node.getDeviceID().getSegments()); + } + + @Test + public void testRelationalInsertRowsNode_deviceIDSkipsMissingTagValue() + throws IllegalPathException { + RelationalInsertRowNode rowNode = + buildRelationalInsertRowNodeWithTwoTags(new Object[] {"tag0"}); + RelationalInsertRowsNode rowsNode = new RelationalInsertRowsNode(new PlanNodeId("test")); + rowsNode.addOneInsertRowNode(rowNode, 0); + + assertArrayEquals(new Object[] {"table1", "tag0"}, rowsNode.getDeviceID(0).getSegments()); + } + // ----------------------------------------------------------------------- // InsertTabletNode // ----------------------------------------------------------------------- @@ -181,6 +202,47 @@ public void testInsertTabletNode_retainedMeasurementWithNullColumnDoesNotCompose assertNull(node.composeLastTimeValuePair(0)); } + @Test + public void testInsertTabletNode_markFailedMeasurementHandlesShortColumns() + throws IllegalPathException { + InsertTabletNode node = buildInsertTabletNode(new String[] {"s0", "s1"}); + node.setColumns(new Object[] {new int[] {1, 2}}); + + node.markFailedMeasurement(1); + + assertTrue(node.isMeasurementFailed(1)); + assertNull(node.getDataType(1)); + } + + @Test + public void testInsertTabletNode_getDataTypeReturnsNullForShortDataTypes() + throws IllegalPathException { + InsertTabletNode node = buildInsertTabletNode(new String[] {"s0", "s1"}); + node.setDataTypes(new TSDataType[] {TSDataType.INT32}); + + assertNotNull(node.getDataType(0)); + assertNull(node.getDataType(1)); + } + + @Test + public void testInsertTabletNode_composeLastTimeValuePairHandlesShortBitMaps() + throws IllegalPathException { + InsertTabletNode node = buildInsertTabletNode(new String[] {"s0", "s1"}); + node.setBitMaps(new BitMap[0]); + + assertNotNull(node.composeLastTimeValuePair(1)); + } + + @Test + public void testInsertTabletNode_equalsHandlesShortDataTypes() throws IllegalPathException { + InsertTabletNode left = buildInsertTabletNode(new String[] {"s0", "s1"}); + InsertTabletNode right = buildInsertTabletNode(new String[] {"s0", "s1"}); + left.setDataTypes(new TSDataType[] {TSDataType.INT32}); + right.setDataTypes(new TSDataType[] {TSDataType.INT32}); + + assertFalse(left.equals(right)); + } + @Test public void testRelationalInsertTabletNode_nonFieldColumnsDoNotComposeLastCacheValue() throws IllegalPathException { @@ -191,6 +253,14 @@ public void testRelationalInsertTabletNode_nonFieldColumnsDoNotComposeLastCacheV assertNotNull(node.composeLastTimeValuePair(2)); } + @Test + public void testRelationalInsertTabletNode_deviceIDSkipsMissingTagColumn() + throws IllegalPathException { + RelationalInsertTabletNode node = buildRelationalInsertTabletNodeWithTwoTags(); + + assertArrayEquals(new Object[] {"table1", "tag0"}, node.getDeviceID(0).getSegments()); + } + // ----------------------------------------------------------------------- // Helpers // ----------------------------------------------------------------------- @@ -249,6 +319,30 @@ private static RelationalInsertRowNode buildRelationalInsertRowNode() }); } + private static RelationalInsertRowNode buildRelationalInsertRowNodeWithTwoTags(Object[] values) + throws IllegalPathException { + String[] measurements = {"tag0", "tag1", "field0"}; + TSDataType[] dataTypes = {TSDataType.STRING, TSDataType.STRING, TSDataType.INT32}; + MeasurementSchema[] schemas = { + new MeasurementSchema("tag0", TSDataType.STRING), + new MeasurementSchema("tag1", TSDataType.STRING), + new MeasurementSchema("field0", TSDataType.INT32) + }; + return new RelationalInsertRowNode( + new PlanNodeId("test"), + new PartialPath("table1", false), + true, + measurements, + dataTypes, + schemas, + 1L, + values, + false, + new TsTableColumnCategory[] { + TsTableColumnCategory.TAG, TsTableColumnCategory.TAG, TsTableColumnCategory.FIELD + }); + } + private static InsertTabletNode buildInsertTabletNode(String[] measurementNames) throws IllegalPathException { int n = measurementNames.length; @@ -306,4 +400,30 @@ private static RelationalInsertTabletNode buildRelationalInsertTabletNode() TsTableColumnCategory.TAG, TsTableColumnCategory.ATTRIBUTE, TsTableColumnCategory.FIELD }); } + + private static RelationalInsertTabletNode buildRelationalInsertTabletNodeWithTwoTags() + throws IllegalPathException { + String[] measurements = {"tag0", "tag1", "field0"}; + TSDataType[] dataTypes = {TSDataType.STRING, TSDataType.STRING, TSDataType.INT32}; + MeasurementSchema[] schemas = { + new MeasurementSchema("tag0", TSDataType.STRING), + new MeasurementSchema("tag1", TSDataType.STRING), + new MeasurementSchema("field0", TSDataType.INT32) + }; + Object[] columns = {new Binary[] {new Binary("tag0".getBytes(StandardCharsets.UTF_8))}}; + return new RelationalInsertTabletNode( + new PlanNodeId("test"), + new PartialPath("table1", false), + true, + measurements, + dataTypes, + schemas, + new long[] {1L}, + null, + columns, + 1, + new TsTableColumnCategory[] { + TsTableColumnCategory.TAG, TsTableColumnCategory.TAG, TsTableColumnCategory.FIELD + }); + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertStatementPartialInsertTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertStatementPartialInsertTest.java index 943c640e83360..b6b26cacc8ecc 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertStatementPartialInsertTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertStatementPartialInsertTest.java @@ -20,17 +20,22 @@ package org.apache.iotdb.db.queryengine.plan.statement.crud; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.queryengine.plan.relational.metadata.ColumnSchema; import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.TypeFactory; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; +import org.apache.tsfile.utils.Pair; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.Assert; import org.junit.Test; import java.nio.charset.StandardCharsets; import java.util.Arrays; +import java.util.List; +import java.util.Map; public class InsertStatementPartialInsertTest { @@ -176,6 +181,112 @@ public void testInsertTabletStatementTableDeviceIDHandlesMissingTagColumn() thro new Object[] {"root.sg.d1"}, statement.getTableDeviceID(0).getSegments()); } + @Test + public void testGetDataTypeReturnsNullForShortTypeArray() throws Exception { + final InsertRowStatement rowStatement = createInsertRowStatement(); + rowStatement.setDataTypes(new TSDataType[] {TSDataType.INT32}); + Assert.assertEquals(TSDataType.INT32, rowStatement.getDataType(0)); + Assert.assertNull(rowStatement.getDataType(1)); + + final InsertTabletStatement tabletStatement = createInsertTabletStatement(); + tabletStatement.setDataTypes(new TSDataType[] {TSDataType.INT32}); + Assert.assertEquals(TSDataType.INT32, tabletStatement.getDataType(0)); + Assert.assertNull(tabletStatement.getDataType(1)); + } + + @Test + public void testBaseSettersExpandShortArrays() throws Exception { + final InsertRowStatement statement = createInsertRowStatement(); + statement.setDataTypes(new TSDataType[] {TSDataType.INT32}); + statement.setMeasurementSchemas( + new MeasurementSchema[] {new MeasurementSchema("s1", TSDataType.INT32)}); + statement.setColumnCategories(new TsTableColumnCategory[] {TsTableColumnCategory.FIELD}); + + statement.setDataType(TSDataType.INT64, 1); + statement.setMeasurementSchema(new MeasurementSchema("s2", TSDataType.INT64), 1); + statement.setColumnCategory(TsTableColumnCategory.TAG, 1); + + Assert.assertEquals(TSDataType.INT64, statement.getDataType(1)); + Assert.assertEquals("s2", statement.getMeasurementSchemas()[1].getMeasurementName()); + Assert.assertEquals(TsTableColumnCategory.TAG, statement.getColumnCategories()[1]); + } + + @Test + public void testInsertColumnWithNullDataTypesKeepsArraysAligned() throws Exception { + final InsertRowStatement statement = createInsertRowStatement(); + statement.setDataTypes(null); + + final ColumnSchema columnSchema = + new ColumnSchema( + "tag1", TypeFactory.getType(TSDataType.STRING), false, TsTableColumnCategory.TAG); + statement.insertColumn(1, columnSchema); + + Assert.assertEquals(3, statement.getMeasurements().length); + Assert.assertEquals(3, statement.getDataTypes().length); + Assert.assertEquals(3, statement.getValues().length); + Assert.assertNull(statement.getDataType(0)); + Assert.assertEquals(TSDataType.STRING, statement.getDataType(1)); + Assert.assertNull(statement.getDataType(2)); + } + + @Test + public void testRebuildArraysAfterExpansionHandlesShortValueArrays() throws Exception { + final InsertRowStatement statement = createInsertRowStatement(); + statement.setDataTypes(new TSDataType[] {TSDataType.INT32}); + statement.setValues(new Object[] {1}); + statement.setColumnCategories( + new TsTableColumnCategory[] {TsTableColumnCategory.FIELD, TsTableColumnCategory.FIELD}); + + statement.rebuildArraysAfterExpansion(new int[] {-1, 0, 1}, new String[] {"tag1", "s1", "s2"}); + + Assert.assertArrayEquals(new String[] {"tag1", "s1", "s2"}, statement.getMeasurements()); + Assert.assertArrayEquals(new Object[] {null, 1, null}, statement.getValues()); + Assert.assertEquals(TSDataType.STRING, statement.getDataType(0)); + Assert.assertEquals(TSDataType.INT32, statement.getDataType(1)); + Assert.assertNull(statement.getDataType(2)); + Assert.assertEquals(TsTableColumnCategory.TAG, statement.getColumnCategories()[0]); + } + + @Test + public void testInsertRowStatementSwapColumnInvalidatesTableDeviceId() throws Exception { + final InsertRowStatement statement = createInsertRowStatement(); + statement.setColumnCategories( + new TsTableColumnCategory[] {TsTableColumnCategory.TAG, TsTableColumnCategory.TAG}); + statement.setValues(new Object[] {"tag1", "tag2"}); + + Assert.assertArrayEquals( + new Object[] {"root.sg.d1", "tag1", "tag2"}, statement.getTableDeviceID().getSegments()); + + statement.swapColumn(0, 1); + + Assert.assertArrayEquals( + new Object[] {"root.sg.d1", "tag2", "tag1"}, statement.getTableDeviceID().getSegments()); + } + + @Test + public void testDeviceMeasurementMapSkipsMissingRowValues() throws Exception { + final InsertRowStatement statement = createInsertRowStatement(); + statement.setValues(new Object[] {1}); + + final Map>> result = + statement.getMapFromDeviceToMeasurementAndIndex(); + + Assert.assertEquals(1, result.get(statement.getDevicePath()).size()); + Assert.assertEquals("s1", result.get(statement.getDevicePath()).get(0).left); + } + + @Test + public void testDeviceMeasurementMapSkipsMissingTabletColumns() throws Exception { + final InsertTabletStatement statement = createInsertTabletStatement(); + statement.setColumns(new Object[] {new int[] {1, 2}}); + + final Map>> result = + statement.getMapFromDeviceToMeasurementAndIndex(); + + Assert.assertEquals(1, result.get(statement.getDevicePath()).size()); + Assert.assertEquals("s1", result.get(statement.getDevicePath()).get(0).left); + } + private static InsertRowStatement createInsertRowStatement() throws Exception { final InsertRowStatement statement = new InsertRowStatement(); statement.setDevicePath(new PartialPath("root.sg.d1")); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InsertNodeMeasurementInfo.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InsertNodeMeasurementInfo.java index b5e5cb58ce33c..271f84ac65283 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InsertNodeMeasurementInfo.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InsertNodeMeasurementInfo.java @@ -135,7 +135,7 @@ public int getMeasurementCount() { } public TSDataType getType(int index) { - if (dataTypes == null) { + if (dataTypes == null || index < 0 || index >= dataTypes.length) { return null; } return dataTypes[index]; diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/schema/table/InsertNodeMeasurementInfoTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/schema/table/InsertNodeMeasurementInfoTest.java index 3767611b564d6..0645ef3bab1d1 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/schema/table/InsertNodeMeasurementInfoTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/schema/table/InsertNodeMeasurementInfoTest.java @@ -19,11 +19,14 @@ package org.apache.iotdb.commons.schema.table; +import org.apache.tsfile.enums.TSDataType; import org.junit.Test; import java.util.concurrent.atomic.AtomicBoolean; import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; public class InsertNodeMeasurementInfoTest { @@ -58,4 +61,23 @@ public void testToLowerCaseUpdatesLocalMeasurementsAndDelegate() { assertArrayEquals(new String[] {"tag1", "attr1", "s1"}, statementMeasurements); assertTrue(delegateCalled.get()); } + + @Test + public void testGetTypeReturnsNullForShortDataTypes() { + final InsertNodeMeasurementInfo measurementInfo = + new InsertNodeMeasurementInfo( + "table1", + null, + new String[] {"s1", "s2"}, + new TSDataType[] {TSDataType.INT32}, + index -> null, + null, + null, + null, + null, + null); + + assertEquals(TSDataType.INT32, measurementInfo.getType(0)); + assertNull(measurementInfo.getType(1)); + } } From ab35850685bd0709251ad9015684ef1e8689ae48 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 10 Jun 2026 12:05:03 +0800 Subject: [PATCH 5/6] Fix failed measurement WAL serialization tests --- .../memtable/AbstractWritableMemChunk.java | 17 ++++++++++++++ .../memtable/AlignedWritableMemChunk.java | 7 ++---- .../dataregion/memtable/WritableMemChunk.java | 7 ++---- .../dataregion/DataRegionTest.java | 22 ++++++++++++------- 4 files changed, 35 insertions(+), 18 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractWritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractWritableMemChunk.java index 873f49c54e217..2a574f0cb7b6e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractWritableMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractWritableMemChunk.java @@ -37,6 +37,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UncheckedIOException; import java.util.Iterator; import java.util.List; import java.util.concurrent.BlockingQueue; @@ -279,6 +282,20 @@ public abstract void encode( @Override public abstract void setEncryptParameter(EncryptParameter encryptParameter); + protected static byte[] serializeSchemaToWALBytes(IMeasurementSchema schema) { + try { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(schema.serializedSize()); + schema.serializeTo(outputStream); + return outputStream.toByteArray(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + protected static int getSerializedSchemaSize(IMeasurementSchema schema) { + return serializeSchemaToWALBytes(schema).length; + } + public synchronized TVList initWorkingListForFlushIfNecessary( TVList workingList, boolean needCloneTimesAndIndicesInWorkingTVList) { if (workingListForFlush == null) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java index b82786a32ed3d..949466a9f979f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java @@ -49,7 +49,6 @@ import java.io.DataInputStream; import java.io.IOException; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.LinkedHashMap; @@ -898,7 +897,7 @@ public int serializedSize() { int size = 0; size += Integer.BYTES; for (IMeasurementSchema schema : schemaList) { - size += schema.serializedSize(); + size += getSerializedSchemaSize(schema); } size += Integer.BYTES; for (AlignedTVList alignedTvList : sortedList) { @@ -912,9 +911,7 @@ public int serializedSize() { public void serializeToWAL(IWALByteBufferView buffer) { WALWriteUtils.write(schemaList.size(), buffer); for (IMeasurementSchema schema : schemaList) { - byte[] bytes = new byte[schema.serializedSize()]; - schema.serializeTo(ByteBuffer.wrap(bytes)); - buffer.put(bytes); + buffer.put(serializeSchemaToWALBytes(schema)); } buffer.putInt(sortedList.size()); for (AlignedTVList alignedTvList : sortedList) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java index 5abd87d07292d..a5a4ce25848c0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java @@ -50,7 +50,6 @@ import java.io.DataInputStream; import java.io.IOException; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -528,7 +527,7 @@ public void release() { @Override public int serializedSize() { - int serializedSize = schema.serializedSize() + list.serializedSize(); + int serializedSize = getSerializedSchemaSize(schema) + list.serializedSize(); serializedSize += Integer.BYTES; for (TVList tvList : sortedList) { serializedSize += tvList.serializedSize(); @@ -538,9 +537,7 @@ public int serializedSize() { @Override public void serializeToWAL(IWALByteBufferView buffer) { - byte[] bytes = new byte[schema.serializedSize()]; - schema.serializeTo(ByteBuffer.wrap(bytes)); - buffer.put(bytes); + buffer.put(serializeSchemaToWALBytes(schema)); buffer.putInt(sortedList.size()); for (TVList tvList : sortedList) { tvList.serializeToWAL(buffer); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java index 3aced7e9fe11f..f5ece33a7787e 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java @@ -456,12 +456,12 @@ public void testIoTDBTabletWriteAndSyncClose() new QueryId("test_write").genPlanNodeId(), new PartialPath("root.vehicle.d0"), false, - measurements, - dataTypes, - measurementSchemas, + measurements.clone(), + dataTypes.clone(), + measurementSchemas.clone(), times, null, - columns, + columns.clone(), times.length); dataRegion.insertTablet(insertTabletNode2); @@ -667,6 +667,8 @@ public void testAllMeasurementsFailedTabletWriteAndSyncClose() null, columns, times.length); + insertTabletNode1.markFailedMeasurement(0); + insertTabletNode1.markFailedMeasurement(1); insertTabletNode1.setFailedMeasurementNumber(2); dataRegion.insertTablet(insertTabletNode1); @@ -683,13 +685,15 @@ public void testAllMeasurementsFailedTabletWriteAndSyncClose() new QueryId("test_write").genPlanNodeId(), new PartialPath("root.vehicle.d0"), false, - measurements, - dataTypes, - measurementSchemas, + measurements.clone(), + dataTypes.clone(), + measurementSchemas.clone(), times, null, - columns, + columns.clone(), times.length); + insertTabletNode2.markFailedMeasurement(0); + insertTabletNode2.markFailedMeasurement(1); insertTabletNode2.setFailedMeasurementNumber(2); dataRegion.insertTablet(insertTabletNode2); @@ -743,6 +747,7 @@ public void testAllMeasurementsFailedRecordSeqAndUnSeqSyncClose() TSRecord record = new TSRecord(deviceId, j); record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId, String.valueOf(j))); InsertRowNode rowNode = buildInsertRowNodeByTSRecord(record); + rowNode.markFailedMeasurement(0); rowNode.setFailedMeasurementNumber(1); dataRegion.insert(rowNode); dataRegion.syncCloseAllWorkingTsFileProcessors(); @@ -752,6 +757,7 @@ public void testAllMeasurementsFailedRecordSeqAndUnSeqSyncClose() TSRecord record = new TSRecord(deviceId, j); record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId, String.valueOf(j))); InsertRowNode rowNode = buildInsertRowNodeByTSRecord(record); + rowNode.markFailedMeasurement(0); rowNode.setFailedMeasurementNumber(1); dataRegion.insert(rowNode); dataRegion.syncCloseAllWorkingTsFileProcessors(); From 86cf3aac9b85425c5f6dfb9d74dc700b9bb281c4 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 11 Jun 2026 11:07:55 +0800 Subject: [PATCH 6/6] Fix all-failed tablet test column reuse --- .../iotdb/db/storageengine/dataregion/DataRegionTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java index f5ece33a7787e..152189332b59d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java @@ -660,12 +660,12 @@ public void testAllMeasurementsFailedTabletWriteAndSyncClose() new QueryId("test_write").genPlanNodeId(), new PartialPath("root.vehicle.d0"), false, - measurements, - dataTypes, - measurementSchemas, + measurements.clone(), + dataTypes.clone(), + measurementSchemas.clone(), times, null, - columns, + columns.clone(), times.length); insertTabletNode1.markFailedMeasurement(0); insertTabletNode1.markFailedMeasurement(1);