From dd112cf35bd759f301d801fa6c09a993cddb654b Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 8 Jun 2026 17:53:51 +0800 Subject: [PATCH 1/7] Add show receivers support --- .../iotdb/db/qp/sql/IdentifierParser.g4 | 3 +- .../apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 6 +- .../org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 6 +- .../protocol/IoTDBConfigNodeReceiver.java | 41 ++- .../protocol/IoTDBConfigRegionAirGapSink.java | 1 + .../thrift/IoTDBDataNodeReceiver.java | 222 +++++++++--- .../IoTDBDataNodeAsyncClientManager.java | 1 + .../airgap/IoTDBDataNodeAirGapSink.java | 1 + .../async/IoTDBDataRegionAsyncSink.java | 2 + .../common/header/DatasetHeaderFactory.java | 4 + .../source/ShowReceiversOperator.java | 161 +++++++++ ...formationSchemaContentSupplierFactory.java | 49 +++ .../db/queryengine/plan/analyze/Analysis.java | 2 + .../plan/analyze/AnalyzeVisitor.java | 29 ++ .../queryengine/plan/parser/ASTVisitor.java | 6 + .../plan/planner/LogicalPlanBuilder.java | 39 +++ .../plan/planner/LogicalPlanVisitor.java | 7 + .../plan/planner/OperatorTreeGenerator.java | 15 + .../SimpleFragmentParallelPlanner.java | 2 + .../node/DataNodePlanNodeDeserializer.java | 3 + .../plan/planner/plan/node/PlanVisitor.java | 5 + .../plan/node/source/ShowReceiversNode.java | 115 +++++++ .../DataNodeLocationSupplierFactory.java | 1 + .../security/TreeAccessCheckVisitor.java | 7 + .../relational/sql/parser/AstBuilder.java | 11 + .../plan/statement/StatementType.java | 1 + .../plan/statement/StatementVisitor.java | 5 + .../statement/sys/ShowReceiversStatement.java | 60 ++++ .../plan/parser/StatementGeneratorTest.java | 11 + .../node/source/SourceNodeSerdeTest.java | 11 + .../informationschema/ShowReceiversTest.java | 53 +++ .../pipe/receiver/IoTDBFileReceiver.java | 100 +++++- .../runtime/PipeReceiverRuntimeRegistry.java | 315 ++++++++++++++++++ .../runtime/PipeReceiverRuntimeSnapshot.java | 118 +++++++ .../pipe/sink/client/IoTDBClientManager.java | 20 ++ .../sink/client/IoTDBSyncClientManager.java | 1 + .../common/PipeTransferHandshakeConstant.java | 2 + .../commons/pipe/sink/protocol/IoTDBSink.java | 15 + .../pipe/sink/protocol/IoTDBSslSyncSink.java | 1 + .../plan/planner/plan/node/PlanNodeType.java | 1 + .../schema/column/ColumnHeaderConstant.java | 44 +++ .../schema/table/InformationSchema.java | 36 ++ .../PipeReceiverRuntimeRegistryTest.java | 127 +++++++ .../relational/grammar/sql/RelationalSql.g4 | 8 +- 44 files changed, 1612 insertions(+), 56 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ShowReceiversOperator.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/source/ShowReceiversNode.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/ShowReceiversStatement.java create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/informationschema/ShowReceiversTest.java create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/runtime/PipeReceiverRuntimeRegistry.java create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/runtime/PipeReceiverRuntimeSnapshot.java create mode 100644 iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/receiver/runtime/PipeReceiverRuntimeRegistryTest.java diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4 index 54d53bab67431..f16bf85bc76a1 100644 --- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4 +++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4 @@ -196,6 +196,7 @@ keyWords | QUERY | QUERYID | QUOTA + | RECEIVERS | RANGE | READONLY | READ @@ -298,4 +299,4 @@ keyWords | OPTION | INF | CURRENT_TIMESTAMP - ; \ No newline at end of file + ; diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 index 3e94667a2faa5..dcad209354cde 100644 --- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 +++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 @@ -55,7 +55,7 @@ ddlStatement // ExternalService | createService | startService | stopService | dropService | showService // Pipe Task - | createPipe | alterPipe | dropPipe | startPipe | stopPipe | showPipes + | createPipe | alterPipe | dropPipe | startPipe | stopPipe | showPipes | showReceivers // Pipe Plugin | createPipePlugin | dropPipePlugin | showPipePlugins // Subscription @@ -701,6 +701,10 @@ showPipes : SHOW ((PIPE pipeName=identifier) | PIPES (WHERE (CONNECTOR | SINK) USED BY pipeName=identifier)?) ; +showReceivers + : SHOW RECEIVERS + ; + // Pipe Plugin ========================================================================================= createPipePlugin : CREATE PIPEPLUGIN (IF NOT EXISTS)? pluginName=identifier AS className=STRING_LITERAL uriClause diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 index 59bac72fc7813..4f96e1f69a374 100644 --- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 +++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 @@ -723,6 +723,10 @@ QUOTA : Q U O T A ; +RECEIVERS + : R E C E I V E R S + ; + RANGE : R A N G E ; @@ -1411,4 +1415,4 @@ fragment V: [vV]; fragment W: [wW]; fragment X: [xX]; fragment Y: [yY]; -fragment Z: [zZ]; \ No newline at end of file +fragment Z: [zZ]; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java index 7de8e8bf78d9d..9d84e448c9f7c 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java @@ -35,6 +35,7 @@ import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern; import org.apache.iotdb.commons.pipe.receiver.IoTDBFileReceiver; import org.apache.iotdb.commons.pipe.receiver.PipeReceiverStatusHandler; +import org.apache.iotdb.commons.pipe.receiver.runtime.PipeReceiverRuntimeRegistry; import org.apache.iotdb.commons.pipe.sink.payload.airgap.AirGapPseudoTPipeTransferRequest; import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeRequestType; import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeTransferCompressedReq; @@ -194,20 +195,22 @@ public TPipeTransferResp receive(final TPipeTransferReq req) { PipeTransferConfigNodeHandshakeV1Req.fromTPipeTransferReq(req)); PipeConfigNodeReceiverMetrics.getInstance() .recordHandshakeConfigNodeV1Timer(System.nanoTime() - startTime); - return resp; + return recordConfigNodeHandshakeIfSuccess(resp, req); case HANDSHAKE_CONFIGNODE_V2: resp = handleTransferHandshakeV2( PipeTransferConfigNodeHandshakeV2Req.fromTPipeTransferReq(req)); - userEntity.setAuditLogOperation(AuditLogOperation.DDL); + if (Objects.nonNull(userEntity)) { + userEntity.setAuditLogOperation(AuditLogOperation.DDL); + } PipeConfigNodeReceiverMetrics.getInstance() .recordHandshakeConfigNodeV2Timer(System.nanoTime() - startTime); - return resp; + return recordConfigNodeHandshakeIfSuccess(resp, req); case TRANSFER_CONFIG_PLAN: resp = handleTransferConfigPlan(PipeTransferConfigPlanReq.fromTPipeTransferReq(req)); PipeConfigNodeReceiverMetrics.getInstance() .recordTransferConfigPlanTimer(System.nanoTime() - startTime); - return resp; + return recordConfigNodeTransferIfSuccess(resp); case TRANSFER_CONFIG_SNAPSHOT_PIECE: resp = handleTransferFilePiece( @@ -216,14 +219,14 @@ public TPipeTransferResp receive(final TPipeTransferReq req) { false); PipeConfigNodeReceiverMetrics.getInstance() .recordTransferConfigSnapshotPieceTimer(System.nanoTime() - startTime); - return resp; + return recordConfigNodeTransferIfSuccess(resp); case TRANSFER_CONFIG_SNAPSHOT_SEAL: resp = handleTransferFileSealV2( PipeTransferConfigSnapshotSealReq.fromTPipeTransferReq(req)); PipeConfigNodeReceiverMetrics.getInstance() .recordTransferConfigSnapshotSealTimer(System.nanoTime() - startTime); - return resp; + return recordConfigNodeTransferIfSuccess(resp); case TRANSFER_COMPRESSED: return receive(PipeTransferCompressedReq.fromTPipeTransferReq(req)); default: @@ -262,6 +265,32 @@ private boolean needHandshake(final PipeRequestType type) { && type != PipeRequestType.HANDSHAKE_CONFIGNODE_V2; } + private TPipeTransferResp recordConfigNodeHandshakeIfSuccess( + final TPipeTransferResp resp, final TPipeTransferReq req) { + if (isSuccess(resp)) { + recordPipeReceiverHandshake( + PipeReceiverRuntimeRegistry.NODE_TYPE_CONFIG_NODE, + ConfigNodeDescriptor.getInstance().getConf().getConfigNodeId(), + getProtocol(req)); + } + return resp; + } + + private TPipeTransferResp recordConfigNodeTransferIfSuccess(final TPipeTransferResp resp) { + if (isSuccess(resp)) { + recordPipeReceiverTransfer(); + } else { + recordPipeReceiverRequest(); + } + return resp; + } + + private static String getProtocol(final TPipeTransferReq req) { + return req instanceof AirGapPseudoTPipeTransferRequest + ? PipeReceiverRuntimeRegistry.PROTOCOL_AIR_GAP + : PipeReceiverRuntimeRegistry.PROTOCOL_THRIFT; + } + private TPipeTransferResp handleTransferConfigPlan(final PipeTransferConfigPlanReq req) throws IOException { return new TPipeTransferResp( diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/sink/protocol/IoTDBConfigRegionAirGapSink.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/sink/protocol/IoTDBConfigRegionAirGapSink.java index c8c2890f5be3e..f8e91993eeb90 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/sink/protocol/IoTDBConfigRegionAirGapSink.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/sink/protocol/IoTDBConfigRegionAirGapSink.java @@ -89,6 +89,7 @@ protected byte[] generateHandShakeV2Payload() throws IOException { Boolean.toString(shouldMarkAsPipeRequest)); params.put( PipeTransferHandshakeConstant.HANDSHAKE_KEY_SKIP_IF, Boolean.toString(skipIfNoPrivileges)); + appendPipeInfoToHandshakeParams(params); return PipeTransferConfigNodeHandshakeV2Req.toTPipeTransferBytes(params); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java index 731c2eb50f0f0..e237e7129bf00 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java @@ -35,8 +35,10 @@ import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern; import org.apache.iotdb.commons.pipe.receiver.IoTDBFileReceiver; import org.apache.iotdb.commons.pipe.receiver.PipeReceiverStatusHandler; +import org.apache.iotdb.commons.pipe.receiver.runtime.PipeReceiverRuntimeRegistry; import org.apache.iotdb.commons.pipe.resource.log.PipeLogger; import org.apache.iotdb.commons.pipe.sink.payload.airgap.AirGapPseudoTPipeTransferRequest; +import org.apache.iotdb.commons.pipe.sink.payload.thrift.common.PipeTransferHandshakeConstant; import org.apache.iotdb.commons.pipe.sink.payload.thrift.common.PipeTransferSliceReqHandler; import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeRequestType; import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeTransferCompressedReq; @@ -115,16 +117,19 @@ import com.google.common.util.concurrent.ListenableFuture; import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.utils.ReadWriteIOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.FileNotFoundException; import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.file.Paths; import java.time.ZoneId; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -174,6 +179,8 @@ public class IoTDBDataNodeReceiver extends IoTDBFileReceiver { // datanode (cluster B). private static final AtomicLong CONFIG_RECEIVER_ID_GENERATOR = new AtomicLong(0); protected final AtomicReference configReceiverId = new AtomicReference<>(); + private final AtomicReference configPipeReceiverRuntimeSessionKey = + new AtomicReference<>(); private final PipeTransferSliceReqHandler sliceReqHandler = new PipeTransferSliceReqHandler(); @@ -217,8 +224,10 @@ public synchronized TPipeTransferResp receive(final TPipeTransferReq req) { TSStatusCode.PIPE_HANDSHAKE_ERROR.getStatusCode(), "The receiver memory is not enough to handle the handshake request from datanode.")); } - return handleTransferHandshakeV1( - PipeTransferDataNodeHandshakeV1Req.fromTPipeTransferReq(req)); + return recordDataNodeHandshakeIfSuccess( + handleTransferHandshakeV1( + PipeTransferDataNodeHandshakeV1Req.fromTPipeTransferReq(req)), + req); } finally { PipeDataNodeReceiverMetrics.getInstance() .recordHandshakeDatanodeV1Timer(System.nanoTime() - startTime); @@ -235,8 +244,10 @@ public synchronized TPipeTransferResp receive(final TPipeTransferReq req) { TSStatusCode.PIPE_HANDSHAKE_ERROR.getStatusCode(), "The receiver memory is not enough to handle the handshake request from datanode.")); } - return handleTransferHandshakeV2( - PipeTransferDataNodeHandshakeV2Req.fromTPipeTransferReq(req)); + return recordDataNodeHandshakeIfSuccess( + handleTransferHandshakeV2( + PipeTransferDataNodeHandshakeV2Req.fromTPipeTransferReq(req)), + req); } finally { PipeDataNodeReceiverMetrics.getInstance() .recordHandshakeDatanodeV2Timer(System.nanoTime() - startTime); @@ -245,8 +256,9 @@ public synchronized TPipeTransferResp receive(final TPipeTransferReq req) { case TRANSFER_TABLET_INSERT_NODE: { try { - return handleTransferTabletInsertNode( - PipeTransferTabletInsertNodeReq.fromTPipeTransferReq(req)); + return recordDataNodeTransferIfSuccess( + handleTransferTabletInsertNode( + PipeTransferTabletInsertNodeReq.fromTPipeTransferReq(req))); } finally { PipeDataNodeReceiverMetrics.getInstance() @@ -256,8 +268,9 @@ public synchronized TPipeTransferResp receive(final TPipeTransferReq req) { case TRANSFER_TABLET_INSERT_NODE_V2: { try { - return handleTransferTabletInsertNode( - PipeTransferTabletInsertNodeReqV2.fromTPipeTransferReq(req)); + return recordDataNodeTransferIfSuccess( + handleTransferTabletInsertNode( + PipeTransferTabletInsertNodeReqV2.fromTPipeTransferReq(req))); } finally { PipeDataNodeReceiverMetrics.getInstance() .recordTransferTabletInsertNodeV2Timer(System.nanoTime() - startTime); @@ -266,7 +279,8 @@ public synchronized TPipeTransferResp receive(final TPipeTransferReq req) { case TRANSFER_TABLET_RAW: { try { - return handleTransferTabletRaw(PipeTransferTabletRawReq.fromTPipeTransferReq(req)); + return recordDataNodeTransferIfSuccess( + handleTransferTabletRaw(PipeTransferTabletRawReq.fromTPipeTransferReq(req))); } finally { PipeDataNodeReceiverMetrics.getInstance() .recordTransferTabletRawTimer(System.nanoTime() - startTime); @@ -275,8 +289,8 @@ public synchronized TPipeTransferResp receive(final TPipeTransferReq req) { case TRANSFER_TABLET_RAW_V2: { try { - return handleTransferTabletRaw( - PipeTransferTabletRawReqV2.fromTPipeTransferReq(req)); + return recordDataNodeTransferIfSuccess( + handleTransferTabletRaw(PipeTransferTabletRawReqV2.fromTPipeTransferReq(req))); } finally { PipeDataNodeReceiverMetrics.getInstance() .recordTransferTabletRawV2Timer(System.nanoTime() - startTime); @@ -285,8 +299,9 @@ public synchronized TPipeTransferResp receive(final TPipeTransferReq req) { case TRANSFER_TABLET_BINARY: { try { - return handleTransferTabletBinary( - PipeTransferTabletBinaryReq.fromTPipeTransferReq(req)); + return recordDataNodeTransferIfSuccess( + handleTransferTabletBinary( + PipeTransferTabletBinaryReq.fromTPipeTransferReq(req))); } finally { PipeDataNodeReceiverMetrics.getInstance() .recordTransferTabletBinaryTimer(System.nanoTime() - startTime); @@ -295,8 +310,9 @@ public synchronized TPipeTransferResp receive(final TPipeTransferReq req) { case TRANSFER_TABLET_BINARY_V2: { try { - return handleTransferTabletBinary( - PipeTransferTabletBinaryReqV2.fromTPipeTransferReq(req)); + return recordDataNodeTransferIfSuccess( + handleTransferTabletBinary( + PipeTransferTabletBinaryReqV2.fromTPipeTransferReq(req))); } finally { PipeDataNodeReceiverMetrics.getInstance() .recordTransferTabletBinaryV2Timer(System.nanoTime() - startTime); @@ -305,8 +321,9 @@ public synchronized TPipeTransferResp receive(final TPipeTransferReq req) { case TRANSFER_TABLET_BATCH: { try { - return handleTransferTabletBatch( - PipeTransferTabletBatchReq.fromTPipeTransferReq(req)); + return recordDataNodeTransferIfSuccess( + handleTransferTabletBatch( + PipeTransferTabletBatchReq.fromTPipeTransferReq(req))); } finally { PipeDataNodeReceiverMetrics.getInstance() .recordTransferTabletBatchTimer(System.nanoTime() - startTime); @@ -315,8 +332,9 @@ public synchronized TPipeTransferResp receive(final TPipeTransferReq req) { case TRANSFER_TABLET_BATCH_V2: { try { - return handleTransferTabletBatchV2( - PipeTransferTabletBatchReqV2.fromTPipeTransferReq(req)); + return recordDataNodeTransferIfSuccess( + handleTransferTabletBatchV2( + PipeTransferTabletBatchReqV2.fromTPipeTransferReq(req))); } finally { PipeDataNodeReceiverMetrics.getInstance() .recordTransferTabletBatchV2Timer(System.nanoTime() - startTime); @@ -325,10 +343,11 @@ public synchronized TPipeTransferResp receive(final TPipeTransferReq req) { case TRANSFER_TS_FILE_PIECE: { try { - return handleTransferFilePiece( - PipeTransferTsFilePieceReq.fromTPipeTransferReq(req), - req instanceof AirGapPseudoTPipeTransferRequest, - true); + return recordDataNodeTransferIfSuccess( + handleTransferFilePiece( + PipeTransferTsFilePieceReq.fromTPipeTransferReq(req), + req instanceof AirGapPseudoTPipeTransferRequest, + true)); } finally { PipeDataNodeReceiverMetrics.getInstance() .recordTransferTsFilePieceTimer(System.nanoTime() - startTime); @@ -337,8 +356,8 @@ public synchronized TPipeTransferResp receive(final TPipeTransferReq req) { case TRANSFER_TS_FILE_SEAL: { try { - return handleTransferFileSealV1( - PipeTransferTsFileSealReq.fromTPipeTransferReq(req)); + return recordDataNodeTransferIfSuccess( + handleTransferFileSealV1(PipeTransferTsFileSealReq.fromTPipeTransferReq(req))); } finally { PipeDataNodeReceiverMetrics.getInstance() .recordTransferTsFileSealTimer(System.nanoTime() - startTime); @@ -347,10 +366,11 @@ public synchronized TPipeTransferResp receive(final TPipeTransferReq req) { case TRANSFER_TS_FILE_PIECE_WITH_MOD: { try { - return handleTransferFilePiece( - PipeTransferTsFilePieceWithModReq.fromTPipeTransferReq(req), - req instanceof AirGapPseudoTPipeTransferRequest, - false); + return recordDataNodeTransferIfSuccess( + handleTransferFilePiece( + PipeTransferTsFilePieceWithModReq.fromTPipeTransferReq(req), + req instanceof AirGapPseudoTPipeTransferRequest, + false)); } finally { PipeDataNodeReceiverMetrics.getInstance() @@ -360,8 +380,9 @@ public synchronized TPipeTransferResp receive(final TPipeTransferReq req) { case TRANSFER_TS_FILE_SEAL_WITH_MOD: { try { - return handleTransferFileSealV2( - PipeTransferTsFileSealWithModReq.fromTPipeTransferReq(req)); + return recordDataNodeTransferIfSuccess( + handleTransferFileSealV2( + PipeTransferTsFileSealWithModReq.fromTPipeTransferReq(req))); } finally { PipeDataNodeReceiverMetrics.getInstance() .recordTransferTsFileSealWithModTimer(System.nanoTime() - startTime); @@ -370,7 +391,8 @@ public synchronized TPipeTransferResp receive(final TPipeTransferReq req) { case TRANSFER_PLAN_NODE: { try { - return handleTransferSchemaPlan(PipeTransferPlanNodeReq.fromTPipeTransferReq(req)); + return recordDataNodeTransferIfSuccess( + handleTransferSchemaPlan(PipeTransferPlanNodeReq.fromTPipeTransferReq(req))); } finally { PipeDataNodeReceiverMetrics.getInstance() .recordTransferSchemaPlanTimer(System.nanoTime() - startTime); @@ -379,10 +401,11 @@ public synchronized TPipeTransferResp receive(final TPipeTransferReq req) { case TRANSFER_SCHEMA_SNAPSHOT_PIECE: { try { - return handleTransferFilePiece( - PipeTransferSchemaSnapshotPieceReq.fromTPipeTransferReq(req), - req instanceof AirGapPseudoTPipeTransferRequest, - false); + return recordDataNodeTransferIfSuccess( + handleTransferFilePiece( + PipeTransferSchemaSnapshotPieceReq.fromTPipeTransferReq(req), + req instanceof AirGapPseudoTPipeTransferRequest, + false)); } finally { PipeDataNodeReceiverMetrics.getInstance() @@ -392,8 +415,9 @@ public synchronized TPipeTransferResp receive(final TPipeTransferReq req) { case TRANSFER_SCHEMA_SNAPSHOT_SEAL: { try { - return handleTransferFileSealV2( - PipeTransferSchemaSnapshotSealReq.fromTPipeTransferReq(req)); + return recordDataNodeTransferIfSuccess( + handleTransferFileSealV2( + PipeTransferSchemaSnapshotSealReq.fromTPipeTransferReq(req))); } finally { PipeDataNodeReceiverMetrics.getInstance() @@ -409,7 +433,7 @@ public synchronized TPipeTransferResp receive(final TPipeTransferReq req) { try { // Config requests will first be received by the DataNode receiver, // then transferred to ConfigNode receiver to execute. - return handleTransferConfigPlan(req); + return recordConfigNodeReceiverRuntimeIfSuccess(handleTransferConfigPlan(req), req); } finally { PipeDataNodeReceiverMetrics.getInstance() .recordTransferConfigPlanTimer(System.nanoTime() - startTime); @@ -760,6 +784,123 @@ private TPipeTransferResp handleTransferSlice(final PipeTransferSliceReq pipeTra return receive(req.get()); } + private TPipeTransferResp recordDataNodeHandshakeIfSuccess( + final TPipeTransferResp resp, final TPipeTransferReq req) { + if (isSuccess(resp)) { + recordPipeReceiverHandshake( + PipeReceiverRuntimeRegistry.NODE_TYPE_DATA_NODE, + IOTDB_CONFIG.getDataNodeId(), + getProtocol(req)); + } + return resp; + } + + private TPipeTransferResp recordDataNodeTransferIfSuccess(final TPipeTransferResp resp) { + if (isSuccess(resp)) { + recordPipeReceiverTransfer(); + } else { + recordPipeReceiverRequest(); + } + return resp; + } + + private TPipeTransferResp recordConfigNodeReceiverRuntimeIfSuccess( + final TPipeTransferResp resp, final TPipeTransferReq req) { + if (!PipeRequestType.isValidatedRequestType(req.getType())) { + return resp; + } + + final PipeRequestType requestType = PipeRequestType.valueOf(req.getType()); + if (requestType == PipeRequestType.HANDSHAKE_CONFIGNODE_V1 + || requestType == PipeRequestType.HANDSHAKE_CONFIGNODE_V2) { + if (isSuccess(resp)) { + recordConfigNodeHandshake(req, requestType); + } + } else { + if (isSuccess(resp)) { + PipeReceiverRuntimeRegistry.getInstance() + .markTransfer(configPipeReceiverRuntimeSessionKey.get(), System.currentTimeMillis()); + } else { + PipeReceiverRuntimeRegistry.getInstance() + .markRequest(configPipeReceiverRuntimeSessionKey.get()); + } + } + return resp; + } + + private void recordConfigNodeHandshake( + final TPipeTransferReq req, final PipeRequestType requestType) { + final String protocol = getProtocol(req); + final String sessionKey = + String.format( + "%s-%s-%s-%s", + PipeReceiverRuntimeRegistry.NODE_TYPE_CONFIG_NODE, -1, protocol, getConfigReceiverId()); + final String oldSessionKey = configPipeReceiverRuntimeSessionKey.getAndSet(sessionKey); + if (!Objects.equals(oldSessionKey, sessionKey)) { + PipeReceiverRuntimeRegistry.getInstance().deregister(oldSessionKey); + } + + final Map params = + requestType == PipeRequestType.HANDSHAKE_CONFIGNODE_V2 + ? parseHandshakeV2Params(req) + : Collections.emptyMap(); + PipeReceiverRuntimeRegistry.getInstance() + .registerOrUpdateSession( + sessionKey, + PipeReceiverRuntimeRegistry.NODE_TYPE_CONFIG_NODE, + -1, + protocol, + getSenderHost(), + parseSenderPort(getSenderPort()), + params.getOrDefault(PipeTransferHandshakeConstant.HANDSHAKE_KEY_USERNAME, username), + params.getOrDefault( + PipeTransferHandshakeConstant.HANDSHAKE_KEY_CLUSTER_ID, + PipeReceiverRuntimeRegistry.UNKNOWN), + params.get(PipeTransferHandshakeConstant.HANDSHAKE_KEY_PIPE_NAME), + parsePipeCreationTime( + params.get(PipeTransferHandshakeConstant.HANDSHAKE_KEY_PIPE_CREATION_TIME)), + System.currentTimeMillis()); + } + + private static Map parseHandshakeV2Params(final TPipeTransferReq req) { + final Map params = new HashMap<>(); + if (req.getBody() == null) { + return params; + } + final ByteBuffer body = req.getBody().duplicate(); + body.rewind(); + final int size = ReadWriteIOUtils.readInt(body); + for (int i = 0; i < size; ++i) { + params.put(ReadWriteIOUtils.readString(body), ReadWriteIOUtils.readString(body)); + } + return params; + } + + private static String getProtocol(final TPipeTransferReq req) { + return req instanceof AirGapPseudoTPipeTransferRequest + ? PipeReceiverRuntimeRegistry.PROTOCOL_AIR_GAP + : PipeReceiverRuntimeRegistry.PROTOCOL_THRIFT; + } + + private static int parseSenderPort(final String senderPort) { + try { + return Integer.parseInt(senderPort); + } catch (final Exception e) { + return -1; + } + } + + private static long parsePipeCreationTime(final String pipeCreationTime) { + if (pipeCreationTime == null) { + return Long.MIN_VALUE; + } + try { + return Long.parseLong(pipeCreationTime); + } catch (final NumberFormatException e) { + return Long.MIN_VALUE; + } + } + /** * For {@link InsertRowsStatement} and {@link InsertMultiTabletsStatement}, the returned {@link * TSStatus} will use sub-status to record the endpoint for redirection. Each sub-status records @@ -1157,6 +1298,9 @@ private TSStatus executeStatementForTableModelWithPermissionCheck( @Override public synchronized void handleExit() { + PipeReceiverRuntimeRegistry.getInstance() + .deregister(configPipeReceiverRuntimeSessionKey.getAndSet(null)); + if (Objects.nonNull(configReceiverId.get())) { try { ClusterConfigTaskExecutor.getInstance().handlePipeConfigClientExit(configReceiverId.get()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/client/IoTDBDataNodeAsyncClientManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/client/IoTDBDataNodeAsyncClientManager.java index 5c38c3a85402f..69efcf39fb249 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/client/IoTDBDataNodeAsyncClientManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/client/IoTDBDataNodeAsyncClientManager.java @@ -319,6 +319,7 @@ public void onError(final Exception e) { params.put( PipeTransferHandshakeConstant.HANDSHAKE_KEY_SKIP_IF, Boolean.toString(skipIfNoPrivileges)); + appendPipeInfoToHandshakeParams(params); client.setTimeoutDynamically(PipeConfig.getInstance().getPipeSinkHandshakeTimeoutMs()); client.pipeTransfer(PipeTransferDataNodeHandshakeV2Req.toTPipeTransferReq(params), callback); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/airgap/IoTDBDataNodeAirGapSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/airgap/IoTDBDataNodeAirGapSink.java index b593df5661206..9e05c240a155e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/airgap/IoTDBDataNodeAirGapSink.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/airgap/IoTDBDataNodeAirGapSink.java @@ -72,6 +72,7 @@ protected byte[] generateHandShakeV2Payload() throws IOException { Boolean.toString(shouldMarkAsPipeRequest)); params.put( PipeTransferHandshakeConstant.HANDSHAKE_KEY_SKIP_IF, Boolean.toString(skipIfNoPrivileges)); + appendPipeInfoToHandshakeParams(params); return PipeTransferDataNodeHandshakeV2Req.toTPipeTransferBytes(params); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/IoTDBDataRegionAsyncSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/IoTDBDataRegionAsyncSink.java index b8b169b1f6abe..a96b3d58c3665 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/IoTDBDataRegionAsyncSink.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/IoTDBDataRegionAsyncSink.java @@ -172,6 +172,7 @@ public void customize( shouldMarkAsPipeRequest, false, skipIfNoPrivileges); + clientManager.setPipeInfo(pipeName, creationTime); transferTsFileClientManager = new IoTDBDataNodeAsyncClientManager( @@ -188,6 +189,7 @@ public void customize( shouldMarkAsPipeRequest, isSplitTSFileBatchModeEnabled, skipIfNoPrivileges); + transferTsFileClientManager.setPipeInfo(pipeName, creationTime); if (isTabletBatchModeEnabled) { tabletBatchBuilder = new PipeTransferBatchReqBuilder(parameters); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/DatasetHeaderFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/DatasetHeaderFactory.java index 18f15eea8f397..45c3670cd168f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/DatasetHeaderFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/DatasetHeaderFactory.java @@ -207,6 +207,10 @@ public static DatasetHeader getShowQueriesHeader() { return new DatasetHeader(ColumnHeaderConstant.showQueriesColumnHeaders, false); } + public static DatasetHeader getShowReceiversHeader() { + return new DatasetHeader(ColumnHeaderConstant.showReceiversColumnHeaders, false); + } + public static DatasetHeader getShowDiskUsageHeader() { return new DatasetHeader(ColumnHeaderConstant.showDiskUsageColumnHeaders, true); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ShowReceiversOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ShowReceiversOperator.java new file mode 100644 index 0000000000000..c9cf62317f5f6 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ShowReceiversOperator.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.source; + +import org.apache.iotdb.commons.pipe.receiver.runtime.PipeReceiverRuntimeRegistry; +import org.apache.iotdb.commons.pipe.receiver.runtime.PipeReceiverRuntimeSnapshot; +import org.apache.iotdb.commons.queryengine.execution.MemoryEstimationHelper; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.commons.queryengine.utils.DateTimeUtils; +import org.apache.iotdb.commons.queryengine.utils.TimestampPrecisionUtils; +import org.apache.iotdb.db.queryengine.common.header.DatasetHeaderFactory; +import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext; + +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.common.conf.TSFileDescriptor; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.block.TsBlockBuilder; +import org.apache.tsfile.read.common.block.column.TimeColumnBuilder; +import org.apache.tsfile.utils.BytesUtils; +import org.apache.tsfile.utils.RamUsageEstimator; + +import java.util.List; +import java.util.concurrent.TimeUnit; + +public class ShowReceiversOperator implements SourceOperator { + + private final OperatorContext operatorContext; + private final PlanNodeId sourceId; + + private TsBlock tsBlock; + private boolean hasConsumed; + + private static final int DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES = + TSFileDescriptor.getInstance().getConfig().getMaxTsBlockSizeInBytes(); + + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(ShowReceiversOperator.class); + + public ShowReceiversOperator(OperatorContext operatorContext, PlanNodeId sourceId) { + this.operatorContext = operatorContext; + this.sourceId = sourceId; + } + + @Override + public OperatorContext getOperatorContext() { + return operatorContext; + } + + @Override + public TsBlock next() { + TsBlock res = tsBlock; + hasConsumed = true; + tsBlock = null; + return res; + } + + @Override + public boolean hasNext() { + if (hasConsumed) { + return false; + } + if (tsBlock == null) { + tsBlock = buildTsBlock(); + } + return true; + } + + @Override + public boolean isFinished() { + return hasConsumed; + } + + @Override + public void close() { + // do nothing + } + + @Override + public long calculateMaxPeekMemory() { + return DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES; + } + + @Override + public long calculateMaxReturnSize() { + return DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES; + } + + @Override + public long calculateRetainedSizeAfterCallingNext() { + return 0; + } + + @Override + public PlanNodeId getSourceId() { + return sourceId; + } + + private TsBlock buildTsBlock() { + final List outputDataTypes = + DatasetHeaderFactory.getShowReceiversHeader().getRespDataTypes(); + final TsBlockBuilder builder = new TsBlockBuilder(outputDataTypes); + final TimeColumnBuilder timeColumnBuilder = builder.getTimeColumnBuilder(); + final ColumnBuilder[] columnBuilders = builder.getValueColumnBuilders(); + final long currentTime = + TimestampPrecisionUtils.convertToCurrPrecision( + System.currentTimeMillis(), TimeUnit.MILLISECONDS); + + for (PipeReceiverRuntimeSnapshot snapshot : + PipeReceiverRuntimeRegistry.getInstance().snapshot()) { + timeColumnBuilder.writeLong(currentTime); + columnBuilders[0].writeBinary(BytesUtils.valueOf(snapshot.getReceiverNodeType())); + columnBuilders[1].writeInt(snapshot.getReceiverNodeId()); + columnBuilders[2].writeBinary(BytesUtils.valueOf(snapshot.getProtocol())); + columnBuilders[3].writeBinary(BytesUtils.valueOf(snapshot.getSenderAddress())); + columnBuilders[4].writeBinary(BytesUtils.valueOf(snapshot.getSenderPorts())); + columnBuilders[5].writeInt(snapshot.getConnectionCount()); + columnBuilders[6].writeInt(snapshot.getPipeCount()); + columnBuilders[7].writeBinary(BytesUtils.valueOf(snapshot.getPipeIds())); + columnBuilders[8].writeBinary(BytesUtils.valueOf(snapshot.getUserName())); + columnBuilders[9].writeBinary(BytesUtils.valueOf(snapshot.getSenderClusterId())); + columnBuilders[10].writeBinary( + BytesUtils.valueOf(formatTime(snapshot.getLastHandshakeTime()))); + columnBuilders[11].writeBinary( + BytesUtils.valueOf(formatTime(snapshot.getLastTransferTime()))); + columnBuilders[12].writeLong(snapshot.getRequestNum()); + builder.declarePosition(); + } + return builder.build(); + } + + private static String formatTime(long timestampInMillis) { + return timestampInMillis <= 0 + ? PipeReceiverRuntimeRegistry.UNKNOWN + : DateTimeUtils.convertLongToDate(timestampInMillis, "ms"); + } + + @Override + public long ramBytesUsed() { + return INSTANCE_SIZE + + MemoryEstimationHelper.getEstimatedSizeOfAccountableObject(operatorContext) + + MemoryEstimationHelper.getEstimatedSizeOfAccountableObject(sourceId); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java index ca9c09ecb9548..e84348851cefb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java @@ -28,12 +28,15 @@ import org.apache.iotdb.common.rpc.thrift.TExternalServiceEntry; import org.apache.iotdb.common.rpc.thrift.TExternalServiceListResp; import org.apache.iotdb.commons.audit.UserEntity; +import org.apache.iotdb.commons.auth.entity.PrivilegeType; import org.apache.iotdb.commons.client.exception.ClientManagerException; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.exception.IoTDBRuntimeException; import org.apache.iotdb.commons.exception.auth.AccessDeniedException; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.BuiltinPipePlugin; import org.apache.iotdb.commons.pipe.agent.plugin.meta.PipePluginMeta; +import org.apache.iotdb.commons.pipe.receiver.runtime.PipeReceiverRuntimeRegistry; +import org.apache.iotdb.commons.pipe.receiver.runtime.PipeReceiverRuntimeSnapshot; import org.apache.iotdb.commons.queryengine.common.ConnectionInfo; import org.apache.iotdb.commons.queryengine.common.SqlDialect; import org.apache.iotdb.commons.queryengine.plan.relational.function.TableBuiltinTableFunction; @@ -184,6 +187,8 @@ public static IInformationSchemaContentSupplier getSupplier( return new RegionSupplier(dataTypes, userEntity); case InformationSchema.PIPES: return new PipeSupplier(dataTypes, userEntity.getUsername()); + case InformationSchema.RECEIVERS: + return new ReceiversSupplier(dataTypes, userEntity); case InformationSchema.PIPE_PLUGINS: return new PipePluginSupplier(dataTypes, userEntity); case InformationSchema.TOPICS: @@ -710,6 +715,50 @@ public boolean hasNext() { } } + private static class ReceiversSupplier extends TsBlockSupplier { + private final Iterator iterator; + + private ReceiversSupplier(final List dataTypes, final UserEntity userEntity) { + super(dataTypes); + accessControl.checkMissingPrivileges( + userEntity.getUsername(), Collections.singletonList(PrivilegeType.USE_PIPE), userEntity); + iterator = PipeReceiverRuntimeRegistry.getInstance().snapshot().iterator(); + } + + @Override + protected void constructLine() { + final PipeReceiverRuntimeSnapshot snapshot = iterator.next(); + columnBuilders[0].writeBinary(BytesUtils.valueOf(snapshot.getReceiverNodeType())); + columnBuilders[1].writeInt(snapshot.getReceiverNodeId()); + columnBuilders[2].writeBinary(BytesUtils.valueOf(snapshot.getProtocol())); + columnBuilders[3].writeBinary(BytesUtils.valueOf(snapshot.getSenderAddress())); + columnBuilders[4].writeBinary(BytesUtils.valueOf(snapshot.getSenderPorts())); + columnBuilders[5].writeInt(snapshot.getConnectionCount()); + columnBuilders[6].writeInt(snapshot.getPipeCount()); + columnBuilders[7].writeBinary(BytesUtils.valueOf(snapshot.getPipeIds())); + columnBuilders[8].writeBinary(BytesUtils.valueOf(snapshot.getUserName())); + columnBuilders[9].writeBinary(BytesUtils.valueOf(snapshot.getSenderClusterId())); + writeTimestamp(columnBuilders[10], snapshot.getLastHandshakeTime()); + writeTimestamp(columnBuilders[11], snapshot.getLastTransferTime()); + columnBuilders[12].writeLong(snapshot.getRequestNum()); + resultBuilder.declarePosition(); + } + + private static void writeTimestamp(ColumnBuilder columnBuilder, long timestampInMillis) { + if (timestampInMillis <= 0) { + columnBuilder.appendNull(); + return; + } + columnBuilder.writeLong( + TimestampPrecisionUtils.convertToCurrPrecision(timestampInMillis, TimeUnit.MILLISECONDS)); + } + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + } + private static class PipePluginSupplier extends TsBlockSupplier { private final Iterator iterator; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java index be20d218e6a70..d5539bfe93d4c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java @@ -60,6 +60,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.ExplainAnalyzeStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowDiskUsageStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowReceiversStatement; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; @@ -488,6 +489,7 @@ private boolean hasDataSource() { return (dataPartition != null && !dataPartition.isEmpty()) || (schemaPartition != null && !schemaPartition.isEmpty()) || statement instanceof ShowQueriesStatement + || statement instanceof ShowReceiversStatement || statement instanceof ShowDiskUsageStatement || (statement instanceof QueryStatement && ((QueryStatement) statement).isAggregationQuery()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java index 22d5b0518ac55..27a9778802a8d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java @@ -145,6 +145,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.ExplainStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowDiskUsageStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowReceiversStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowVersionStatement; import org.apache.iotdb.db.schemaengine.schemaregion.view.visitor.TransformToExpressionVisitor; import org.apache.iotdb.rpc.RpcUtils; @@ -3822,6 +3823,34 @@ public Analysis visitShowQueries( return analysis; } + @Override + public Analysis visitShowReceivers( + ShowReceiversStatement showReceiversStatement, MPPQueryContext context) { + Analysis analysis = new Analysis(); + analysis.setRealStatement(showReceiversStatement); + analysis.setRespDatasetHeader(DatasetHeaderFactory.getShowReceiversHeader()); + analysis.setVirtualSource(true); + + List allReadableDataNodeLocations = getReadableDataNodeLocations(); + if (allReadableDataNodeLocations.isEmpty()) { + throw new StatementAnalyzeException(DataNodeQueryMessages.NO_RUNNING_DATANODES); + } + analysis.setReadableDataNodeLocations(allReadableDataNodeLocations); + + Set sourceExpressions = new HashSet<>(); + for (ColumnHeader columnHeader : analysis.getRespDatasetHeader().getColumnHeaders()) { + sourceExpressions.add( + TimeSeriesOperand.constructColumnHeaderExpression( + columnHeader.getColumnName(), columnHeader.getColumnType())); + } + analysis.setSourceExpressions(sourceExpressions); + sourceExpressions.forEach(expression -> analyzeExpressionType(analysis, expression)); + + analysis.setMergeOrderParameter(new OrderByParameter(showReceiversStatement.getSortItemList())); + + return analysis; + } + @Override public Analysis visitShowDiskUsage( ShowDiskUsageStatement showDiskUsageStatement, MPPQueryContext context) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java index 49f79a6a923c0..feca397e7491c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java @@ -250,6 +250,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowCurrentUserStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowDiskUsageStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowReceiversStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowVersionStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.StartRepairDataStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.StopRepairDataStatement; @@ -4412,6 +4413,11 @@ public Statement visitShowPipes(IoTDBSqlParser.ShowPipesContext ctx) { return showPipesStatement; } + @Override + public Statement visitShowReceivers(IoTDBSqlParser.ShowReceiversContext ctx) { + return new ShowReceiversStatement(); + } + @Override public Statement visitCreateTopic(IoTDBSqlParser.CreateTopicContext ctx) { final CreateTopicStatement createTopicStatement = new CreateTopicStatement(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java index 26d361c1f210f..36e522e5d1368 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java @@ -90,6 +90,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.SeriesSourceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.ShowDiskUsageNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.ShowQueriesNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.ShowReceiversNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.TimeseriesRegionScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.AggregationDescriptor; import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.AggregationStep; @@ -1328,6 +1329,44 @@ private LogicalPlanBuilder planSingleShowQueries( return this; } + public LogicalPlanBuilder planShowReceivers(Analysis analysis) { + List dataNodeLocations = analysis.getReadableDataNodeLocations(); + if (dataNodeLocations.size() == 1) { + this.root = + planSingleShowReceivers(dataNodeLocations.get(0)) + .planSort(analysis.getMergeOrderParameter()) + .getRoot(); + } else { + List outputColumns = new ArrayList<>(); + MergeSortNode mergeSortNode = + new MergeSortNode( + context.getQueryId().genPlanNodeId(), + analysis.getMergeOrderParameter(), + outputColumns); + + dataNodeLocations.forEach( + dataNodeLocation -> + mergeSortNode.addChild( + this.planSingleShowReceivers(dataNodeLocation) + .planSort(analysis.getMergeOrderParameter()) + .getRoot())); + outputColumns.addAll(mergeSortNode.getChildren().get(0).getOutputColumnNames()); + this.root = mergeSortNode; + } + + ColumnHeaderConstant.showReceiversColumnHeaders.forEach( + columnHeader -> + context + .getTypeProvider() + .setTreeModelType(columnHeader.getColumnName(), columnHeader.getColumnType())); + return this; + } + + private LogicalPlanBuilder planSingleShowReceivers(TDataNodeLocation dataNodeLocation) { + this.root = new ShowReceiversNode(context.getQueryId().genPlanNodeId(), dataNodeLocation); + return this; + } + public LogicalPlanBuilder planShowDiskUsage(Analysis analysis, PartialPath pathPattern) { List dataNodeLocations = analysis.getReadableDataNodeLocations(); if (dataNodeLocations.size() == 1) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java index 2c9304ce0b98c..f04dc1c9263ce 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java @@ -92,6 +92,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.ExplainAnalyzeStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowDiskUsageStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowReceiversStatement; import org.apache.iotdb.db.schemaengine.SchemaEngineMode; import org.apache.tsfile.enums.TSDataType; @@ -1017,6 +1018,12 @@ public PlanNode visitShowQueries( return planBuilder.getRoot(); } + @Override + public PlanNode visitShowReceivers( + ShowReceiversStatement showReceiversStatement, MPPQueryContext context) { + return new LogicalPlanBuilder(analysis, context).planShowReceivers(analysis).getRoot(); + } + @Override public PlanNode visitShowDiskUsage( ShowDiskUsageStatement showDiskUsageStatement, MPPQueryContext context) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java index 327dbecdda26d..6e32f55b7322b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java @@ -140,6 +140,7 @@ import org.apache.iotdb.db.queryengine.execution.operator.source.SeriesScanOperator; import org.apache.iotdb.db.queryengine.execution.operator.source.ShowDiskUsageOperator; import org.apache.iotdb.db.queryengine.execution.operator.source.ShowQueriesOperator; +import org.apache.iotdb.db.queryengine.execution.operator.source.ShowReceiversOperator; import org.apache.iotdb.db.queryengine.execution.operator.window.ConditionWindowParameter; import org.apache.iotdb.db.queryengine.execution.operator.window.CountWindowParameter; import org.apache.iotdb.db.queryengine.execution.operator.window.SessionWindowParameter; @@ -224,6 +225,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.SeriesScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.ShowDiskUsageNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.ShowQueriesNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.ShowReceiversNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.TimeseriesRegionScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.AggregationDescriptor; import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.CrossSeriesAggregationDescriptor; @@ -2478,6 +2480,19 @@ public Operator visitShowQueries(ShowQueriesNode node, LocalExecutionPlanContext node.getAllowedUsername()); } + @Override + public Operator visitShowReceivers(ShowReceiversNode node, LocalExecutionPlanContext context) { + OperatorContext operatorContext = + context + .getDriverContext() + .addOperatorContext( + context.getNextOperatorId(), + node.getPlanNodeId(), + ShowReceiversOperator.class.getSimpleName()); + + return new ShowReceiversOperator(operatorContext, node.getPlanNodeId()); + } + @Override public Operator visitShowDiskUsage(ShowDiskUsageNode node, LocalExecutionPlanContext context) { OperatorContext operatorContext = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/SimpleFragmentParallelPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/SimpleFragmentParallelPlanner.java index 4e34d36147168..cd9a9fe44c81d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/SimpleFragmentParallelPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/SimpleFragmentParallelPlanner.java @@ -39,6 +39,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.ExplainAnalyzeStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowDiskUsageStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowReceiversStatement; import org.apache.tsfile.read.common.Path; import org.apache.tsfile.utils.Pair; @@ -159,6 +160,7 @@ private void produceFragmentInstance(PlanFragment fragment) { if (analysis.getTreeStatement() instanceof QueryStatement || analysis.getTreeStatement() instanceof ExplainAnalyzeStatement || analysis.getTreeStatement() instanceof ShowQueriesStatement + || analysis.getTreeStatement() instanceof ShowReceiversStatement || analysis.getTreeStatement() instanceof ShowDiskUsageStatement || (analysis.getTreeStatement() instanceof ShowTimeSeriesStatement && (((ShowTimeSeriesStatement) analysis.getTreeStatement()).isOrderByHeat() diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/DataNodePlanNodeDeserializer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/DataNodePlanNodeDeserializer.java index 2a9b9b6905c77..59de11d98d5d4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/DataNodePlanNodeDeserializer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/DataNodePlanNodeDeserializer.java @@ -109,6 +109,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.SeriesScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.ShowDiskUsageNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.ShowQueriesNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.ShowReceiversNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.TimeseriesRegionScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ContinuousSameSearchIndexSeparatorNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; @@ -415,6 +416,8 @@ public PlanNode deserialize(ByteBuffer buffer, short nodeType) { return ShowDiskUsageNode.deserialize(buffer); case 108: return CollectNode.deserialize(buffer); + case 110: + return ShowReceiversNode.deserialize(buffer); case 902: return CreateOrUpdateTableDeviceNode.deserialize(buffer); case 903: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java index 54ee9d17a9420..418f46ec8659b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java @@ -108,6 +108,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.SeriesScanSourceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.ShowDiskUsageNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.ShowQueriesNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.ShowReceiversNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.TimeseriesRegionScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertMultiTabletsNode; @@ -348,6 +349,10 @@ default R visitShowQueries(ShowQueriesNode node, C context) { return visitPlan(node, context); } + default R visitShowReceivers(ShowReceiversNode node, C context) { + return visitPlan(node, context); + } + default R visitShowDiskUsage(ShowDiskUsageNode node, C context) { return visitPlan(node, context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/source/ShowReceiversNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/source/ShowReceiversNode.java new file mode 100644 index 0000000000000..1cf76707c560b --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/source/ShowReceiversNode.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.planner.plan.node.source; + +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.IPlanVisitor; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeType; +import org.apache.iotdb.commons.schema.column.ColumnHeader; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; + +import com.google.common.collect.ImmutableList; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Objects; + +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.showReceiversColumnHeaders; + +public class ShowReceiversNode extends VirtualSourceNode { + + public static final List SHOW_RECEIVERS_HEADER_COLUMNS = + showReceiversColumnHeaders.stream() + .map(ColumnHeader::getColumnName) + .collect(ImmutableList.toImmutableList()); + + public ShowReceiversNode(PlanNodeId id, TDataNodeLocation dataNodeLocation) { + super(id, dataNodeLocation); + } + + @Override + public List getChildren() { + return ImmutableList.of(); + } + + @Override + public void addChild(PlanNode child) { + throw new UnsupportedOperationException("no child is allowed for ShowReceiversNode"); + } + + @Override + public PlanNodeType getType() { + return PlanNodeType.SHOW_RECEIVERS; + } + + @Override + public PlanNode clone() { + return new ShowReceiversNode(getPlanNodeId(), getDataNodeLocation()); + } + + @Override + public int allowedChildCount() { + return NO_CHILD_ALLOWED; + } + + @Override + public List getOutputColumnNames() { + return SHOW_RECEIVERS_HEADER_COLUMNS; + } + + @Override + public R accept(IPlanVisitor visitor, C context) { + return ((PlanVisitor) visitor).visitShowReceivers(this, context); + } + + @Override + protected void serializeAttributes(ByteBuffer byteBuffer) { + PlanNodeType.SHOW_RECEIVERS.serialize(byteBuffer); + } + + @Override + protected void serializeAttributes(DataOutputStream stream) throws IOException { + PlanNodeType.SHOW_RECEIVERS.serialize(stream); + } + + public static ShowReceiversNode deserialize(ByteBuffer byteBuffer) { + PlanNodeId planNodeId = PlanNodeId.deserialize(byteBuffer); + return new ShowReceiversNode(planNodeId, null); + } + + @Override + public boolean equals(Object o) { + return super.equals(o); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode()); + } + + @Override + public String toString() { + return "ShowReceiversNode-" + this.getPlanNodeId(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/DataNodeLocationSupplierFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/DataNodeLocationSupplierFactory.java index d0b3cb330e10c..2d3f6cea91042 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/DataNodeLocationSupplierFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/DataNodeLocationSupplierFactory.java @@ -123,6 +123,7 @@ public List getDataNodeLocations(final String tableName) { case InformationSchema.CONNECTIONS: case InformationSchema.CURRENT_QUERIES: case InformationSchema.QUERIES_COSTS_HISTOGRAM: + case InformationSchema.RECEIVERS: return getReadableDataNodeLocations(); case InformationSchema.DATABASES: case InformationSchema.TABLES: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/TreeAccessCheckVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/TreeAccessCheckVisitor.java index fd9757c5f8902..7f0a0e93dfac3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/TreeAccessCheckVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/TreeAccessCheckVisitor.java @@ -159,6 +159,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowCurrentUserStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowDiskUsageStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowReceiversStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowVersionStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.StartRepairDataStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.StopRepairDataStatement; @@ -848,6 +849,12 @@ public TSStatus visitShowPipes(ShowPipesStatement statement, TreeAccessCheckCont return StatusUtils.OK; } + @Override + public TSStatus visitShowReceivers( + ShowReceiversStatement statement, TreeAccessCheckContext context) { + return checkPipeManagement(context.setAuditLogOperation(AuditLogOperation.QUERY), () -> ""); + } + @Override public TSStatus visitDropPipe(DropPipeStatement statement, TreeAccessCheckContext context) { return checkPipeManagement( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index 25d68eba6beb5..22a9caf548eeb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -1722,6 +1722,17 @@ public Node visitShowQueriesStatement(RelationalSqlParser.ShowQueriesStatementCo limit); } + @Override + public Node visitShowReceiversStatement(RelationalSqlParser.ShowReceiversStatementContext ctx) { + return new ShowStatement( + getLocation(ctx), + InformationSchema.RECEIVERS, + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty()); + } + @Override public Node visitKillQueryStatement(RelationalSqlParser.KillQueryStatementContext ctx) { if (ctx.queryId == null) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementType.java index b40c6444816fc..32f4c301601c4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementType.java @@ -201,4 +201,5 @@ public enum StatementType { SHOW_EXTERNAL_SERVICE, SHOW_DISK_USAGE, + SHOW_RECEIVERS, } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java index 847e850c52172..e9a64c5981ceb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java @@ -147,6 +147,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowCurrentUserStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowDiskUsageStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowReceiversStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowVersionStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.StartRepairDataStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.StopRepairDataStatement; @@ -543,6 +544,10 @@ public R visitShowQueries(ShowQueriesStatement showQueriesStatement, C context) return visitStatement(showQueriesStatement, context); } + public R visitShowReceivers(ShowReceiversStatement showReceiversStatement, C context) { + return visitStatement(showReceiversStatement, context); + } + public R visitShowDiskUsage(ShowDiskUsageStatement showDiskUsageStatement, C context) { return visitStatement(showDiskUsageStatement, context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/ShowReceiversStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/ShowReceiversStatement.java new file mode 100644 index 0000000000000..1e8a87fd92a59 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/ShowReceiversStatement.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.statement.sys; + +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; +import org.apache.iotdb.db.queryengine.plan.statement.StatementType; +import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; +import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; +import org.apache.iotdb.db.queryengine.plan.statement.component.SortItem; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowStatement; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +public class ShowReceiversStatement extends ShowStatement { + + private static final List DEFAULT_SORT_ITEMS = + ImmutableList.of( + new SortItem(ColumnHeaderConstant.RECEIVER_NODE_TYPE, Ordering.ASC), + new SortItem(ColumnHeaderConstant.RECEIVER_NODE_ID, Ordering.ASC), + new SortItem(ColumnHeaderConstant.PROTOCOL, Ordering.ASC), + new SortItem(ColumnHeaderConstant.SENDER_ADDRESS, Ordering.ASC), + new SortItem(ColumnHeaderConstant.RECEIVER_USER_NAME, Ordering.ASC)); + + public ShowReceiversStatement() { + this.statementType = StatementType.SHOW_RECEIVERS; + } + + @Override + public boolean isQuery() { + return true; + } + + @Override + public R accept(StatementVisitor visitor, C context) { + return visitor.visitShowReceivers(this, context); + } + + public List getSortItemList() { + return DEFAULT_SORT_ITEMS; + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java index b98f34a2484d9..06851d8c99619 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java @@ -65,6 +65,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.AuthorStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowDiskUsageStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowReceiversStatement; import org.apache.iotdb.isession.template.TemplateNode; import org.apache.iotdb.rpc.StatementExecutionException; import org.apache.iotdb.service.rpc.thrift.TSAggregationQueryReq; @@ -181,6 +182,16 @@ public void testShowQueries() { "show queries order by a", ZonedDateTime.now().getOffset())); } + @Test + public void testShowReceivers() { + final Statement showReceivers = + StatementGenerator.createStatement("show receivers", ZonedDateTime.now().getOffset()); + Assert.assertTrue(showReceivers instanceof ShowReceiversStatement); + Assert.assertEquals( + new SortItem("ReceiverNodeType", Ordering.ASC), + ((ShowReceiversStatement) showReceivers).getSortItemList().get(0)); + } + @Test public void testRawDataQuery() throws IllegalPathException { TSRawDataQueryReq req = diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/source/SourceNodeSerdeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/source/SourceNodeSerdeTest.java index 5597a6ca8e600..97fbb5cc502e6 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/source/SourceNodeSerdeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/source/SourceNodeSerdeTest.java @@ -31,6 +31,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.LastQueryScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.ShowDiskUsageNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.ShowQueriesNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.ShowReceiversNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableDiskUsageInformationSchemaTableScanNode; import org.apache.tsfile.enums.TSDataType; @@ -117,6 +118,16 @@ public void testShowQueriesNode() throws IllegalPathException { assertEquals(PlanNodeDeserializeHelper.deserialize(byteBuffer), node); } + @Test + public void testShowReceiversNode() { + ShowReceiversNode node = new ShowReceiversNode(new PlanNodeId("test"), null); + + ByteBuffer byteBuffer = ByteBuffer.allocate(2048); + node.serialize(byteBuffer); + byteBuffer.flip(); + assertEquals(PlanNodeDeserializeHelper.deserialize(byteBuffer), node); + } + @Test public void testTableDiskUsageInformationTableScanNode() throws IllegalPathException { List symbols = Arrays.asList(new Symbol("database"), new Symbol("size_in_bytes")); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/informationschema/ShowReceiversTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/informationschema/ShowReceiversTest.java new file mode 100644 index 0000000000000..318a84bd483c5 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/informationschema/ShowReceiversTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.informationschema; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.LogicalQueryPlan; +import org.apache.iotdb.db.queryengine.plan.relational.planner.PlanTester; + +import org.junit.Test; + +import java.util.Optional; + +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanAssert.assertPlan; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.infoSchemaTableScan; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.output; + +public class ShowReceiversTest { + + private final PlanTester planTester = new PlanTester(); + + @Test + public void testShowReceiversRewrite() { + final LogicalQueryPlan logicalQueryPlan = planTester.createPlan("show receivers"); + assertPlan( + logicalQueryPlan, + output(infoSchemaTableScan("information_schema.receivers", Optional.empty()))); + } + + @Test + public void testSelectReceivers() { + final LogicalQueryPlan logicalQueryPlan = + planTester.createPlan("select * from information_schema.receivers"); + assertPlan( + logicalQueryPlan, + output(infoSchemaTableScan("information_schema.receivers", Optional.empty()))); + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/IoTDBFileReceiver.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/IoTDBFileReceiver.java index 8304161c9d0db..fca1a857b0eac 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/IoTDBFileReceiver.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/IoTDBFileReceiver.java @@ -27,6 +27,7 @@ import org.apache.iotdb.commons.i18n.PipeMessages; import org.apache.iotdb.commons.pipe.config.PipeConfig; import org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant; +import org.apache.iotdb.commons.pipe.receiver.runtime.PipeReceiverRuntimeRegistry; import org.apache.iotdb.commons.pipe.resource.log.PipeLogger; import org.apache.iotdb.commons.pipe.sink.payload.thrift.common.PipeTransferHandshakeConstant; import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.IoTDBSinkRequestVersion; @@ -100,12 +101,21 @@ public abstract class IoTDBFileReceiver implements IoTDBReceiver { protected final AtomicBoolean shouldMarkAsPipeRequest = new AtomicBoolean(true); protected final AtomicBoolean skipIfNoPrivileges = new AtomicBoolean(false); + protected String senderClusterId = PipeReceiverRuntimeRegistry.UNKNOWN; + protected String receiverPipeName; + protected long receiverPipeCreationTime = Long.MIN_VALUE; + private final AtomicReference pipeReceiverRuntimeSessionKey = new AtomicReference<>(); + @Override public IoTDBSinkRequestVersion getVersion() { return IoTDBSinkRequestVersion.VERSION_1; } protected TPipeTransferResp handleTransferHandshakeV1(final PipeTransferHandshakeV1Req req) { + senderClusterId = PipeReceiverRuntimeRegistry.UNKNOWN; + receiverPipeName = null; + receiverPipeCreationTime = Long.MIN_VALUE; + if (!CommonDescriptor.getInstance() .getConfig() .getTimestampPrecision() @@ -340,16 +350,28 @@ protected TPipeTransferResp handleTransferHandshakeV2(final PipeTransferHandshak req.getParams() .getOrDefault(PipeTransferHandshakeConstant.HANDSHAKE_KEY_SKIP_IF, "false"))); + final String pipeNameString = + req.getParams().get(PipeTransferHandshakeConstant.HANDSHAKE_KEY_PIPE_NAME); + final String pipeCreationTimeString = + req.getParams().get(PipeTransferHandshakeConstant.HANDSHAKE_KEY_PIPE_CREATION_TIME); + // Handle the handshake request as a v1 request. // Here we construct a fake "dataNode" request to valid from v1 validation logic, though // it may not require the actual type of the v1 request. - return handleTransferHandshakeV1( - new PipeTransferHandshakeV1Req() { - @Override - protected PipeRequestType getPlanType() { - return PipeRequestType.HANDSHAKE_DATANODE_V1; - } - }.convertToTPipeTransferReq(timestampPrecision)); + final TPipeTransferResp resp = + handleTransferHandshakeV1( + new PipeTransferHandshakeV1Req() { + @Override + protected PipeRequestType getPlanType() { + return PipeRequestType.HANDSHAKE_DATANODE_V1; + } + }.convertToTPipeTransferReq(timestampPrecision)); + if (isSuccess(resp)) { + senderClusterId = clusterIdFromHandshakeRequest; + receiverPipeName = pipeNameString; + receiverPipeCreationTime = parsePipeCreationTime(pipeCreationTimeString); + } + return resp; } protected abstract String getClusterId(); @@ -873,8 +895,72 @@ protected abstract TSStatus loadFileV2( final PipeTransferFileSealReqV2 req, final List fileAbsolutePaths) throws IOException, IllegalPathException; + protected void recordPipeReceiverHandshake( + final String receiverNodeType, final int receiverNodeId, final String protocol) { + final String sessionKey = + String.format("%s-%s-%s-%s", receiverNodeType, receiverNodeId, protocol, receiverId.get()); + final String oldSessionKey = pipeReceiverRuntimeSessionKey.getAndSet(sessionKey); + if (!Objects.equals(oldSessionKey, sessionKey)) { + PipeReceiverRuntimeRegistry.getInstance().deregister(oldSessionKey); + } + PipeReceiverRuntimeRegistry.getInstance() + .registerOrUpdateSession( + sessionKey, + receiverNodeType, + receiverNodeId, + protocol, + getSenderHost(), + parseSenderPort(getSenderPort()), + username, + senderClusterId, + receiverPipeName, + receiverPipeCreationTime, + System.currentTimeMillis()); + } + + protected void recordPipeReceiverTransfer() { + PipeReceiverRuntimeRegistry.getInstance() + .markTransfer(pipeReceiverRuntimeSessionKey.get(), System.currentTimeMillis()); + } + + protected void recordPipeReceiverRequest() { + PipeReceiverRuntimeRegistry.getInstance().markRequest(pipeReceiverRuntimeSessionKey.get()); + } + + protected void clearPipeReceiverRuntime() { + PipeReceiverRuntimeRegistry.getInstance() + .deregister(pipeReceiverRuntimeSessionKey.getAndSet(null)); + } + + protected static boolean isSuccess(final TPipeTransferResp resp) { + return resp != null + && resp.getStatus() != null + && resp.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode(); + } + + private static int parseSenderPort(final String senderPort) { + try { + return Integer.parseInt(senderPort); + } catch (final Exception e) { + return -1; + } + } + + private static long parsePipeCreationTime(final String pipeCreationTime) { + if (pipeCreationTime == null) { + return Long.MIN_VALUE; + } + try { + return Long.parseLong(pipeCreationTime); + } catch (final NumberFormatException e) { + return Long.MIN_VALUE; + } + } + @Override public synchronized void handleExit() { + clearPipeReceiverRuntime(); + if (writingFileWriter != null) { try { writingFileWriter.close(); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/runtime/PipeReceiverRuntimeRegistry.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/runtime/PipeReceiverRuntimeRegistry.java new file mode 100644 index 0000000000000..4aaa24c834966 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/runtime/PipeReceiverRuntimeRegistry.java @@ -0,0 +1,315 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.receiver.runtime; + +import org.apache.iotdb.commons.queryengine.utils.DateTimeUtils; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.StringJoiner; +import java.util.TreeSet; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicLong; + +public class PipeReceiverRuntimeRegistry { + + public static final String UNKNOWN = "Unknown"; + public static final String NODE_TYPE_DATA_NODE = "DataNode"; + public static final String NODE_TYPE_CONFIG_NODE = "ConfigNode"; + public static final String PROTOCOL_THRIFT = "thrift"; + public static final String PROTOCOL_AIR_GAP = "air_gap"; + + private static final PipeReceiverRuntimeRegistry INSTANCE = new PipeReceiverRuntimeRegistry(); + + private final ConcurrentMap sessionInfoMap = + new ConcurrentHashMap<>(); + + private PipeReceiverRuntimeRegistry() {} + + public static PipeReceiverRuntimeRegistry getInstance() { + return INSTANCE; + } + + public void registerOrUpdateSession( + String connectionKey, + String receiverNodeType, + int receiverNodeId, + String protocol, + String senderAddress, + int senderPort, + String userName, + String senderClusterId, + String pipeName, + long pipeCreationTime, + long handshakeTime) { + if (isBlank(connectionKey)) { + return; + } + + sessionInfoMap.compute( + connectionKey, + (key, oldSession) -> { + final SessionRuntimeInfo session = + oldSession == null ? new SessionRuntimeInfo(connectionKey) : oldSession; + synchronized (session) { + session.receiverNodeType = normalize(receiverNodeType); + session.receiverNodeId = receiverNodeId; + session.protocol = normalize(protocol); + session.senderAddress = normalize(senderAddress); + session.senderPort = senderPort; + session.userName = normalize(userName); + session.senderClusterId = normalize(senderClusterId); + session.lastHandshakeTime = handshakeTime; + session.lastTransferTime = Math.max(session.lastTransferTime, handshakeTime); + session.requestNum.incrementAndGet(); + if (!isBlank(pipeName)) { + session.pipeIds.add(formatPipeId(pipeName, pipeCreationTime)); + } + } + return session; + }); + } + + public void markTransfer(String connectionKey, long transferTime) { + final SessionRuntimeInfo session = sessionInfoMap.get(connectionKey); + if (session == null) { + return; + } + synchronized (session) { + session.lastTransferTime = Math.max(session.lastTransferTime, transferTime); + session.requestNum.incrementAndGet(); + } + } + + public void markRequest(String connectionKey) { + final SessionRuntimeInfo session = sessionInfoMap.get(connectionKey); + if (session == null) { + return; + } + session.requestNum.incrementAndGet(); + } + + public void deregister(String connectionKey) { + if (!isBlank(connectionKey)) { + sessionInfoMap.remove(connectionKey); + } + } + + public List snapshot() { + final Map aggregatedInfoMap = new HashMap<>(); + for (SessionRuntimeInfo session : sessionInfoMap.values()) { + synchronized (session) { + final GroupKey groupKey = + new GroupKey( + session.receiverNodeType, + session.receiverNodeId, + session.protocol, + session.senderAddress, + session.userName); + AggregatedRuntimeInfo aggregatedInfo = aggregatedInfoMap.get(groupKey); + if (aggregatedInfo == null) { + aggregatedInfo = new AggregatedRuntimeInfo(groupKey); + aggregatedInfoMap.put(groupKey, aggregatedInfo); + } + aggregatedInfo.senderPorts.add(session.senderPort); + aggregatedInfo.connectionCount++; + aggregatedInfo.pipeIds.addAll(session.pipeIds); + aggregatedInfo.senderClusterIds.add(session.senderClusterId); + aggregatedInfo.lastHandshakeTime = + Math.max(aggregatedInfo.lastHandshakeTime, session.lastHandshakeTime); + aggregatedInfo.lastTransferTime = + Math.max(aggregatedInfo.lastTransferTime, session.lastTransferTime); + aggregatedInfo.requestNum += session.requestNum.get(); + } + } + + final List aggregatedInfos = new ArrayList<>(aggregatedInfoMap.values()); + aggregatedInfos.sort(Comparator.comparing(AggregatedRuntimeInfo::getGroupKey)); + + final List snapshots = new ArrayList<>(aggregatedInfos.size()); + for (AggregatedRuntimeInfo aggregatedInfo : aggregatedInfos) { + snapshots.add(aggregatedInfo.toSnapshot()); + } + return snapshots; + } + + public void clear() { + sessionInfoMap.clear(); + } + + private static String normalize(String value) { + return isBlank(value) ? UNKNOWN : value; + } + + private static boolean isBlank(String value) { + return value == null || value.trim().isEmpty(); + } + + private static String formatPipeId(String pipeName, long pipeCreationTime) { + if (pipeCreationTime < 0) { + return pipeName + "@" + UNKNOWN; + } + return pipeName + "@" + DateTimeUtils.convertLongToDate(pipeCreationTime, "ms"); + } + + private static class SessionRuntimeInfo { + private final String connectionKey; + private String receiverNodeType = UNKNOWN; + private int receiverNodeId = -1; + private String protocol = UNKNOWN; + private String senderAddress = UNKNOWN; + private int senderPort = -1; + private String userName = UNKNOWN; + private String senderClusterId = UNKNOWN; + private long lastHandshakeTime; + private long lastTransferTime; + private final AtomicLong requestNum = new AtomicLong(); + private final TreeSet pipeIds = new TreeSet<>(); + + private SessionRuntimeInfo(String connectionKey) { + this.connectionKey = connectionKey; + } + + @Override + public int hashCode() { + return Objects.hash(connectionKey); + } + } + + private static class GroupKey implements Comparable { + private final String receiverNodeType; + private final int receiverNodeId; + private final String protocol; + private final String senderAddress; + private final String userName; + + private GroupKey( + String receiverNodeType, + int receiverNodeId, + String protocol, + String senderAddress, + String userName) { + this.receiverNodeType = receiverNodeType; + this.receiverNodeId = receiverNodeId; + this.protocol = protocol; + this.senderAddress = senderAddress; + this.userName = userName; + } + + @Override + public int compareTo(GroupKey other) { + int result = receiverNodeType.compareTo(other.receiverNodeType); + if (result != 0) { + return result; + } + result = Integer.compare(receiverNodeId, other.receiverNodeId); + if (result != 0) { + return result; + } + result = protocol.compareTo(other.protocol); + if (result != 0) { + return result; + } + result = senderAddress.compareTo(other.senderAddress); + if (result != 0) { + return result; + } + return userName.compareTo(other.userName); + } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (!(object instanceof GroupKey)) { + return false; + } + final GroupKey groupKey = (GroupKey) object; + return receiverNodeId == groupKey.receiverNodeId + && Objects.equals(receiverNodeType, groupKey.receiverNodeType) + && Objects.equals(protocol, groupKey.protocol) + && Objects.equals(senderAddress, groupKey.senderAddress) + && Objects.equals(userName, groupKey.userName); + } + + @Override + public int hashCode() { + return Objects.hash(receiverNodeType, receiverNodeId, protocol, senderAddress, userName); + } + } + + private static class AggregatedRuntimeInfo { + private final GroupKey groupKey; + private final TreeSet senderPorts = new TreeSet<>(); + private final TreeSet pipeIds = new TreeSet<>(); + private final TreeSet senderClusterIds = new TreeSet<>(); + private int connectionCount; + private long lastHandshakeTime; + private long lastTransferTime; + private long requestNum; + + private AggregatedRuntimeInfo(GroupKey groupKey) { + this.groupKey = groupKey; + } + + private GroupKey getGroupKey() { + return groupKey; + } + + private PipeReceiverRuntimeSnapshot toSnapshot() { + return new PipeReceiverRuntimeSnapshot( + groupKey.receiverNodeType, + groupKey.receiverNodeId, + groupKey.protocol, + groupKey.senderAddress, + joinIntegerSet(senderPorts), + connectionCount, + pipeIds.size(), + pipeIds.isEmpty() ? UNKNOWN : joinStringSet(pipeIds), + groupKey.userName, + senderClusterIds.isEmpty() ? UNKNOWN : joinStringSet(senderClusterIds), + lastHandshakeTime, + lastTransferTime, + requestNum); + } + } + + private static String joinIntegerSet(TreeSet values) { + final StringJoiner joiner = new StringJoiner(","); + for (Integer value : values) { + joiner.add(String.valueOf(value)); + } + return joiner.toString(); + } + + private static String joinStringSet(TreeSet values) { + final StringJoiner joiner = new StringJoiner(";"); + for (String value : values) { + joiner.add(value); + } + return joiner.toString(); + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/runtime/PipeReceiverRuntimeSnapshot.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/runtime/PipeReceiverRuntimeSnapshot.java new file mode 100644 index 0000000000000..58a56333286a3 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/runtime/PipeReceiverRuntimeSnapshot.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.receiver.runtime; + +public class PipeReceiverRuntimeSnapshot { + + private final String receiverNodeType; + private final int receiverNodeId; + private final String protocol; + private final String senderAddress; + private final String senderPorts; + private final int connectionCount; + private final int pipeCount; + private final String pipeIds; + private final String userName; + private final String senderClusterId; + private final long lastHandshakeTime; + private final long lastTransferTime; + private final long requestNum; + + public PipeReceiverRuntimeSnapshot( + String receiverNodeType, + int receiverNodeId, + String protocol, + String senderAddress, + String senderPorts, + int connectionCount, + int pipeCount, + String pipeIds, + String userName, + String senderClusterId, + long lastHandshakeTime, + long lastTransferTime, + long requestNum) { + this.receiverNodeType = receiverNodeType; + this.receiverNodeId = receiverNodeId; + this.protocol = protocol; + this.senderAddress = senderAddress; + this.senderPorts = senderPorts; + this.connectionCount = connectionCount; + this.pipeCount = pipeCount; + this.pipeIds = pipeIds; + this.userName = userName; + this.senderClusterId = senderClusterId; + this.lastHandshakeTime = lastHandshakeTime; + this.lastTransferTime = lastTransferTime; + this.requestNum = requestNum; + } + + public String getReceiverNodeType() { + return receiverNodeType; + } + + public int getReceiverNodeId() { + return receiverNodeId; + } + + public String getProtocol() { + return protocol; + } + + public String getSenderAddress() { + return senderAddress; + } + + public String getSenderPorts() { + return senderPorts; + } + + public int getConnectionCount() { + return connectionCount; + } + + public int getPipeCount() { + return pipeCount; + } + + public String getPipeIds() { + return pipeIds; + } + + public String getUserName() { + return userName; + } + + public String getSenderClusterId() { + return senderClusterId; + } + + public long getLastHandshakeTime() { + return lastHandshakeTime; + } + + public long getLastTransferTime() { + return lastTransferTime; + } + + public long getRequestNum() { + return requestNum; + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/client/IoTDBClientManager.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/client/IoTDBClientManager.java index 1f76f5d2453f5..9d7c85d975afb 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/client/IoTDBClientManager.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/client/IoTDBClientManager.java @@ -22,12 +22,14 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.commons.audit.UserEntity; import org.apache.iotdb.commons.pipe.config.PipeConfig; +import org.apache.iotdb.commons.pipe.sink.payload.thrift.common.PipeTransferHandshakeConstant; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.net.SocketTimeoutException; import java.util.List; +import java.util.Map; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; @@ -51,6 +53,9 @@ public abstract class IoTDBClientManager { protected final boolean shouldMarkAsPipeRequest; protected final boolean skipIfNoPrivileges; + protected volatile String pipeName; + protected volatile long pipeCreationTime = Long.MIN_VALUE; + // This flag indicates whether the receiver supports mods transferring if // it is a DataNode receiver. The flag is useless for configNode receiver. protected boolean supportModsIfIsDataNodeReceiver = true; @@ -89,6 +94,21 @@ public boolean supportModsIfIsDataNodeReceiver() { return supportModsIfIsDataNodeReceiver; } + public void setPipeInfo(final String pipeName, final long pipeCreationTime) { + this.pipeName = pipeName; + this.pipeCreationTime = pipeCreationTime; + } + + protected void appendPipeInfoToHandshakeParams(final Map params) { + if (pipeName == null) { + return; + } + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_PIPE_NAME, pipeName); + params.put( + PipeTransferHandshakeConstant.HANDSHAKE_KEY_PIPE_CREATION_TIME, + String.valueOf(pipeCreationTime)); + } + public void adjustTimeoutIfNecessary(Throwable e) { do { if (e instanceof SocketTimeoutException || e instanceof TimeoutException) { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/client/IoTDBSyncClientManager.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/client/IoTDBSyncClientManager.java index ff62dbf477b7a..e92d93ca64bca 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/client/IoTDBSyncClientManager.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/client/IoTDBSyncClientManager.java @@ -249,6 +249,7 @@ public void sendHandshakeReq(final Pair clientAndStatu params.put( PipeTransferHandshakeConstant.HANDSHAKE_KEY_SKIP_IF, Boolean.toString(skipIfNoPrivileges)); + appendPipeInfoToHandshakeParams(params); // Try to handshake by PipeTransferHandshakeV2Req. TPipeTransferResp resp = client.pipeTransfer(buildHandshakeV2Req(params)); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/common/PipeTransferHandshakeConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/common/PipeTransferHandshakeConstant.java index 710fca828e938..3f084a73dc8d7 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/common/PipeTransferHandshakeConstant.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/common/PipeTransferHandshakeConstant.java @@ -29,6 +29,8 @@ public class PipeTransferHandshakeConstant { public static final String HANDSHAKE_KEY_USERNAME = "username"; public static final String HANDSHAKE_KEY_CLI_HOSTNAME = "cliHostname"; public static final String HANDSHAKE_KEY_PASSWORD = "password"; + public static final String HANDSHAKE_KEY_PIPE_NAME = "pipeName"; + public static final String HANDSHAKE_KEY_PIPE_CREATION_TIME = "pipeCreationTime"; public static final String HANDSHAKE_KEY_VALIDATE_TSFILE = "validateTsFile"; public static final String HANDSHAKE_KEY_MARK_AS_PIPE_REQUEST = "markAsPipeRequest"; public static final String HANDSHAKE_KEY_SKIP_IF = "skipIf"; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSink.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSink.java index b5662aeec2ce9..9a1fef61014fd 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSink.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSink.java @@ -30,6 +30,7 @@ import org.apache.iotdb.commons.pipe.sink.compressor.PipeCompressorFactory; import org.apache.iotdb.commons.pipe.sink.limiter.GlobalRPCRateLimiter; import org.apache.iotdb.commons.pipe.sink.limiter.PipeEndPointRateLimiter; +import org.apache.iotdb.commons.pipe.sink.payload.thrift.common.PipeTransferHandshakeConstant; import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeTransferCompressedReq; import org.apache.iotdb.commons.utils.NodeUrlUtils; import org.apache.iotdb.metrics.type.Histogram; @@ -190,6 +191,8 @@ public abstract class IoTDBSink implements PipeConnector, PipeConnectorWithEvent private final AtomicLong totalCompressedSize = new AtomicLong(0); protected String attributeSortedString; protected String sinkTaskId; + protected String pipeName; + protected long creationTime = Long.MIN_VALUE; protected Timer compressionTimer; protected boolean isRealtimeFirst; @@ -396,6 +399,8 @@ public void customize( (PipeTaskSinkRuntimeEnvironment) environment; attributeSortedString = sinkEnvironment.getAttributeSortedString(); sinkTaskId = sinkEnvironment.getSinkTaskId(); + pipeName = sinkEnvironment.getPipeName(); + creationTime = sinkEnvironment.getCreationTime(); } nodeUrls.clear(); @@ -621,6 +626,16 @@ public long getTotalUncompressedSize() { return totalUncompressedSize.get(); } + protected void appendPipeInfoToHandshakeParams(final Map params) { + if (pipeName == null) { + return; + } + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_PIPE_NAME, pipeName); + params.put( + PipeTransferHandshakeConstant.HANDSHAKE_KEY_PIPE_CREATION_TIME, + String.valueOf(creationTime)); + } + public void rateLimitIfNeeded( final String pipeName, final long creationTime, diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSslSyncSink.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSslSyncSink.java index 6d3c05d3b4f21..064b596024598 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSslSyncSink.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSslSyncSink.java @@ -143,6 +143,7 @@ public void customize( loadTsFileValidation, shouldMarkAsPipeRequest, skipIfNoPrivileges); + clientManager.setPipeInfo(pipeName, creationTime); } protected abstract IoTDBSyncClientManager constructClient( diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/planner/plan/node/PlanNodeType.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/planner/plan/node/PlanNodeType.java index 71eb2238f15f0..790b1f9c19294 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/planner/plan/node/PlanNodeType.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/planner/plan/node/PlanNodeType.java @@ -142,6 +142,7 @@ public enum PlanNodeType { SHOW_DISK_USAGE((short) 107), TREE_COLLECT((short) 108), LOAD_TSFILE_OBJECT_PIECE((short) 109), + SHOW_RECEIVERS((short) 110), CREATE_OR_UPDATE_TABLE_DEVICE((short) 902), TABLE_DEVICE_QUERY_SCAN((short) 903), diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java index 186f7daa6846d..9109e417163eb 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java @@ -197,6 +197,21 @@ private ColumnHeaderConstant() { public static final String REMAINING_EVENT_COUNT = "RemainingEventCount"; public static final String ESTIMATED_REMAINING_SECONDS = "EstimatedRemainingSeconds"; + // column names for show receivers + public static final String RECEIVER_NODE_TYPE = "ReceiverNodeType"; + public static final String RECEIVER_NODE_ID = "ReceiverNodeId"; + public static final String PROTOCOL = "Protocol"; + public static final String SENDER_ADDRESS = "SenderAddress"; + public static final String SENDER_PORTS = "SenderPorts"; + public static final String CONNECTION_COUNT = "ConnectionCount"; + public static final String PIPE_COUNT = "PipeCount"; + public static final String PIPE_IDS = "PipeIDs"; + public static final String RECEIVER_USER_NAME = "UserName"; + public static final String SENDER_CLUSTER_ID = "SenderClusterId"; + public static final String LAST_HANDSHAKE_TIME = "LastHandshakeTime"; + public static final String LAST_TRANSFER_TIME = "LastTransferTime"; + public static final String REQUEST_NUM = "RequestNum"; + // column names for select into public static final String SOURCE_DEVICE = "SourceDevice"; public static final String SOURCE_COLUMN = "SourceColumn"; @@ -270,6 +285,19 @@ private ColumnHeaderConstant() { public static final String ESTIMATED_REMAINING_SECONDS_TABLE_MODEL = "estimated_remaining_seconds"; + public static final String RECEIVER_NODE_TYPE_TABLE_MODEL = "receiver_node_type"; + public static final String RECEIVER_NODE_ID_TABLE_MODEL = "receiver_node_id"; + public static final String PROTOCOL_TABLE_MODEL = "protocol"; + public static final String SENDER_ADDRESS_TABLE_MODEL = "sender_address"; + public static final String SENDER_PORTS_TABLE_MODEL = "sender_ports"; + public static final String CONNECTION_COUNT_TABLE_MODEL = "connection_count"; + public static final String PIPE_COUNT_TABLE_MODEL = "pipe_count"; + public static final String PIPE_IDS_TABLE_MODEL = "pipe_ids"; + public static final String SENDER_CLUSTER_ID_TABLE_MODEL = "sender_cluster_id"; + public static final String LAST_HANDSHAKE_TIME_TABLE_MODEL = "last_handshake_time"; + public static final String LAST_TRANSFER_TIME_TABLE_MODEL = "last_transfer_time"; + public static final String REQUEST_NUM_TABLE_MODEL = "request_num"; + public static final String PLUGIN_NAME_TABLE_MODEL = "plugin_name"; public static final String PLUGIN_TYPE_TABLE_MODEL = "plugin_type"; public static final String CLASS_NAME_TABLE_MODEL = "class_name"; @@ -664,6 +692,22 @@ private ColumnHeaderConstant() { new ColumnHeader(CLIENT_IP_TREE_MODEL, TSDataType.STRING), new ColumnHeader(TIMEOUT, TSDataType.INT64)); + public static final List showReceiversColumnHeaders = + ImmutableList.of( + new ColumnHeader(RECEIVER_NODE_TYPE, TSDataType.TEXT), + new ColumnHeader(RECEIVER_NODE_ID, TSDataType.INT32), + new ColumnHeader(PROTOCOL, TSDataType.TEXT), + new ColumnHeader(SENDER_ADDRESS, TSDataType.TEXT), + new ColumnHeader(SENDER_PORTS, TSDataType.TEXT), + new ColumnHeader(CONNECTION_COUNT, TSDataType.INT32), + new ColumnHeader(PIPE_COUNT, TSDataType.INT32), + new ColumnHeader(PIPE_IDS, TSDataType.TEXT), + new ColumnHeader(RECEIVER_USER_NAME, TSDataType.TEXT), + new ColumnHeader(SENDER_CLUSTER_ID, TSDataType.TEXT), + new ColumnHeader(LAST_HANDSHAKE_TIME, TSDataType.TEXT), + new ColumnHeader(LAST_TRANSFER_TIME, TSDataType.TEXT), + new ColumnHeader(REQUEST_NUM, TSDataType.INT64)); + public static final List showDiskUsageColumnHeaders = ImmutableList.of( new ColumnHeader(DATABASE, TSDataType.TEXT), diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java index c4001ecbb1e82..fd91c0f892154 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java @@ -47,6 +47,7 @@ public class InformationSchema { public static final String COLUMNS = "columns"; public static final String REGIONS = "regions"; public static final String PIPES = "pipes"; + public static final String RECEIVERS = "receivers"; public static final String PIPE_PLUGINS = "pipe_plugins"; public static final String TOPICS = "topics"; public static final String SUBSCRIPTIONS = "subscriptions"; @@ -221,6 +222,41 @@ public class InformationSchema { ColumnHeaderConstant.ESTIMATED_REMAINING_SECONDS_TABLE_MODEL, TSDataType.DOUBLE)); schemaTables.put(PIPES, pipeTable); + final TsTable receiversTable = new TsTable(RECEIVERS); + receiversTable.addColumnSchema( + new TagColumnSchema( + ColumnHeaderConstant.RECEIVER_NODE_TYPE_TABLE_MODEL, TSDataType.STRING)); + receiversTable.addColumnSchema( + new TagColumnSchema(ColumnHeaderConstant.RECEIVER_NODE_ID_TABLE_MODEL, TSDataType.INT32)); + receiversTable.addColumnSchema( + new TagColumnSchema(ColumnHeaderConstant.PROTOCOL_TABLE_MODEL, TSDataType.STRING)); + receiversTable.addColumnSchema( + new TagColumnSchema(ColumnHeaderConstant.SENDER_ADDRESS_TABLE_MODEL, TSDataType.STRING)); + receiversTable.addColumnSchema( + new AttributeColumnSchema( + ColumnHeaderConstant.SENDER_PORTS_TABLE_MODEL, TSDataType.STRING)); + receiversTable.addColumnSchema( + new AttributeColumnSchema( + ColumnHeaderConstant.CONNECTION_COUNT_TABLE_MODEL, TSDataType.INT32)); + receiversTable.addColumnSchema( + new AttributeColumnSchema(ColumnHeaderConstant.PIPE_COUNT_TABLE_MODEL, TSDataType.INT32)); + receiversTable.addColumnSchema( + new AttributeColumnSchema(ColumnHeaderConstant.PIPE_IDS_TABLE_MODEL, TSDataType.STRING)); + receiversTable.addColumnSchema( + new TagColumnSchema(ColumnHeaderConstant.USER_NAME, TSDataType.STRING)); + receiversTable.addColumnSchema( + new AttributeColumnSchema( + ColumnHeaderConstant.SENDER_CLUSTER_ID_TABLE_MODEL, TSDataType.STRING)); + receiversTable.addColumnSchema( + new AttributeColumnSchema( + ColumnHeaderConstant.LAST_HANDSHAKE_TIME_TABLE_MODEL, TSDataType.TIMESTAMP)); + receiversTable.addColumnSchema( + new AttributeColumnSchema( + ColumnHeaderConstant.LAST_TRANSFER_TIME_TABLE_MODEL, TSDataType.TIMESTAMP)); + receiversTable.addColumnSchema( + new AttributeColumnSchema(ColumnHeaderConstant.REQUEST_NUM_TABLE_MODEL, TSDataType.INT64)); + schemaTables.put(RECEIVERS, receiversTable); + final TsTable pipePluginTable = new TsTable(PIPE_PLUGINS); pipePluginTable.addColumnSchema( new TagColumnSchema(ColumnHeaderConstant.PLUGIN_NAME_TABLE_MODEL, TSDataType.STRING)); diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/receiver/runtime/PipeReceiverRuntimeRegistryTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/receiver/runtime/PipeReceiverRuntimeRegistryTest.java new file mode 100644 index 0000000000000..4445c8e9f5848 --- /dev/null +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/receiver/runtime/PipeReceiverRuntimeRegistryTest.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.receiver.runtime; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class PipeReceiverRuntimeRegistryTest { + + private final PipeReceiverRuntimeRegistry registry = PipeReceiverRuntimeRegistry.getInstance(); + + @Before + public void setUp() { + registry.clear(); + } + + @After + public void tearDown() { + registry.clear(); + } + + @Test + public void testAggregateAndSortSnapshots() { + registry.registerOrUpdateSession( + "data-1", + PipeReceiverRuntimeRegistry.NODE_TYPE_DATA_NODE, + 2, + PipeReceiverRuntimeRegistry.PROTOCOL_THRIFT, + "127.0.0.1", + 9001, + "root", + "cluster-a", + "pipe-a", + 1, + 100); + registry.markTransfer("data-1", 200); + registry.markRequest("data-1"); + registry.registerOrUpdateSession( + "data-2", + PipeReceiverRuntimeRegistry.NODE_TYPE_DATA_NODE, + 2, + PipeReceiverRuntimeRegistry.PROTOCOL_THRIFT, + "127.0.0.1", + 9002, + "root", + "cluster-b", + "pipe-b", + 2, + 150); + registry.registerOrUpdateSession( + "config-1", + PipeReceiverRuntimeRegistry.NODE_TYPE_CONFIG_NODE, + 1, + PipeReceiverRuntimeRegistry.PROTOCOL_AIR_GAP, + "127.0.0.2", + 9003, + "root", + PipeReceiverRuntimeRegistry.UNKNOWN, + null, + Long.MIN_VALUE, + 300); + + final List snapshots = registry.snapshot(); + + assertEquals(2, snapshots.size()); + assertEquals( + PipeReceiverRuntimeRegistry.NODE_TYPE_CONFIG_NODE, snapshots.get(0).getReceiverNodeType()); + assertEquals( + PipeReceiverRuntimeRegistry.NODE_TYPE_DATA_NODE, snapshots.get(1).getReceiverNodeType()); + + final PipeReceiverRuntimeSnapshot dataSnapshot = snapshots.get(1); + assertEquals(2, dataSnapshot.getConnectionCount()); + assertEquals("9001,9002", dataSnapshot.getSenderPorts()); + assertEquals(2, dataSnapshot.getPipeCount()); + assertTrue(dataSnapshot.getPipeIds().contains("pipe-a@")); + assertTrue(dataSnapshot.getPipeIds().contains("pipe-b@")); + assertEquals("cluster-a;cluster-b", dataSnapshot.getSenderClusterId()); + assertEquals(150, dataSnapshot.getLastHandshakeTime()); + assertEquals(200, dataSnapshot.getLastTransferTime()); + assertEquals(4, dataSnapshot.getRequestNum()); + } + + @Test + public void testDeregisterSession() { + registry.registerOrUpdateSession( + "data-1", + PipeReceiverRuntimeRegistry.NODE_TYPE_DATA_NODE, + 1, + PipeReceiverRuntimeRegistry.PROTOCOL_THRIFT, + "127.0.0.1", + 9001, + "root", + "cluster-a", + "pipe-a", + 1, + 100); + + assertEquals(1, registry.snapshot().size()); + + registry.deregister("data-1"); + + assertTrue(registry.snapshot().isEmpty()); + } +} diff --git a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 index f984e825fc7ad..e7a02f269abcd 100644 --- a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 +++ b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 @@ -102,6 +102,7 @@ statement | startPipeStatement | stopPipeStatement | showPipesStatement + | showReceiversStatement | createPipePluginStatement | dropPipePluginStatement | showPipePluginsStatement @@ -519,6 +520,10 @@ showPipesStatement : SHOW ((PIPE pipeName=identifier) | PIPES (WHERE (CONNECTOR | SINK) USED BY pipeName=identifier)?) ; +showReceiversStatement + : SHOW RECEIVERS + ; + createPipePluginStatement : CREATE PIPEPLUGIN (IF NOT EXISTS)? pluginName=identifier AS className=string uriClause ; @@ -1504,7 +1509,7 @@ nonReserved | OBJECT | OF | OFFSET | OMIT | ONE | ONLY | OPTION | ORDINALITY | OUTPUT | OVER | OVERFLOW | PARTITION | PARTITIONS | PASSING | PAST | PATH | PATTERN | PER | PERIOD | PERMUTE | PIPE | PIPEPLUGIN | PIPEPLUGINS | PIPES | PLAN | POSITION | PRECEDING | PRECISION | PRIVILEGES | PREVIOUS | PROCESSLIST | PROCESSOR | PROPERTIES | PRUNE | QUERIES | QUERY | QUOTES - | RANGE | READ | READONLY | RECONSTRUCT | REFRESH | REGION | REGIONID | REGIONS | REMOVE | RENAME | REPAIR | REPEAT | REPEATABLE | REPLACE | RESET | RESPECT | RESTRICT | RETURN | RETURNING | RETURNS | REVOKE | ROLE | ROLES | ROLLBACK | ROOT | ROW | ROWS | RPR_FIRST | RPR_LAST | RUNNING + | RANGE | READ | READONLY | RECEIVERS | RECONSTRUCT | REFRESH | REGION | REGIONID | REGIONS | REMOVE | RENAME | REPAIR | REPEAT | REPEATABLE | REPLACE | RESET | RESPECT | RESTRICT | RETURN | RETURNING | RETURNS | REVOKE | ROLE | ROLES | ROLLBACK | ROOT | ROW | ROWS | RPR_FIRST | RPR_LAST | RUNNING | SERIESSLOTID | SERVICE | SERVICES | SCALAR | SCHEMA | SCHEMAS | SECOND | SECURITY | SEEK | SERIALIZABLE | SESSION | SET | SETS | SECURITY | SHOW | SINK | SOME | SOURCE | START | STATS | STOP | SUBSCRIPTION | SUBSCRIPTIONS | SUBSET | SUBSTRING | SYSTEM | TABLES | TABLESAMPLE | TAG | TAGS | TEXT | TEXT_STRING | TIES | TIME | TIMEPARTITION | TIMER | TIMER_XL | TIMESERIES | TIMESLOTID | TIMESTAMP | TO | TOPIC | TOPICS | TRAILING | TRANSACTION | TRUNCATE | TRY_CAST | TYPE @@ -1794,6 +1799,7 @@ QUOTES: 'QUOTES'; RANGE: 'RANGE'; READ: 'READ'; READONLY: 'READONLY'; +RECEIVERS: 'RECEIVERS'; RECONSTRUCT: 'RECONSTRUCT'; RECURSIVE: 'RECURSIVE'; REFRESH: 'REFRESH'; From 7544ec8f21925e688c4b236da5747db31efa1360 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 8 Jun 2026 18:41:24 +0800 Subject: [PATCH 2/7] Fix show receivers runtime edge cases --- .../source/ShowReceiversOperator.java | 11 +- ...formationSchemaContentSupplierFactory.java | 11 +- .../source/ShowReceiversOperatorTest.java | 71 +++++++++ .../runtime/PipeReceiverRuntimeRegistry.java | 18 ++- .../runtime/PipeReceiverRuntimeSnapshot.java | 4 + .../PipeReceiverRuntimeRegistryTest.java | 146 ++++++++++++++++++ 6 files changed, 254 insertions(+), 7 deletions(-) create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/ShowReceiversOperatorTest.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ShowReceiversOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ShowReceiversOperator.java index c9cf62317f5f6..f8badc76352d3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ShowReceiversOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ShowReceiversOperator.java @@ -127,7 +127,7 @@ private TsBlock buildTsBlock() { PipeReceiverRuntimeRegistry.getInstance().snapshot()) { timeColumnBuilder.writeLong(currentTime); columnBuilders[0].writeBinary(BytesUtils.valueOf(snapshot.getReceiverNodeType())); - columnBuilders[1].writeInt(snapshot.getReceiverNodeId()); + writeReceiverNodeId(columnBuilders[1], snapshot); columnBuilders[2].writeBinary(BytesUtils.valueOf(snapshot.getProtocol())); columnBuilders[3].writeBinary(BytesUtils.valueOf(snapshot.getSenderAddress())); columnBuilders[4].writeBinary(BytesUtils.valueOf(snapshot.getSenderPorts())); @@ -146,6 +146,15 @@ private TsBlock buildTsBlock() { return builder.build(); } + private static void writeReceiverNodeId( + ColumnBuilder columnBuilder, PipeReceiverRuntimeSnapshot snapshot) { + if (snapshot.isReceiverNodeIdKnown()) { + columnBuilder.writeInt(snapshot.getReceiverNodeId()); + } else { + columnBuilder.appendNull(); + } + } + private static String formatTime(long timestampInMillis) { return timestampInMillis <= 0 ? PipeReceiverRuntimeRegistry.UNKNOWN diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java index e84348851cefb..6b70c72471d96 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java @@ -729,7 +729,7 @@ private ReceiversSupplier(final List dataTypes, final UserEntity use protected void constructLine() { final PipeReceiverRuntimeSnapshot snapshot = iterator.next(); columnBuilders[0].writeBinary(BytesUtils.valueOf(snapshot.getReceiverNodeType())); - columnBuilders[1].writeInt(snapshot.getReceiverNodeId()); + writeReceiverNodeId(columnBuilders[1], snapshot); columnBuilders[2].writeBinary(BytesUtils.valueOf(snapshot.getProtocol())); columnBuilders[3].writeBinary(BytesUtils.valueOf(snapshot.getSenderAddress())); columnBuilders[4].writeBinary(BytesUtils.valueOf(snapshot.getSenderPorts())); @@ -744,6 +744,15 @@ protected void constructLine() { resultBuilder.declarePosition(); } + private static void writeReceiverNodeId( + ColumnBuilder columnBuilder, PipeReceiverRuntimeSnapshot snapshot) { + if (snapshot.isReceiverNodeIdKnown()) { + columnBuilder.writeInt(snapshot.getReceiverNodeId()); + } else { + columnBuilder.appendNull(); + } + } + private static void writeTimestamp(ColumnBuilder columnBuilder, long timestampInMillis) { if (timestampInMillis <= 0) { columnBuilder.appendNull(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/ShowReceiversOperatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/ShowReceiversOperatorTest.java new file mode 100644 index 0000000000000..c3f7266bbbbf7 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/ShowReceiversOperatorTest.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.source; + +import org.apache.iotdb.commons.pipe.receiver.runtime.PipeReceiverRuntimeRegistry; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; + +import org.apache.tsfile.read.common.block.TsBlock; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class ShowReceiversOperatorTest { + + private final PipeReceiverRuntimeRegistry registry = PipeReceiverRuntimeRegistry.getInstance(); + + @Before + public void setUp() { + registry.clear(); + } + + @After + public void tearDown() { + registry.clear(); + } + + @Test + public void testUnknownReceiverNodeIdIsNull() { + registry.registerOrUpdateSession( + "config-1", + PipeReceiverRuntimeRegistry.NODE_TYPE_CONFIG_NODE, + -1, + PipeReceiverRuntimeRegistry.PROTOCOL_THRIFT, + "127.0.0.1", + 9001, + "root", + "cluster-a", + "pipe-a", + 1, + 100); + + final ShowReceiversOperator operator = + new ShowReceiversOperator(null, new PlanNodeId("show-receivers")); + + assertTrue(operator.hasNext()); + final TsBlock tsBlock = operator.next(); + + assertEquals(1, tsBlock.getPositionCount()); + assertTrue(tsBlock.getColumn(1).isNull(0)); + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/runtime/PipeReceiverRuntimeRegistry.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/runtime/PipeReceiverRuntimeRegistry.java index 4aaa24c834966..90dac99236322 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/runtime/PipeReceiverRuntimeRegistry.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/runtime/PipeReceiverRuntimeRegistry.java @@ -84,9 +84,7 @@ public void registerOrUpdateSession( session.lastHandshakeTime = handshakeTime; session.lastTransferTime = Math.max(session.lastTransferTime, handshakeTime); session.requestNum.incrementAndGet(); - if (!isBlank(pipeName)) { - session.pipeIds.add(formatPipeId(pipeName, pipeCreationTime)); - } + session.pipeId = isBlank(pipeName) ? null : formatPipeId(pipeName, pipeCreationTime); } return session; }); @@ -135,7 +133,9 @@ public List snapshot() { } aggregatedInfo.senderPorts.add(session.senderPort); aggregatedInfo.connectionCount++; - aggregatedInfo.pipeIds.addAll(session.pipeIds); + if (session.pipeId != null) { + aggregatedInfo.pipeIds.add(session.pipeId); + } aggregatedInfo.senderClusterIds.add(session.senderClusterId); aggregatedInfo.lastHandshakeTime = Math.max(aggregatedInfo.lastHandshakeTime, session.lastHandshakeTime); @@ -186,7 +186,7 @@ private static class SessionRuntimeInfo { private long lastHandshakeTime; private long lastTransferTime; private final AtomicLong requestNum = new AtomicLong(); - private final TreeSet pipeIds = new TreeSet<>(); + private String pipeId; private SessionRuntimeInfo(String connectionKey) { this.connectionKey = connectionKey; @@ -299,7 +299,15 @@ private PipeReceiverRuntimeSnapshot toSnapshot() { private static String joinIntegerSet(TreeSet values) { final StringJoiner joiner = new StringJoiner(","); + boolean hasUnknown = false; for (Integer value : values) { + if (value < 0) { + if (!hasUnknown) { + joiner.add(UNKNOWN); + hasUnknown = true; + } + continue; + } joiner.add(String.valueOf(value)); } return joiner.toString(); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/runtime/PipeReceiverRuntimeSnapshot.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/runtime/PipeReceiverRuntimeSnapshot.java index 58a56333286a3..d1aa752e96363 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/runtime/PipeReceiverRuntimeSnapshot.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/runtime/PipeReceiverRuntimeSnapshot.java @@ -72,6 +72,10 @@ public int getReceiverNodeId() { return receiverNodeId; } + public boolean isReceiverNodeIdKnown() { + return receiverNodeId >= 0; + } + public String getProtocol() { return protocol; } diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/receiver/runtime/PipeReceiverRuntimeRegistryTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/receiver/runtime/PipeReceiverRuntimeRegistryTest.java index 4445c8e9f5848..5624cd6ff18e4 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/receiver/runtime/PipeReceiverRuntimeRegistryTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/receiver/runtime/PipeReceiverRuntimeRegistryTest.java @@ -26,6 +26,7 @@ import java.util.List; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; public class PipeReceiverRuntimeRegistryTest { @@ -124,4 +125,149 @@ public void testDeregisterSession() { assertTrue(registry.snapshot().isEmpty()); } + + @Test + public void testRegisterOrUpdateSessionReplacesPipeId() { + registry.registerOrUpdateSession( + "config-1", + PipeReceiverRuntimeRegistry.NODE_TYPE_CONFIG_NODE, + -1, + PipeReceiverRuntimeRegistry.PROTOCOL_THRIFT, + "127.0.0.1", + 9001, + "root", + "cluster-a", + "pipe-a", + 1, + 100); + registry.registerOrUpdateSession( + "config-1", + PipeReceiverRuntimeRegistry.NODE_TYPE_CONFIG_NODE, + -1, + PipeReceiverRuntimeRegistry.PROTOCOL_THRIFT, + "127.0.0.1", + 9001, + "root", + "cluster-a", + "pipe-b", + 2, + 200); + + final List snapshots = registry.snapshot(); + + assertEquals(1, snapshots.size()); + assertEquals(1, snapshots.get(0).getPipeCount()); + assertFalse(snapshots.get(0).getPipeIds().contains("pipe-a@")); + assertTrue(snapshots.get(0).getPipeIds().contains("pipe-b@")); + } + + @Test + public void testRegisterOrUpdateSessionClearsPipeId() { + registry.registerOrUpdateSession( + "config-1", + PipeReceiverRuntimeRegistry.NODE_TYPE_CONFIG_NODE, + -1, + PipeReceiverRuntimeRegistry.PROTOCOL_THRIFT, + "127.0.0.1", + 9001, + "root", + "cluster-a", + "pipe-a", + 1, + 100); + registry.registerOrUpdateSession( + "config-1", + PipeReceiverRuntimeRegistry.NODE_TYPE_CONFIG_NODE, + -1, + PipeReceiverRuntimeRegistry.PROTOCOL_THRIFT, + "127.0.0.1", + 9001, + "root", + PipeReceiverRuntimeRegistry.UNKNOWN, + null, + Long.MIN_VALUE, + 200); + + final List snapshots = registry.snapshot(); + + assertEquals(1, snapshots.size()); + assertEquals(0, snapshots.get(0).getPipeCount()); + assertEquals(PipeReceiverRuntimeRegistry.UNKNOWN, snapshots.get(0).getPipeIds()); + } + + @Test + public void testUnknownSenderPort() { + registry.registerOrUpdateSession( + "data-1", + PipeReceiverRuntimeRegistry.NODE_TYPE_DATA_NODE, + 1, + PipeReceiverRuntimeRegistry.PROTOCOL_THRIFT, + "127.0.0.1", + -1, + "root", + "cluster-a", + "pipe-a", + 1, + 100); + + final List snapshots = registry.snapshot(); + + assertEquals(1, snapshots.size()); + assertEquals(PipeReceiverRuntimeRegistry.UNKNOWN, snapshots.get(0).getSenderPorts()); + } + + @Test + public void testDuplicateUnknownSenderPortsAreDeduplicated() { + registry.registerOrUpdateSession( + "data-1", + PipeReceiverRuntimeRegistry.NODE_TYPE_DATA_NODE, + 1, + PipeReceiverRuntimeRegistry.PROTOCOL_THRIFT, + "127.0.0.1", + -1, + "root", + "cluster-a", + "pipe-a", + 1, + 100); + registry.registerOrUpdateSession( + "data-2", + PipeReceiverRuntimeRegistry.NODE_TYPE_DATA_NODE, + 1, + PipeReceiverRuntimeRegistry.PROTOCOL_THRIFT, + "127.0.0.1", + -2, + "root", + "cluster-b", + "pipe-b", + 2, + 200); + + final List snapshots = registry.snapshot(); + + assertEquals(1, snapshots.size()); + assertEquals(PipeReceiverRuntimeRegistry.UNKNOWN, snapshots.get(0).getSenderPorts()); + } + + @Test + public void testUnknownReceiverNodeId() { + registry.registerOrUpdateSession( + "config-1", + PipeReceiverRuntimeRegistry.NODE_TYPE_CONFIG_NODE, + -1, + PipeReceiverRuntimeRegistry.PROTOCOL_THRIFT, + "127.0.0.1", + 9001, + "root", + "cluster-a", + "pipe-a", + 1, + 100); + + final List snapshots = registry.snapshot(); + + assertEquals(1, snapshots.size()); + assertEquals(-1, snapshots.get(0).getReceiverNodeId()); + assertFalse(snapshots.get(0).isReceiverNodeIdKnown()); + } } From 018840a8c3ffe0ce9a27eeeec9fc1f3d056d09fa Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 10 Jun 2026 17:40:18 +0800 Subject: [PATCH 3/7] Fix receiver handshake body duplication --- .../db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java index e237e7129bf00..4fbed68cdb044 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java @@ -867,7 +867,7 @@ private static Map parseHandshakeV2Params(final TPipeTransferReq if (req.getBody() == null) { return params; } - final ByteBuffer body = req.getBody().duplicate(); + final ByteBuffer body = req.body.duplicate(); body.rewind(); final int size = ReadWriteIOUtils.readInt(body); for (int i = 0; i < size; ++i) { From 824f642026e2bb907385e66c4bbf45e93f69724e Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 10 Jun 2026 18:35:11 +0800 Subject: [PATCH 4/7] Fix show receivers schema alignment --- .../protocol/IoTDBConfigNodeReceiver.java | 2 - .../thrift/IoTDBDataNodeReceiver.java | 23 +++--- .../db/protocol/client/ConfigNodeClient.java | 5 ++ .../db/protocol/client/ConfigNodeInfo.java | 48 ++++++++++++ .../source/ShowReceiversOperator.java | 1 - ...formationSchemaContentSupplierFactory.java | 1 - .../executor/ClusterConfigTaskExecutor.java | 21 +++-- .../org/apache/iotdb/db/service/DataNode.java | 6 +- .../protocol/client/ConfigNodeInfoTest.java | 76 +++++++++++++++++++ .../source/ShowReceiversOperatorTest.java | 34 +++++++++ .../informationschema/ShowReceiversTest.java | 36 ++++++++- .../pipe/receiver/IoTDBFileReceiver.java | 4 - .../runtime/PipeReceiverRuntimeRegistry.java | 17 +---- .../runtime/PipeReceiverRuntimeSnapshot.java | 9 +-- .../schema/column/ColumnHeaderConstant.java | 5 +- .../schema/table/InformationSchema.java | 2 - .../PipeReceiverRuntimeRegistryTest.java | 2 - 17 files changed, 228 insertions(+), 64 deletions(-) create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/client/ConfigNodeInfoTest.java diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java index 9d84e448c9f7c..8cb3b68ab9ceb 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java @@ -279,8 +279,6 @@ private TPipeTransferResp recordConfigNodeHandshakeIfSuccess( private TPipeTransferResp recordConfigNodeTransferIfSuccess(final TPipeTransferResp resp) { if (isSuccess(resp)) { recordPipeReceiverTransfer(); - } else { - recordPipeReceiverRequest(); } return resp; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java index 4fbed68cdb044..18afa1d693b03 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java @@ -747,9 +747,9 @@ private TPipeTransferResp handleTransferSchemaPlan(final PipeTransferPlanNodeReq null)); } - private TPipeTransferResp handleTransferConfigPlan(final TPipeTransferReq req) { + private Pair handleTransferConfigPlan(final TPipeTransferReq req) { return ClusterConfigTaskExecutor.getInstance() - .handleTransferConfigPlan(getConfigReceiverId(), req); + .handleTransferConfigPlanAndGetReceiverNodeId(getConfigReceiverId(), req); } /** Used to identify the sender client */ @@ -798,14 +798,13 @@ private TPipeTransferResp recordDataNodeHandshakeIfSuccess( private TPipeTransferResp recordDataNodeTransferIfSuccess(final TPipeTransferResp resp) { if (isSuccess(resp)) { recordPipeReceiverTransfer(); - } else { - recordPipeReceiverRequest(); } return resp; } private TPipeTransferResp recordConfigNodeReceiverRuntimeIfSuccess( - final TPipeTransferResp resp, final TPipeTransferReq req) { + final Pair respWithReceiverNodeId, final TPipeTransferReq req) { + final TPipeTransferResp resp = respWithReceiverNodeId.left; if (!PipeRequestType.isValidatedRequestType(req.getType())) { return resp; } @@ -814,27 +813,27 @@ private TPipeTransferResp recordConfigNodeReceiverRuntimeIfSuccess( if (requestType == PipeRequestType.HANDSHAKE_CONFIGNODE_V1 || requestType == PipeRequestType.HANDSHAKE_CONFIGNODE_V2) { if (isSuccess(resp)) { - recordConfigNodeHandshake(req, requestType); + recordConfigNodeHandshake(req, requestType, respWithReceiverNodeId.right); } } else { if (isSuccess(resp)) { PipeReceiverRuntimeRegistry.getInstance() .markTransfer(configPipeReceiverRuntimeSessionKey.get(), System.currentTimeMillis()); - } else { - PipeReceiverRuntimeRegistry.getInstance() - .markRequest(configPipeReceiverRuntimeSessionKey.get()); } } return resp; } private void recordConfigNodeHandshake( - final TPipeTransferReq req, final PipeRequestType requestType) { + final TPipeTransferReq req, final PipeRequestType requestType, final int receiverNodeId) { final String protocol = getProtocol(req); final String sessionKey = String.format( "%s-%s-%s-%s", - PipeReceiverRuntimeRegistry.NODE_TYPE_CONFIG_NODE, -1, protocol, getConfigReceiverId()); + PipeReceiverRuntimeRegistry.NODE_TYPE_CONFIG_NODE, + receiverNodeId, + protocol, + getConfigReceiverId()); final String oldSessionKey = configPipeReceiverRuntimeSessionKey.getAndSet(sessionKey); if (!Objects.equals(oldSessionKey, sessionKey)) { PipeReceiverRuntimeRegistry.getInstance().deregister(oldSessionKey); @@ -848,7 +847,7 @@ private void recordConfigNodeHandshake( .registerOrUpdateSession( sessionKey, PipeReceiverRuntimeRegistry.NODE_TYPE_CONFIG_NODE, - -1, + receiverNodeId, protocol, getSenderHost(), parseSenderPort(getSenderPort()), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java index 1f0b09f0b8689..82b118c29ac22 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java @@ -358,6 +358,10 @@ public TTransport getTransport() { return transport; } + public TEndPoint getConfigNode() { + return configNode; + } + public void syncLatestConfigNodeList() { configNodes = ConfigNodeInfo.getInstance().getLatestConfigNodes(); cursor = 0; @@ -491,6 +495,7 @@ public TDataNodeRegisterResp registerDataNode(TDataNodeRegisterReq req) throws T for (TConfigNodeLocation configNodeLocation : resp.getConfigNodeList()) { newConfigNodes.add(configNodeLocation.getInternalEndPoint()); } + ConfigNodeInfo.getInstance().updateConfigNodeLocations(resp.getConfigNodeList()); configNodes = newConfigNodes; } catch (TException e) { String message = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeInfo.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeInfo.java index 235a1c9d86e64..a0f9c007434f2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeInfo.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeInfo.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.protocol.client; +import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation; import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.commons.consensus.ConfigRegionId; import org.apache.iotdb.commons.exception.BadNodeUrlException; @@ -33,8 +34,10 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -49,6 +52,8 @@ public class ConfigNodeInfo { /** latest config nodes. */ private final Set onlineConfigNodes; + private final Map configNodeIdMap; + public static final ConfigRegionId CONFIG_REGION_ID = new ConfigRegionId(0); SystemPropertiesHandler systemPropertiesHandler = DataNodeSystemPropertiesHandler.getInstance(); @@ -56,6 +61,7 @@ public class ConfigNodeInfo { private ConfigNodeInfo() { this.configNodeInfoReadWriteLock = new ReentrantReadWriteLock(); this.onlineConfigNodes = new HashSet<>(); + this.configNodeIdMap = new HashMap<>(); } public static void reinitializeStatics() { @@ -81,6 +87,7 @@ public boolean updateConfigNodeList(List latestConfigNodes) { try { onlineConfigNodes.clear(); onlineConfigNodes.addAll(latestConfigNodes); + configNodeIdMap.keySet().retainAll(onlineConfigNodes); storeConfigNodeList(); long endTime = System.currentTimeMillis(); logger.info( @@ -96,6 +103,35 @@ public boolean updateConfigNodeList(List latestConfigNodes) { return true; } + public boolean updateConfigNodeLocations(List latestConfigNodeLocations) { + if (latestConfigNodeLocations == null) { + return false; + } + final List latestConfigNodes = new ArrayList<>(); + final Map latestConfigNodeIdMap = new HashMap<>(); + for (TConfigNodeLocation configNodeLocation : latestConfigNodeLocations) { + if (configNodeLocation == null || configNodeLocation.getInternalEndPoint() == null) { + continue; + } + final TEndPoint internalEndPoint = configNodeLocation.getInternalEndPoint(); + latestConfigNodes.add(internalEndPoint); + latestConfigNodeIdMap.put(internalEndPoint, configNodeLocation.getConfigNodeId()); + } + + if (!updateConfigNodeList(latestConfigNodes)) { + return false; + } + + configNodeInfoReadWriteLock.writeLock().lock(); + try { + configNodeIdMap.putAll(latestConfigNodeIdMap); + configNodeIdMap.keySet().retainAll(onlineConfigNodes); + } finally { + configNodeInfoReadWriteLock.writeLock().unlock(); + } + return true; + } + /** * Call this method to store config node list. * @@ -152,6 +188,18 @@ public List getLatestConfigNodes() { return result; } + public int getConfigNodeId(TEndPoint internalEndPoint) { + if (internalEndPoint == null) { + return -1; + } + configNodeInfoReadWriteLock.readLock().lock(); + try { + return configNodeIdMap.getOrDefault(internalEndPoint, -1); + } finally { + configNodeInfoReadWriteLock.readLock().unlock(); + } + } + private static class ConfigNodeInfoHolder { private static ConfigNodeInfo INSTANCE = new ConfigNodeInfo(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ShowReceiversOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ShowReceiversOperator.java index f8badc76352d3..9dc35890b1da4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ShowReceiversOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ShowReceiversOperator.java @@ -140,7 +140,6 @@ private TsBlock buildTsBlock() { BytesUtils.valueOf(formatTime(snapshot.getLastHandshakeTime()))); columnBuilders[11].writeBinary( BytesUtils.valueOf(formatTime(snapshot.getLastTransferTime()))); - columnBuilders[12].writeLong(snapshot.getRequestNum()); builder.declarePosition(); } return builder.build(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java index 6b70c72471d96..bcd3950aa1181 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java @@ -740,7 +740,6 @@ protected void constructLine() { columnBuilders[9].writeBinary(BytesUtils.valueOf(snapshot.getSenderClusterId())); writeTimestamp(columnBuilders[10], snapshot.getLastHandshakeTime()); writeTimestamp(columnBuilders[11], snapshot.getLastTransferTime()); - columnBuilders[12].writeLong(snapshot.getRequestNum()); resultBuilder.declarePosition(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index 15a538b98b42b..ba83324f7b513 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -357,6 +357,7 @@ import org.apache.tsfile.external.commons.codec.digest.DigestUtils; import org.apache.tsfile.read.common.block.TsBlockBuilder; import org.apache.tsfile.read.common.block.column.TimeColumnBuilder; +import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -4094,6 +4095,11 @@ public TSpaceQuotaResp getSpaceQuota() { @Override public TPipeTransferResp handleTransferConfigPlan( final String clientId, final TPipeTransferReq req) { + return handleTransferConfigPlanAndGetReceiverNodeId(clientId, req).left; + } + + public Pair handleTransferConfigPlanAndGetReceiverNodeId( + final String clientId, final TPipeTransferReq req) { final TPipeConfigTransferReq configTransferReq = new TPipeConfigTransferReq( req.version, @@ -4106,18 +4112,23 @@ public TPipeTransferResp handleTransferConfigPlan( CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { final TPipeConfigTransferResp pipeConfigTransferResp = configNodeClient.handleTransferConfigPlan(configTransferReq); + final int receiverNodeId = + ConfigNodeInfo.getInstance().getConfigNodeId(configNodeClient.getConfigNode()); if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != pipeConfigTransferResp.getStatus().getCode()) { LOGGER.warn( DataNodeQueryMessages.FAILED_TO_HANDLETRANSFERCONFIGPLAN_STATUS_IS, pipeConfigTransferResp); } - return new TPipeTransferResp(pipeConfigTransferResp.status) - .setBody(pipeConfigTransferResp.body); + return new Pair<>( + new TPipeTransferResp(pipeConfigTransferResp.status).setBody(pipeConfigTransferResp.body), + receiverNodeId); } catch (Exception e) { - return new TPipeTransferResp( - new TSStatus(TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode()) - .setMessage(e.toString())); + return new Pair<>( + new TPipeTransferResp( + new TSStatus(TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode()) + .setMessage(e.toString())), + -1); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java index 09a71ad0b8986..3ce09f4047935 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java @@ -487,11 +487,7 @@ protected void storeRuntimeConfigurations( List configNodeLocations, TRuntimeConfiguration runtimeConfiguration) throws StartupException { /* Store ConfigNodeList */ - List configNodeList = new ArrayList<>(); - for (TConfigNodeLocation configNodeLocation : configNodeLocations) { - configNodeList.add(configNodeLocation.getInternalEndPoint()); - } - ConfigNodeInfo.getInstance().updateConfigNodeList(configNodeList); + ConfigNodeInfo.getInstance().updateConfigNodeLocations(configNodeLocations); /* Store templateSetInfo */ ClusterTemplateManager.getInstance() diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/client/ConfigNodeInfoTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/client/ConfigNodeInfoTest.java new file mode 100644 index 0000000000000..3807856640c49 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/client/ConfigNodeInfoTest.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.protocol.client; + +import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.commons.file.SystemPropertiesHandler; + +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class ConfigNodeInfoTest { + + @Before + public void setUp() { + ConfigNodeInfo.reinitializeStatics(); + ConfigNodeInfo.getInstance().systemPropertiesHandler = new NoopSystemPropertiesHandler(); + } + + @Test + public void testUpdateConfigNodeLocationsCachesNodeIds() { + final ConfigNodeInfo configNodeInfo = ConfigNodeInfo.getInstance(); + final TEndPoint firstInternalEndPoint = new TEndPoint("127.0.0.1", 10710); + final TEndPoint secondInternalEndPoint = new TEndPoint("127.0.0.2", 10710); + + assertTrue( + configNodeInfo.updateConfigNodeLocations( + Arrays.asList( + new TConfigNodeLocation( + 1, firstInternalEndPoint, new TEndPoint("127.0.0.1", 10720)), + new TConfigNodeLocation( + 2, secondInternalEndPoint, new TEndPoint("127.0.0.2", 10720))))); + assertEquals(1, configNodeInfo.getConfigNodeId(new TEndPoint("127.0.0.1", 10710))); + assertEquals(2, configNodeInfo.getConfigNodeId(new TEndPoint("127.0.0.2", 10710))); + + assertTrue( + configNodeInfo.updateConfigNodeList(Collections.singletonList(firstInternalEndPoint))); + assertEquals(1, configNodeInfo.getConfigNodeId(new TEndPoint("127.0.0.1", 10710))); + assertEquals(-1, configNodeInfo.getConfigNodeId(new TEndPoint("127.0.0.2", 10710))); + } + + private static class NoopSystemPropertiesHandler extends SystemPropertiesHandler { + + private NoopSystemPropertiesHandler() { + super("target/noop-system.properties"); + } + + @Override + public boolean fileExist() { + return false; + } + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/ShowReceiversOperatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/ShowReceiversOperatorTest.java index c3f7266bbbbf7..29f3f45d553fa 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/ShowReceiversOperatorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/ShowReceiversOperatorTest.java @@ -21,12 +21,26 @@ import org.apache.iotdb.commons.pipe.receiver.runtime.PipeReceiverRuntimeRegistry; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.common.header.DatasetHeaderFactory; +import com.google.common.collect.ImmutableList; import org.apache.tsfile.read.common.block.TsBlock; import org.junit.After; import org.junit.Before; import org.junit.Test; +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.CONNECTION_COUNT; +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.LAST_HANDSHAKE_TIME; +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.LAST_TRANSFER_TIME; +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.PIPE_COUNT; +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.PIPE_IDS; +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.PROTOCOL; +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.RECEIVER_NODE_ID; +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.RECEIVER_NODE_TYPE; +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.RECEIVER_USER_NAME; +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.SENDER_ADDRESS; +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.SENDER_CLUSTER_ID; +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.SENDER_PORTS; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -44,6 +58,25 @@ public void tearDown() { registry.clear(); } + @Test + public void testShowReceiversHeaderColumns() { + assertEquals( + ImmutableList.of( + RECEIVER_NODE_TYPE, + RECEIVER_NODE_ID, + PROTOCOL, + SENDER_ADDRESS, + SENDER_PORTS, + CONNECTION_COUNT, + PIPE_COUNT, + PIPE_IDS, + RECEIVER_USER_NAME, + SENDER_CLUSTER_ID, + LAST_HANDSHAKE_TIME, + LAST_TRANSFER_TIME), + DatasetHeaderFactory.getShowReceiversHeader().getRespColumns()); + } + @Test public void testUnknownReceiverNodeIdIsNull() { registry.registerOrUpdateSession( @@ -66,6 +99,7 @@ public void testUnknownReceiverNodeIdIsNull() { final TsBlock tsBlock = operator.next(); assertEquals(1, tsBlock.getPositionCount()); + assertEquals(12, tsBlock.getValueColumnCount()); assertTrue(tsBlock.getColumn(1).isNull(0)); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/informationschema/ShowReceiversTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/informationschema/ShowReceiversTest.java index 318a84bd483c5..a37fafc32fe8e 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/informationschema/ShowReceiversTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/informationschema/ShowReceiversTest.java @@ -22,16 +22,44 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.LogicalQueryPlan; import org.apache.iotdb.db.queryengine.plan.relational.planner.PlanTester; +import com.google.common.collect.ImmutableList; import org.junit.Test; import java.util.Optional; +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.CONNECTION_COUNT_TABLE_MODEL; +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.LAST_HANDSHAKE_TIME_TABLE_MODEL; +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.LAST_TRANSFER_TIME_TABLE_MODEL; +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.PIPE_COUNT_TABLE_MODEL; +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.PIPE_IDS_TABLE_MODEL; +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.PROTOCOL_TABLE_MODEL; +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.RECEIVER_NODE_ID_TABLE_MODEL; +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.RECEIVER_NODE_TYPE_TABLE_MODEL; +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.SENDER_ADDRESS_TABLE_MODEL; +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.SENDER_CLUSTER_ID_TABLE_MODEL; +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.SENDER_PORTS_TABLE_MODEL; +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.USER_NAME; import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanAssert.assertPlan; import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.infoSchemaTableScan; import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.output; public class ShowReceiversTest { + private static final ImmutableList RECEIVERS_COLUMNS = + ImmutableList.of( + RECEIVER_NODE_TYPE_TABLE_MODEL, + RECEIVER_NODE_ID_TABLE_MODEL, + PROTOCOL_TABLE_MODEL, + SENDER_ADDRESS_TABLE_MODEL, + SENDER_PORTS_TABLE_MODEL, + CONNECTION_COUNT_TABLE_MODEL, + PIPE_COUNT_TABLE_MODEL, + PIPE_IDS_TABLE_MODEL, + USER_NAME, + SENDER_CLUSTER_ID_TABLE_MODEL, + LAST_HANDSHAKE_TIME_TABLE_MODEL, + LAST_TRANSFER_TIME_TABLE_MODEL); + private final PlanTester planTester = new PlanTester(); @Test @@ -39,7 +67,9 @@ public void testShowReceiversRewrite() { final LogicalQueryPlan logicalQueryPlan = planTester.createPlan("show receivers"); assertPlan( logicalQueryPlan, - output(infoSchemaTableScan("information_schema.receivers", Optional.empty()))); + output( + infoSchemaTableScan( + "information_schema.receivers", Optional.empty(), RECEIVERS_COLUMNS))); } @Test @@ -48,6 +78,8 @@ public void testSelectReceivers() { planTester.createPlan("select * from information_schema.receivers"); assertPlan( logicalQueryPlan, - output(infoSchemaTableScan("information_schema.receivers", Optional.empty()))); + output( + infoSchemaTableScan( + "information_schema.receivers", Optional.empty(), RECEIVERS_COLUMNS))); } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/IoTDBFileReceiver.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/IoTDBFileReceiver.java index fca1a857b0eac..55f2712cf5e73 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/IoTDBFileReceiver.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/IoTDBFileReceiver.java @@ -923,10 +923,6 @@ protected void recordPipeReceiverTransfer() { .markTransfer(pipeReceiverRuntimeSessionKey.get(), System.currentTimeMillis()); } - protected void recordPipeReceiverRequest() { - PipeReceiverRuntimeRegistry.getInstance().markRequest(pipeReceiverRuntimeSessionKey.get()); - } - protected void clearPipeReceiverRuntime() { PipeReceiverRuntimeRegistry.getInstance() .deregister(pipeReceiverRuntimeSessionKey.getAndSet(null)); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/runtime/PipeReceiverRuntimeRegistry.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/runtime/PipeReceiverRuntimeRegistry.java index 90dac99236322..e71aadcb2e22b 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/runtime/PipeReceiverRuntimeRegistry.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/runtime/PipeReceiverRuntimeRegistry.java @@ -31,7 +31,6 @@ import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicLong; public class PipeReceiverRuntimeRegistry { @@ -83,7 +82,6 @@ public void registerOrUpdateSession( session.senderClusterId = normalize(senderClusterId); session.lastHandshakeTime = handshakeTime; session.lastTransferTime = Math.max(session.lastTransferTime, handshakeTime); - session.requestNum.incrementAndGet(); session.pipeId = isBlank(pipeName) ? null : formatPipeId(pipeName, pipeCreationTime); } return session; @@ -97,18 +95,9 @@ public void markTransfer(String connectionKey, long transferTime) { } synchronized (session) { session.lastTransferTime = Math.max(session.lastTransferTime, transferTime); - session.requestNum.incrementAndGet(); } } - public void markRequest(String connectionKey) { - final SessionRuntimeInfo session = sessionInfoMap.get(connectionKey); - if (session == null) { - return; - } - session.requestNum.incrementAndGet(); - } - public void deregister(String connectionKey) { if (!isBlank(connectionKey)) { sessionInfoMap.remove(connectionKey); @@ -141,7 +130,6 @@ public List snapshot() { Math.max(aggregatedInfo.lastHandshakeTime, session.lastHandshakeTime); aggregatedInfo.lastTransferTime = Math.max(aggregatedInfo.lastTransferTime, session.lastTransferTime); - aggregatedInfo.requestNum += session.requestNum.get(); } } @@ -185,7 +173,6 @@ private static class SessionRuntimeInfo { private String senderClusterId = UNKNOWN; private long lastHandshakeTime; private long lastTransferTime; - private final AtomicLong requestNum = new AtomicLong(); private String pipeId; private SessionRuntimeInfo(String connectionKey) { @@ -269,7 +256,6 @@ private static class AggregatedRuntimeInfo { private int connectionCount; private long lastHandshakeTime; private long lastTransferTime; - private long requestNum; private AggregatedRuntimeInfo(GroupKey groupKey) { this.groupKey = groupKey; @@ -292,8 +278,7 @@ private PipeReceiverRuntimeSnapshot toSnapshot() { groupKey.userName, senderClusterIds.isEmpty() ? UNKNOWN : joinStringSet(senderClusterIds), lastHandshakeTime, - lastTransferTime, - requestNum); + lastTransferTime); } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/runtime/PipeReceiverRuntimeSnapshot.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/runtime/PipeReceiverRuntimeSnapshot.java index d1aa752e96363..f6effeeafd2b5 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/runtime/PipeReceiverRuntimeSnapshot.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/runtime/PipeReceiverRuntimeSnapshot.java @@ -33,7 +33,6 @@ public class PipeReceiverRuntimeSnapshot { private final String senderClusterId; private final long lastHandshakeTime; private final long lastTransferTime; - private final long requestNum; public PipeReceiverRuntimeSnapshot( String receiverNodeType, @@ -47,8 +46,7 @@ public PipeReceiverRuntimeSnapshot( String userName, String senderClusterId, long lastHandshakeTime, - long lastTransferTime, - long requestNum) { + long lastTransferTime) { this.receiverNodeType = receiverNodeType; this.receiverNodeId = receiverNodeId; this.protocol = protocol; @@ -61,7 +59,6 @@ public PipeReceiverRuntimeSnapshot( this.senderClusterId = senderClusterId; this.lastHandshakeTime = lastHandshakeTime; this.lastTransferTime = lastTransferTime; - this.requestNum = requestNum; } public String getReceiverNodeType() { @@ -115,8 +112,4 @@ public long getLastHandshakeTime() { public long getLastTransferTime() { return lastTransferTime; } - - public long getRequestNum() { - return requestNum; - } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java index 9109e417163eb..cecc6226cd471 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java @@ -210,7 +210,6 @@ private ColumnHeaderConstant() { public static final String SENDER_CLUSTER_ID = "SenderClusterId"; public static final String LAST_HANDSHAKE_TIME = "LastHandshakeTime"; public static final String LAST_TRANSFER_TIME = "LastTransferTime"; - public static final String REQUEST_NUM = "RequestNum"; // column names for select into public static final String SOURCE_DEVICE = "SourceDevice"; @@ -296,7 +295,6 @@ private ColumnHeaderConstant() { public static final String SENDER_CLUSTER_ID_TABLE_MODEL = "sender_cluster_id"; public static final String LAST_HANDSHAKE_TIME_TABLE_MODEL = "last_handshake_time"; public static final String LAST_TRANSFER_TIME_TABLE_MODEL = "last_transfer_time"; - public static final String REQUEST_NUM_TABLE_MODEL = "request_num"; public static final String PLUGIN_NAME_TABLE_MODEL = "plugin_name"; public static final String PLUGIN_TYPE_TABLE_MODEL = "plugin_type"; @@ -705,8 +703,7 @@ private ColumnHeaderConstant() { new ColumnHeader(RECEIVER_USER_NAME, TSDataType.TEXT), new ColumnHeader(SENDER_CLUSTER_ID, TSDataType.TEXT), new ColumnHeader(LAST_HANDSHAKE_TIME, TSDataType.TEXT), - new ColumnHeader(LAST_TRANSFER_TIME, TSDataType.TEXT), - new ColumnHeader(REQUEST_NUM, TSDataType.INT64)); + new ColumnHeader(LAST_TRANSFER_TIME, TSDataType.TEXT)); public static final List showDiskUsageColumnHeaders = ImmutableList.of( diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java index fd91c0f892154..e44f34ed95ef8 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java @@ -253,8 +253,6 @@ public class InformationSchema { receiversTable.addColumnSchema( new AttributeColumnSchema( ColumnHeaderConstant.LAST_TRANSFER_TIME_TABLE_MODEL, TSDataType.TIMESTAMP)); - receiversTable.addColumnSchema( - new AttributeColumnSchema(ColumnHeaderConstant.REQUEST_NUM_TABLE_MODEL, TSDataType.INT64)); schemaTables.put(RECEIVERS, receiversTable); final TsTable pipePluginTable = new TsTable(PIPE_PLUGINS); diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/receiver/runtime/PipeReceiverRuntimeRegistryTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/receiver/runtime/PipeReceiverRuntimeRegistryTest.java index 5624cd6ff18e4..e5cf1da20a21d 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/receiver/runtime/PipeReceiverRuntimeRegistryTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/receiver/runtime/PipeReceiverRuntimeRegistryTest.java @@ -58,7 +58,6 @@ public void testAggregateAndSortSnapshots() { 1, 100); registry.markTransfer("data-1", 200); - registry.markRequest("data-1"); registry.registerOrUpdateSession( "data-2", PipeReceiverRuntimeRegistry.NODE_TYPE_DATA_NODE, @@ -101,7 +100,6 @@ public void testAggregateAndSortSnapshots() { assertEquals("cluster-a;cluster-b", dataSnapshot.getSenderClusterId()); assertEquals(150, dataSnapshot.getLastHandshakeTime()); assertEquals(200, dataSnapshot.getLastTransferTime()); - assertEquals(4, dataSnapshot.getRequestNum()); } @Test From de01c8b21d7c580ef8abdf43554ed8db5c4eb705 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 11 Jun 2026 12:10:14 +0800 Subject: [PATCH 5/7] Add show receivers visibility and HA tests --- .../pipe/it/single/IoTDBShowReceiversIT.java | 200 +++++++++++++ .../PipeReceiverRuntimeSnapshotFilter.java | 63 ++++ .../source/ShowReceiversOperator.java | 10 +- ...formationSchemaContentSupplierFactory.java | 7 +- .../plan/planner/OperatorTreeGenerator.java | 13 +- .../security/TreeAccessCheckVisitor.java | 3 +- .../source/ShowReceiversOperatorTest.java | 112 ++++++++ ...nformationSchemaReceiversSupplierTest.java | 240 ++++++++++++++++ .../node/source/SourceNodeSerdeTest.java | 2 +- .../plan/relational/analyzer/AuthTest.java | 43 +++ .../PipeReceiverRuntimeRegistryTest.java | 270 ++++++++++++++++++ 11 files changed, 954 insertions(+), 9 deletions(-) create mode 100644 integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBShowReceiversIT.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/PipeReceiverRuntimeSnapshotFilter.java create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaReceiversSupplierTest.java diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBShowReceiversIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBShowReceiversIT.java new file mode 100644 index 0000000000000..1081ec37297fa --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBShowReceiversIT.java @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.single; + +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.commons.cluster.NodeStatus; +import org.apache.iotdb.confignode.rpc.thrift.TDataNodeInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowDataNodesResp; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.isession.SessionConfig; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT1; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.awaitility.Awaitility; +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.Collections; +import java.util.concurrent.TimeUnit; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT1.class}) +public class IoTDBShowReceiversIT extends AbstractPipeSingleIT { + + @Test + public void testShowReceiversInTreeAndTableModel() { + createWriteBackPipe("root.show_receivers", "show_receivers_pipe"); + + assertShowReceivers("show receivers", BaseEnv.TREE_SQL_DIALECT); + assertShowReceivers("select * from information_schema.receivers", BaseEnv.TABLE_SQL_DIALECT); + } + + @Test + public void testShowReceiversWithStoppedDataNode() throws Exception { + Assert.assertTrue(env.getDataNodeWrapperList().size() >= 3); + createWriteBackPipe("root.show_receivers_ha", "show_receivers_ha_pipe"); + + assertShowReceivers("show receivers", BaseEnv.TREE_SQL_DIALECT, "show_receivers_ha_pipe"); + assertShowReceivers( + "select * from information_schema.receivers", + BaseEnv.TABLE_SQL_DIALECT, + "show_receivers_ha_pipe"); + + final int stoppedDataNodeIndex = 0; + final DataNodeWrapper stoppedDataNode = env.getDataNodeWrapper(stoppedDataNodeIndex); + final int stoppedDataNodeId = getDataNodeId(stoppedDataNode); + final DataNodeWrapper queryDataNode = env.getDataNodeWrapper(1); + + env.shutdownDataNode(stoppedDataNodeIndex); + env.ensureNodeStatus( + Collections.singletonList(stoppedDataNode), Collections.singletonList(NodeStatus.Unknown)); + + assertShowReceiversWithoutDataNode( + "show receivers", BaseEnv.TREE_SQL_DIALECT, queryDataNode, stoppedDataNodeId); + assertShowReceiversWithoutDataNode( + "select * from information_schema.receivers", + BaseEnv.TABLE_SQL_DIALECT, + queryDataNode, + stoppedDataNodeId); + } + + private void createWriteBackPipe(final String database, final String pipeName) { + TestUtils.executeNonQueries( + env, + Arrays.asList( + "create database " + database, + "create timeseries " + database + ".d1.s1 with datatype=INT32, encoding=PLAIN", + "create pipe " + + pipeName + + " with source ('pattern'='" + + database + + "') with sink ('sink'='write-back-sink')", + "insert into " + database + ".d1(time, s1) values (1, 1)", + "flush"), + null); + } + + private void assertShowReceivers(final String sql, final String sqlDialect) { + assertShowReceivers(sql, sqlDialect, "show_receivers_pipe"); + } + + private void assertShowReceivers( + final String sql, final String sqlDialect, final String pipeName) { + Awaitility.await() + .pollInSameThread() + .pollDelay(1L, TimeUnit.SECONDS) + .pollInterval(1L, TimeUnit.SECONDS) + .atMost(60L, TimeUnit.SECONDS) + .untilAsserted(() -> Assert.assertTrue(hasExpectedReceiver(sql, sqlDialect, pipeName))); + } + + private boolean hasExpectedReceiver( + final String sql, final String sqlDialect, final String pipeName) throws SQLException { + try (final Connection connection = env.getConnection(sqlDialect); + final Statement statement = connection.createStatement(); + final ResultSet resultSet = statement.executeQuery(sql)) { + while (resultSet.next()) { + if ("DataNode".equals(resultSet.getString(1)) + && "thrift".equals(resultSet.getString(3)) + && resultSet.getString(4) != null + && !resultSet.getString(4).isEmpty() + && resultSet.getString(5) != null + && !resultSet.getString(5).isEmpty() + && resultSet.getInt(6) >= 1 + && resultSet.getInt(7) >= 1 + && resultSet.getString(8).contains(pipeName + "@") + && "root".equals(resultSet.getString(9)) + && resultSet.getString(10) != null + && !resultSet.getString(10).isEmpty() + && resultSet.getString(11) != null + && resultSet.getString(12) != null) { + return true; + } + } + return false; + } + } + + private int getDataNodeId(final DataNodeWrapper targetDataNode) throws Exception { + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) env.getLeaderConfigNodeConnection()) { + final TShowDataNodesResp response = client.showDataNodes(); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), response.getStatus().getCode()); + for (final TDataNodeInfo dataNodeInfo : response.getDataNodesInfoList()) { + if (targetDataNode.getIp().equals(dataNodeInfo.getRpcAddresss()) + && targetDataNode.getPort() == dataNodeInfo.getRpcPort()) { + return dataNodeInfo.getDataNodeId(); + } + } + } + throw new AssertionError("Cannot find DataNodeId for " + targetDataNode.getIpAndPortString()); + } + + private void assertShowReceiversWithoutDataNode( + final String sql, + final String sqlDialect, + final DataNodeWrapper queryDataNode, + final int excludedDataNodeId) { + Awaitility.await() + .pollInSameThread() + .pollDelay(1L, TimeUnit.SECONDS) + .pollInterval(1L, TimeUnit.SECONDS) + .atMost(60L, TimeUnit.SECONDS) + .untilAsserted( + () -> + assertQueryResultDoesNotContainDataNode( + sql, sqlDialect, queryDataNode, excludedDataNodeId)); + } + + private void assertQueryResultDoesNotContainDataNode( + final String sql, + final String sqlDialect, + final DataNodeWrapper queryDataNode, + final int excludedDataNodeId) + throws SQLException { + try (final Connection connection = + env.getConnection( + queryDataNode, + SessionConfig.DEFAULT_USER, + SessionConfig.DEFAULT_PASSWORD, + sqlDialect); + final Statement statement = connection.createStatement(); + final ResultSet resultSet = statement.executeQuery(sql)) { + while (resultSet.next()) { + final int receiverDataNodeId = resultSet.getInt(2); + if (!resultSet.wasNull()) { + Assert.assertNotEquals(excludedDataNodeId, receiverDataNodeId); + } + } + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/PipeReceiverRuntimeSnapshotFilter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/PipeReceiverRuntimeSnapshotFilter.java new file mode 100644 index 0000000000000..3300c29ddf674 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/PipeReceiverRuntimeSnapshotFilter.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.source; + +import org.apache.iotdb.commons.audit.UserEntity; +import org.apache.iotdb.commons.auth.entity.PrivilegeType; +import org.apache.iotdb.commons.pipe.receiver.runtime.PipeReceiverRuntimeRegistry; +import org.apache.iotdb.commons.pipe.receiver.runtime.PipeReceiverRuntimeSnapshot; +import org.apache.iotdb.db.auth.AuthorityChecker; + +import java.util.List; +import java.util.stream.Collectors; + +public final class PipeReceiverRuntimeSnapshotFilter { + + private PipeReceiverRuntimeSnapshotFilter() { + // utility class + } + + public static List visibleSnapshots(final UserEntity userEntity) { + final List snapshots = + PipeReceiverRuntimeRegistry.getInstance().snapshot(); + if (canSeeAllReceivers(userEntity)) { + return snapshots; + } + final String userName = userEntity.getUsername(); + return snapshots.stream() + .filter(snapshot -> userName.equals(snapshot.getUserName())) + .collect(Collectors.toList()); + } + + private static boolean canSeeAllReceivers(final UserEntity userEntity) { + if (userEntity == null || userEntity.getUsername() == null) { + return true; + } + if (AuthorityChecker.SUPER_USER_ID == userEntity.getUserId() + || AuthorityChecker.SUPER_USER.equals(userEntity.getUsername())) { + return true; + } + try { + return AuthorityChecker.checkSystemPermission(userEntity.getUsername(), PrivilegeType.SYSTEM); + } catch (final Exception e) { + return false; + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ShowReceiversOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ShowReceiversOperator.java index 9dc35890b1da4..c59f86e37bd79 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ShowReceiversOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ShowReceiversOperator.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.queryengine.execution.operator.source; +import org.apache.iotdb.commons.audit.UserEntity; import org.apache.iotdb.commons.pipe.receiver.runtime.PipeReceiverRuntimeRegistry; import org.apache.iotdb.commons.pipe.receiver.runtime.PipeReceiverRuntimeSnapshot; import org.apache.iotdb.commons.queryengine.execution.MemoryEstimationHelper; @@ -44,6 +45,7 @@ public class ShowReceiversOperator implements SourceOperator { private final OperatorContext operatorContext; private final PlanNodeId sourceId; + private final UserEntity userEntity; private TsBlock tsBlock; private boolean hasConsumed; @@ -55,8 +57,14 @@ public class ShowReceiversOperator implements SourceOperator { RamUsageEstimator.shallowSizeOfInstance(ShowReceiversOperator.class); public ShowReceiversOperator(OperatorContext operatorContext, PlanNodeId sourceId) { + this(operatorContext, sourceId, null); + } + + public ShowReceiversOperator( + OperatorContext operatorContext, PlanNodeId sourceId, UserEntity userEntity) { this.operatorContext = operatorContext; this.sourceId = sourceId; + this.userEntity = userEntity; } @Override @@ -124,7 +132,7 @@ private TsBlock buildTsBlock() { System.currentTimeMillis(), TimeUnit.MILLISECONDS); for (PipeReceiverRuntimeSnapshot snapshot : - PipeReceiverRuntimeRegistry.getInstance().snapshot()) { + PipeReceiverRuntimeSnapshotFilter.visibleSnapshots(userEntity)) { timeColumnBuilder.writeLong(currentTime); columnBuilders[0].writeBinary(BytesUtils.valueOf(snapshot.getReceiverNodeType())); writeReceiverNodeId(columnBuilders[1], snapshot); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java index bcd3950aa1181..18f64ee0167d1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java @@ -28,14 +28,12 @@ import org.apache.iotdb.common.rpc.thrift.TExternalServiceEntry; import org.apache.iotdb.common.rpc.thrift.TExternalServiceListResp; import org.apache.iotdb.commons.audit.UserEntity; -import org.apache.iotdb.commons.auth.entity.PrivilegeType; import org.apache.iotdb.commons.client.exception.ClientManagerException; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.exception.IoTDBRuntimeException; import org.apache.iotdb.commons.exception.auth.AccessDeniedException; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.BuiltinPipePlugin; import org.apache.iotdb.commons.pipe.agent.plugin.meta.PipePluginMeta; -import org.apache.iotdb.commons.pipe.receiver.runtime.PipeReceiverRuntimeRegistry; import org.apache.iotdb.commons.pipe.receiver.runtime.PipeReceiverRuntimeSnapshot; import org.apache.iotdb.commons.queryengine.common.ConnectionInfo; import org.apache.iotdb.commons.queryengine.common.SqlDialect; @@ -85,6 +83,7 @@ import org.apache.iotdb.db.queryengine.common.QueryId; import org.apache.iotdb.db.queryengine.execution.QueryState; import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext; +import org.apache.iotdb.db.queryengine.execution.operator.source.PipeReceiverRuntimeSnapshotFilter; import org.apache.iotdb.db.queryengine.plan.Coordinator; import org.apache.iotdb.db.queryengine.plan.execution.IQueryExecution; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.ShowCreateViewTask; @@ -720,9 +719,7 @@ private static class ReceiversSupplier extends TsBlockSupplier { private ReceiversSupplier(final List dataTypes, final UserEntity userEntity) { super(dataTypes); - accessControl.checkMissingPrivileges( - userEntity.getUsername(), Collections.singletonList(PrivilegeType.USE_PIPE), userEntity); - iterator = PipeReceiverRuntimeRegistry.getInstance().snapshot().iterator(); + iterator = PipeReceiverRuntimeSnapshotFilter.visibleSnapshots(userEntity).iterator(); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java index 6e32f55b7322b..bf5683c7f0530 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java @@ -40,6 +40,7 @@ import org.apache.iotdb.calc.transformation.dag.column.leaf.LeafColumnTransformer; import org.apache.iotdb.common.rpc.thrift.TAggregationType; import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.commons.audit.UserEntity; import org.apache.iotdb.commons.model.ModelInformation; import org.apache.iotdb.commons.path.AlignedFullPath; import org.apache.iotdb.commons.path.AlignedPath; @@ -327,6 +328,15 @@ public class OperatorTreeGenerator implements PlanVisitor DESC_BINARY_COMPARATOR = Comparator.reverseOrder(); + private static UserEntity getCurrentUserEntity(final LocalExecutionPlanContext context) { + if (context.getDriverContext() == null + || context.getDriverContext().getFragmentInstanceContext() == null + || context.getDriverContext().getFragmentInstanceContext().getSessionInfo() == null) { + return null; + } + return context.getDriverContext().getFragmentInstanceContext().getSessionInfo().getUserEntity(); + } + @Override public Operator visitPlan(PlanNode node, LocalExecutionPlanContext context) { throw new UnsupportedOperationException( @@ -2490,7 +2500,8 @@ public Operator visitShowReceivers(ShowReceiversNode node, LocalExecutionPlanCon node.getPlanNodeId(), ShowReceiversOperator.class.getSimpleName()); - return new ShowReceiversOperator(operatorContext, node.getPlanNodeId()); + return new ShowReceiversOperator( + operatorContext, node.getPlanNodeId(), getCurrentUserEntity(context)); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/TreeAccessCheckVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/TreeAccessCheckVisitor.java index 7f0a0e93dfac3..8d0a6728a5e0d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/TreeAccessCheckVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/TreeAccessCheckVisitor.java @@ -852,7 +852,8 @@ public TSStatus visitShowPipes(ShowPipesStatement statement, TreeAccessCheckCont @Override public TSStatus visitShowReceivers( ShowReceiversStatement statement, TreeAccessCheckContext context) { - return checkPipeManagement(context.setAuditLogOperation(AuditLogOperation.QUERY), () -> ""); + // This query follows SHOW PIPES: it cannot be rejected here. + return StatusUtils.OK; } @Override diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/ShowReceiversOperatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/ShowReceiversOperatorTest.java index 29f3f45d553fa..3d3173968a0c9 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/ShowReceiversOperatorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/ShowReceiversOperatorTest.java @@ -19,8 +19,11 @@ package org.apache.iotdb.db.queryengine.execution.operator.source; +import org.apache.iotdb.commons.audit.UserEntity; import org.apache.iotdb.commons.pipe.receiver.runtime.PipeReceiverRuntimeRegistry; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.commons.queryengine.utils.DateTimeUtils; +import org.apache.iotdb.db.auth.AuthorityChecker; import org.apache.iotdb.db.queryengine.common.header.DatasetHeaderFactory; import com.google.common.collect.ImmutableList; @@ -42,6 +45,7 @@ import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.SENDER_CLUSTER_ID; import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.SENDER_PORTS; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; public class ShowReceiversOperatorTest { @@ -77,6 +81,44 @@ public void testShowReceiversHeaderColumns() { DatasetHeaderFactory.getShowReceiversHeader().getRespColumns()); } + @Test + public void testSingleSenderSingleDataNodeSingleConnectionSinglePipeOutput() { + registry.registerOrUpdateSession( + "data-1", + PipeReceiverRuntimeRegistry.NODE_TYPE_DATA_NODE, + 1, + PipeReceiverRuntimeRegistry.PROTOCOL_THRIFT, + "10.0.0.1", + 9001, + "root", + "cluster-a", + "pipe-a", + 1, + 100); + registry.markTransfer("data-1", 200); + + final ShowReceiversOperator operator = + new ShowReceiversOperator(null, new PlanNodeId("show-receivers")); + + assertTrue(operator.hasNext()); + final TsBlock tsBlock = operator.next(); + + assertEquals(1, tsBlock.getPositionCount()); + assertEquals(12, tsBlock.getValueColumnCount()); + assertEquals(PipeReceiverRuntimeRegistry.NODE_TYPE_DATA_NODE, getText(tsBlock, 0)); + assertEquals(1, tsBlock.getColumn(1).getInt(0)); + assertEquals(PipeReceiverRuntimeRegistry.PROTOCOL_THRIFT, getText(tsBlock, 2)); + assertEquals("10.0.0.1", getText(tsBlock, 3)); + assertEquals("9001", getText(tsBlock, 4)); + assertEquals(1, tsBlock.getColumn(5).getInt(0)); + assertEquals(1, tsBlock.getColumn(6).getInt(0)); + assertTrue(getText(tsBlock, 7).contains("pipe-a@")); + assertEquals("root", getText(tsBlock, 8)); + assertEquals("cluster-a", getText(tsBlock, 9)); + assertEquals(DateTimeUtils.convertLongToDate(100, "ms"), getText(tsBlock, 10)); + assertEquals(DateTimeUtils.convertLongToDate(200, "ms"), getText(tsBlock, 11)); + } + @Test public void testUnknownReceiverNodeIdIsNull() { registry.registerOrUpdateSession( @@ -102,4 +144,74 @@ public void testUnknownReceiverNodeIdIsNull() { assertEquals(12, tsBlock.getValueColumnCount()); assertTrue(tsBlock.getColumn(1).isNull(0)); } + + @Test + public void testNormalUserOnlySeesOwnReceiverSnapshots() { + registerUserSession("data-user1", "10.0.0.1", 9001, "user1", "cluster-a", "pipe-a", 1, 100); + registerUserSession("data-user2", "10.0.0.2", 9002, "user2", "cluster-b", "pipe-b", 2, 200); + + final ShowReceiversOperator operator = + new ShowReceiversOperator( + null, new PlanNodeId("show-receivers"), new UserEntity(1L, "user1", "127.0.0.1")); + + assertTrue(operator.hasNext()); + final TsBlock tsBlock = operator.next(); + + assertEquals(1, tsBlock.getPositionCount()); + assertEquals("10.0.0.1", getText(tsBlock, 3)); + assertEquals("user1", getText(tsBlock, 8)); + assertFalse(operator.hasNext()); + } + + @Test + public void testRootUserSeesAllReceiverSnapshots() { + registerUserSession("data-user1", "10.0.0.1", 9001, "user1", "cluster-a", "pipe-a", 1, 100); + registerUserSession("data-user2", "10.0.0.2", 9002, "user2", "cluster-b", "pipe-b", 2, 200); + + final ShowReceiversOperator operator = + new ShowReceiversOperator( + null, + new PlanNodeId("show-receivers"), + new UserEntity( + AuthorityChecker.SUPER_USER_ID, AuthorityChecker.SUPER_USER, "127.0.0.1")); + + assertTrue(operator.hasNext()); + final TsBlock tsBlock = operator.next(); + + assertEquals(2, tsBlock.getPositionCount()); + assertEquals("user1", getText(tsBlock, 8, 0)); + assertEquals("user2", getText(tsBlock, 8, 1)); + assertFalse(operator.hasNext()); + } + + private void registerUserSession( + String connectionKey, + String senderAddress, + int senderPort, + String userName, + String senderClusterId, + String pipeName, + long pipeCreationTime, + long handshakeTime) { + registry.registerOrUpdateSession( + connectionKey, + PipeReceiverRuntimeRegistry.NODE_TYPE_DATA_NODE, + 1, + PipeReceiverRuntimeRegistry.PROTOCOL_THRIFT, + senderAddress, + senderPort, + userName, + senderClusterId, + pipeName, + pipeCreationTime, + handshakeTime); + } + + private static String getText(final TsBlock tsBlock, final int columnIndex) { + return getText(tsBlock, columnIndex, 0); + } + + private static String getText(final TsBlock tsBlock, final int columnIndex, final int position) { + return tsBlock.getColumn(columnIndex).getBinary(position).toString(); + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaReceiversSupplierTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaReceiversSupplierTest.java new file mode 100644 index 0000000000000..570677e184a92 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaReceiversSupplierTest.java @@ -0,0 +1,240 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.source.relational; + +import org.apache.iotdb.commons.audit.UserEntity; +import org.apache.iotdb.commons.pipe.receiver.runtime.PipeReceiverRuntimeRegistry; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.commons.queryengine.plan.relational.metadata.ColumnSchema; +import org.apache.iotdb.commons.queryengine.plan.relational.metadata.QualifiedObjectName; +import org.apache.iotdb.commons.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.commons.schema.table.InformationSchema; +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; +import org.apache.iotdb.db.auth.AuthorityChecker; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.InformationSchemaTableScanNode; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.type.TypeFactory; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class InformationSchemaReceiversSupplierTest { + + private final PipeReceiverRuntimeRegistry registry = PipeReceiverRuntimeRegistry.getInstance(); + + @Before + public void setUp() { + registry.clear(); + } + + @After + public void tearDown() { + registry.clear(); + } + + @Test + public void testReceiversSupplierBuildsTableModelRows() { + registry.registerOrUpdateSession( + "data-1", + PipeReceiverRuntimeRegistry.NODE_TYPE_DATA_NODE, + 1, + PipeReceiverRuntimeRegistry.PROTOCOL_THRIFT, + "10.0.0.1", + 9001, + "root", + "cluster-a", + "pipe-a", + 1, + 100); + registry.markTransfer("data-1", 200); + + final InformationSchemaContentSupplierFactory.IInformationSchemaContentSupplier supplier = + InformationSchemaContentSupplierFactory.getSupplier( + null, dataTypes(), rootUser(), receiversScanNode()); + + assertTrue(supplier.hasNext()); + final TsBlock tsBlock = supplier.next(); + + assertEquals(1, tsBlock.getPositionCount()); + assertEquals(PipeReceiverRuntimeRegistry.NODE_TYPE_DATA_NODE, getText(tsBlock, 0)); + assertEquals(1, tsBlock.getColumn(1).getInt(0)); + assertEquals(PipeReceiverRuntimeRegistry.PROTOCOL_THRIFT, getText(tsBlock, 2)); + assertEquals("10.0.0.1", getText(tsBlock, 3)); + assertEquals("9001", getText(tsBlock, 4)); + assertEquals(1, tsBlock.getColumn(5).getInt(0)); + assertEquals(1, tsBlock.getColumn(6).getInt(0)); + assertTrue(getText(tsBlock, 7).contains("pipe-a@")); + assertEquals("root", getText(tsBlock, 8)); + assertEquals("cluster-a", getText(tsBlock, 9)); + assertEquals(100L, tsBlock.getColumn(10).getLong(0)); + assertEquals(200L, tsBlock.getColumn(11).getLong(0)); + assertFalse(supplier.hasNext()); + } + + @Test + public void testReceiversSupplierWritesUnknownFieldsAsNulls() { + registry.registerOrUpdateSession( + "config-1", + PipeReceiverRuntimeRegistry.NODE_TYPE_CONFIG_NODE, + -1, + PipeReceiverRuntimeRegistry.PROTOCOL_AIR_GAP, + "10.0.0.2", + 9002, + "root", + "cluster-b", + null, + Long.MIN_VALUE, + 0); + + final InformationSchemaContentSupplierFactory.IInformationSchemaContentSupplier supplier = + InformationSchemaContentSupplierFactory.getSupplier( + null, dataTypes(), rootUser(), receiversScanNode()); + + assertTrue(supplier.hasNext()); + final TsBlock tsBlock = supplier.next(); + + assertEquals(1, tsBlock.getPositionCount()); + assertEquals(PipeReceiverRuntimeRegistry.NODE_TYPE_CONFIG_NODE, getText(tsBlock, 0)); + assertTrue(tsBlock.getColumn(1).isNull(0)); + assertEquals(PipeReceiverRuntimeRegistry.PROTOCOL_AIR_GAP, getText(tsBlock, 2)); + assertEquals(0, tsBlock.getColumn(6).getInt(0)); + assertEquals(PipeReceiverRuntimeRegistry.UNKNOWN, getText(tsBlock, 7)); + assertTrue(tsBlock.getColumn(10).isNull(0)); + assertTrue(tsBlock.getColumn(11).isNull(0)); + assertFalse(supplier.hasNext()); + } + + @Test + public void testReceiversSupplierFiltersByNormalUser() { + registerUserSession("data-user1", "10.0.0.1", 9001, "user1", "cluster-a", "pipe-a", 1, 100); + registerUserSession("data-user2", "10.0.0.2", 9002, "user2", "cluster-b", "pipe-b", 2, 200); + + final InformationSchemaContentSupplierFactory.IInformationSchemaContentSupplier supplier = + InformationSchemaContentSupplierFactory.getSupplier( + null, dataTypes(), new UserEntity(1L, "user1", "127.0.0.1"), receiversScanNode()); + + assertTrue(supplier.hasNext()); + final TsBlock tsBlock = supplier.next(); + + assertEquals(1, tsBlock.getPositionCount()); + assertEquals("10.0.0.1", getText(tsBlock, 3)); + assertEquals("user1", getText(tsBlock, 8)); + assertFalse(supplier.hasNext()); + } + + @Test + public void testReceiversSupplierRootSeesAllReceiverSnapshots() { + registerUserSession("data-user1", "10.0.0.1", 9001, "user1", "cluster-a", "pipe-a", 1, 100); + registerUserSession("data-user2", "10.0.0.2", 9002, "user2", "cluster-b", "pipe-b", 2, 200); + + final InformationSchemaContentSupplierFactory.IInformationSchemaContentSupplier supplier = + InformationSchemaContentSupplierFactory.getSupplier( + null, dataTypes(), rootUser(), receiversScanNode()); + + assertTrue(supplier.hasNext()); + final TsBlock tsBlock = supplier.next(); + + assertEquals(2, tsBlock.getPositionCount()); + assertEquals("user1", getText(tsBlock, 8, 0)); + assertEquals("user2", getText(tsBlock, 8, 1)); + assertFalse(supplier.hasNext()); + } + + private static InformationSchemaTableScanNode receiversScanNode() { + final List outputSymbols = new ArrayList<>(); + final Map assignments = new LinkedHashMap<>(); + for (TsTableColumnSchema columnSchema : receiversTable().getColumnList()) { + final Symbol symbol = new Symbol(columnSchema.getColumnName()); + outputSymbols.add(symbol); + assignments.put( + symbol, + new ColumnSchema( + columnSchema.getColumnName(), + TypeFactory.getType(columnSchema.getDataType()), + false, + columnSchema.getColumnCategory())); + } + return new InformationSchemaTableScanNode( + new PlanNodeId("information-schema-receivers"), + new QualifiedObjectName( + InformationSchema.INFORMATION_DATABASE, InformationSchema.RECEIVERS), + outputSymbols, + assignments); + } + + private static List dataTypes() { + final List dataTypes = new ArrayList<>(); + for (TsTableColumnSchema columnSchema : receiversTable().getColumnList()) { + dataTypes.add(columnSchema.getDataType()); + } + return dataTypes; + } + + private static TsTable receiversTable() { + return InformationSchema.getSchemaTables().get(InformationSchema.RECEIVERS); + } + + private void registerUserSession( + String connectionKey, + String senderAddress, + int senderPort, + String userName, + String senderClusterId, + String pipeName, + long pipeCreationTime, + long handshakeTime) { + registry.registerOrUpdateSession( + connectionKey, + PipeReceiverRuntimeRegistry.NODE_TYPE_DATA_NODE, + 1, + PipeReceiverRuntimeRegistry.PROTOCOL_THRIFT, + senderAddress, + senderPort, + userName, + senderClusterId, + pipeName, + pipeCreationTime, + handshakeTime); + } + + private static UserEntity rootUser() { + return new UserEntity(AuthorityChecker.SUPER_USER_ID, AuthorityChecker.SUPER_USER, "127.0.0.1"); + } + + private static String getText(final TsBlock tsBlock, final int columnIndex) { + return getText(tsBlock, columnIndex, 0); + } + + private static String getText(final TsBlock tsBlock, final int columnIndex, final int position) { + return tsBlock.getColumn(columnIndex).getBinary(position).toString(); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/source/SourceNodeSerdeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/source/SourceNodeSerdeTest.java index 97fbb5cc502e6..9df412f2b64f9 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/source/SourceNodeSerdeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/source/SourceNodeSerdeTest.java @@ -119,7 +119,7 @@ public void testShowQueriesNode() throws IllegalPathException { } @Test - public void testShowReceiversNode() { + public void testShowReceiversNode() throws IllegalPathException { ShowReceiversNode node = new ShowReceiversNode(new PlanNodeId("test"), null); ByteBuffer byteBuffer = ByteBuffer.allocate(2048); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AuthTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AuthTest.java index e758e49962827..9c0754bb5c477 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AuthTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AuthTest.java @@ -19,21 +19,27 @@ package org.apache.iotdb.db.queryengine.plan.relational.analyzer; +import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.exception.auth.AccessDeniedException; import org.apache.iotdb.commons.queryengine.common.SessionInfo; import org.apache.iotdb.commons.queryengine.common.SqlDialect; import org.apache.iotdb.commons.queryengine.plan.relational.metadata.QualifiedObjectName; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Statement; import org.apache.iotdb.commons.queryengine.plan.relational.type.InternalTypeManager; +import org.apache.iotdb.commons.schema.table.InformationSchema; +import org.apache.iotdb.commons.utils.StatusUtils; import org.apache.iotdb.db.protocol.session.IClientSession; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.plan.execution.config.TableConfigTaskVisitor; import org.apache.iotdb.db.queryengine.plan.relational.security.AccessControlImpl; import org.apache.iotdb.db.queryengine.plan.relational.security.ITableAuthChecker; import org.apache.iotdb.db.queryengine.plan.relational.security.TableModelPrivilege; +import org.apache.iotdb.db.queryengine.plan.relational.security.TreeAccessCheckContext; import org.apache.iotdb.db.queryengine.plan.relational.security.TreeAccessCheckVisitor; import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.SqlParser; import org.apache.iotdb.db.queryengine.plan.relational.sql.rewrite.StatementRewrite; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.pipe.ShowPipesStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowReceiversStatement; import org.junit.Test; import org.mockito.Mockito; @@ -109,6 +115,43 @@ public void testQueryRelatedAuth() { } } + @Test + public void testInformationSchemaReceiversUseSameLocalAuthAsPipes() { + final ITableAuthChecker authChecker = Mockito.mock(ITableAuthChecker.class); + + try { + analyzeSQL( + String.format( + "SELECT * FROM %s.%s", + InformationSchema.INFORMATION_DATABASE, InformationSchema.PIPES), + user1, + authChecker); + analyzeSQL( + String.format( + "SELECT * FROM %s.%s", + InformationSchema.INFORMATION_DATABASE, InformationSchema.RECEIVERS), + user1, + authChecker); + } catch (Exception e) { + fail("Unexpected exception : " + e.getMessage()); + } + + Mockito.verify(authChecker, Mockito.never()).checkGlobalPrivileges(any(), any(), any()); + } + + @Test + public void testTreeShowReceiversUsesSameLocalAuthAsShowPipes() { + final TreeAccessCheckVisitor visitor = new TreeAccessCheckVisitor(); + final TreeAccessCheckContext context = new TreeAccessCheckContext(1, user1, "127.0.0.1"); + + final TSStatus showPipesStatus = visitor.visitShowPipes(new ShowPipesStatement(), context); + final TSStatus showReceiversStatus = + visitor.visitShowReceivers(new ShowReceiversStatement(), context); + + assertEquals(StatusUtils.OK.getCode(), showPipesStatus.getCode()); + assertEquals(showPipesStatus.getCode(), showReceiversStatus.getCode()); + } + @Test public void testDatabaseManagementRelatedAuth() { diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/receiver/runtime/PipeReceiverRuntimeRegistryTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/receiver/runtime/PipeReceiverRuntimeRegistryTest.java index e5cf1da20a21d..dd968c69bbff8 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/receiver/runtime/PipeReceiverRuntimeRegistryTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/receiver/runtime/PipeReceiverRuntimeRegistryTest.java @@ -24,9 +24,11 @@ import org.junit.Test; import java.util.List; +import java.util.concurrent.TimeUnit; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; public class PipeReceiverRuntimeRegistryTest { @@ -43,6 +45,93 @@ public void tearDown() { registry.clear(); } + @Test + public void testSingleSenderSingleDataNodeSingleConnectionSinglePipeBasicQuery() { + registerDataSession("data-1", 1, "10.0.0.1", 9001, "root", "cluster-a", "pipe-a", 1, 100); + registry.markTransfer("data-1", 200); + + final List snapshots = registry.snapshot(); + + assertEquals(1, snapshots.size()); + final PipeReceiverRuntimeSnapshot snapshot = snapshots.get(0); + assertEquals(PipeReceiverRuntimeRegistry.NODE_TYPE_DATA_NODE, snapshot.getReceiverNodeType()); + assertEquals(1, snapshot.getReceiverNodeId()); + assertEquals(PipeReceiverRuntimeRegistry.PROTOCOL_THRIFT, snapshot.getProtocol()); + assertEquals("10.0.0.1", snapshot.getSenderAddress()); + assertEquals("9001", snapshot.getSenderPorts()); + assertEquals(1, snapshot.getConnectionCount()); + assertEquals(1, snapshot.getPipeCount()); + assertTrue(snapshot.getPipeIds().contains("pipe-a@")); + assertEquals("root", snapshot.getUserName()); + assertEquals("cluster-a", snapshot.getSenderClusterId()); + assertEquals(100, snapshot.getLastHandshakeTime()); + assertEquals(200, snapshot.getLastTransferTime()); + } + + @Test + public void testMultipleSendersAndMultipleDataNodesClusterAggregation() { + registerDataSession("data-1", 1, "10.0.0.1", 9001, "root", "cluster-a", "pipe-a", 1, 100); + registerDataSession("data-2", 2, "10.0.0.1", 9002, "root", "cluster-a", "pipe-b", 2, 200); + registerDataSession("data-3", 2, "10.0.0.2", 9003, "root", "cluster-b", "pipe-c", 3, 300); + + final List snapshots = registry.snapshot(); + + assertEquals(3, snapshots.size()); + assertNotNull( + findSnapshot( + snapshots, + PipeReceiverRuntimeRegistry.NODE_TYPE_DATA_NODE, + 1, + PipeReceiverRuntimeRegistry.PROTOCOL_THRIFT, + "10.0.0.1")); + assertNotNull( + findSnapshot( + snapshots, + PipeReceiverRuntimeRegistry.NODE_TYPE_DATA_NODE, + 2, + PipeReceiverRuntimeRegistry.PROTOCOL_THRIFT, + "10.0.0.1")); + assertNotNull( + findSnapshot( + snapshots, + PipeReceiverRuntimeRegistry.NODE_TYPE_DATA_NODE, + 2, + PipeReceiverRuntimeRegistry.PROTOCOL_THRIFT, + "10.0.0.2")); + } + + @Test + public void testConfigNodeReceiversAreShownWithDataNodeResultsAndAirGapProtocol() { + registerDataSession("data-1", 1, "10.0.0.1", 9001, "root", "cluster-a", "pipe-a", 1, 100); + registry.registerOrUpdateSession( + "config-1", + PipeReceiverRuntimeRegistry.NODE_TYPE_CONFIG_NODE, + 0, + PipeReceiverRuntimeRegistry.PROTOCOL_AIR_GAP, + "10.0.0.2", + 9002, + "root", + "cluster-b", + "pipe-b", + 2, + 200); + + final List snapshots = registry.snapshot(); + + assertEquals(2, snapshots.size()); + final PipeReceiverRuntimeSnapshot configSnapshot = + findSnapshot( + snapshots, + PipeReceiverRuntimeRegistry.NODE_TYPE_CONFIG_NODE, + 0, + PipeReceiverRuntimeRegistry.PROTOCOL_AIR_GAP, + "10.0.0.2"); + assertNotNull(configSnapshot); + assertEquals(PipeReceiverRuntimeRegistry.PROTOCOL_AIR_GAP, configSnapshot.getProtocol()); + assertEquals(1, configSnapshot.getConnectionCount()); + assertEquals(1, configSnapshot.getPipeCount()); + } + @Test public void testAggregateAndSortSnapshots() { registry.registerOrUpdateSession( @@ -124,6 +213,146 @@ public void testDeregisterSession() { assertTrue(registry.snapshot().isEmpty()); } + @Test + public void testPipeStopOrDropUpdatesPipeIdsAndPipeCount() { + registerDataSession("data-1", 1, "10.0.0.1", 9001, "root", "cluster-a", "pipe-a", 1, 100); + registerDataSession("data-2", 1, "10.0.0.1", 9002, "root", "cluster-a", "pipe-b", 2, 200); + + List snapshots = registry.snapshot(); + assertEquals(1, snapshots.size()); + assertEquals(2, snapshots.get(0).getConnectionCount()); + assertEquals(2, snapshots.get(0).getPipeCount()); + + registry.registerOrUpdateSession( + "data-1", + PipeReceiverRuntimeRegistry.NODE_TYPE_DATA_NODE, + 1, + PipeReceiverRuntimeRegistry.PROTOCOL_THRIFT, + "10.0.0.1", + 9001, + "root", + "cluster-a", + null, + Long.MIN_VALUE, + 300); + + snapshots = registry.snapshot(); + assertEquals(1, snapshots.size()); + assertEquals(2, snapshots.get(0).getConnectionCount()); + assertEquals(1, snapshots.get(0).getPipeCount()); + assertFalse(snapshots.get(0).getPipeIds().contains("pipe-a@")); + assertTrue(snapshots.get(0).getPipeIds().contains("pipe-b@")); + + registry.deregister("data-2"); + + snapshots = registry.snapshot(); + assertEquals(1, snapshots.size()); + assertEquals(1, snapshots.get(0).getConnectionCount()); + assertEquals(0, snapshots.get(0).getPipeCount()); + assertEquals(PipeReceiverRuntimeRegistry.UNKNOWN, snapshots.get(0).getPipeIds()); + } + + @Test + public void testMixedVersionPipeIdsUnknownCompatibility() { + registerDataSession( + "data-1", + 1, + "10.0.0.1", + 9001, + "root", + PipeReceiverRuntimeRegistry.UNKNOWN, + null, + Long.MIN_VALUE, + 100); + + final List snapshots = registry.snapshot(); + + assertEquals(1, snapshots.size()); + assertEquals("10.0.0.1", snapshots.get(0).getSenderAddress()); + assertEquals("9001", snapshots.get(0).getSenderPorts()); + assertEquals(1, snapshots.get(0).getConnectionCount()); + assertEquals(0, snapshots.get(0).getPipeCount()); + assertEquals(PipeReceiverRuntimeRegistry.UNKNOWN, snapshots.get(0).getPipeIds()); + } + + @Test + public void testReceiverRestartClearsRuntimeAndAllowsReconnect() { + registerDataSession("data-1", 1, "10.0.0.1", 9001, "root", "cluster-a", "pipe-a", 1, 100); + + assertEquals(1, registry.snapshot().size()); + + registry.clear(); + + assertTrue(registry.snapshot().isEmpty()); + + registerDataSession("data-2", 1, "10.0.0.1", 9002, "root", "cluster-a", "pipe-b", 2, 200); + + final List snapshots = registry.snapshot(); + assertEquals(1, snapshots.size()); + assertEquals("9002", snapshots.get(0).getSenderPorts()); + assertTrue(snapshots.get(0).getPipeIds().contains("pipe-b@")); + } + + @Test + public void testLargeActiveSessionsSnapshotPerformanceAndTransferHotPathOverhead() { + final int activeConnectionCount = 1000; + final int pipeLifecycleOperationCount = 5000; + for (int i = 0; i < activeConnectionCount; i++) { + registerDataSession( + "data-" + i, + i % 10, + "10.0." + (i / 100) + "." + (i % 100), + 9000 + i, + "root", + "cluster-" + (i % 5), + "pipe-" + i, + i, + i); + } + + for (int i = 0; i < pipeLifecycleOperationCount; i++) { + final int connectionIndex = i % activeConnectionCount; + registerDataSession( + "data-" + connectionIndex, + connectionIndex % 10, + "10.0." + (connectionIndex / 100) + "." + (connectionIndex % 100), + 9000 + connectionIndex, + "root", + "cluster-" + (connectionIndex % 5), + "pipe-update-" + i, + i, + 10_000L + i); + } + + final long transferStartTime = System.nanoTime(); + for (int i = 0; i < pipeLifecycleOperationCount; i++) { + registry.markTransfer("data-" + (i % activeConnectionCount), 20_000L + i); + } + final long transferDurationNanos = System.nanoTime() - transferStartTime; + + final long snapshotStartTime = System.nanoTime(); + final List snapshots = registry.snapshot(); + final long snapshotDurationNanos = System.nanoTime() - snapshotStartTime; + + assertEquals( + activeConnectionCount, + snapshots.stream().mapToInt(PipeReceiverRuntimeSnapshot::getConnectionCount).sum()); + assertEquals( + 20_000L + pipeLifecycleOperationCount - 1, + snapshots.stream() + .mapToLong(PipeReceiverRuntimeSnapshot::getLastTransferTime) + .max() + .orElse(0)); + assertTrue( + "transfer updates should stay lightweight, duration ms: " + + TimeUnit.NANOSECONDS.toMillis(transferDurationNanos), + TimeUnit.NANOSECONDS.toMillis(transferDurationNanos) < 3000); + assertTrue( + "snapshot should be built from in-memory state, duration ms: " + + TimeUnit.NANOSECONDS.toMillis(snapshotDurationNanos), + TimeUnit.NANOSECONDS.toMillis(snapshotDurationNanos) < 3000); + } + @Test public void testRegisterOrUpdateSessionReplacesPipeId() { registry.registerOrUpdateSession( @@ -268,4 +497,45 @@ public void testUnknownReceiverNodeId() { assertEquals(-1, snapshots.get(0).getReceiverNodeId()); assertFalse(snapshots.get(0).isReceiverNodeIdKnown()); } + + private void registerDataSession( + String connectionKey, + int receiverNodeId, + String senderAddress, + int senderPort, + String userName, + String senderClusterId, + String pipeName, + long pipeCreationTime, + long handshakeTime) { + registry.registerOrUpdateSession( + connectionKey, + PipeReceiverRuntimeRegistry.NODE_TYPE_DATA_NODE, + receiverNodeId, + PipeReceiverRuntimeRegistry.PROTOCOL_THRIFT, + senderAddress, + senderPort, + userName, + senderClusterId, + pipeName, + pipeCreationTime, + handshakeTime); + } + + private static PipeReceiverRuntimeSnapshot findSnapshot( + List snapshots, + String receiverNodeType, + int receiverNodeId, + String protocol, + String senderAddress) { + for (PipeReceiverRuntimeSnapshot snapshot : snapshots) { + if (receiverNodeType.equals(snapshot.getReceiverNodeType()) + && receiverNodeId == snapshot.getReceiverNodeId() + && protocol.equals(snapshot.getProtocol()) + && senderAddress.equals(snapshot.getSenderAddress())) { + return snapshot; + } + } + return null; + } } From 01d1b8bf3cdf14a9756457447d8232249b4b3e81 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 11 Jun 2026 14:17:13 +0800 Subject: [PATCH 6/7] Add show receivers edge case tests --- .../source/ShowReceiversOperatorTest.java | 28 +++++++ ...nformationSchemaReceiversSupplierTest.java | 75 +++++++++++++++++++ .../informationschema/ShowReceiversTest.java | 19 +++++ .../PipeReceiverRuntimeRegistryTest.java | 60 +++++++++++++++ 4 files changed, 182 insertions(+) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/ShowReceiversOperatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/ShowReceiversOperatorTest.java index 3d3173968a0c9..3bed8d8ace236 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/ShowReceiversOperatorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/ShowReceiversOperatorTest.java @@ -20,6 +20,8 @@ package org.apache.iotdb.db.queryengine.execution.operator.source; import org.apache.iotdb.commons.audit.UserEntity; +import org.apache.iotdb.commons.auth.entity.PrivilegeType; +import org.apache.iotdb.commons.auth.entity.User; import org.apache.iotdb.commons.pipe.receiver.runtime.PipeReceiverRuntimeRegistry; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.commons.queryengine.utils.DateTimeUtils; @@ -55,11 +57,13 @@ public class ShowReceiversOperatorTest { @Before public void setUp() { registry.clear(); + AuthorityChecker.getAuthorityFetcher().getAuthorCache().invalidAllCache(); } @After public void tearDown() { registry.clear(); + AuthorityChecker.getAuthorityFetcher().getAuthorCache().invalidAllCache(); } @Test @@ -184,6 +188,30 @@ public void testRootUserSeesAllReceiverSnapshots() { assertFalse(operator.hasNext()); } + @Test + public void testSystemUserSeesAllReceiverSnapshots() { + registerUserSession("data-user1", "10.0.0.1", 9001, "user1", "cluster-a", "pipe-a", 1, 100); + registerUserSession("data-user2", "10.0.0.2", 9002, "user2", "cluster-b", "pipe-b", 2, 200); + + final User systemUser = new User("system_user", "password"); + systemUser.grantSysPrivilege(PrivilegeType.SYSTEM, false); + AuthorityChecker.getAuthorityFetcher() + .getAuthorCache() + .putUserCache(systemUser.getName(), systemUser); + + final ShowReceiversOperator operator = + new ShowReceiversOperator( + null, new PlanNodeId("show-receivers"), new UserEntity(2L, "system_user", "127.0.0.1")); + + assertTrue(operator.hasNext()); + final TsBlock tsBlock = operator.next(); + + assertEquals(2, tsBlock.getPositionCount()); + assertEquals("user1", getText(tsBlock, 8, 0)); + assertEquals("user2", getText(tsBlock, 8, 1)); + assertFalse(operator.hasNext()); + } + private void registerUserSession( String connectionKey, String senderAddress, diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaReceiversSupplierTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaReceiversSupplierTest.java index 570677e184a92..449cb18cf6780 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaReceiversSupplierTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaReceiversSupplierTest.java @@ -20,6 +20,8 @@ package org.apache.iotdb.db.queryengine.execution.operator.source.relational; import org.apache.iotdb.commons.audit.UserEntity; +import org.apache.iotdb.commons.auth.entity.PrivilegeType; +import org.apache.iotdb.commons.auth.entity.User; import org.apache.iotdb.commons.pipe.receiver.runtime.PipeReceiverRuntimeRegistry; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.commons.queryengine.plan.relational.metadata.ColumnSchema; @@ -29,6 +31,7 @@ import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.db.auth.AuthorityChecker; +import org.apache.iotdb.db.queryengine.execution.operator.source.ShowReceiversOperator; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.InformationSchemaTableScanNode; import org.apache.tsfile.enums.TSDataType; @@ -54,11 +57,13 @@ public class InformationSchemaReceiversSupplierTest { @Before public void setUp() { registry.clear(); + AuthorityChecker.getAuthorityFetcher().getAuthorCache().invalidAllCache(); } @After public void tearDown() { registry.clear(); + AuthorityChecker.getAuthorityFetcher().getAuthorCache().invalidAllCache(); } @Test @@ -169,6 +174,76 @@ public void testReceiversSupplierRootSeesAllReceiverSnapshots() { assertFalse(supplier.hasNext()); } + @Test + public void testReceiversSupplierSystemUserSeesAllReceiverSnapshots() { + registerUserSession("data-user1", "10.0.0.1", 9001, "user1", "cluster-a", "pipe-a", 1, 100); + registerUserSession("data-user2", "10.0.0.2", 9002, "user2", "cluster-b", "pipe-b", 2, 200); + + final User systemUser = new User("system_user", "password"); + systemUser.grantSysPrivilege(PrivilegeType.SYSTEM, false); + AuthorityChecker.getAuthorityFetcher() + .getAuthorCache() + .putUserCache(systemUser.getName(), systemUser); + + final InformationSchemaContentSupplierFactory.IInformationSchemaContentSupplier supplier = + InformationSchemaContentSupplierFactory.getSupplier( + null, dataTypes(), new UserEntity(2L, "system_user", "127.0.0.1"), receiversScanNode()); + + assertTrue(supplier.hasNext()); + final TsBlock tsBlock = supplier.next(); + + assertEquals(2, tsBlock.getPositionCount()); + assertEquals("user1", getText(tsBlock, 8, 0)); + assertEquals("user2", getText(tsBlock, 8, 1)); + assertFalse(supplier.hasNext()); + } + + @Test + public void testReceiversSupplierMatchesShowReceiversOutput() { + registry.registerOrUpdateSession( + "data-1", + PipeReceiverRuntimeRegistry.NODE_TYPE_DATA_NODE, + 1, + PipeReceiverRuntimeRegistry.PROTOCOL_THRIFT, + "10.0.0.1", + 9001, + "root", + "cluster-a", + "pipe-a", + 1, + 100); + registry.markTransfer("data-1", 200); + + final ShowReceiversOperator showReceiversOperator = + new ShowReceiversOperator(null, new PlanNodeId("show-receivers"), rootUser()); + assertTrue(showReceiversOperator.hasNext()); + final TsBlock showReceiversTsBlock = showReceiversOperator.next(); + final InformationSchemaContentSupplierFactory.IInformationSchemaContentSupplier supplier = + InformationSchemaContentSupplierFactory.getSupplier( + null, dataTypes(), rootUser(), receiversScanNode()); + assertTrue(supplier.hasNext()); + final TsBlock informationSchemaTsBlock = supplier.next(); + + assertEquals( + showReceiversTsBlock.getPositionCount(), informationSchemaTsBlock.getPositionCount()); + assertEquals(1, informationSchemaTsBlock.getPositionCount()); + for (int columnIndex : new int[] {0, 2, 3, 4, 7, 8, 9}) { + assertEquals( + getText(showReceiversTsBlock, columnIndex), + getText(informationSchemaTsBlock, columnIndex)); + } + assertEquals( + showReceiversTsBlock.getColumn(1).getInt(0), + informationSchemaTsBlock.getColumn(1).getInt(0)); + assertEquals( + showReceiversTsBlock.getColumn(5).getInt(0), + informationSchemaTsBlock.getColumn(5).getInt(0)); + assertEquals( + showReceiversTsBlock.getColumn(6).getInt(0), + informationSchemaTsBlock.getColumn(6).getInt(0)); + assertFalse(supplier.hasNext()); + } + private static InformationSchemaTableScanNode receiversScanNode() { final List outputSymbols = new ArrayList<>(); final Map assignments = new LinkedHashMap<>(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/informationschema/ShowReceiversTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/informationschema/ShowReceiversTest.java index a37fafc32fe8e..13df05f2c88b5 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/informationschema/ShowReceiversTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/informationschema/ShowReceiversTest.java @@ -19,6 +19,9 @@ package org.apache.iotdb.db.queryengine.plan.relational.planner.informationschema; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.ComparisonExpression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.StringLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.SymbolReference; import org.apache.iotdb.db.queryengine.plan.planner.plan.LogicalQueryPlan; import org.apache.iotdb.db.queryengine.plan.relational.planner.PlanTester; @@ -27,6 +30,7 @@ import java.util.Optional; +import static org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.ComparisonExpression.Operator.EQUAL; import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.CONNECTION_COUNT_TABLE_MODEL; import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.LAST_HANDSHAKE_TIME_TABLE_MODEL; import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.LAST_TRANSFER_TIME_TABLE_MODEL; @@ -40,6 +44,7 @@ import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.SENDER_PORTS_TABLE_MODEL; import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.USER_NAME; import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanAssert.assertPlan; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.filter; import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.infoSchemaTableScan; import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.output; @@ -82,4 +87,18 @@ public void testSelectReceivers() { infoSchemaTableScan( "information_schema.receivers", Optional.empty(), RECEIVERS_COLUMNS))); } + + @Test + public void testSelectReceiversWithProtocolFilter() { + final LogicalQueryPlan logicalQueryPlan = + planTester.createPlan("select * from information_schema.receivers where protocol='thrift'"); + assertPlan( + logicalQueryPlan, + output( + filter( + new ComparisonExpression( + EQUAL, new SymbolReference(PROTOCOL_TABLE_MODEL), new StringLiteral("thrift")), + infoSchemaTableScan( + "information_schema.receivers", Optional.empty(), RECEIVERS_COLUMNS)))); + } } diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/receiver/runtime/PipeReceiverRuntimeRegistryTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/receiver/runtime/PipeReceiverRuntimeRegistryTest.java index dd968c69bbff8..3bd346d71f13a 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/receiver/runtime/PipeReceiverRuntimeRegistryTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/receiver/runtime/PipeReceiverRuntimeRegistryTest.java @@ -388,6 +388,66 @@ public void testRegisterOrUpdateSessionReplacesPipeId() { assertTrue(snapshots.get(0).getPipeIds().contains("pipe-b@")); } + @Test + public void testMultipleConnectionsAndDuplicatePipeAggregationDoesNotInflatePipeCount() { + registerDataSession("data-1", 1, "10.0.0.1", 9001, "root", "cluster-a", "pipe-a", 1, 100); + registerDataSession("data-2", 1, "10.0.0.1", 9002, "root", "cluster-a", "pipe-a", 1, 200); + registerDataSession("data-3", 1, "10.0.0.1", -1, "root", "cluster-a", "pipe-b", 2, 150); + + registry.markTransfer("data-1", 300); + registry.markTransfer("data-2", 250); + registry.markTransfer("data-1", 50); + + final List snapshots = registry.snapshot(); + + assertEquals(1, snapshots.size()); + final PipeReceiverRuntimeSnapshot snapshot = snapshots.get(0); + assertEquals(3, snapshot.getConnectionCount()); + assertEquals("Unknown,9001,9002", snapshot.getSenderPorts()); + assertEquals(2, snapshot.getPipeCount()); + assertTrue(snapshot.getPipeIds().contains("pipe-a@")); + assertTrue(snapshot.getPipeIds().contains("pipe-b@")); + assertEquals(200, snapshot.getLastHandshakeTime()); + assertEquals(300, snapshot.getLastTransferTime()); + } + + @Test + public void testBlankAndMalformedRuntimeFieldsFallbackToUnknown() { + registry.registerOrUpdateSession( + " ", + PipeReceiverRuntimeRegistry.NODE_TYPE_DATA_NODE, + 1, + PipeReceiverRuntimeRegistry.PROTOCOL_THRIFT, + "10.0.0.1", + 9001, + "root", + "cluster-a", + "pipe-a", + 1, + 100); + assertTrue(registry.snapshot().isEmpty()); + + registry.registerOrUpdateSession( + "data-blank", " ", -1, "\t", "", -1, "\n", " ", "pipe-legacy", Long.MIN_VALUE, 0); + + final List snapshots = registry.snapshot(); + + assertEquals(1, snapshots.size()); + final PipeReceiverRuntimeSnapshot snapshot = snapshots.get(0); + assertEquals(PipeReceiverRuntimeRegistry.UNKNOWN, snapshot.getReceiverNodeType()); + assertEquals(-1, snapshot.getReceiverNodeId()); + assertFalse(snapshot.isReceiverNodeIdKnown()); + assertEquals(PipeReceiverRuntimeRegistry.UNKNOWN, snapshot.getProtocol()); + assertEquals(PipeReceiverRuntimeRegistry.UNKNOWN, snapshot.getSenderAddress()); + assertEquals(PipeReceiverRuntimeRegistry.UNKNOWN, snapshot.getSenderPorts()); + assertEquals(PipeReceiverRuntimeRegistry.UNKNOWN, snapshot.getUserName()); + assertEquals(PipeReceiverRuntimeRegistry.UNKNOWN, snapshot.getSenderClusterId()); + assertEquals(1, snapshot.getPipeCount()); + assertEquals("pipe-legacy@Unknown", snapshot.getPipeIds()); + assertEquals(0, snapshot.getLastHandshakeTime()); + assertEquals(0, snapshot.getLastTransferTime()); + } + @Test public void testRegisterOrUpdateSessionClearsPipeId() { registry.registerOrUpdateSession( From 00a52f82a150f0232c6dd2de1fb29ac17c48e9a6 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 11 Jun 2026 14:45:25 +0800 Subject: [PATCH 7/7] Add show receivers schema filter tests --- .../pipe/it/single/IoTDBShowReceiversIT.java | 13 ++++++ ...nformationSchemaReceiversSupplierTest.java | 42 +++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBShowReceiversIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBShowReceiversIT.java index 1081ec37297fa..61326210fce1c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBShowReceiversIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBShowReceiversIT.java @@ -57,6 +57,19 @@ public void testShowReceiversInTreeAndTableModel() { assertShowReceivers("select * from information_schema.receivers", BaseEnv.TABLE_SQL_DIALECT); } + @Test + public void testInformationSchemaReceiversWithProtocolFilter() { + createWriteBackPipe("root.show_receivers_filter", "show_receivers_filter_pipe"); + + assertShowReceivers( + "select receiver_node_type, receiver_node_id, protocol, sender_address, sender_ports, " + + "connection_count, pipe_count, pipe_ids, user_name, sender_cluster_id, " + + "last_handshake_time, last_transfer_time " + + "from information_schema.receivers where protocol = 'thrift'", + BaseEnv.TABLE_SQL_DIALECT, + "show_receivers_filter_pipe"); + } + @Test public void testShowReceiversWithStoppedDataNode() throws Exception { Assert.assertTrue(env.getDataNodeWrapperList().size() >= 3); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaReceiversSupplierTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaReceiversSupplierTest.java index 449cb18cf6780..e767798650c6f 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaReceiversSupplierTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaReceiversSupplierTest.java @@ -29,6 +29,7 @@ import org.apache.iotdb.commons.queryengine.plan.relational.planner.Symbol; import org.apache.iotdb.commons.schema.table.InformationSchema; import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.db.auth.AuthorityChecker; import org.apache.iotdb.db.queryengine.execution.operator.source.ShowReceiversOperator; @@ -66,6 +67,37 @@ public void tearDown() { AuthorityChecker.getAuthorityFetcher().getAuthorCache().invalidAllCache(); } + @Test + public void testReceiversInformationSchemaTableDefinition() { + final List columns = receiversTable().getColumnList(); + + assertEquals(12, columns.size()); + assertColumn( + columns.get(0), "receiver_node_type", TSDataType.STRING, TsTableColumnCategory.TAG); + assertColumn(columns.get(1), "receiver_node_id", TSDataType.INT32, TsTableColumnCategory.TAG); + assertColumn(columns.get(2), "protocol", TSDataType.STRING, TsTableColumnCategory.TAG); + assertColumn(columns.get(3), "sender_address", TSDataType.STRING, TsTableColumnCategory.TAG); + assertColumn( + columns.get(4), "sender_ports", TSDataType.STRING, TsTableColumnCategory.ATTRIBUTE); + assertColumn( + columns.get(5), "connection_count", TSDataType.INT32, TsTableColumnCategory.ATTRIBUTE); + assertColumn(columns.get(6), "pipe_count", TSDataType.INT32, TsTableColumnCategory.ATTRIBUTE); + assertColumn(columns.get(7), "pipe_ids", TSDataType.STRING, TsTableColumnCategory.ATTRIBUTE); + assertColumn(columns.get(8), "user_name", TSDataType.STRING, TsTableColumnCategory.TAG); + assertColumn( + columns.get(9), "sender_cluster_id", TSDataType.STRING, TsTableColumnCategory.ATTRIBUTE); + assertColumn( + columns.get(10), + "last_handshake_time", + TSDataType.TIMESTAMP, + TsTableColumnCategory.ATTRIBUTE); + assertColumn( + columns.get(11), + "last_transfer_time", + TSDataType.TIMESTAMP, + TsTableColumnCategory.ATTRIBUTE); + } + @Test public void testReceiversSupplierBuildsTableModelRows() { registry.registerOrUpdateSession( @@ -278,6 +310,16 @@ private static TsTable receiversTable() { return InformationSchema.getSchemaTables().get(InformationSchema.RECEIVERS); } + private static void assertColumn( + final TsTableColumnSchema column, + final String columnName, + final TSDataType dataType, + final TsTableColumnCategory columnCategory) { + assertEquals(columnName, column.getColumnName()); + assertEquals(dataType, column.getDataType()); + assertEquals(columnCategory, column.getColumnCategory()); + } + private void registerUserSession( String connectionKey, String senderAddress,