From b75f6dc30e26204e30f334d0eeb8f6616f2f93a1 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Tue, 26 May 2026 15:14:59 -0400 Subject: [PATCH 1/2] HDDS-15195. Rename FUTURE_VERSION to UNKNOWN_VERSION and version terminology Co-authored-by: Cursor --- .../org/apache/hadoop/hdds/HDDSVersion.java | 8 ++--- .../apache/hadoop/ozone/ClientVersion.java | 2 +- .../hadoop/ozone/OzoneManagerVersion.java | 8 ++--- .../apache/hadoop/hdds/TestHDDSVersion.java | 2 +- .../hadoop/hdds/TestOzoneManagerVersion.java | 2 +- .../container/common/DatanodeStorage.java | 15 +++++---- .../endpoint/HeartbeatEndpointTask.java | 11 +++---- .../ozone/container/upgrade/UpgradeUtils.java | 17 +++++++--- ...atanodeProtocolServerSideTranslatorPB.java | 11 +++---- .../upgrade/TestDatanodeVersionManager.java | 2 +- .../hdds/upgrade/HDDSVersionManager.java | 2 +- .../hdds/upgrade/TestHDDSLayoutFeature.java | 2 +- .../hadoop/hdds/scm/node/DatanodeInfo.java | 18 +++++------ .../hadoop/hdds/scm/node/NodeManager.java | 10 +++--- .../hdds/scm/node/NodeStateManager.java | 11 +++---- .../hadoop/hdds/scm/node/SCMNodeManager.java | 30 +++++++++--------- .../SCMDatanodeHeartbeatDispatcher.java | 11 +++---- .../hdds/scm/container/MockNodeManager.java | 10 +++--- .../scm/container/SimpleMockNodeManager.java | 8 +++-- .../TestContainerPlacementFactory.java | 2 +- .../TestSCMContainerPlacementCapacity.java | 2 +- .../TestSCMContainerPlacementRackAware.java | 4 +-- .../TestSCMContainerPlacementRackScatter.java | 4 +-- .../TestSCMContainerPlacementRandom.java | 4 +-- .../hdds/scm/node/TestNodeStateManager.java | 31 ++++++++++--------- .../hdds/scm/node/TestSCMNodeManager.java | 22 ++++++------- .../scm/node/states/TestNodeStateMap.java | 4 ++- .../TestPipelinePlacementFactory.java | 2 +- .../ozone/container/common/TestEndPoint.java | 4 +-- .../hadoop/ozone/client/rpc/RpcClient.java | 2 +- .../ozone/client/rpc/TestRpcClient.java | 28 ++++++++--------- .../ozone/om/upgrade/OMVersionManager.java | 2 +- .../ozone/om/upgrade/TestOMLayoutFeature.java | 2 +- .../om/upgrade/TestOMVersionManager.java | 2 +- .../hadoop/ozone/recon/api/TestEndpoints.java | 10 +++--- .../recon/api/TestOpenContainerCount.java | 6 ++-- .../api/TestStorageDistributionEndpoint.java | 3 +- .../ozone/freon/SCMThroughputBenchmark.java | 4 +-- 38 files changed, 164 insertions(+), 154 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HDDSVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HDDSVersion.java index 51d4229a7484..345f76a7aa65 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HDDSVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HDDSVersion.java @@ -41,8 +41,8 @@ public enum HDDSVersion implements ComponentVersion { ZDU(100, "Version that supports zero downtime upgrade"), - FUTURE_VERSION(-1, "Used internally in the client when the server side is " - + " newer and an unknown server version has arrived to the client."); + UNKNOWN_VERSION(-1, "Used when a version cannot be deserialized to any version recognized by this" + + " component, which may indicate it came from a component in a newer version"); ////////////////////////////// ////////////////////////////// @@ -85,11 +85,11 @@ public int serialize() { /** * @param value The serialized version to convert. - * @return The version corresponding to this serialized value, or {@link #FUTURE_VERSION} if no matching version is + * @return The version corresponding to this serialized value, or {@link #UNKNOWN_VERSION} if no matching version is * found. */ public static HDDSVersion deserialize(int value) { - return BY_VALUE.getOrDefault(value, FUTURE_VERSION); + return BY_VALUE.getOrDefault(value, UNKNOWN_VERSION); } @Override diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java index dd451ab052cb..36dbbc74e0d8 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java @@ -42,7 +42,7 @@ public enum ClientVersion implements ComponentVersion { "This client version has support for Object Store and File " + "System Optimized Bucket Layouts."), - FUTURE_VERSION(-1, "Used internally when the server side is older and an" + FUTURE_VERSION(-1, "Used internally by the server when the server side is older and an" + " unknown client version has arrived from the client."); private static final SortedMap BY_VALUE = diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java index 8e91d72d7981..55ec6b77f7a0 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java @@ -64,8 +64,8 @@ public enum OzoneManagerVersion implements ComponentVersion { ZDU(100, "OzoneManager version that supports zero downtime upgrade"), - FUTURE_VERSION(-1, "Used internally in the client when the server side is " - + " newer and an unknown server version has arrived to the client."); + UNKNOWN_VERSION(-1, "Used when a version cannot be deserialized to any version recognized by this" + + " component, which may indicate it came from a component in a newer version"); private static final SortedMap BY_VALUE = Arrays.stream(values()) @@ -93,11 +93,11 @@ public int serialize() { /** * @param value The serialized version to convert. - * @return The version corresponding to this serialized value, or {@link #FUTURE_VERSION} if no matching version is + * @return The version corresponding to this serialized value, or {@link #UNKNOWN_VERSION} if no matching version is * found. */ public static OzoneManagerVersion deserialize(int value) { - return BY_VALUE.getOrDefault(value, FUTURE_VERSION); + return BY_VALUE.getOrDefault(value, UNKNOWN_VERSION); } diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHDDSVersion.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHDDSVersion.java index 89d24d3c82ea..d5bb06571be2 100644 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHDDSVersion.java +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHDDSVersion.java @@ -38,7 +38,7 @@ protected ComponentVersion getDefaultVersion() { @Override protected ComponentVersion getFutureVersion() { - return HDDSVersion.FUTURE_VERSION; + return HDDSVersion.UNKNOWN_VERSION; } @Override diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestOzoneManagerVersion.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestOzoneManagerVersion.java index bb94ac938fde..cb5bd94198c9 100644 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestOzoneManagerVersion.java +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestOzoneManagerVersion.java @@ -39,7 +39,7 @@ protected ComponentVersion getDefaultVersion() { @Override protected ComponentVersion getFutureVersion() { - return OzoneManagerVersion.FUTURE_VERSION; + return OzoneManagerVersion.UNKNOWN_VERSION; } @Override diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DatanodeStorage.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DatanodeStorage.java index 1b1be8df8b60..fe2677834378 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DatanodeStorage.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DatanodeStorage.java @@ -17,12 +17,12 @@ package org.apache.hadoop.ozone.container.common; -import static org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager.maxLayoutVersion; import static org.apache.hadoop.ozone.OzoneConsts.DATANODE_LAYOUT_VERSION_DIR; import java.io.File; import java.io.IOException; import java.util.Properties; +import org.apache.hadoop.hdds.HDDSVersion; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType; @@ -43,7 +43,7 @@ public class DatanodeStorage extends Storage { public DatanodeStorage(ConfigurationSource conf, String dataNodeId) throws IOException { super(NodeType.DATANODE, ServerUtils.getOzoneMetaDirPath(conf), - DATANODE_LAYOUT_VERSION_DIR, dataNodeId, getDefaultLayoutVersion(conf)); + DATANODE_LAYOUT_VERSION_DIR, dataNodeId, getDefaultApparentVersion(conf)); } public DatanodeStorage(OzoneConfiguration conf, String dataNodeId, @@ -56,7 +56,7 @@ public DatanodeStorage(OzoneConfiguration conf, String dataNodeId, public DatanodeStorage(ConfigurationSource conf) throws IOException { super(NodeType.DATANODE, ServerUtils.getOzoneMetaDirPath(conf), - DATANODE_LAYOUT_VERSION_DIR, getDefaultLayoutVersion(conf)); + DATANODE_LAYOUT_VERSION_DIR, getDefaultApparentVersion(conf)); } @Override @@ -94,15 +94,14 @@ public void setClusterId(String clusterId) throws IOException { * @return The layout version that should be used for the datanode if no * layout version is found on disk. */ - private static int getDefaultLayoutVersion(ConfigurationSource conf) { - int defaultLayoutVersion = maxLayoutVersion(); + private static int getDefaultApparentVersion(ConfigurationSource conf) { + int defaultApparentVersion = HDDSVersion.SOFTWARE_VERSION.serialize(); File dnIdFile = new File(HddsServerUtil.getDatanodeIdFilePath(conf)); if (dnIdFile.exists()) { - defaultLayoutVersion = - HDDSLayoutFeature.INITIAL_VERSION.layoutVersion(); + defaultApparentVersion = HDDSLayoutFeature.INITIAL_VERSION.layoutVersion(); } - return defaultLayoutVersion; + return defaultApparentVersion; } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java index c40fd317360d..e35192daa098 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java @@ -21,8 +21,7 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_ACTION_MAX_LIMIT_DEFAULT; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PIPELINE_ACTION_MAX_LIMIT; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PIPELINE_ACTION_MAX_LIMIT_DEFAULT; -import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type.finalizeNewLayoutVersionCommand; -import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.toLayoutVersionProto; +import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.toVersionProto; import com.google.common.base.Preconditions; import com.google.protobuf.Descriptors; @@ -129,13 +128,13 @@ public EndpointStateMachine.EndPointStates call() throws Exception { try { Preconditions.checkState(this.datanodeDetailsProto != null); - LayoutVersionProto layoutinfo = toLayoutVersionProto( - versionManager.getApparentVersion().serialize(), - versionManager.getSoftwareVersion().serialize()); + LayoutVersionProto versionInfo = toVersionProto( + versionManager.getApparentVersion(), + versionManager.getSoftwareVersion()); requestBuilder = SCMHeartbeatRequestProto.newBuilder() .setDatanodeDetails(datanodeDetailsProto) - .setDataNodeLayoutVersion(layoutinfo); + .setDataNodeLayoutVersion(versionInfo); addReports(requestBuilder); addContainerActions(requestBuilder); addPipelineActions(requestBuilder); diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/UpgradeUtils.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/UpgradeUtils.java index d96aab53b5e4..4b2cb3f8c9c9 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/UpgradeUtils.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/UpgradeUtils.java @@ -17,6 +17,7 @@ package org.apache.hadoop.ozone.container.upgrade; +import org.apache.hadoop.hdds.ComponentVersion; import org.apache.hadoop.hdds.HDDSVersion; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto; @@ -28,16 +29,24 @@ public final class UpgradeUtils { private UpgradeUtils() { } - public static LayoutVersionProto defaultLayoutVersionProto() { + public static LayoutVersionProto defaultVersionProto() { int softwareVersion = HDDSVersion.SOFTWARE_VERSION.serialize(); return LayoutVersionProto.newBuilder() .setMetadataLayoutVersion(softwareVersion) .setSoftwareLayoutVersion(softwareVersion).build(); } - public static LayoutVersionProto toLayoutVersionProto(int mLv, int sLv) { + public static LayoutVersionProto toVersionProto(ComponentVersion apparentVersion, ComponentVersion softwareVersion) { return LayoutVersionProto.newBuilder() - .setMetadataLayoutVersion(mLv) - .setSoftwareLayoutVersion(sLv).build(); + .setMetadataLayoutVersion(apparentVersion.serialize()) + .setSoftwareLayoutVersion(softwareVersion.serialize()) + .build(); + } + + public static LayoutVersionProto toVersionProto(int metadataLayoutVersion, int softwareLayoutVersion) { + return LayoutVersionProto.newBuilder() + .setMetadataLayoutVersion(metadataLayoutVersion) + .setSoftwareLayoutVersion(softwareLayoutVersion) + .build(); } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java index 633500f9431c..3f91c277bf79 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java @@ -18,7 +18,7 @@ package org.apache.hadoop.ozone.protocolPB; import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.INITIAL_VERSION; -import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.toLayoutVersionProto; +import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.toVersionProto; import com.google.protobuf.RpcController; import com.google.protobuf.ServiceException; @@ -71,17 +71,16 @@ public SCMRegisteredResponseProto register( .getContainerReport(); NodeReportProto dnNodeReport = request.getNodeReport(); PipelineReportsProto pipelineReport = request.getPipelineReports(); - LayoutVersionProto layoutInfo = null; + LayoutVersionProto versionInfo = null; if (request.hasDataNodeLayoutVersion()) { - layoutInfo = request.getDataNodeLayoutVersion(); + versionInfo = request.getDataNodeLayoutVersion(); } else { // Backward compatibility to make sure old Datanodes can still talk to // SCM. - layoutInfo = toLayoutVersionProto(INITIAL_VERSION.layoutVersion(), - INITIAL_VERSION.layoutVersion()); + versionInfo = toVersionProto(INITIAL_VERSION, INITIAL_VERSION); } return impl.register(request.getExtendedDatanodeDetails(), dnNodeReport, - containerRequestProto, pipelineReport, layoutInfo); + containerRequestProto, pipelineReport, versionInfo); } @Override diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeVersionManager.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeVersionManager.java index 93d33ca9aea2..840fcf2aee0f 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeVersionManager.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeVersionManager.java @@ -84,7 +84,7 @@ class TestDatanodeVersionManager extends AbstractComponentVersionManagerTest { for (HDDSVersion version : HDDSVersion.values()) { // Add all defined versions after and including ZDU to get the complete version list. - if (HDDSVersion.ZDU.isSupportedBy(version) && version != HDDSVersion.FUTURE_VERSION) { + if (HDDSVersion.ZDU.isSupportedBy(version) && version != HDDSVersion.UNKNOWN_VERSION) { ALL_VERSIONS.add(version); } } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSVersionManager.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSVersionManager.java index 9d01346bcd35..d9dcb36e6702 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSVersionManager.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSVersionManager.java @@ -43,7 +43,7 @@ protected HDDSVersionManager(Storage storage) throws IOException { private static ComponentVersion computeApparentVersion(int serializedApparentVersion) throws IOException { if (serializedApparentVersion >= HDDSVersion.ZDU.serialize()) { HDDSVersion fromHdds = HDDSVersion.deserialize(serializedApparentVersion); - if (fromHdds != HDDSVersion.FUTURE_VERSION) { + if (fromHdds != HDDSVersion.UNKNOWN_VERSION) { return fromHdds; } } else { diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSLayoutFeature.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSLayoutFeature.java index fbb89220e976..e1853c0b4bbc 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSLayoutFeature.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSLayoutFeature.java @@ -101,7 +101,7 @@ public void testIsSupportedByFeatureBoundary() { public void testAllLayoutFeaturesAreSupportedByFutureVersions() { for (HDDSLayoutFeature feature : HDDSLayoutFeature.values()) { assertSupportedBy(feature, HDDSVersion.ZDU); - assertSupportedBy(feature, HDDSVersion.FUTURE_VERSION); + assertSupportedBy(feature, HDDSVersion.UNKNOWN_VERSION); // No ComponentVersion instance represents an arbitrary future version. assertTrue(feature.isSupportedBy(Integer.MAX_VALUE)); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeInfo.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeInfo.java index cb4cb1db6967..59f0ec22e524 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeInfo.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeInfo.java @@ -17,7 +17,7 @@ package org.apache.hadoop.hdds.scm.node; -import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.toLayoutVersionProto; +import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.toVersionProto; import com.google.common.annotations.VisibleForTesting; import java.util.Collections; @@ -67,16 +67,16 @@ public class DatanodeInfo extends DatanodeDetails { * Constructs DatanodeInfo from DatanodeDetails. * * @param datanodeDetails Details about the datanode - * @param layoutInfo Details about the LayoutVersionProto + * @param versionInfo Details about the LayoutVersionProto */ public DatanodeInfo(DatanodeDetails datanodeDetails, NodeStatus nodeStatus, - LayoutVersionProto layoutInfo, long containerRollIntervalMs) { + LayoutVersionProto versionInfo, long containerRollIntervalMs) { super(datanodeDetails); this.lock = new ReentrantReadWriteLock(); this.lastHeartbeatTime = Time.monotonicNow(); - lastKnownLayoutVersion = toLayoutVersionProto( - layoutInfo != null ? layoutInfo.getMetadataLayoutVersion() : 0, - layoutInfo != null ? layoutInfo.getSoftwareLayoutVersion() : 0); + lastKnownLayoutVersion = toVersionProto( + versionInfo != null ? versionInfo.getMetadataLayoutVersion() : 0, + versionInfo != null ? versionInfo.getSoftwareLayoutVersion() : 0); this.storageReports = Collections.emptyList(); this.nodeStatus = nodeStatus; this.metadataStorageReports = Collections.emptyList(); @@ -108,15 +108,15 @@ public void updateLastHeartbeatTime(long milliSecondsSinceEpoch) { } /** - * Updates the last LayoutVersion. + * Updates the last known version reported by this datanode. */ - public void updateLastKnownLayoutVersion(LayoutVersionProto version) { + public void updateLastKnownVersions(LayoutVersionProto version) { if (version == null) { return; } try { lock.writeLock().lock(); - lastKnownLayoutVersion = toLayoutVersionProto( + lastKnownLayoutVersion = toVersionProto( version.getMetadataLayoutVersion(), version.getSoftwareLayoutVersion()); } finally { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java index 69b62dec4d49..b4e52a586aee 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java @@ -17,7 +17,7 @@ package org.apache.hadoop.hdds.scm.node; -import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.defaultLayoutVersionProto; +import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.defaultVersionProto; import jakarta.annotation.Nullable; import java.io.Closeable; @@ -91,7 +91,7 @@ default RegisteredCommand register( DatanodeDetails datanodeDetails, NodeReportProto nodeReport, PipelineReportsProto pipelineReportsProto) { return register(datanodeDetails, nodeReport, pipelineReportsProto, - defaultLayoutVersionProto()); + defaultVersionProto()); } /** @@ -372,8 +372,8 @@ void processNodeReport(DatanodeDetails datanodeDetails, * @param datanodeDetails * @param layoutReport */ - void processLayoutVersionReport(DatanodeDetails datanodeDetails, - LayoutVersionProto layoutReport); + void processVersionReport(DatanodeDetails datanodeDetails, + LayoutVersionProto layoutReport); /** * Get the number of commands of the given type queued on the datanode at the @@ -476,7 +476,7 @@ default Collection getPeerList(DatanodeDetails dn) { default HDDSLayoutVersionManager getLayoutVersionManager() { return null; } - + /** * This API allows removal of only DECOMMISSIONED, IN_MAINTENANCE and DEAD nodes * from NodeManager data structures and cleanup memory. diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java index eb341cb21d00..968eb75513ae 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java @@ -234,7 +234,7 @@ public void addNode(DatanodeDetails datanodeDetails, LayoutVersionProto layoutInfo) throws NodeAlreadyExistsException { nodeStateMap.addNode(newDatanodeInfo(datanodeDetails, layoutInfo)); try { - updateLastKnownLayoutVersion(datanodeDetails, layoutInfo); + updateLastKnownVersionInfo(datanodeDetails, layoutInfo); } catch (NodeNotFoundException ex) { throw new IllegalStateException("Inconsistent NodeStateMap! Datanode " + datanodeDetails.getID() + " was added but not found in map: " + nodeStateMap); @@ -318,11 +318,10 @@ public void updateLastHeartbeatTime(DatanodeDetails datanodeDetails) * * @throws NodeNotFoundException if the node is not present */ - public void updateLastKnownLayoutVersion(DatanodeDetails datanodeDetails, - LayoutVersionProto layoutInfo) - throws NodeNotFoundException { + public void updateLastKnownVersionInfo(DatanodeDetails datanodeDetails, + LayoutVersionProto layoutInfo) throws NodeNotFoundException { nodeStateMap.getNodeInfo(datanodeDetails.getID()) - .updateLastKnownLayoutVersion(layoutInfo); + .updateLastKnownVersions(layoutInfo); } /** @@ -339,7 +338,7 @@ public void updateNode(DatanodeDetails datanodeDetails, final DatanodeInfo oldInfo = nodeStateMap.updateNode(newInfo); LOG.info("Updated datanode {} {} to {} {}", oldInfo, oldInfo.getNodeStatus(), newInfo, newInfo.getNodeStatus()); - updateLastKnownLayoutVersion(datanodeDetails, layoutInfo); + updateLastKnownVersionInfo(datanodeDetails, layoutInfo); } /** diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java index 22fd52835d9e..44bdf7a16383 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java @@ -407,8 +407,8 @@ public RegisteredCommand register( public RegisteredCommand register( DatanodeDetails datanodeDetails, NodeReportProto nodeReport, PipelineReportsProto pipelineReportsProto, - LayoutVersionProto layoutInfo) { - int dnSlvRegister = layoutInfo.getSoftwareLayoutVersion(); + LayoutVersionProto dnVersionInfo) { + int dnSlvRegister = dnVersionInfo.getSoftwareLayoutVersion(); int scmSlvRegister = scmLayoutVersionManager.getSoftwareLayoutVersion(); if (shouldFenceDatanode(dnSlvRegister, scmSlvRegister)) { return RegisteredCommand.newBuilder() @@ -440,7 +440,7 @@ public RegisteredCommand register( if (!isNodeRegistered(datanodeDetails)) { try { clusterMap.add(datanodeDetails); - nodeStateManager.addNode(datanodeDetails, layoutInfo); + nodeStateManager.addNode(datanodeDetails, dnVersionInfo); // Check that datanode in nodeStateManager has topology parent set DatanodeDetails dn = nodeStateManager.getNode(datanodeDetails); Preconditions.checkState(dn.getParent() != null); @@ -466,7 +466,7 @@ public RegisteredCommand register( hostName, ipAddress, dnId)) { LOG.info("Updating datanode from {} to {}", oldNode, datanodeDetails); clusterMap.update(oldNode, datanodeDetails); - nodeStateManager.updateNode(datanodeDetails, layoutInfo); + nodeStateManager.updateNode(datanodeDetails, dnVersionInfo); DatanodeDetails dn = nodeStateManager.getNode(datanodeDetails); Preconditions.checkState(dn.getParent() != null); processNodeReport(datanodeDetails, nodeReport); @@ -476,7 +476,7 @@ public RegisteredCommand register( LOG.info("Update the version for registered datanode {}, " + "oldVersion = {}, newVersion = {}.", datanodeDetails, oldNode.getVersion(), datanodeDetails.getVersion()); - nodeStateManager.updateNode(datanodeDetails, layoutInfo); + nodeStateManager.updateNode(datanodeDetails, dnVersionInfo); } } catch (NodeNotFoundException e) { LOG.error("Cannot find datanode {} from nodeStateManager", @@ -735,11 +735,11 @@ public void processNodeReport(DatanodeDetails datanodeDetails, * Process Layout Version report. * * @param datanodeDetails - * @param layoutVersionReport + * @param versionReport */ @Override - public void processLayoutVersionReport(DatanodeDetails datanodeDetails, - LayoutVersionProto layoutVersionReport) { + public void processVersionReport(DatanodeDetails datanodeDetails, + LayoutVersionProto versionReport) { if (LOG.isDebugEnabled()) { LOG.debug("Processing Layout Version report from [datanode={}]", datanodeDetails.getHostName()); @@ -747,27 +747,27 @@ public void processLayoutVersionReport(DatanodeDetails datanodeDetails, if (LOG.isTraceEnabled()) { LOG.trace("HB is received from [datanode={}]: {}", datanodeDetails.getHostName(), - layoutVersionReport.toString().replaceAll("\n", "\\\\n")); + versionReport.toString().replaceAll("\n", "\\\\n")); } try { - nodeStateManager.updateLastKnownLayoutVersion(datanodeDetails, - layoutVersionReport); + nodeStateManager.updateLastKnownVersionInfo(datanodeDetails, + versionReport); } catch (NodeNotFoundException e) { LOG.error("SCM trying to process Layout Version from an " + "unregistered node {}.", datanodeDetails); return; } - sendFinalizeToDatanodeIfNeeded(datanodeDetails, layoutVersionReport); + sendFinalizeToDatanodeIfNeeded(datanodeDetails, versionReport); } protected void sendFinalizeToDatanodeIfNeeded(DatanodeDetails datanodeDetails, - LayoutVersionProto layoutVersionReport) { + LayoutVersionProto versionReport) { // Software layout version is hardcoded to the SCM. int scmSlv = scmLayoutVersionManager.getSoftwareLayoutVersion(); - int dnSlv = layoutVersionReport.getSoftwareLayoutVersion(); - int dnMlv = layoutVersionReport.getMetadataLayoutVersion(); + int dnSlv = versionReport.getSoftwareLayoutVersion(); + int dnMlv = versionReport.getMetadataLayoutVersion(); // A datanode with a larger software layout version is from a future // version of ozone. It should not have been added to the cluster. diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java index 1641a5353a34..efb7a5b81166 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java @@ -25,7 +25,7 @@ import static org.apache.hadoop.hdds.scm.events.SCMEvents.PIPELINE_ACTIONS; import static org.apache.hadoop.hdds.scm.events.SCMEvents.PIPELINE_REPORT; import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.INITIAL_VERSION; -import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.toLayoutVersionProto; +import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.toVersionProto; import com.google.protobuf.Message; import java.util.List; @@ -97,18 +97,17 @@ public List> dispatch(SCMHeartbeatRequestProto heartbeat) { } else { - LayoutVersionProto layoutVersion = null; + LayoutVersionProto versionInfo = null; if (!heartbeat.hasDataNodeLayoutVersion()) { // Backward compatibility to make sure old Datanodes can still talk to // SCM. - layoutVersion = toLayoutVersionProto(INITIAL_VERSION.layoutVersion(), - INITIAL_VERSION.layoutVersion()); + versionInfo = toVersionProto(INITIAL_VERSION, INITIAL_VERSION); } else { - layoutVersion = heartbeat.getDataNodeLayoutVersion(); + versionInfo = heartbeat.getDataNodeLayoutVersion(); } LOG.debug("Processing DataNode Layout Report."); - nodeManager.processLayoutVersionReport(datanodeDetails, layoutVersion); + nodeManager.processVersionReport(datanodeDetails, versionInfo); CommandQueueReportProto commandQueueReport = null; if (heartbeat.hasCommandQueueReport()) { diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java index 57d38ece3dd6..afc564e6edde 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java @@ -268,7 +268,7 @@ public List getNodes( List healthyNodesWithInfo = new ArrayList<>(); for (DatanodeDetails dd : healthyNodes) { DatanodeInfo di = new DatanodeInfo(dd, NodeStatus.inServiceHealthy(), - UpgradeUtils.defaultLayoutVersionProto(), HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT); + UpgradeUtils.defaultVersionProto(), HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT); long capacity = nodeMetricMap.get(dd).getCapacity().get(); long used = nodeMetricMap.get(dd).getScmUsed().get(); @@ -347,7 +347,7 @@ public List getAllNodes() { nodeStatus = NodeStatus.inServiceDead(); } DatanodeInfo di = new DatanodeInfo(entry.getKey(), nodeStatus, - UpgradeUtils.defaultLayoutVersionProto(), HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT); + UpgradeUtils.defaultVersionProto(), HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT); long capacity = entry.getValue().getCapacity().get(); long used = entry.getValue().getScmUsed().get(); @@ -436,7 +436,7 @@ public DatanodeInfo getDatanodeInfo(DatanodeDetails dd) { } DatanodeInfo di = new DatanodeInfo(dd, NodeStatus.inServiceHealthy(), - UpgradeUtils.defaultLayoutVersionProto(), HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT); + UpgradeUtils.defaultVersionProto(), HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT); long capacity = nodeMetricMap.get(dd).getCapacity().get(); long used = nodeMetricMap.get(dd).getScmUsed().get(); long remaining = nodeMetricMap.get(dd).getRemaining().get(); @@ -626,8 +626,8 @@ public void processNodeReport(DatanodeDetails dnUuid, * @param layoutReport */ @Override - public void processLayoutVersionReport(DatanodeDetails dnUuid, - LayoutVersionProto layoutReport) { + public void processVersionReport(DatanodeDetails dnUuid, + LayoutVersionProto layoutReport) { // do nothing } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java index f2da8fd2878b..7d386913c422 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java @@ -47,6 +47,7 @@ import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineID; import org.apache.hadoop.hdds.server.events.EventPublisher; +import org.apache.hadoop.ozone.container.upgrade.UpgradeUtils; import org.apache.hadoop.ozone.protocol.VersionResponse; import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode; import org.apache.hadoop.ozone.protocol.commands.RegisteredCommand; @@ -67,7 +68,8 @@ public class SimpleMockNodeManager implements NodeManager { public void register(DatanodeDetails dd, NodeStatus status) { dd.setPersistedOpState(status.getOperationalState()); dd.setPersistedOpStateExpiryEpochSec(status.getOpStateExpiryEpochSeconds()); - nodeMap.put(dd.getID(), new DatanodeInfo(dd, status, null, HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT)); + nodeMap.put(dd.getID(), new DatanodeInfo(dd, status, UpgradeUtils.defaultVersionProto(), + HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT)); } public void setNodeStatus(DatanodeDetails dd, NodeStatus status) { @@ -301,8 +303,8 @@ public void processNodeReport(DatanodeDetails datanodeDetails, } @Override - public void processLayoutVersionReport(DatanodeDetails datanodeDetails, - LayoutVersionProto layoutReport) { + public void processVersionReport(DatanodeDetails datanodeDetails, + LayoutVersionProto layoutReport) { } /** diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java index ed4e96b8de56..863da78884d1 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java @@ -99,7 +99,7 @@ public void testRackAwarePolicy() throws IOException { .createDatanodeDetails(hostname + i, rack + (i / 5)); DatanodeInfo datanodeInfo = new DatanodeInfo( datanodeDetails, NodeStatus.inServiceHealthy(), - UpgradeUtils.defaultLayoutVersionProto(), HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT); + UpgradeUtils.defaultVersionProto(), HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT); StorageReportProto storage1 = HddsTestUtils.createStorageReport( datanodeInfo.getID(), "/data1-" + datanodeInfo.getID(), diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementCapacity.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementCapacity.java index 1fb3f53504d3..44d37f359c04 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementCapacity.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementCapacity.java @@ -64,7 +64,7 @@ public void chooseDatanodes() throws SCMException { DatanodeInfo datanodeInfo = new DatanodeInfo( MockDatanodeDetails.randomDatanodeDetails(), NodeStatus.inServiceHealthy(), - UpgradeUtils.defaultLayoutVersionProto(), + UpgradeUtils.defaultVersionProto(), HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT); StorageReportProto storage1 = HddsTestUtils.createStorageReport( diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java index beb6ad82877c..549a3b7ad35e 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java @@ -112,7 +112,7 @@ private void setup(int datanodeCount) { cluster.add(datanodeDetails); DatanodeInfo datanodeInfo = new DatanodeInfo( datanodeDetails, NodeStatus.inServiceHealthy(), - UpgradeUtils.defaultLayoutVersionProto(), + UpgradeUtils.defaultVersionProto(), HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT); StorageReportProto storage1 = HddsTestUtils.createStorageReport( @@ -456,7 +456,7 @@ public void testDatanodeWithDefaultNetworkLocation(int datanodeCount) hostname + i, null); DatanodeInfo dnInfo = new DatanodeInfo( dn, NodeStatus.inServiceHealthy(), - UpgradeUtils.defaultLayoutVersionProto(), + UpgradeUtils.defaultVersionProto(), HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT); StorageReportProto storage1 = HddsTestUtils.createStorageReport( diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackScatter.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackScatter.java index e015b93c1e31..6403fc7c56b3 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackScatter.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackScatter.java @@ -180,7 +180,7 @@ private void setupDatanode(DatanodeDetails datanodeDetails) { cluster.add(datanodeDetails); DatanodeInfo datanodeInfo = new DatanodeInfo( datanodeDetails, NodeStatus.inServiceHealthy(), - UpgradeUtils.defaultLayoutVersionProto(), + UpgradeUtils.defaultVersionProto(), HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT); StorageReportProto storage1 = HddsTestUtils.createStorageReport( @@ -491,7 +491,7 @@ public void testDatanodeWithDefaultNetworkLocation(int datanodeCount) hostname + i, null); DatanodeInfo dnInfo = new DatanodeInfo( dn, NodeStatus.inServiceHealthy(), - UpgradeUtils.defaultLayoutVersionProto(), + UpgradeUtils.defaultVersionProto(), HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT); StorageReportProto storage1 = HddsTestUtils.createStorageReport( diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRandom.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRandom.java index 47602a385fd6..ed1b349d5ecb 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRandom.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRandom.java @@ -61,7 +61,7 @@ public void chooseDatanodes() throws SCMException { DatanodeInfo datanodeInfo = new DatanodeInfo( MockDatanodeDetails.randomDatanodeDetails(), NodeStatus.inServiceHealthy(), - UpgradeUtils.defaultLayoutVersionProto(), + UpgradeUtils.defaultVersionProto(), HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT); StorageReportProto storage1 = HddsTestUtils.createStorageReport( @@ -168,7 +168,7 @@ public void testIsValidNode() throws SCMException { DatanodeInfo datanodeInfo = new DatanodeInfo( MockDatanodeDetails.randomDatanodeDetails(), NodeStatus.inServiceHealthy(), - UpgradeUtils.defaultLayoutVersionProto(), + UpgradeUtils.defaultVersionProto(), HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT); StorageReportProto storage1 = HddsTestUtils.createStorageReport( diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeStateManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeStateManager.java index db0425fed745..de49a6bc7680 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeStateManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeStateManager.java @@ -17,7 +17,7 @@ package org.apache.hadoop.hdds.scm.node; -import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.defaultLayoutVersionProto; +import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.defaultVersionProto; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; @@ -40,6 +40,7 @@ import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; import org.apache.hadoop.hdds.server.events.Event; import org.apache.hadoop.hdds.server.events.EventPublisher; +import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature; import org.apache.hadoop.hdds.utils.HddsServerUtil; import org.apache.hadoop.ozone.container.upgrade.UpgradeUtils; import org.apache.hadoop.util.Time; @@ -86,7 +87,7 @@ public void testNodeCanBeAddedAndRetrieved() throws NodeAlreadyExistsException, NodeNotFoundException { // Create a datanode, then add and retrieve it DatanodeDetails dn = generateDatanode(); - nsm.addNode(dn, UpgradeUtils.defaultLayoutVersionProto()); + nsm.addNode(dn, UpgradeUtils.defaultVersionProto()); assertEquals(dn.getUuid(), nsm.getNode(dn).getUuid()); // Now get the status of the newly added node and it should be // IN_SERVICE and HEALTHY @@ -98,9 +99,9 @@ public void testNodeCanBeAddedAndRetrieved() public void testGetAllNodesReturnsCorrectly() throws NodeAlreadyExistsException { DatanodeDetails dn = generateDatanode(); - nsm.addNode(dn, UpgradeUtils.defaultLayoutVersionProto()); + nsm.addNode(dn, UpgradeUtils.defaultVersionProto()); dn = generateDatanode(); - nsm.addNode(dn, UpgradeUtils.defaultLayoutVersionProto()); + nsm.addNode(dn, UpgradeUtils.defaultVersionProto()); assertEquals(2, nsm.getAllNodes().size()); assertEquals(2, nsm.getTotalNodeCount()); } @@ -109,7 +110,7 @@ public void testGetAllNodesReturnsCorrectly() public void testGetNodeCountReturnsCorrectly() throws NodeAlreadyExistsException { DatanodeDetails dn = generateDatanode(); - nsm.addNode(dn, UpgradeUtils.defaultLayoutVersionProto()); + nsm.addNode(dn, UpgradeUtils.defaultVersionProto()); assertEquals(1, nsm.getNodes(NodeStatus.inServiceHealthy()).size()); assertEquals(0, nsm.getNodes(NodeStatus.inServiceStale()).size()); } @@ -117,7 +118,7 @@ public void testGetNodeCountReturnsCorrectly() @Test public void testGetNodeCount() throws NodeAlreadyExistsException { DatanodeDetails dn = generateDatanode(); - nsm.addNode(dn, UpgradeUtils.defaultLayoutVersionProto()); + nsm.addNode(dn, UpgradeUtils.defaultVersionProto()); assertEquals(1, nsm.getNodeCount(NodeStatus.inServiceHealthy())); assertEquals(0, nsm.getNodeCount(NodeStatus.inServiceStale())); } @@ -132,15 +133,15 @@ public void testNodesMarkedDeadAndStale() long deadLimit = HddsServerUtil.getDeadNodeInterval(conf) + 1000; DatanodeDetails staleDn = generateDatanode(); - nsm.addNode(staleDn, defaultLayoutVersionProto()); + nsm.addNode(staleDn, defaultVersionProto()); nsm.getNode(staleDn).updateLastHeartbeatTime(now - staleLimit); DatanodeDetails deadDn = generateDatanode(); - nsm.addNode(deadDn, defaultLayoutVersionProto()); + nsm.addNode(deadDn, defaultVersionProto()); nsm.getNode(deadDn).updateLastHeartbeatTime(now - deadLimit); DatanodeDetails healthyDn = generateDatanode(); - nsm.addNode(healthyDn, defaultLayoutVersionProto()); + nsm.addNode(healthyDn, defaultVersionProto()); nsm.getNode(healthyDn).updateLastHeartbeatTime(); nsm.checkNodesHealth(); @@ -165,7 +166,7 @@ public void testNodeCanTransitionThroughHealthStatesAndFiresEvents() long deadLimit = HddsServerUtil.getDeadNodeInterval(conf) + 1000; DatanodeDetails dn = generateDatanode(); - nsm.addNode(dn, defaultLayoutVersionProto()); + nsm.addNode(dn, defaultVersionProto()); DatanodeInfo dni = nsm.getNode(dn); dni.updateLastHeartbeatTime(); @@ -209,7 +210,7 @@ public void testNodeCanTransitionThroughHealthStatesAndFiresEvents() public void testNodeOpStateCanBeSet() throws NodeAlreadyExistsException, NodeNotFoundException { DatanodeDetails dn = generateDatanode(); - nsm.addNode(dn, UpgradeUtils.defaultLayoutVersionProto()); + nsm.addNode(dn, UpgradeUtils.defaultVersionProto()); nsm.setNodeOperationalState(dn, HddsProtos.NodeOperationalState.DECOMMISSIONED); @@ -223,7 +224,7 @@ public void testNodeOpStateCanBeSet() public void testContainerCanBeAddedAndRemovedFromDN() throws NodeAlreadyExistsException, NodeNotFoundException { DatanodeDetails dn = generateDatanode(); - nsm.addNode(dn, UpgradeUtils.defaultLayoutVersionProto()); + nsm.addNode(dn, UpgradeUtils.defaultVersionProto()); nsm.addContainer(dn.getID(), ContainerID.valueOf(1)); nsm.addContainer(dn.getID(), ContainerID.valueOf(2)); @@ -244,7 +245,7 @@ public void testContainerCanBeAddedAndRemovedFromDN() public void testHealthEventsFiredWhenOpStateChanged() throws NodeAlreadyExistsException, NodeNotFoundException { DatanodeDetails dn = generateDatanode(); - nsm.addNode(dn, UpgradeUtils.defaultLayoutVersionProto()); + nsm.addNode(dn, UpgradeUtils.defaultVersionProto()); // First set the node to decommissioned, then run through all op states in // order and ensure the unhealthy_to_healthy event gets fired @@ -297,7 +298,7 @@ public void testUpdateNode() throws NodeAlreadyExistsException, String hostName = "test-host"; StorageContainerDatanodeProtocolProtos.LayoutVersionProto layoutVersionProto = - UpgradeUtils.toLayoutVersionProto(1, 2); + UpgradeUtils.toVersionProto(HDDSLayoutFeature.INITIAL_VERSION, HDDSLayoutFeature.INITIAL_VERSION); DatanodeDetails dn = DatanodeDetails.newBuilder() .setUuid(dnUuid) .setIpAddress(ipAddress) @@ -309,7 +310,7 @@ public void testUpdateNode() throws NodeAlreadyExistsException, String newIpAddress = "2.3.4.5"; String newHostName = "new-host"; StorageContainerDatanodeProtocolProtos.LayoutVersionProto - newLayoutVersionProto = UpgradeUtils.defaultLayoutVersionProto(); + newLayoutVersionProto = UpgradeUtils.defaultVersionProto(); DatanodeDetails newDn = DatanodeDetails.newBuilder() .setUuid(dnUuid) .setIpAddress(newIpAddress) diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java index 543b97530cd0..7e29939cd1e2 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java @@ -38,7 +38,7 @@ import static org.apache.hadoop.hdds.scm.events.SCMEvents.DATANODE_COMMAND; import static org.apache.hadoop.hdds.scm.events.SCMEvents.DATANODE_COMMAND_COUNT_UPDATED; import static org.apache.hadoop.hdds.scm.events.SCMEvents.NEW_NODE; -import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.toLayoutVersionProto; +import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.toVersionProto; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; @@ -139,17 +139,17 @@ public class TestSCMNodeManager { private static final int MAX_SOFTWARE_VERSION = HDDSLayoutVersionManager.maxLayoutVersion(); private static final LayoutVersionProto LARGER_SOFTWARE_PROTO = - toLayoutVersionProto(MAX_SOFTWARE_VERSION, MAX_SOFTWARE_VERSION + 1); + toVersionProto(MAX_SOFTWARE_VERSION, MAX_SOFTWARE_VERSION + 1); private static final LayoutVersionProto SMALLER_APPARENT_VERSION_PROTO = - toLayoutVersionProto(MAX_SOFTWARE_VERSION - 1, MAX_SOFTWARE_VERSION); + toVersionProto(MAX_SOFTWARE_VERSION - 1, MAX_SOFTWARE_VERSION); // In a real cluster, startup is disallowed if MLV is larger than SLV, so // increase both numbers to test smaller SLV or larger MLV. private static final LayoutVersionProto SMALLER_ALL_VERSIONS_PROTO = - toLayoutVersionProto(MAX_SOFTWARE_VERSION - 1, MAX_SOFTWARE_VERSION - 1); + toVersionProto(MAX_SOFTWARE_VERSION - 1, MAX_SOFTWARE_VERSION - 1); private static final LayoutVersionProto LARGER_ALL_VERSIONS_PROTO = - toLayoutVersionProto(MAX_SOFTWARE_VERSION + 1, MAX_SOFTWARE_VERSION + 1); + toVersionProto(MAX_SOFTWARE_VERSION + 1, MAX_SOFTWARE_VERSION + 1); private static final LayoutVersionProto MATCHING_VERSION_PROTO = - toLayoutVersionProto(MAX_SOFTWARE_VERSION, MAX_SOFTWARE_VERSION); + toVersionProto(MAX_SOFTWARE_VERSION, MAX_SOFTWARE_VERSION); @BeforeEach public void setup() { @@ -271,7 +271,7 @@ public void testGetLastHeartbeatTimeDiff() throws Exception { */ private DatanodeDetails registerWithCapacity(SCMNodeManager nodeManager) { return registerWithCapacity(nodeManager, - UpgradeUtils.defaultLayoutVersionProto(), success); + UpgradeUtils.defaultVersionProto(), success); } /** @@ -753,7 +753,7 @@ public void testDatanodeFinalizedCounterTracksVersionReports() nodeManager.getLayoutVersionManager().getSoftwareLayoutVersion(); int metadataVersion = nodeManager.getLayoutVersionManager().getMetadataLayoutVersion(); - nodeManager.processLayoutVersionReport(node, + nodeManager.processVersionReport(node, LayoutVersionProto.newBuilder() .setMetadataLayoutVersion(metadataVersion - 1) .setSoftwareLayoutVersion(softwareVersion) @@ -762,7 +762,7 @@ public void testDatanodeFinalizedCounterTracksVersionReports() .getNumFinalizedDatanodes(), "Lower metadata layout version should decrement finalized count"); - nodeManager.processLayoutVersionReport(node, + nodeManager.processVersionReport(node, LayoutVersionProto.newBuilder() .setMetadataLayoutVersion(metadataVersion) .setSoftwareLayoutVersion(softwareVersion) @@ -910,7 +910,7 @@ scmStorageConfig, eventPublisher, new NetworkTopologyImpl(conf), nodeManager.getLayoutVersionManager().getMetadataLayoutVersion(); int scmSlv = nodeManager.getLayoutVersionManager().getSoftwareLayoutVersion(); - nodeManager.processLayoutVersionReport(node1, + nodeManager.processVersionReport(node1, LayoutVersionProto.newBuilder() .setMetadataLayoutVersion(scmMlv + 1) .setSoftwareLayoutVersion(scmSlv + 1) @@ -942,7 +942,7 @@ scmStorageConfig, eventPublisher, new NetworkTopologyImpl(conf), times(1)).fireEvent(NEW_NODE, node1); int scmMlv = nodeManager.getLayoutVersionManager().getMetadataLayoutVersion(); - nodeManager.processLayoutVersionReport(node1, + nodeManager.processVersionReport(node1, LayoutVersionProto.newBuilder() .setMetadataLayoutVersion(scmMlv - 1) .setSoftwareLayoutVersion(scmMlv) diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/states/TestNodeStateMap.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/states/TestNodeStateMap.java index e9f13ee82c8c..d0b9c53d0e75 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/states/TestNodeStateMap.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/states/TestNodeStateMap.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.node.DatanodeInfo; import org.apache.hadoop.hdds.scm.node.NodeStatus; +import org.apache.hadoop.ozone.container.upgrade.UpgradeUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -53,7 +54,8 @@ void addNode(NodeStatus status) throws NodeAlreadyExistsException { } void addNode(DatanodeDetails datanode, NodeStatus status) throws NodeAlreadyExistsException { - map.addNode(new DatanodeInfo(datanode, status, null, HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT)); + map.addNode(new DatanodeInfo(datanode, status, UpgradeUtils.defaultVersionProto(), + HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT)); } @BeforeEach diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementFactory.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementFactory.java index 84afb74f0a5a..3739581a825a 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementFactory.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementFactory.java @@ -103,7 +103,7 @@ private void setupRacks(int datanodeCount, int nodesPerRack, cluster.add(datanodeDetails); DatanodeInfo datanodeInfo = new DatanodeInfo( datanodeDetails, NodeStatus.inServiceHealthy(), - UpgradeUtils.defaultLayoutVersionProto(), + UpgradeUtils.defaultVersionProto(), HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT); StorageContainerDatanodeProtocolProtos.StorageReportProto storage1 = diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java index c9a66f7c56c0..feb244edb56a 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java @@ -19,7 +19,7 @@ import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails; import static org.apache.hadoop.ozone.container.common.ContainerTestUtils.createEndpoint; -import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.defaultLayoutVersionProto; +import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.defaultVersionProto; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertInstanceOf; @@ -398,7 +398,7 @@ public void testRegister() throws Exception { nodeToRegister.getID()))), HddsTestUtils.getRandomContainerReports(10), HddsTestUtils.getRandomPipelineReports(), - defaultLayoutVersionProto()); + defaultVersionProto()); assertNotNull(responseProto); assertEquals(nodeToRegister.getUuidString(), responseProto.getDatanodeUUID()); assertNotNull(responseProto.getClusterID()); diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 3421ca9ec000..5c3a3328782d 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -361,7 +361,7 @@ public static OzoneManagerVersion getOmVersion(ServiceInfoEx info) { static boolean validateOmVersion(OzoneManagerVersion minimumVersion, List serviceInfoList) { - if (minimumVersion == OzoneManagerVersion.FUTURE_VERSION) { + if (minimumVersion == OzoneManagerVersion.UNKNOWN_VERSION) { // A FUTURE_VERSION should not be expected ever. throw new IllegalArgumentException("Configuration error, expected " + "OzoneManager version config evaluates to a future version."); diff --git a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/rpc/TestRpcClient.java b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/rpc/TestRpcClient.java index b6b501f67741..c6d996971113 100644 --- a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/rpc/TestRpcClient.java +++ b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/rpc/TestRpcClient.java @@ -59,13 +59,13 @@ private enum ValidateOmVersionTestCases { NULL_EXPECTED_ONE_CURRENT_ONE_FUTURE_OM( null, OzoneManagerVersion.SOFTWARE_VERSION, - OzoneManagerVersion.FUTURE_VERSION, + OzoneManagerVersion.UNKNOWN_VERSION, true ), NULL_EXPECTED_TWO_FUTURE_OM( null, - OzoneManagerVersion.FUTURE_VERSION, - OzoneManagerVersion.FUTURE_VERSION, + OzoneManagerVersion.UNKNOWN_VERSION, + OzoneManagerVersion.UNKNOWN_VERSION, true ), @@ -86,7 +86,7 @@ private enum ValidateOmVersionTestCases { true), DEFAULT_EXPECTED_ONE_FUTURE_OM( OzoneManagerVersion.DEFAULT_VERSION, - OzoneManagerVersion.FUTURE_VERSION, + OzoneManagerVersion.UNKNOWN_VERSION, null, true), DEFAULT_EXPECTED_TWO_DEFAULT_OM( @@ -101,8 +101,8 @@ private enum ValidateOmVersionTestCases { true), DEFAULT_EXPECTED_TWO_FUTURE_OM( OzoneManagerVersion.DEFAULT_VERSION, - OzoneManagerVersion.FUTURE_VERSION, - OzoneManagerVersion.FUTURE_VERSION, + OzoneManagerVersion.UNKNOWN_VERSION, + OzoneManagerVersion.UNKNOWN_VERSION, true), DEFAULT_EXPECTED_ONE_DEFAULT_ONE_CURRENT_OM( OzoneManagerVersion.DEFAULT_VERSION, @@ -112,12 +112,12 @@ private enum ValidateOmVersionTestCases { DEFAULT_EXPECTED_ONE_DEFAULT_ONE_FUTURE_OM( OzoneManagerVersion.DEFAULT_VERSION, OzoneManagerVersion.DEFAULT_VERSION, - OzoneManagerVersion.FUTURE_VERSION, + OzoneManagerVersion.UNKNOWN_VERSION, true), DEFAULT_EXPECTED_ONE_CURRENT_ONE_FUTURE_OM( OzoneManagerVersion.DEFAULT_VERSION, OzoneManagerVersion.SOFTWARE_VERSION, - OzoneManagerVersion.FUTURE_VERSION, + OzoneManagerVersion.UNKNOWN_VERSION, true), CURRENT_EXPECTED_NO_OM( @@ -137,7 +137,7 @@ private enum ValidateOmVersionTestCases { true), CURRENT_EXPECTED_ONE_FUTURE_OM( OzoneManagerVersion.SOFTWARE_VERSION, - OzoneManagerVersion.FUTURE_VERSION, + OzoneManagerVersion.UNKNOWN_VERSION, null, true), CURRENT_EXPECTED_TWO_DEFAULT_OM( @@ -152,8 +152,8 @@ private enum ValidateOmVersionTestCases { true), CURRENT_EXPECTED_TWO_FUTURE_OM( OzoneManagerVersion.SOFTWARE_VERSION, - OzoneManagerVersion.FUTURE_VERSION, - OzoneManagerVersion.FUTURE_VERSION, + OzoneManagerVersion.UNKNOWN_VERSION, + OzoneManagerVersion.UNKNOWN_VERSION, true), CURRENT_EXPECTED_ONE_DEFAULT_ONE_CURRENT_OM( OzoneManagerVersion.SOFTWARE_VERSION, @@ -163,12 +163,12 @@ private enum ValidateOmVersionTestCases { CURRENT_EXPECTED_ONE_DEFAULT_ONE_FUTURE_OM( OzoneManagerVersion.SOFTWARE_VERSION, OzoneManagerVersion.DEFAULT_VERSION, - OzoneManagerVersion.FUTURE_VERSION, + OzoneManagerVersion.UNKNOWN_VERSION, false), CURRENT_EXPECTED_ONE_CURRENT_ONE_FUTURE_OM( OzoneManagerVersion.SOFTWARE_VERSION, OzoneManagerVersion.SOFTWARE_VERSION, - OzoneManagerVersion.FUTURE_VERSION, + OzoneManagerVersion.UNKNOWN_VERSION, true); private final OzoneManagerVersion expectedVersion; @@ -213,6 +213,6 @@ public void testValidateOmVersion(ValidateOmVersionTestCases testCase) { public void testFutureVersionShouldNotBeAnExpectedVersion() { assertThrows( IllegalArgumentException.class, - () -> validateOmVersion(OzoneManagerVersion.FUTURE_VERSION, null)); + () -> validateOmVersion(OzoneManagerVersion.UNKNOWN_VERSION, null)); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMVersionManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMVersionManager.java index 99bbab840e9a..fffc76f53223 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMVersionManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMVersionManager.java @@ -116,7 +116,7 @@ private static ComponentVersion getApparentVersionInDB(OMMetadataManager metadat private static ComponentVersion computeApparentVersion(int serializedApparentVersion) throws IOException { if (serializedApparentVersion >= OzoneManagerVersion.ZDU.serialize()) { OzoneManagerVersion fromOm = OzoneManagerVersion.deserialize(serializedApparentVersion); - if (fromOm != OzoneManagerVersion.FUTURE_VERSION) { + if (fromOm != OzoneManagerVersion.UNKNOWN_VERSION) { return fromOm; } } else { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutFeature.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutFeature.java index 9e98935b5b3d..9534c0ccba19 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutFeature.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutFeature.java @@ -101,7 +101,7 @@ public void testIsSupportedByFeatureBoundary() { public void testAllLayoutFeaturesAreSupportedByFutureVersions() { for (OMLayoutFeature feature : OMLayoutFeature.values()) { assertSupportedBy(feature, OzoneManagerVersion.ZDU); - assertSupportedBy(feature, OzoneManagerVersion.FUTURE_VERSION); + assertSupportedBy(feature, OzoneManagerVersion.UNKNOWN_VERSION); // No ComponentVersion instance represents an arbitrary unknown future version. assertTrue(feature.isSupportedBy(Integer.MAX_VALUE)); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMVersionManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMVersionManager.java index 2565109a73f6..6d232e85be9d 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMVersionManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMVersionManager.java @@ -83,7 +83,7 @@ class TestOMVersionManager extends AbstractComponentVersionManagerTest { ALL_VERSIONS = new ArrayList<>(Arrays.asList(OMLayoutFeature.values())); for (OzoneManagerVersion version : OzoneManagerVersion.values()) { // Add all defined versions after and including ZDU to get the complete version list. - if (ZDU.isSupportedBy(version) && version != OzoneManagerVersion.FUTURE_VERSION) { + if (ZDU.isSupportedBy(version) && version != OzoneManagerVersion.UNKNOWN_VERSION) { ALL_VERSIONS.add(version); } } diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java index 330bd3adbe21..04cdb2751aa9 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java @@ -18,7 +18,7 @@ package org.apache.hadoop.ozone.recon.api; import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails; -import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.defaultLayoutVersionProto; +import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.defaultVersionProto; import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getRandomPipeline; import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager; import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.initializeNewOmMetadataManager; @@ -427,7 +427,7 @@ public void setUp() throws Exception { NodeReportProto.newBuilder() .addStorageReport(storageReportProto3) .addStorageReport(storageReportProto4).build(); - LayoutVersionProto layoutInfo = defaultLayoutVersionProto(); + LayoutVersionProto layoutInfo = defaultVersionProto(); DatanodeDetailsProto datanodeDetailsProto3 = DatanodeDetailsProto.newBuilder() @@ -471,12 +471,12 @@ public void setUp() throws Exception { .register(extendedDatanodeDetailsProto2, nodeReportProto2, ContainerReportsProto.newBuilder().build(), PipelineReportsProto.newBuilder().build(), - defaultLayoutVersionProto()); + defaultVersionProto()); reconScm.getDatanodeProtocolServer() .register(extendedDatanodeDetailsProto3, nodeReportProto3, ContainerReportsProto.newBuilder().build(), PipelineReportsProto.newBuilder().build(), - defaultLayoutVersionProto()); + defaultVersionProto()); // Process all events in the event queue reconScm.getEventQueue().processAll(1000); }); @@ -1276,7 +1276,7 @@ private void waitAndCheckConditionAfterHeartbeat(Callable check) .setContainerReport(containerReportsProto) .setDatanodeDetails(extendedDatanodeDetailsProto .getDatanodeDetails()) - .setDataNodeLayoutVersion(defaultLayoutVersionProto()) + .setDataNodeLayoutVersion(defaultVersionProto()) .build(); reconScm.getDatanodeProtocolServer().sendHeartbeat(heartbeatRequestProto); LambdaTestUtils.await(30000, 1000, check); diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOpenContainerCount.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOpenContainerCount.java index be2fd9bd8247..fb8f4b66d7b9 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOpenContainerCount.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOpenContainerCount.java @@ -18,7 +18,7 @@ package org.apache.hadoop.ozone.recon.api; import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails; -import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.defaultLayoutVersionProto; +import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.defaultVersionProto; import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getRandomPipeline; import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager; import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.initializeNewOmMetadataManager; @@ -321,7 +321,7 @@ public void setUp() throws Exception { reconScm.getDatanodeProtocolServer() .register(extendedDatanodeDetailsProto, nodeReportProto, containerReportsProto, pipelineReportsProto, - defaultLayoutVersionProto()); + defaultVersionProto()); // Process all events in the event queue reconScm.getEventQueue().processAll(1000); }); @@ -411,7 +411,7 @@ private void updateContainerReport(long containerId) { reconScm.getDatanodeProtocolServer() .register(extendedDatanodeDetailsProto, nodeReportProto, containerReportsProto, pipelineReportsProto, - defaultLayoutVersionProto()); + defaultVersionProto()); // Process all events in the event queue reconScm.getEventQueue().processAll(1000); }); diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestStorageDistributionEndpoint.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestStorageDistributionEndpoint.java index d62aae159139..1b646e2aa20e 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestStorageDistributionEndpoint.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestStorageDistributionEndpoint.java @@ -17,6 +17,7 @@ package org.apache.hadoop.ozone.recon.api; +import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.defaultVersionProto; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -225,7 +226,7 @@ private List mockStorageDistributionData(int numNodes) throws Exception .build(); pendingDeletionMetrics.add(new DatanodePendingDeletionMetrics(hostName, uuid.toString(), PENDING_DELETION_SIZE)); - dataNodes.add(new DatanodeInfo(datanode, NodeStatus.inServiceHealthy(), null, 5 * 60 * 1000)); + dataNodes.add(new DatanodeInfo(datanode, NodeStatus.inServiceHealthy(), defaultVersionProto(), 5 * 60 * 1000)); when(nodeManager.getNodeStat(datanode)) .thenReturn(new SCMNodeMetric(OZONE_CAPACITY, OZONE_USED, OZONE_REMAINING, COMMITTED, MIN_FREE_SPACE, RESERVED)); diff --git a/hadoop-ozone/vapor/src/main/java/org/apache/hadoop/ozone/freon/SCMThroughputBenchmark.java b/hadoop-ozone/vapor/src/main/java/org/apache/hadoop/ozone/freon/SCMThroughputBenchmark.java index e575576428b3..dfbf0ded77d0 100644 --- a/hadoop-ozone/vapor/src/main/java/org/apache/hadoop/ozone/freon/SCMThroughputBenchmark.java +++ b/hadoop-ozone/vapor/src/main/java/org/apache/hadoop/ozone/freon/SCMThroughputBenchmark.java @@ -799,7 +799,7 @@ public void register() throws IOException { createNodeReport(datanodeDetails.getUuid()), createContainerReport(), createPipelineReport(), - UpgradeUtils.defaultLayoutVersionProto()); + UpgradeUtils.defaultVersionProto()); if (response.hasHostname() && response.hasIpAddress()) { datanodeDetails.setHostName(response.getHostname()); @@ -816,7 +816,7 @@ public void sendHeartbeat() throws IOException, TimeoutException { .newBuilder() .setDatanodeDetails(datanodeDetails.getProtoBufMessage()) .setContainerReport(containerReport) - .setDataNodeLayoutVersion(UpgradeUtils.defaultLayoutVersionProto()) + .setDataNodeLayoutVersion(UpgradeUtils.defaultVersionProto()) .build(); datanodeScmClient.sendHeartbeat(heartbeatRequest); // scm commands are ignored From 9e166a946e7c0a6e2975a089607f3413fcb4902c Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Wed, 27 May 2026 11:21:37 -0400 Subject: [PATCH 2/2] Switch default version which was not intended in this change. --- .../apache/hadoop/ozone/container/common/DatanodeStorage.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DatanodeStorage.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DatanodeStorage.java index fe2677834378..53b62afc3e16 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DatanodeStorage.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DatanodeStorage.java @@ -17,12 +17,12 @@ package org.apache.hadoop.ozone.container.common; +import static org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager.maxLayoutVersion; import static org.apache.hadoop.ozone.OzoneConsts.DATANODE_LAYOUT_VERSION_DIR; import java.io.File; import java.io.IOException; import java.util.Properties; -import org.apache.hadoop.hdds.HDDSVersion; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType; @@ -95,7 +95,7 @@ public void setClusterId(String clusterId) throws IOException { * layout version is found on disk. */ private static int getDefaultApparentVersion(ConfigurationSource conf) { - int defaultApparentVersion = HDDSVersion.SOFTWARE_VERSION.serialize(); + int defaultApparentVersion = maxLayoutVersion(); File dnIdFile = new File(HddsServerUtil.getDatanodeIdFilePath(conf)); if (dnIdFile.exists()) {