HDDS-15386. Remove LayoutFeature interface

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HDDSVersion.java

@@ -41,8 +41,8 @@ public enum HDDSVersion implements ComponentVersion {
 
   ZDU(100, "Version that supports zero downtime upgrade"),
 
-  FUTURE_VERSION(-1, "Used internally in the client when the server side is "
-      + " newer and an unknown server version has arrived to the client.");
+  UNKNOWN_VERSION(-1, "Used when a version cannot be deserialized to any version recognized by this" +
+      " component, which may indicate it came from a component in a newer version");
 
   //////////////////////////////  //////////////////////////////
 
@@ -85,11 +85,11 @@ public int serialize() {
 
   /**
    * @param value The serialized version to convert.
-   * @return The version corresponding to this serialized value, or {@link #FUTURE_VERSION} if no matching version is
+   * @return The version corresponding to this serialized value, or {@link #UNKNOWN_VERSION} if no matching version is
    *    found.
    */
   public static HDDSVersion deserialize(int value) {
-    return BY_VALUE.getOrDefault(value, FUTURE_VERSION);
+    return BY_VALUE.getOrDefault(value, UNKNOWN_VERSION);
   }
 
   @Override

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java

@@ -42,7 +42,7 @@ public enum ClientVersion implements ComponentVersion {
       "This client version has support for Object Store and File " +
           "System Optimized Bucket Layouts."),
 
-  FUTURE_VERSION(-1, "Used internally when the server side is older and an"
+  FUTURE_VERSION(-1, "Used internally by the server when the server side is older and an"
       + " unknown client version has arrived from the client.");
 
   private static final SortedMap<Integer, ClientVersion> BY_VALUE =

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java

@@ -64,8 +64,8 @@ public enum OzoneManagerVersion implements ComponentVersion {
 
   ZDU(100, "OzoneManager version that supports zero downtime upgrade"),
 
-  FUTURE_VERSION(-1, "Used internally in the client when the server side is "
-      + " newer and an unknown server version has arrived to the client.");
+  UNKNOWN_VERSION(-1, "Used when a version cannot be deserialized to any version recognized by this" +
+      " component, which may indicate it came from a component in a newer version");
 
   private static final SortedMap<Integer, OzoneManagerVersion> BY_VALUE =
       Arrays.stream(values())
@@ -93,11 +93,11 @@ public int serialize() {
 
   /**
    * @param value The serialized version to convert.
-   * @return The version corresponding to this serialized value, or {@link #FUTURE_VERSION} if no matching version is
+   * @return The version corresponding to this serialized value, or {@link #UNKNOWN_VERSION} if no matching version is
    *    found.
    */
   public static OzoneManagerVersion deserialize(int value) {
-    return BY_VALUE.getOrDefault(value, FUTURE_VERSION);
+    return BY_VALUE.getOrDefault(value, UNKNOWN_VERSION);
   }
 
 

hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHDDSVersion.java

@@ -38,7 +38,7 @@ protected ComponentVersion getDefaultVersion() {
 
   @Override
   protected ComponentVersion getFutureVersion() {
-    return HDDSVersion.FUTURE_VERSION;
+    return HDDSVersion.UNKNOWN_VERSION;
   }
 
   @Override

hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestOzoneManagerVersion.java

@@ -39,7 +39,7 @@ protected ComponentVersion getDefaultVersion() {
 
   @Override
   protected ComponentVersion getFutureVersion() {
-    return OzoneManagerVersion.FUTURE_VERSION;
+    return OzoneManagerVersion.UNKNOWN_VERSION;
   }
 
   @Override

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DatanodeStorage.java

@@ -17,12 +17,12 @@
 
 package org.apache.hadoop.ozone.container.common;
 
-import static org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager.maxLayoutVersion;
 import static org.apache.hadoop.ozone.OzoneConsts.DATANODE_LAYOUT_VERSION_DIR;
 
 import java.io.File;
 import java.io.IOException;
 import java.util.Properties;
+import org.apache.hadoop.hdds.HDDSVersion;
 import org.apache.hadoop.hdds.conf.ConfigurationSource;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType;
@@ -43,7 +43,7 @@ public class DatanodeStorage extends Storage {
   public DatanodeStorage(ConfigurationSource conf, String dataNodeId)
       throws IOException {
     super(NodeType.DATANODE, ServerUtils.getOzoneMetaDirPath(conf),
-        DATANODE_LAYOUT_VERSION_DIR, dataNodeId, getDefaultLayoutVersion(conf));
+        DATANODE_LAYOUT_VERSION_DIR, dataNodeId, getDefaultApparentVersion(conf));
   }
 
   public DatanodeStorage(OzoneConfiguration conf, String dataNodeId,
@@ -56,7 +56,7 @@ public DatanodeStorage(OzoneConfiguration conf, String dataNodeId,
   public DatanodeStorage(ConfigurationSource conf)
       throws IOException {
     super(NodeType.DATANODE, ServerUtils.getOzoneMetaDirPath(conf),
-        DATANODE_LAYOUT_VERSION_DIR, getDefaultLayoutVersion(conf));
+        DATANODE_LAYOUT_VERSION_DIR, getDefaultApparentVersion(conf));
   }
 
   @Override
@@ -94,15 +94,14 @@ public void setClusterId(String clusterId) throws IOException {
    * @return The layout version that should be used for the datanode if no
    * layout version is found on disk.
    */
-  private static int getDefaultLayoutVersion(ConfigurationSource conf) {
-    int defaultLayoutVersion = maxLayoutVersion();
+  private static int getDefaultApparentVersion(ConfigurationSource conf) {
+    int defaultApparentVersion = HDDSVersion.SOFTWARE_VERSION.serialize();
 
     File dnIdFile = new File(HddsServerUtil.getDatanodeIdFilePath(conf));
     if (dnIdFile.exists()) {
-      defaultLayoutVersion =
-          HDDSLayoutFeature.INITIAL_VERSION.layoutVersion();
+      defaultApparentVersion = HDDSLayoutFeature.INITIAL_VERSION.serialize();
     }
 
-    return defaultLayoutVersion;
+    return defaultApparentVersion;
   }
 }

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java

@@ -21,8 +21,7 @@
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_ACTION_MAX_LIMIT_DEFAULT;
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PIPELINE_ACTION_MAX_LIMIT;
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PIPELINE_ACTION_MAX_LIMIT_DEFAULT;
-import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type.finalizeNewLayoutVersionCommand;
-import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.toLayoutVersionProto;
+import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.toVersionProto;
 
 import com.google.common.base.Preconditions;
 import com.google.protobuf.Descriptors;
@@ -129,13 +128,13 @@ public EndpointStateMachine.EndPointStates call() throws Exception {
     try {
       Preconditions.checkState(this.datanodeDetailsProto != null);
 
-      LayoutVersionProto layoutinfo = toLayoutVersionProto(
-          versionManager.getApparentVersion().serialize(),
-          versionManager.getSoftwareVersion().serialize());
+      LayoutVersionProto versionInfo = toVersionProto(
+          versionManager.getApparentVersion(),
+          versionManager.getSoftwareVersion());
 
       requestBuilder = SCMHeartbeatRequestProto.newBuilder()
           .setDatanodeDetails(datanodeDetailsProto)
-          .setDataNodeLayoutVersion(layoutinfo);
+          .setDataNodeLayoutVersion(versionInfo);
       addReports(requestBuilder);
       addContainerActions(requestBuilder);
       addPipelineActions(requestBuilder);

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DatanodeVersionManager.java

@@ -21,18 +21,20 @@
 import java.io.IOException;
 import java.util.Map;
 import org.apache.hadoop.hdds.ComponentVersion;
+import org.apache.hadoop.hdds.HDDSVersion;
 import org.apache.hadoop.hdds.upgrade.DatanodeUpgradeAction;
 import org.apache.hadoop.hdds.upgrade.DatanodeUpgradeActionProvider;
-import org.apache.hadoop.hdds.upgrade.HDDSVersionManager;
+import org.apache.hadoop.hdds.upgrade.HDDSVersionUtils;
 import org.apache.hadoop.ozone.container.common.DatanodeStorage;
 import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
 import org.apache.hadoop.ozone.upgrade.ComponentUpgradeActionProvider;
+import org.apache.hadoop.ozone.upgrade.ComponentVersionManager;
 import org.apache.hadoop.ozone.upgrade.UpgradeException;
 
 /**
  * Datanode-specific version manager that wires upgrade actions internally.
  */
-public class DatanodeVersionManager extends HDDSVersionManager {
+public class DatanodeVersionManager extends ComponentVersionManager {
 
   private final Map<ComponentVersion, DatanodeUpgradeAction> upgradeActions;
   private final DatanodeStateMachine upgradeActionArg;
@@ -44,7 +46,9 @@ public DatanodeVersionManager(DatanodeStorage storage, DatanodeStateMachine upgr
   @VisibleForTesting
   public DatanodeVersionManager(DatanodeStorage storage, DatanodeStateMachine upgradeActionArg,
       ComponentUpgradeActionProvider<DatanodeUpgradeAction> upgradeActionProvider) throws IOException {
-    super(storage);
+    super(storage,
+        HDDSVersionUtils.deserializedPersistedApparentVersion(storage.getApparentVersion()),
+        HDDSVersion.SOFTWARE_VERSION);
     this.upgradeActionArg = upgradeActionArg;
     upgradeActions = upgradeActionProvider.load();
   }

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/UpgradeUtils.java

@@ -17,6 +17,7 @@
 
 package org.apache.hadoop.ozone.container.upgrade;
 
+import org.apache.hadoop.hdds.ComponentVersion;
 import org.apache.hadoop.hdds.HDDSVersion;
 import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto;
 
@@ -28,16 +29,17 @@ public final class UpgradeUtils {
   private UpgradeUtils() {
   }
 
-  public static LayoutVersionProto defaultLayoutVersionProto() {
+  public static LayoutVersionProto defaultVersionProto() {
     int softwareVersion = HDDSVersion.SOFTWARE_VERSION.serialize();
     return LayoutVersionProto.newBuilder()
         .setMetadataLayoutVersion(softwareVersion)
         .setSoftwareLayoutVersion(softwareVersion).build();
   }
 
-  public static LayoutVersionProto toLayoutVersionProto(int mLv, int sLv) {
+  public static LayoutVersionProto toVersionProto(ComponentVersion apparentVersion, ComponentVersion softwareVersion) {
     return LayoutVersionProto.newBuilder()
-        .setMetadataLayoutVersion(mLv)
-        .setSoftwareLayoutVersion(sLv).build();
+        .setMetadataLayoutVersion(apparentVersion.serialize())
+        .setSoftwareLayoutVersion(softwareVersion.serialize())
+        .build();
   }
 }

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java

@@ -18,7 +18,7 @@
 package org.apache.hadoop.ozone.protocolPB;
 
 import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.INITIAL_VERSION;
-import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.toLayoutVersionProto;
+import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.toVersionProto;
 
 import com.google.protobuf.RpcController;
 import com.google.protobuf.ServiceException;
@@ -71,17 +71,16 @@ public SCMRegisteredResponseProto register(
         .getContainerReport();
     NodeReportProto dnNodeReport = request.getNodeReport();
     PipelineReportsProto pipelineReport = request.getPipelineReports();
-    LayoutVersionProto layoutInfo = null;
+    LayoutVersionProto versionInfo = null;
     if (request.hasDataNodeLayoutVersion()) {
-      layoutInfo = request.getDataNodeLayoutVersion();
+      versionInfo = request.getDataNodeLayoutVersion();
     } else {
       // Backward compatibility to make sure old Datanodes can still talk to
       // SCM.
-      layoutInfo = toLayoutVersionProto(INITIAL_VERSION.layoutVersion(),
-          INITIAL_VERSION.layoutVersion());
+      versionInfo = toVersionProto(INITIAL_VERSION, INITIAL_VERSION);
     }
     return impl.register(request.getExtendedDatanodeDetails(), dnNodeReport,
-        containerRequestProto, pipelineReport, layoutInfo);
+        containerRequestProto, pipelineReport, versionInfo);
   }
 
   @Override

hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java

@@ -231,7 +231,7 @@ public void testDatanodeStateContext() throws IOException,
 
       DatanodeStorage layoutStorage = new DatanodeStorage(conf,
           UUID.randomUUID().toString(),
-          HDDSLayoutFeature.DATANODE_SCHEMA_V3.layoutVersion());
+          HDDSLayoutFeature.DATANODE_SCHEMA_V3.serialize());
       layoutStorage.initialize();
 
       // This execute will invoke getVersion calls against all SCM endpoints

hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/helpers/TestDatanodeIdYaml.java

@@ -62,7 +62,7 @@ void testWriteReadBeforeRatisDatastreamPortLayoutVersion(@TempDir File dir)
     conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, dir.toString());
     DatanodeStorage layoutStorage = new DatanodeStorage(conf,
         UUID.randomUUID().toString(),
-        HDDSLayoutFeature.DATANODE_SCHEMA_V3.layoutVersion());
+        HDDSLayoutFeature.DATANODE_SCHEMA_V3.serialize());
     layoutStorage.initialize();
 
     DatanodeIdYaml.createDatanodeIdFile(original, file, conf);
@@ -86,7 +86,7 @@ void testWriteReadAfterRatisDatastreamPortLayoutVersion(@TempDir File dir)
     DatanodeStorage layoutStorage = new DatanodeStorage(conf,
         UUID.randomUUID().toString(),
         HDDSLayoutFeature.RATIS_DATASTREAM_PORT_IN_DATANODEDETAILS
-            .layoutVersion());
+            .serialize());
     layoutStorage.initialize();
 
     DatanodeIdYaml.createDatanodeIdFile(original, file, conf);
@@ -106,7 +106,7 @@ void testWriteReadBeforeWebUIPortLayoutVersion(@TempDir File dir)
     conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, dir.toString());
     DatanodeStorage layoutStorage = new DatanodeStorage(conf,
         UUID.randomUUID().toString(),
-        HDDSLayoutFeature.DATANODE_SCHEMA_V3.layoutVersion());
+        HDDSLayoutFeature.DATANODE_SCHEMA_V3.serialize());
     layoutStorage.initialize();
 
     DatanodeIdYaml.createDatanodeIdFile(original, file, conf);
@@ -127,7 +127,7 @@ void testWriteReadAfterWebUIPortLayoutVersion(@TempDir File dir)
     conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, dir.toString());
     DatanodeStorage layoutStorage = new DatanodeStorage(conf,
         UUID.randomUUID().toString(),
-        HDDSLayoutFeature.WEBUI_PORTS_IN_DATANODEDETAILS.layoutVersion());
+        HDDSLayoutFeature.WEBUI_PORTS_IN_DATANODEDETAILS.serialize());
     layoutStorage.initialize();
 
     DatanodeIdYaml.createDatanodeIdFile(original, file, conf);

hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeStartupInvalidApparentVersion.java

@@ -48,7 +48,7 @@ public class TestDatanodeStartupInvalidApparentVersion {
   public void testStartupFailsWhenApparentVersionBetweenLastLayoutFeatureAndZdu()
       throws Exception {
     assertStartupFailsWithComponentVersionMessage(
-        HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION.layoutVersion() + 1);
+        HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION.serialize() + 1);
   }
 
   @Test

hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeUpgradeToContainerIdsTable.java

@@ -85,7 +85,7 @@ public void testContainerTableAccessBeforeAndAfterUpgrade() throws Exception {
     InetSocketAddress address = scmRpcServer.getListenerAddress();
     UpgradeTestHelper.addHddsVolume(conf, tempFolder);
     dsm = UpgradeTestHelper.startPreFinalizedDatanode(conf, tempFolder, dsm, address,
-        HDDSLayoutFeature.HBASE_SUPPORT.layoutVersion());
+        HDDSLayoutFeature.HBASE_SUPPORT.serialize());
     ContainerDispatcher dispatcher = dsm.getContainer().getDispatcher();
     final Pipeline pipeline = MockPipeline.createPipeline(Collections.singletonList(dsm.getDatanodeDetails()));
 
@@ -122,7 +122,7 @@ public void testContainerTableFinalizeRetry() throws Exception {
     InetSocketAddress address = scmRpcServer.getListenerAddress();
     UpgradeTestHelper.addHddsVolume(conf, tempFolder);
     dsm = UpgradeTestHelper.startPreFinalizedDatanode(conf, tempFolder, dsm, address,
-        HDDSLayoutFeature.HBASE_SUPPORT.layoutVersion());
+        HDDSLayoutFeature.HBASE_SUPPORT.serialize());
     ContainerDispatcher dispatcher = dsm.getContainer().getDispatcher();
     final Pipeline pipeline = MockPipeline.createPipeline(Collections.singletonList(dsm.getDatanodeDetails()));
 

hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeUpgradeToHBaseSupport.java

@@ -83,7 +83,7 @@ public void testIncrementalChunkListBeforeAndAfterUpgrade() throws Exception {
     InetSocketAddress address = scmRpcServer.getListenerAddress();
     UpgradeTestHelper.addHddsVolume(conf, tempFolder);
     dsm = UpgradeTestHelper.startPreFinalizedDatanode(conf, tempFolder, dsm, address,
-        HDDSLayoutFeature.HADOOP_PRC_PORTS_IN_DATANODEDETAILS.layoutVersion());
+        HDDSLayoutFeature.HADOOP_PRC_PORTS_IN_DATANODEDETAILS.serialize());
     ContainerDispatcher dispatcher = dsm.getContainer().getDispatcher();
     final Pipeline pipeline = MockPipeline.createPipeline(
         Collections.singletonList(dsm.getDatanodeDetails()));
@@ -118,7 +118,7 @@ public void testBlockFinalizationBeforeAndAfterUpgrade() throws Exception {
     InetSocketAddress address = scmRpcServer.getListenerAddress();
     UpgradeTestHelper.addHddsVolume(conf, tempFolder);
     dsm = UpgradeTestHelper.startPreFinalizedDatanode(conf, tempFolder, dsm, address,
-        HDDSLayoutFeature.HADOOP_PRC_PORTS_IN_DATANODEDETAILS.layoutVersion());
+        HDDSLayoutFeature.HADOOP_PRC_PORTS_IN_DATANODEDETAILS.serialize());
     ContainerDispatcher dispatcher = dsm.getContainer().getDispatcher();
     final Pipeline pipeline = MockPipeline.createPipeline(
         Collections.singletonList(dsm.getDatanodeDetails()));