HDDS-15379. Remove unused classes from the old upgrade framework

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HDDSVersion.java

@@ -41,8 +41,8 @@ public enum HDDSVersion implements ComponentVersion {
 
   ZDU(100, "Version that supports zero downtime upgrade"),
 
-  FUTURE_VERSION(-1, "Used internally in the client when the server side is "
-      + " newer and an unknown server version has arrived to the client.");
+  UNKNOWN_VERSION(-1, "Used when a version cannot be deserialized to any version recognized by this" +
+      " component, which may indicate it came from a component in a newer version");
 
   //////////////////////////////  //////////////////////////////
 
@@ -85,11 +85,11 @@ public int serialize() {
 
   /**
    * @param value The serialized version to convert.
-   * @return The version corresponding to this serialized value, or {@link #FUTURE_VERSION} if no matching version is
+   * @return The version corresponding to this serialized value, or {@link #UNKNOWN_VERSION} if no matching version is
    *    found.
    */
   public static HDDSVersion deserialize(int value) {
-    return BY_VALUE.getOrDefault(value, FUTURE_VERSION);
+    return BY_VALUE.getOrDefault(value, UNKNOWN_VERSION);
   }
 
   @Override

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java

@@ -42,7 +42,7 @@ public enum ClientVersion implements ComponentVersion {
       "This client version has support for Object Store and File " +
           "System Optimized Bucket Layouts."),
 
-  FUTURE_VERSION(-1, "Used internally when the server side is older and an"
+  FUTURE_VERSION(-1, "Used internally by the server when the server side is older and an"
       + " unknown client version has arrived from the client.");
 
   private static final SortedMap<Integer, ClientVersion> BY_VALUE =

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java

@@ -64,8 +64,8 @@ public enum OzoneManagerVersion implements ComponentVersion {
 
   ZDU(100, "OzoneManager version that supports zero downtime upgrade"),
 
-  FUTURE_VERSION(-1, "Used internally in the client when the server side is "
-      + " newer and an unknown server version has arrived to the client.");
+  UNKNOWN_VERSION(-1, "Used when a version cannot be deserialized to any version recognized by this" +
+      " component, which may indicate it came from a component in a newer version");
 
   private static final SortedMap<Integer, OzoneManagerVersion> BY_VALUE =
       Arrays.stream(values())
@@ -93,11 +93,11 @@ public int serialize() {
 
   /**
    * @param value The serialized version to convert.
-   * @return The version corresponding to this serialized value, or {@link #FUTURE_VERSION} if no matching version is
+   * @return The version corresponding to this serialized value, or {@link #UNKNOWN_VERSION} if no matching version is
    *    found.
    */
   public static OzoneManagerVersion deserialize(int value) {
-    return BY_VALUE.getOrDefault(value, FUTURE_VERSION);
+    return BY_VALUE.getOrDefault(value, UNKNOWN_VERSION);
   }
 
 

hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHDDSVersion.java

@@ -38,7 +38,7 @@ protected ComponentVersion getDefaultVersion() {
 
   @Override
   protected ComponentVersion getFutureVersion() {
-    return HDDSVersion.FUTURE_VERSION;
+    return HDDSVersion.UNKNOWN_VERSION;
   }
 
   @Override

hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestOzoneManagerVersion.java

@@ -39,7 +39,7 @@ protected ComponentVersion getDefaultVersion() {
 
   @Override
   protected ComponentVersion getFutureVersion() {
-    return OzoneManagerVersion.FUTURE_VERSION;
+    return OzoneManagerVersion.UNKNOWN_VERSION;
   }
 
   @Override

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DatanodeStorage.java

@@ -17,12 +17,12 @@
 
 package org.apache.hadoop.ozone.container.common;
 
-import static org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager.maxLayoutVersion;
 import static org.apache.hadoop.ozone.OzoneConsts.DATANODE_LAYOUT_VERSION_DIR;
 
 import java.io.File;
 import java.io.IOException;
 import java.util.Properties;
+import org.apache.hadoop.hdds.HDDSVersion;
 import org.apache.hadoop.hdds.conf.ConfigurationSource;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType;
@@ -43,7 +43,7 @@ public class DatanodeStorage extends Storage {
   public DatanodeStorage(ConfigurationSource conf, String dataNodeId)
       throws IOException {
     super(NodeType.DATANODE, ServerUtils.getOzoneMetaDirPath(conf),
-        DATANODE_LAYOUT_VERSION_DIR, dataNodeId, getDefaultLayoutVersion(conf));
+        DATANODE_LAYOUT_VERSION_DIR, dataNodeId, getDefaultApparentVersion(conf));
   }
 
   public DatanodeStorage(OzoneConfiguration conf, String dataNodeId,
@@ -56,7 +56,7 @@ public DatanodeStorage(OzoneConfiguration conf, String dataNodeId,
   public DatanodeStorage(ConfigurationSource conf)
       throws IOException {
     super(NodeType.DATANODE, ServerUtils.getOzoneMetaDirPath(conf),
-        DATANODE_LAYOUT_VERSION_DIR, getDefaultLayoutVersion(conf));
+        DATANODE_LAYOUT_VERSION_DIR, getDefaultApparentVersion(conf));
   }
 
   @Override
@@ -94,15 +94,14 @@ public void setClusterId(String clusterId) throws IOException {
    * @return The layout version that should be used for the datanode if no
    * layout version is found on disk.
    */
-  private static int getDefaultLayoutVersion(ConfigurationSource conf) {
-    int defaultLayoutVersion = maxLayoutVersion();
+  private static int getDefaultApparentVersion(ConfigurationSource conf) {
+    int defaultApparentVersion = HDDSVersion.SOFTWARE_VERSION.serialize();
 
     File dnIdFile = new File(HddsServerUtil.getDatanodeIdFilePath(conf));
     if (dnIdFile.exists()) {
-      defaultLayoutVersion =
-          HDDSLayoutFeature.INITIAL_VERSION.layoutVersion();
+      defaultApparentVersion = HDDSLayoutFeature.INITIAL_VERSION.layoutVersion();
     }
 
-    return defaultLayoutVersion;
+    return defaultApparentVersion;
   }
 }

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java

@@ -21,8 +21,7 @@
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_ACTION_MAX_LIMIT_DEFAULT;
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PIPELINE_ACTION_MAX_LIMIT;
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PIPELINE_ACTION_MAX_LIMIT_DEFAULT;
-import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type.finalizeNewLayoutVersionCommand;
-import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.toLayoutVersionProto;
+import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.toVersionProto;
 
 import com.google.common.base.Preconditions;
 import com.google.protobuf.Descriptors;
@@ -129,13 +128,13 @@ public EndpointStateMachine.EndPointStates call() throws Exception {
     try {
       Preconditions.checkState(this.datanodeDetailsProto != null);
 
-      LayoutVersionProto layoutinfo = toLayoutVersionProto(
-          versionManager.getApparentVersion().serialize(),
-          versionManager.getSoftwareVersion().serialize());
+      LayoutVersionProto versionInfo = toVersionProto(
+          versionManager.getApparentVersion(),
+          versionManager.getSoftwareVersion());
 
       requestBuilder = SCMHeartbeatRequestProto.newBuilder()
           .setDatanodeDetails(datanodeDetailsProto)
-          .setDataNodeLayoutVersion(layoutinfo);
+          .setDataNodeLayoutVersion(versionInfo);
       addReports(requestBuilder);
       addContainerActions(requestBuilder);
       addPipelineActions(requestBuilder);

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DatanodeVersionManager.java

@@ -21,18 +21,20 @@
 import java.io.IOException;
 import java.util.Map;
 import org.apache.hadoop.hdds.ComponentVersion;
+import org.apache.hadoop.hdds.HDDSVersion;
 import org.apache.hadoop.hdds.upgrade.DatanodeUpgradeAction;
 import org.apache.hadoop.hdds.upgrade.DatanodeUpgradeActionProvider;
-import org.apache.hadoop.hdds.upgrade.HDDSVersionManager;
+import org.apache.hadoop.hdds.upgrade.HDDSVersionUtils;
 import org.apache.hadoop.ozone.container.common.DatanodeStorage;
 import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
 import org.apache.hadoop.ozone.upgrade.ComponentUpgradeActionProvider;
+import org.apache.hadoop.ozone.upgrade.ComponentVersionManager;
 import org.apache.hadoop.ozone.upgrade.UpgradeException;
 
 /**
  * Datanode-specific version manager that wires upgrade actions internally.
  */
-public class DatanodeVersionManager extends HDDSVersionManager {
+public class DatanodeVersionManager extends ComponentVersionManager {
 
   private final Map<ComponentVersion, DatanodeUpgradeAction> upgradeActions;
   private final DatanodeStateMachine upgradeActionArg;
@@ -44,7 +46,9 @@ public DatanodeVersionManager(DatanodeStorage storage, DatanodeStateMachine upgr
   @VisibleForTesting
   public DatanodeVersionManager(DatanodeStorage storage, DatanodeStateMachine upgradeActionArg,
       ComponentUpgradeActionProvider<DatanodeUpgradeAction> upgradeActionProvider) throws IOException {
-    super(storage);
+    super(storage,
+        HDDSVersionUtils.deserializedPersistedApparentVersion(storage.getApparentVersion()),
+        HDDSVersion.SOFTWARE_VERSION);
     this.upgradeActionArg = upgradeActionArg;
     upgradeActions = upgradeActionProvider.load();
   }

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/UpgradeUtils.java

@@ -17,6 +17,7 @@
 
 package org.apache.hadoop.ozone.container.upgrade;
 
+import org.apache.hadoop.hdds.ComponentVersion;
 import org.apache.hadoop.hdds.HDDSVersion;
 import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto;
 
@@ -28,16 +29,17 @@ public final class UpgradeUtils {
   private UpgradeUtils() {
   }
 
-  public static LayoutVersionProto defaultLayoutVersionProto() {
+  public static LayoutVersionProto defaultVersionProto() {
     int softwareVersion = HDDSVersion.SOFTWARE_VERSION.serialize();
     return LayoutVersionProto.newBuilder()
         .setMetadataLayoutVersion(softwareVersion)
         .setSoftwareLayoutVersion(softwareVersion).build();
   }
 
-  public static LayoutVersionProto toLayoutVersionProto(int mLv, int sLv) {
+  public static LayoutVersionProto toVersionProto(ComponentVersion apparentVersion, ComponentVersion softwareVersion) {
     return LayoutVersionProto.newBuilder()
-        .setMetadataLayoutVersion(mLv)
-        .setSoftwareLayoutVersion(sLv).build();
+        .setMetadataLayoutVersion(apparentVersion.serialize())
+        .setSoftwareLayoutVersion(softwareVersion.serialize())
+        .build();
   }
 }

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java

@@ -18,7 +18,7 @@
 package org.apache.hadoop.ozone.protocolPB;
 
 import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.INITIAL_VERSION;
-import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.toLayoutVersionProto;
+import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.toVersionProto;
 
 import com.google.protobuf.RpcController;
 import com.google.protobuf.ServiceException;
@@ -71,17 +71,16 @@ public SCMRegisteredResponseProto register(
         .getContainerReport();
     NodeReportProto dnNodeReport = request.getNodeReport();
     PipelineReportsProto pipelineReport = request.getPipelineReports();
-    LayoutVersionProto layoutInfo = null;
+    LayoutVersionProto versionInfo = null;
     if (request.hasDataNodeLayoutVersion()) {
-      layoutInfo = request.getDataNodeLayoutVersion();
+      versionInfo = request.getDataNodeLayoutVersion();
     } else {
       // Backward compatibility to make sure old Datanodes can still talk to
       // SCM.
-      layoutInfo = toLayoutVersionProto(INITIAL_VERSION.layoutVersion(),
-          INITIAL_VERSION.layoutVersion());
+      versionInfo = toVersionProto(INITIAL_VERSION, INITIAL_VERSION);
     }
     return impl.register(request.getExtendedDatanodeDetails(), dnNodeReport,
-        containerRequestProto, pipelineReport, layoutInfo);
+        containerRequestProto, pipelineReport, versionInfo);
   }
 
   @Override

hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeVersionManager.java

@@ -84,7 +84,7 @@ class TestDatanodeVersionManager extends AbstractComponentVersionManagerTest {
 
     for (HDDSVersion version : HDDSVersion.values()) {
       // Add all defined versions after and including ZDU to get the complete version list.
-      if (HDDSVersion.ZDU.isSupportedBy(version) && version != HDDSVersion.FUTURE_VERSION) {
+      if (HDDSVersion.ZDU.isSupportedBy(version) && version != HDDSVersion.UNKNOWN_VERSION) {
         ALL_VERSIONS.add(version);
       }
     }

hadoop-hdds/docs/content/design/upgrade-dev-primer.md

@@ -40,7 +40,9 @@ Class to add  a new layout feature being brought in. Layout version is typically
 
 **Ozone Manager** uses [`org.apache.hadoop.ozone.om.upgrade.OMVersionManager`](https://github.com/apache/ozone/blob/master/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMVersionManager.java) ([`org.apache.hadoop.ozone.upgrade.ComponentVersionManager`](https://github.com/apache/ozone/blob/master/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentVersionManager.java)), with upgrade actions discovered via [`org.apache.hadoop.ozone.om.upgrade.OMUpgradeActionProvider`](https://github.com/apache/ozone/blob/master/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeActionProvider.java). It exposes apparent/software `ComponentVersion` and `isAllowed(ComponentVersion)` for gating.
 
-**SCM / DataNode** continue to use [`org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager`](https://github.com/apache/ozone/blob/master/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutVersionManager.java) ([`org.apache.hadoop.ozone.upgrade.AbstractLayoutVersionManager`](https://github.com/apache/ozone/blob/master/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/AbstractLayoutVersionManager.java)), which provides metadata/software layout integers and `isAllowed(LayoutFeature)`.
+**SCM** uses [`org.apache.hadoop.hdds.scm.server.upgrade.ScmVersionManager`](https://github.com/apache/ozone/blob/master/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/ScmVersionManager.java) ([`org.apache.hadoop.ozone.upgrade.ComponentVersionManager`](https://github.com/apache/ozone/blob/master/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentVersionManager.java)), with upgrade actions via [`org.apache.hadoop.hdds.upgrade.ScmUpgradeActionProvider`](https://github.com/apache/ozone/blob/master/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/upgrade/ScmUpgradeActionProvider.java). Cluster-wide SCM finalization uses [`org.apache.hadoop.hdds.scm.server.upgrade.FinalizationManager`](https://github.com/apache/ozone/blob/master/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationManager.java).
+
+**DataNode** uses [`org.apache.hadoop.ozone.container.upgrade.DatanodeVersionManager`](https://github.com/apache/ozone/blob/master/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DatanodeVersionManager.java) with upgrade actions via [`org.apache.hadoop.ozone.container.upgrade.DatanodeUpgradeActionProvider`](https://github.com/apache/ozone/blob/master/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DatanodeUpgradeActionProvider.java). Both SCM and DataNode expose apparent/software `ComponentVersion` and `isAllowed(ComponentVersion)` for gating (including legacy [`HDDSLayoutFeature`](https://github.com/apache/ozone/blob/master/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java) checks where still used).
 
 ## @DisallowedUntilLayoutVersion Annotation
 Method level annotation used to "disallow" an API if current layout version does not include the associated layout feature. Currently it is added only to the OM module, but can easily be moved down to a common module based on need on the HDDS layer.

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java

@@ -21,13 +21,11 @@
 import static java.util.stream.Collectors.toMap;
 
 import java.util.Arrays;
-import java.util.Optional;
 import java.util.SortedMap;
 import java.util.TreeMap;
 import org.apache.hadoop.hdds.ComponentVersion;
 import org.apache.hadoop.hdds.HDDSVersion;
 import org.apache.hadoop.ozone.upgrade.LayoutFeature;
-import org.apache.hadoop.ozone.upgrade.UpgradeAction;
 
 /**
  * List of HDDS Layout Features. All version management has been migrated to {@link HDDSVersion} and no new additions
@@ -65,38 +63,12 @@ public enum HDDSLayoutFeature implements LayoutFeature {
 
   private final int layoutVersion;
   private final String description;
-  private UpgradeAction<?> scmAction;
-  private UpgradeAction<?> datanodeAction;
 
   HDDSLayoutFeature(final int layoutVersion, String description) {
     this.layoutVersion = layoutVersion;
     this.description = description;
   }
 
-  /**
-   * Associates an SCM upgrade action with this feature. Only the first upgrade action registered will be used.
-   *
-   * @param action The upgrade action to associate with this feature.
-   */
-  public void addScmAction(UpgradeAction<?> action) {
-    // Required by SpotBugs since this setter exists in an enum.
-    if (this.scmAction == null) {
-      this.scmAction = action;
-    }
-  }
-
-  /**
-   * Associates a Datanode upgrade action with this feature. Only the first upgrade action registered will be used.
-   *
-   * @param action The upgrade action to associate with this feature.
-   */
-  public void addDatanodeAction(UpgradeAction<?> action) {
-    // Required by SpotBugs since this setter exists in an enum.
-    if (this.datanodeAction == null) {
-      this.datanodeAction = action;
-    }
-  }
-
   @Override
   public int layoutVersion() {
     return layoutVersion;
@@ -135,12 +107,4 @@ public static HDDSLayoutFeature deserialize(int version) {
   public String toString() {
     return name() + " (" + serialize() + ")";
   }
-
-  public Optional<UpgradeAction<?>> scmAction() {
-    return Optional.ofNullable(scmAction);
-  }
-
-  public Optional<UpgradeAction<?>> datanodeAction() {
-    return Optional.ofNullable(datanodeAction);
-  }
 }