HDDS-15377. Switch SCM to use new ScmVersionManager

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HDDSVersion.java

@@ -41,8 +41,8 @@ public enum HDDSVersion implements ComponentVersion {
 
   ZDU(100, "Version that supports zero downtime upgrade"),
 
-  FUTURE_VERSION(-1, "Used internally in the client when the server side is "
-      + " newer and an unknown server version has arrived to the client.");
+  UNKNOWN_VERSION(-1, "Used when a version cannot be deserialized to any version recognized by this" +
+      " component, which may indicate it came from a component in a newer version");
 
   //////////////////////////////  //////////////////////////////
 
@@ -85,11 +85,11 @@ public int serialize() {
 
   /**
    * @param value The serialized version to convert.
-   * @return The version corresponding to this serialized value, or {@link #FUTURE_VERSION} if no matching version is
+   * @return The version corresponding to this serialized value, or {@link #UNKNOWN_VERSION} if no matching version is
    *    found.
    */
   public static HDDSVersion deserialize(int value) {
-    return BY_VALUE.getOrDefault(value, FUTURE_VERSION);
+    return BY_VALUE.getOrDefault(value, UNKNOWN_VERSION);
   }
 
   @Override

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java

@@ -42,7 +42,7 @@ public enum ClientVersion implements ComponentVersion {
       "This client version has support for Object Store and File " +
           "System Optimized Bucket Layouts."),
 
-  FUTURE_VERSION(-1, "Used internally when the server side is older and an"
+  FUTURE_VERSION(-1, "Used internally by the server when the server side is older and an"
       + " unknown client version has arrived from the client.");
 
   private static final SortedMap<Integer, ClientVersion> BY_VALUE =

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java

@@ -64,8 +64,8 @@ public enum OzoneManagerVersion implements ComponentVersion {
 
   ZDU(100, "OzoneManager version that supports zero downtime upgrade"),
 
-  FUTURE_VERSION(-1, "Used internally in the client when the server side is "
-      + " newer and an unknown server version has arrived to the client.");
+  UNKNOWN_VERSION(-1, "Used when a version cannot be deserialized to any version recognized by this" +
+      " component, which may indicate it came from a component in a newer version");
 
   private static final SortedMap<Integer, OzoneManagerVersion> BY_VALUE =
       Arrays.stream(values())
@@ -93,11 +93,11 @@ public int serialize() {
 
   /**
    * @param value The serialized version to convert.
-   * @return The version corresponding to this serialized value, or {@link #FUTURE_VERSION} if no matching version is
+   * @return The version corresponding to this serialized value, or {@link #UNKNOWN_VERSION} if no matching version is
    *    found.
    */
   public static OzoneManagerVersion deserialize(int value) {
-    return BY_VALUE.getOrDefault(value, FUTURE_VERSION);
+    return BY_VALUE.getOrDefault(value, UNKNOWN_VERSION);
   }
 
 

hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHDDSVersion.java

@@ -38,7 +38,7 @@ protected ComponentVersion getDefaultVersion() {
 
   @Override
   protected ComponentVersion getFutureVersion() {
-    return HDDSVersion.FUTURE_VERSION;
+    return HDDSVersion.UNKNOWN_VERSION;
   }
 
   @Override

hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestOzoneManagerVersion.java

@@ -39,7 +39,7 @@ protected ComponentVersion getDefaultVersion() {
 
   @Override
   protected ComponentVersion getFutureVersion() {
-    return OzoneManagerVersion.FUTURE_VERSION;
+    return OzoneManagerVersion.UNKNOWN_VERSION;
   }
 
   @Override

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DatanodeStorage.java

@@ -17,12 +17,12 @@
 
 package org.apache.hadoop.ozone.container.common;
 
-import static org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager.maxLayoutVersion;
 import static org.apache.hadoop.ozone.OzoneConsts.DATANODE_LAYOUT_VERSION_DIR;
 
 import java.io.File;
 import java.io.IOException;
 import java.util.Properties;
+import org.apache.hadoop.hdds.HDDSVersion;
 import org.apache.hadoop.hdds.conf.ConfigurationSource;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType;
@@ -43,7 +43,7 @@ public class DatanodeStorage extends Storage {
   public DatanodeStorage(ConfigurationSource conf, String dataNodeId)
       throws IOException {
     super(NodeType.DATANODE, ServerUtils.getOzoneMetaDirPath(conf),
-        DATANODE_LAYOUT_VERSION_DIR, dataNodeId, getDefaultLayoutVersion(conf));
+        DATANODE_LAYOUT_VERSION_DIR, dataNodeId, getDefaultApparentVersion(conf));
   }
 
   public DatanodeStorage(OzoneConfiguration conf, String dataNodeId,
@@ -56,7 +56,7 @@ public DatanodeStorage(OzoneConfiguration conf, String dataNodeId,
   public DatanodeStorage(ConfigurationSource conf)
       throws IOException {
     super(NodeType.DATANODE, ServerUtils.getOzoneMetaDirPath(conf),
-        DATANODE_LAYOUT_VERSION_DIR, getDefaultLayoutVersion(conf));
+        DATANODE_LAYOUT_VERSION_DIR, getDefaultApparentVersion(conf));
   }
 
   @Override
@@ -94,15 +94,14 @@ public void setClusterId(String clusterId) throws IOException {
    * @return The layout version that should be used for the datanode if no
    * layout version is found on disk.
    */
-  private static int getDefaultLayoutVersion(ConfigurationSource conf) {
-    int defaultLayoutVersion = maxLayoutVersion();
+  private static int getDefaultApparentVersion(ConfigurationSource conf) {
+    int defaultApparentVersion = HDDSVersion.SOFTWARE_VERSION.serialize();
 
     File dnIdFile = new File(HddsServerUtil.getDatanodeIdFilePath(conf));
     if (dnIdFile.exists()) {
-      defaultLayoutVersion =
-          HDDSLayoutFeature.INITIAL_VERSION.layoutVersion();
+      defaultApparentVersion = HDDSLayoutFeature.INITIAL_VERSION.layoutVersion();
     }
 
-    return defaultLayoutVersion;
+    return defaultApparentVersion;
   }
 }

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java

@@ -21,8 +21,7 @@
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_ACTION_MAX_LIMIT_DEFAULT;
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PIPELINE_ACTION_MAX_LIMIT;
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PIPELINE_ACTION_MAX_LIMIT_DEFAULT;
-import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type.finalizeNewLayoutVersionCommand;
-import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.toLayoutVersionProto;
+import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.toVersionProto;
 
 import com.google.common.base.Preconditions;
 import com.google.protobuf.Descriptors;
@@ -129,13 +128,13 @@ public EndpointStateMachine.EndPointStates call() throws Exception {
     try {
       Preconditions.checkState(this.datanodeDetailsProto != null);
 
-      LayoutVersionProto layoutinfo = toLayoutVersionProto(
-          versionManager.getApparentVersion().serialize(),
-          versionManager.getSoftwareVersion().serialize());
+      LayoutVersionProto versionInfo = toVersionProto(
+          versionManager.getApparentVersion(),
+          versionManager.getSoftwareVersion());
 
       requestBuilder = SCMHeartbeatRequestProto.newBuilder()
           .setDatanodeDetails(datanodeDetailsProto)
-          .setDataNodeLayoutVersion(layoutinfo);
+          .setDataNodeLayoutVersion(versionInfo);
       addReports(requestBuilder);
       addContainerActions(requestBuilder);
       addPipelineActions(requestBuilder);

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DatanodeVersionManager.java

@@ -21,18 +21,20 @@
 import java.io.IOException;
 import java.util.Map;
 import org.apache.hadoop.hdds.ComponentVersion;
+import org.apache.hadoop.hdds.HDDSVersion;
 import org.apache.hadoop.hdds.upgrade.DatanodeUpgradeAction;
 import org.apache.hadoop.hdds.upgrade.DatanodeUpgradeActionProvider;
-import org.apache.hadoop.hdds.upgrade.HDDSVersionManager;
+import org.apache.hadoop.hdds.upgrade.HDDSVersionUtils;
 import org.apache.hadoop.ozone.container.common.DatanodeStorage;
 import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
 import org.apache.hadoop.ozone.upgrade.ComponentUpgradeActionProvider;
+import org.apache.hadoop.ozone.upgrade.ComponentVersionManager;
 import org.apache.hadoop.ozone.upgrade.UpgradeException;
 
 /**
  * Datanode-specific version manager that wires upgrade actions internally.
  */
-public class DatanodeVersionManager extends HDDSVersionManager {
+public class DatanodeVersionManager extends ComponentVersionManager {
 
   private final Map<ComponentVersion, DatanodeUpgradeAction> upgradeActions;
   private final DatanodeStateMachine upgradeActionArg;
@@ -44,7 +46,9 @@ public DatanodeVersionManager(DatanodeStorage storage, DatanodeStateMachine upgr
   @VisibleForTesting
   public DatanodeVersionManager(DatanodeStorage storage, DatanodeStateMachine upgradeActionArg,
       ComponentUpgradeActionProvider<DatanodeUpgradeAction> upgradeActionProvider) throws IOException {
-    super(storage);
+    super(storage,
+        HDDSVersionUtils.deserializedPersistedApparentVersion(storage.getApparentVersion()),
+        HDDSVersion.SOFTWARE_VERSION);
     this.upgradeActionArg = upgradeActionArg;
     upgradeActions = upgradeActionProvider.load();
   }

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/UpgradeUtils.java

@@ -17,6 +17,7 @@
 
 package org.apache.hadoop.ozone.container.upgrade;
 
+import org.apache.hadoop.hdds.ComponentVersion;
 import org.apache.hadoop.hdds.HDDSVersion;
 import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto;
 
@@ -28,16 +29,17 @@ public final class UpgradeUtils {
   private UpgradeUtils() {
   }
 
-  public static LayoutVersionProto defaultLayoutVersionProto() {
+  public static LayoutVersionProto defaultVersionProto() {
     int softwareVersion = HDDSVersion.SOFTWARE_VERSION.serialize();
     return LayoutVersionProto.newBuilder()
         .setMetadataLayoutVersion(softwareVersion)
         .setSoftwareLayoutVersion(softwareVersion).build();
   }
 
-  public static LayoutVersionProto toLayoutVersionProto(int mLv, int sLv) {
+  public static LayoutVersionProto toVersionProto(ComponentVersion apparentVersion, ComponentVersion softwareVersion) {
     return LayoutVersionProto.newBuilder()
-        .setMetadataLayoutVersion(mLv)
-        .setSoftwareLayoutVersion(sLv).build();
+        .setMetadataLayoutVersion(apparentVersion.serialize())
+        .setSoftwareLayoutVersion(softwareVersion.serialize())
+        .build();
   }
 }

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java

@@ -18,7 +18,7 @@
 package org.apache.hadoop.ozone.protocolPB;
 
 import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.INITIAL_VERSION;
-import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.toLayoutVersionProto;
+import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.toVersionProto;
 
 import com.google.protobuf.RpcController;
 import com.google.protobuf.ServiceException;
@@ -71,17 +71,16 @@ public SCMRegisteredResponseProto register(
         .getContainerReport();
     NodeReportProto dnNodeReport = request.getNodeReport();
     PipelineReportsProto pipelineReport = request.getPipelineReports();
-    LayoutVersionProto layoutInfo = null;
+    LayoutVersionProto versionInfo = null;
     if (request.hasDataNodeLayoutVersion()) {
-      layoutInfo = request.getDataNodeLayoutVersion();
+      versionInfo = request.getDataNodeLayoutVersion();
     } else {
       // Backward compatibility to make sure old Datanodes can still talk to
       // SCM.
-      layoutInfo = toLayoutVersionProto(INITIAL_VERSION.layoutVersion(),
-          INITIAL_VERSION.layoutVersion());
+      versionInfo = toVersionProto(INITIAL_VERSION, INITIAL_VERSION);
     }
     return impl.register(request.getExtendedDatanodeDetails(), dnNodeReport,
-        containerRequestProto, pipelineReport, layoutInfo);
+        containerRequestProto, pipelineReport, versionInfo);
   }
 
   @Override

hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeVersionManager.java

@@ -84,7 +84,7 @@ class TestDatanodeVersionManager extends AbstractComponentVersionManagerTest {
 
     for (HDDSVersion version : HDDSVersion.values()) {
       // Add all defined versions after and including ZDU to get the complete version list.
-      if (HDDSVersion.ZDU.isSupportedBy(version) && version != HDDSVersion.FUTURE_VERSION) {
+      if (HDDSVersion.ZDU.isSupportedBy(version) && version != HDDSVersion.UNKNOWN_VERSION) {
         ALL_VERSIONS.add(version);
       }
     }

hadoop-hdds/docs/content/design/upgrade-dev-primer.md

@@ -40,7 +40,9 @@ Class to add  a new layout feature being brought in. Layout version is typically
 
 **Ozone Manager** uses [`org.apache.hadoop.ozone.om.upgrade.OMVersionManager`](https://github.com/apache/ozone/blob/master/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMVersionManager.java) ([`org.apache.hadoop.ozone.upgrade.ComponentVersionManager`](https://github.com/apache/ozone/blob/master/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentVersionManager.java)), with upgrade actions discovered via [`org.apache.hadoop.ozone.om.upgrade.OMUpgradeActionProvider`](https://github.com/apache/ozone/blob/master/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeActionProvider.java). It exposes apparent/software `ComponentVersion` and `isAllowed(ComponentVersion)` for gating.
 
-**SCM / DataNode** continue to use [`org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager`](https://github.com/apache/ozone/blob/master/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutVersionManager.java) ([`org.apache.hadoop.ozone.upgrade.AbstractLayoutVersionManager`](https://github.com/apache/ozone/blob/master/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/AbstractLayoutVersionManager.java)), which provides metadata/software layout integers and `isAllowed(LayoutFeature)`.
+**SCM** uses [`org.apache.hadoop.hdds.scm.server.upgrade.ScmVersionManager`](https://github.com/apache/ozone/blob/master/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/ScmVersionManager.java) ([`org.apache.hadoop.ozone.upgrade.ComponentVersionManager`](https://github.com/apache/ozone/blob/master/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentVersionManager.java)), with upgrade actions via [`org.apache.hadoop.hdds.upgrade.ScmUpgradeActionProvider`](https://github.com/apache/ozone/blob/master/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/upgrade/ScmUpgradeActionProvider.java). Cluster-wide SCM finalization uses [`org.apache.hadoop.hdds.scm.server.upgrade.FinalizationManager`](https://github.com/apache/ozone/blob/master/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationManager.java).
+
+**DataNode** uses [`org.apache.hadoop.ozone.container.upgrade.DatanodeVersionManager`](https://github.com/apache/ozone/blob/master/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DatanodeVersionManager.java) with upgrade actions via [`org.apache.hadoop.ozone.container.upgrade.DatanodeUpgradeActionProvider`](https://github.com/apache/ozone/blob/master/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DatanodeUpgradeActionProvider.java). Both SCM and DataNode expose apparent/software `ComponentVersion` and `isAllowed(ComponentVersion)` for gating (including legacy [`HDDSLayoutFeature`](https://github.com/apache/ozone/blob/master/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java) checks where still used).
 
 ## @DisallowedUntilLayoutVersion Annotation
 Method level annotation used to "disallow" an API if current layout version does not include the associated layout feature. Currently it is added only to the OM module, but can easily be moved down to a common module based on need on the HDDS layer.

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSVersionManager.java → hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSVersionUtils.java

@@ -20,43 +20,52 @@
 import java.io.IOException;
 import org.apache.hadoop.hdds.ComponentVersion;
 import org.apache.hadoop.hdds.HDDSVersion;
-import org.apache.hadoop.ozone.common.Storage;
-import org.apache.hadoop.ozone.upgrade.ComponentVersionManager;
-import org.apache.hadoop.ozone.upgrade.UpgradeException;
 
 /**
  * Component version manager for HDDS (Datanodes and SCM).
  */
-public abstract class HDDSVersionManager extends ComponentVersionManager {
-  protected HDDSVersionManager(Storage storage) throws IOException {
-    super(storage, computeApparentVersion(storage.getApparentVersion()), HDDSVersion.SOFTWARE_VERSION);
+public final class HDDSVersionUtils {
+  private HDDSVersionUtils() {
   }
 
   /**
    * If the apparent version stored on the disk is >= {@link HDDSVersion#ZDU} serialized, the apparent version is
-   * resolved via {@link HDDSVersion#deserialize(int)}. Values with no matching {@link HDDSVersion} fail startup with
-   * the persisted integer in the exception message.
+   * resolved via {@link HDDSVersion#deserialize(int)}.
    * If the value is below that threshold, the apparent version is resolved as a {@link HDDSLayoutFeature}. Integers in
-   * the gap between the largest {@link HDDSLayoutFeature} and ZDU are not valid legacy layout values; startup fails
-   * with the persisted integer in the exception message.
+   * the gap between the largest {@link HDDSLayoutFeature} and ZDU are not valid legacy layout values.
+   *
+   * If the serialized version does not match any of these known versions, {@link HDDSVersion#UNKNOWN_VERSION} is
+   * returned.
    */
-  private static ComponentVersion computeApparentVersion(int serializedApparentVersion) throws IOException {
-    if (serializedApparentVersion >= HDDSVersion.ZDU.serialize()) {
-      HDDSVersion fromHdds = HDDSVersion.deserialize(serializedApparentVersion);
-      if (fromHdds != HDDSVersion.FUTURE_VERSION) {
-        return fromHdds;
-      }
+  public static ComponentVersion deserializeHDDSVersionOrLayoutVersion(int serializedVersion) {
+    if (serializedVersion >= HDDSVersion.ZDU.serialize()) {
+      return HDDSVersion.deserialize(serializedVersion);
     } else {
-      ComponentVersion fromLayout = HDDSLayoutFeature.deserialize(serializedApparentVersion);
+      ComponentVersion fromLayout = HDDSLayoutFeature.deserialize(serializedVersion);
       if (fromLayout != null) {
         return fromLayout;
+      } else {
+        return HDDSVersion.UNKNOWN_VERSION;
       }
     }
-    throw new IOException("Initialization failed. Disk contains unknown apparent version " + serializedApparentVersion +
-        " for software version " + HDDSVersion.SOFTWARE_VERSION + ". Make sure this component was not downgraded" +
-        " after finalization");
   }
 
-  @Override
-  protected abstract void runUpgradeAction(ComponentVersion version) throws UpgradeException;
+  /**
+   * If the apparent version stored on the disk is >= {@link HDDSVersion#ZDU} serialized, the apparent version is
+   * resolved via {@link HDDSVersion#deserialize(int)}. Values with no matching {@link HDDSVersion} fail startup with
+   * the persisted integer in the exception message.
+   * If the value is below that threshold, the apparent version is resolved as a {@link HDDSLayoutFeature}. Integers in
+   * the gap between the largest {@link HDDSLayoutFeature} and ZDU are not valid legacy layout values; startup fails
+   * with the persisted integer in the exception message.
+   */
+  public static ComponentVersion deserializedPersistedApparentVersion(int serializedApparentVersion)
+      throws IOException {
+    ComponentVersion persistedVersion = deserializeHDDSVersionOrLayoutVersion(serializedApparentVersion);
+    if (persistedVersion == HDDSVersion.UNKNOWN_VERSION) {
+      throw new IOException("Initialization failed. Disk contains unknown apparent version "
+          + serializedApparentVersion + " for software version " + HDDSVersion.SOFTWARE_VERSION
+          + ". Make sure this component was not downgraded after finalization");
+    }
+    return persistedVersion;
+  }
 }