diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HDDSVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HDDSVersion.java index 51d4229a7484..345f76a7aa65 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HDDSVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HDDSVersion.java @@ -41,8 +41,8 @@ public enum HDDSVersion implements ComponentVersion { ZDU(100, "Version that supports zero downtime upgrade"), - FUTURE_VERSION(-1, "Used internally in the client when the server side is " - + " newer and an unknown server version has arrived to the client."); + UNKNOWN_VERSION(-1, "Used when a version cannot be deserialized to any version recognized by this" + + " component, which may indicate it came from a component in a newer version"); ////////////////////////////// ////////////////////////////// @@ -85,11 +85,11 @@ public int serialize() { /** * @param value The serialized version to convert. - * @return The version corresponding to this serialized value, or {@link #FUTURE_VERSION} if no matching version is + * @return The version corresponding to this serialized value, or {@link #UNKNOWN_VERSION} if no matching version is * found. */ public static HDDSVersion deserialize(int value) { - return BY_VALUE.getOrDefault(value, FUTURE_VERSION); + return BY_VALUE.getOrDefault(value, UNKNOWN_VERSION); } @Override diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java index dd451ab052cb..36dbbc74e0d8 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java @@ -42,7 +42,7 @@ public enum ClientVersion implements ComponentVersion { "This client version has support for Object Store and File " + "System Optimized Bucket Layouts."), - FUTURE_VERSION(-1, "Used internally when the server side is older and an" + FUTURE_VERSION(-1, "Used internally by the server when the server side is older and an" + " unknown client version has arrived from the client."); private static final SortedMap BY_VALUE = diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java index 8e91d72d7981..55ec6b77f7a0 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java @@ -64,8 +64,8 @@ public enum OzoneManagerVersion implements ComponentVersion { ZDU(100, "OzoneManager version that supports zero downtime upgrade"), - FUTURE_VERSION(-1, "Used internally in the client when the server side is " - + " newer and an unknown server version has arrived to the client."); + UNKNOWN_VERSION(-1, "Used when a version cannot be deserialized to any version recognized by this" + + " component, which may indicate it came from a component in a newer version"); private static final SortedMap BY_VALUE = Arrays.stream(values()) @@ -93,11 +93,11 @@ public int serialize() { /** * @param value The serialized version to convert. - * @return The version corresponding to this serialized value, or {@link #FUTURE_VERSION} if no matching version is + * @return The version corresponding to this serialized value, or {@link #UNKNOWN_VERSION} if no matching version is * found. */ public static OzoneManagerVersion deserialize(int value) { - return BY_VALUE.getOrDefault(value, FUTURE_VERSION); + return BY_VALUE.getOrDefault(value, UNKNOWN_VERSION); } diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHDDSVersion.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHDDSVersion.java index 89d24d3c82ea..d5bb06571be2 100644 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHDDSVersion.java +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHDDSVersion.java @@ -38,7 +38,7 @@ protected ComponentVersion getDefaultVersion() { @Override protected ComponentVersion getFutureVersion() { - return HDDSVersion.FUTURE_VERSION; + return HDDSVersion.UNKNOWN_VERSION; } @Override diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestOzoneManagerVersion.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestOzoneManagerVersion.java index bb94ac938fde..cb5bd94198c9 100644 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestOzoneManagerVersion.java +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestOzoneManagerVersion.java @@ -39,7 +39,7 @@ protected ComponentVersion getDefaultVersion() { @Override protected ComponentVersion getFutureVersion() { - return OzoneManagerVersion.FUTURE_VERSION; + return OzoneManagerVersion.UNKNOWN_VERSION; } @Override diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DatanodeStorage.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DatanodeStorage.java index 1b1be8df8b60..fe2677834378 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DatanodeStorage.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DatanodeStorage.java @@ -17,12 +17,12 @@ package org.apache.hadoop.ozone.container.common; -import static org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager.maxLayoutVersion; import static org.apache.hadoop.ozone.OzoneConsts.DATANODE_LAYOUT_VERSION_DIR; import java.io.File; import java.io.IOException; import java.util.Properties; +import org.apache.hadoop.hdds.HDDSVersion; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType; @@ -43,7 +43,7 @@ public class DatanodeStorage extends Storage { public DatanodeStorage(ConfigurationSource conf, String dataNodeId) throws IOException { super(NodeType.DATANODE, ServerUtils.getOzoneMetaDirPath(conf), - DATANODE_LAYOUT_VERSION_DIR, dataNodeId, getDefaultLayoutVersion(conf)); + DATANODE_LAYOUT_VERSION_DIR, dataNodeId, getDefaultApparentVersion(conf)); } public DatanodeStorage(OzoneConfiguration conf, String dataNodeId, @@ -56,7 +56,7 @@ public DatanodeStorage(OzoneConfiguration conf, String dataNodeId, public DatanodeStorage(ConfigurationSource conf) throws IOException { super(NodeType.DATANODE, ServerUtils.getOzoneMetaDirPath(conf), - DATANODE_LAYOUT_VERSION_DIR, getDefaultLayoutVersion(conf)); + DATANODE_LAYOUT_VERSION_DIR, getDefaultApparentVersion(conf)); } @Override @@ -94,15 +94,14 @@ public void setClusterId(String clusterId) throws IOException { * @return The layout version that should be used for the datanode if no * layout version is found on disk. */ - private static int getDefaultLayoutVersion(ConfigurationSource conf) { - int defaultLayoutVersion = maxLayoutVersion(); + private static int getDefaultApparentVersion(ConfigurationSource conf) { + int defaultApparentVersion = HDDSVersion.SOFTWARE_VERSION.serialize(); File dnIdFile = new File(HddsServerUtil.getDatanodeIdFilePath(conf)); if (dnIdFile.exists()) { - defaultLayoutVersion = - HDDSLayoutFeature.INITIAL_VERSION.layoutVersion(); + defaultApparentVersion = HDDSLayoutFeature.INITIAL_VERSION.layoutVersion(); } - return defaultLayoutVersion; + return defaultApparentVersion; } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java index c40fd317360d..e35192daa098 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java @@ -21,8 +21,7 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_ACTION_MAX_LIMIT_DEFAULT; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PIPELINE_ACTION_MAX_LIMIT; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PIPELINE_ACTION_MAX_LIMIT_DEFAULT; -import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type.finalizeNewLayoutVersionCommand; -import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.toLayoutVersionProto; +import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.toVersionProto; import com.google.common.base.Preconditions; import com.google.protobuf.Descriptors; @@ -129,13 +128,13 @@ public EndpointStateMachine.EndPointStates call() throws Exception { try { Preconditions.checkState(this.datanodeDetailsProto != null); - LayoutVersionProto layoutinfo = toLayoutVersionProto( - versionManager.getApparentVersion().serialize(), - versionManager.getSoftwareVersion().serialize()); + LayoutVersionProto versionInfo = toVersionProto( + versionManager.getApparentVersion(), + versionManager.getSoftwareVersion()); requestBuilder = SCMHeartbeatRequestProto.newBuilder() .setDatanodeDetails(datanodeDetailsProto) - .setDataNodeLayoutVersion(layoutinfo); + .setDataNodeLayoutVersion(versionInfo); addReports(requestBuilder); addContainerActions(requestBuilder); addPipelineActions(requestBuilder); diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DatanodeVersionManager.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DatanodeVersionManager.java index 17f92c19bcbf..ec763575c227 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DatanodeVersionManager.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DatanodeVersionManager.java @@ -21,18 +21,20 @@ import java.io.IOException; import java.util.Map; import org.apache.hadoop.hdds.ComponentVersion; +import org.apache.hadoop.hdds.HDDSVersion; import org.apache.hadoop.hdds.upgrade.DatanodeUpgradeAction; import org.apache.hadoop.hdds.upgrade.DatanodeUpgradeActionProvider; -import org.apache.hadoop.hdds.upgrade.HDDSVersionManager; +import org.apache.hadoop.hdds.upgrade.HDDSVersionUtils; import org.apache.hadoop.ozone.container.common.DatanodeStorage; import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine; import org.apache.hadoop.ozone.upgrade.ComponentUpgradeActionProvider; +import org.apache.hadoop.ozone.upgrade.ComponentVersionManager; import org.apache.hadoop.ozone.upgrade.UpgradeException; /** * Datanode-specific version manager that wires upgrade actions internally. */ -public class DatanodeVersionManager extends HDDSVersionManager { +public class DatanodeVersionManager extends ComponentVersionManager { private final Map upgradeActions; private final DatanodeStateMachine upgradeActionArg; @@ -44,7 +46,9 @@ public DatanodeVersionManager(DatanodeStorage storage, DatanodeStateMachine upgr @VisibleForTesting public DatanodeVersionManager(DatanodeStorage storage, DatanodeStateMachine upgradeActionArg, ComponentUpgradeActionProvider upgradeActionProvider) throws IOException { - super(storage); + super(storage, + HDDSVersionUtils.deserializedPersistedApparentVersion(storage.getApparentVersion()), + HDDSVersion.SOFTWARE_VERSION); this.upgradeActionArg = upgradeActionArg; upgradeActions = upgradeActionProvider.load(); } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/UpgradeUtils.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/UpgradeUtils.java index d96aab53b5e4..29024891ee41 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/UpgradeUtils.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/UpgradeUtils.java @@ -17,6 +17,7 @@ package org.apache.hadoop.ozone.container.upgrade; +import org.apache.hadoop.hdds.ComponentVersion; import org.apache.hadoop.hdds.HDDSVersion; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto; @@ -28,16 +29,17 @@ public final class UpgradeUtils { private UpgradeUtils() { } - public static LayoutVersionProto defaultLayoutVersionProto() { + public static LayoutVersionProto defaultVersionProto() { int softwareVersion = HDDSVersion.SOFTWARE_VERSION.serialize(); return LayoutVersionProto.newBuilder() .setMetadataLayoutVersion(softwareVersion) .setSoftwareLayoutVersion(softwareVersion).build(); } - public static LayoutVersionProto toLayoutVersionProto(int mLv, int sLv) { + public static LayoutVersionProto toVersionProto(ComponentVersion apparentVersion, ComponentVersion softwareVersion) { return LayoutVersionProto.newBuilder() - .setMetadataLayoutVersion(mLv) - .setSoftwareLayoutVersion(sLv).build(); + .setMetadataLayoutVersion(apparentVersion.serialize()) + .setSoftwareLayoutVersion(softwareVersion.serialize()) + .build(); } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java index 633500f9431c..3f91c277bf79 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java @@ -18,7 +18,7 @@ package org.apache.hadoop.ozone.protocolPB; import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.INITIAL_VERSION; -import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.toLayoutVersionProto; +import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.toVersionProto; import com.google.protobuf.RpcController; import com.google.protobuf.ServiceException; @@ -71,17 +71,16 @@ public SCMRegisteredResponseProto register( .getContainerReport(); NodeReportProto dnNodeReport = request.getNodeReport(); PipelineReportsProto pipelineReport = request.getPipelineReports(); - LayoutVersionProto layoutInfo = null; + LayoutVersionProto versionInfo = null; if (request.hasDataNodeLayoutVersion()) { - layoutInfo = request.getDataNodeLayoutVersion(); + versionInfo = request.getDataNodeLayoutVersion(); } else { // Backward compatibility to make sure old Datanodes can still talk to // SCM. - layoutInfo = toLayoutVersionProto(INITIAL_VERSION.layoutVersion(), - INITIAL_VERSION.layoutVersion()); + versionInfo = toVersionProto(INITIAL_VERSION, INITIAL_VERSION); } return impl.register(request.getExtendedDatanodeDetails(), dnNodeReport, - containerRequestProto, pipelineReport, layoutInfo); + containerRequestProto, pipelineReport, versionInfo); } @Override diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeVersionManager.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeVersionManager.java index 93d33ca9aea2..840fcf2aee0f 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeVersionManager.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeVersionManager.java @@ -84,7 +84,7 @@ class TestDatanodeVersionManager extends AbstractComponentVersionManagerTest { for (HDDSVersion version : HDDSVersion.values()) { // Add all defined versions after and including ZDU to get the complete version list. - if (HDDSVersion.ZDU.isSupportedBy(version) && version != HDDSVersion.FUTURE_VERSION) { + if (HDDSVersion.ZDU.isSupportedBy(version) && version != HDDSVersion.UNKNOWN_VERSION) { ALL_VERSIONS.add(version); } } diff --git a/hadoop-hdds/docs/content/design/upgrade-dev-primer.md b/hadoop-hdds/docs/content/design/upgrade-dev-primer.md index f693c8484c8a..4daf13a097f5 100644 --- a/hadoop-hdds/docs/content/design/upgrade-dev-primer.md +++ b/hadoop-hdds/docs/content/design/upgrade-dev-primer.md @@ -40,7 +40,9 @@ Class to add a new layout feature being brought in. Layout version is typically **Ozone Manager** uses [`org.apache.hadoop.ozone.om.upgrade.OMVersionManager`](https://github.com/apache/ozone/blob/master/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMVersionManager.java) ([`org.apache.hadoop.ozone.upgrade.ComponentVersionManager`](https://github.com/apache/ozone/blob/master/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentVersionManager.java)), with upgrade actions discovered via [`org.apache.hadoop.ozone.om.upgrade.OMUpgradeActionProvider`](https://github.com/apache/ozone/blob/master/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeActionProvider.java). It exposes apparent/software `ComponentVersion` and `isAllowed(ComponentVersion)` for gating. -**SCM / DataNode** continue to use [`org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager`](https://github.com/apache/ozone/blob/master/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutVersionManager.java) ([`org.apache.hadoop.ozone.upgrade.AbstractLayoutVersionManager`](https://github.com/apache/ozone/blob/master/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/AbstractLayoutVersionManager.java)), which provides metadata/software layout integers and `isAllowed(LayoutFeature)`. +**SCM** uses [`org.apache.hadoop.hdds.scm.server.upgrade.ScmVersionManager`](https://github.com/apache/ozone/blob/master/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/ScmVersionManager.java) ([`org.apache.hadoop.ozone.upgrade.ComponentVersionManager`](https://github.com/apache/ozone/blob/master/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentVersionManager.java)), with upgrade actions via [`org.apache.hadoop.hdds.upgrade.ScmUpgradeActionProvider`](https://github.com/apache/ozone/blob/master/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/upgrade/ScmUpgradeActionProvider.java). Cluster-wide SCM finalization uses [`org.apache.hadoop.hdds.scm.server.upgrade.FinalizationManager`](https://github.com/apache/ozone/blob/master/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationManager.java). + +**DataNode** uses [`org.apache.hadoop.ozone.container.upgrade.DatanodeVersionManager`](https://github.com/apache/ozone/blob/master/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DatanodeVersionManager.java) with upgrade actions via [`org.apache.hadoop.ozone.container.upgrade.DatanodeUpgradeActionProvider`](https://github.com/apache/ozone/blob/master/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DatanodeUpgradeActionProvider.java). Both SCM and DataNode expose apparent/software `ComponentVersion` and `isAllowed(ComponentVersion)` for gating (including legacy [`HDDSLayoutFeature`](https://github.com/apache/ozone/blob/master/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java) checks where still used). ## @DisallowedUntilLayoutVersion Annotation Method level annotation used to "disallow" an API if current layout version does not include the associated layout feature. Currently it is added only to the OM module, but can easily be moved down to a common module based on need on the HDDS layer. diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSVersionManager.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSVersionUtils.java similarity index 54% rename from hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSVersionManager.java rename to hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSVersionUtils.java index 9d01346bcd35..fa39b19ecb8c 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSVersionManager.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSVersionUtils.java @@ -20,43 +20,52 @@ import java.io.IOException; import org.apache.hadoop.hdds.ComponentVersion; import org.apache.hadoop.hdds.HDDSVersion; -import org.apache.hadoop.ozone.common.Storage; -import org.apache.hadoop.ozone.upgrade.ComponentVersionManager; -import org.apache.hadoop.ozone.upgrade.UpgradeException; /** * Component version manager for HDDS (Datanodes and SCM). */ -public abstract class HDDSVersionManager extends ComponentVersionManager { - protected HDDSVersionManager(Storage storage) throws IOException { - super(storage, computeApparentVersion(storage.getApparentVersion()), HDDSVersion.SOFTWARE_VERSION); +public final class HDDSVersionUtils { + private HDDSVersionUtils() { } /** * If the apparent version stored on the disk is >= {@link HDDSVersion#ZDU} serialized, the apparent version is - * resolved via {@link HDDSVersion#deserialize(int)}. Values with no matching {@link HDDSVersion} fail startup with - * the persisted integer in the exception message. + * resolved via {@link HDDSVersion#deserialize(int)}. * If the value is below that threshold, the apparent version is resolved as a {@link HDDSLayoutFeature}. Integers in - * the gap between the largest {@link HDDSLayoutFeature} and ZDU are not valid legacy layout values; startup fails - * with the persisted integer in the exception message. + * the gap between the largest {@link HDDSLayoutFeature} and ZDU are not valid legacy layout values. + * + * If the serialized version does not match any of these known versions, {@link HDDSVersion#UNKNOWN_VERSION} is + * returned. */ - private static ComponentVersion computeApparentVersion(int serializedApparentVersion) throws IOException { - if (serializedApparentVersion >= HDDSVersion.ZDU.serialize()) { - HDDSVersion fromHdds = HDDSVersion.deserialize(serializedApparentVersion); - if (fromHdds != HDDSVersion.FUTURE_VERSION) { - return fromHdds; - } + public static ComponentVersion deserializeHDDSVersionOrLayoutVersion(int serializedVersion) { + if (serializedVersion >= HDDSVersion.ZDU.serialize()) { + return HDDSVersion.deserialize(serializedVersion); } else { - ComponentVersion fromLayout = HDDSLayoutFeature.deserialize(serializedApparentVersion); + ComponentVersion fromLayout = HDDSLayoutFeature.deserialize(serializedVersion); if (fromLayout != null) { return fromLayout; + } else { + return HDDSVersion.UNKNOWN_VERSION; } } - throw new IOException("Initialization failed. Disk contains unknown apparent version " + serializedApparentVersion + - " for software version " + HDDSVersion.SOFTWARE_VERSION + ". Make sure this component was not downgraded" + - " after finalization"); } - @Override - protected abstract void runUpgradeAction(ComponentVersion version) throws UpgradeException; + /** + * If the apparent version stored on the disk is >= {@link HDDSVersion#ZDU} serialized, the apparent version is + * resolved via {@link HDDSVersion#deserialize(int)}. Values with no matching {@link HDDSVersion} fail startup with + * the persisted integer in the exception message. + * If the value is below that threshold, the apparent version is resolved as a {@link HDDSLayoutFeature}. Integers in + * the gap between the largest {@link HDDSLayoutFeature} and ZDU are not valid legacy layout values; startup fails + * with the persisted integer in the exception message. + */ + public static ComponentVersion deserializedPersistedApparentVersion(int serializedApparentVersion) + throws IOException { + ComponentVersion persistedVersion = deserializeHDDSVersionOrLayoutVersion(serializedApparentVersion); + if (persistedVersion == HDDSVersion.UNKNOWN_VERSION) { + throw new IOException("Initialization failed. Disk contains unknown apparent version " + + serializedApparentVersion + " for software version " + HDDSVersion.SOFTWARE_VERSION + + ". Make sure this component was not downgraded after finalization"); + } + return persistedVersion; + } } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/RatisBasedVersionManager.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/RatisBasedVersionManager.java new file mode 100644 index 000000000000..74ba51a09da6 --- /dev/null +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/RatisBasedVersionManager.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.upgrade; + +import static org.apache.hadoop.ozone.OzoneConsts.APPARENT_VERSION_KEY; + +import java.io.IOException; +import org.apache.hadoop.hdds.ComponentVersion; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.ozone.common.Storage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Base version manager implementation for ratis-backed component versions. + */ +public abstract class RatisBasedVersionManager extends ComponentVersionManager { + + private static final Logger LOG = LoggerFactory.getLogger(RatisBasedVersionManager.class); + + protected RatisBasedVersionManager(Storage storage, ComponentVersion apparentVersion, + ComponentVersion softwareVersion) { + super(storage, apparentVersion, softwareVersion); + } + + public void validateDBVersion(Table finalizationStore) throws IOException { + ComponentVersion dbVersion = getApparentVersionInDB(finalizationStore); + ComponentVersion apparentVersion = getApparentVersion(); + + if (!apparentVersion.equals(dbVersion)) { + LOG.info("Version file has different apparent version ({}) than DB ({}). That is expected if this " + + "component has never been finalized to a newer version.", apparentVersion, dbVersion); + } + } + + public void finalizeFromSnapshotIfRequired(Table finalizationStore) throws IOException { + ComponentVersion apparentVersionInNewDB = getApparentVersionInDB(finalizationStore); + if (apparentVersionInNewDB != null && !isAllowed(apparentVersionInNewDB)) { + LOG.info("New snapshot received with higher apparent version {}. Attempting to finalize to that version.", + apparentVersionInNewDB); + finalizeUpgrade(); + // Update the apparent version in the DB to match the VERSION file. + // When finalization is not done with a snapshot, this DB value is updated by OMFinalizeUpgradeRequest. + finalizationStore.put(APPARENT_VERSION_KEY, String.valueOf(getApparentVersion().serialize())); + } + } + + protected abstract ComponentVersion computeApparentVersion(int serializedVersion) throws IOException; + + private ComponentVersion getApparentVersionInDB(Table finalizationStore) throws IOException { + String apparentVersion = finalizationStore.get(APPARENT_VERSION_KEY); + return (apparentVersion == null) ? null : computeApparentVersion(Integer.parseInt(apparentVersion)); + } +} diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSLayoutFeature.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSLayoutFeature.java index fbb89220e976..e1853c0b4bbc 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSLayoutFeature.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSLayoutFeature.java @@ -101,7 +101,7 @@ public void testIsSupportedByFeatureBoundary() { public void testAllLayoutFeaturesAreSupportedByFutureVersions() { for (HDDSLayoutFeature feature : HDDSLayoutFeature.values()) { assertSupportedBy(feature, HDDSVersion.ZDU); - assertSupportedBy(feature, HDDSVersion.FUTURE_VERSION); + assertSupportedBy(feature, HDDSVersion.UNKNOWN_VERSION); // No ComponentVersion instance represents an arbitrary future version. assertTrue(feature.isSupportedBy(Integer.MAX_VALUE)); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeInfo.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeInfo.java index cb4cb1db6967..54e996ec3bf2 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeInfo.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeInfo.java @@ -17,8 +17,6 @@ package org.apache.hadoop.hdds.scm.node; -import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.toLayoutVersionProto; - import com.google.common.annotations.VisibleForTesting; import java.util.Collections; import java.util.HashMap; @@ -26,6 +24,7 @@ import java.util.Map; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import org.apache.hadoop.hdds.ComponentVersion; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CommandQueueReportProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto; @@ -33,6 +32,7 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageReportProto; import org.apache.hadoop.hdds.scm.node.PendingContainerTracker.TwoWindowBucket; +import org.apache.hadoop.hdds.upgrade.HDDSVersionUtils; import org.apache.hadoop.util.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -58,7 +58,8 @@ public class DatanodeInfo extends DatanodeDetails { private List storageReports; private List metadataStorageReports; - private LayoutVersionProto lastKnownLayoutVersion; + private ComponentVersion lastKnownSoftwareVersion; + private ComponentVersion lastKnownApparentVersion; private final Map commandCounts; private NodeStatus nodeStatus; @@ -67,16 +68,17 @@ public class DatanodeInfo extends DatanodeDetails { * Constructs DatanodeInfo from DatanodeDetails. * * @param datanodeDetails Details about the datanode - * @param layoutInfo Details about the LayoutVersionProto + * @param versionInfo Details about the LayoutVersionProto */ public DatanodeInfo(DatanodeDetails datanodeDetails, NodeStatus nodeStatus, - LayoutVersionProto layoutInfo, long containerRollIntervalMs) { + LayoutVersionProto versionInfo, long containerRollIntervalMs) { super(datanodeDetails); this.lock = new ReentrantReadWriteLock(); this.lastHeartbeatTime = Time.monotonicNow(); - lastKnownLayoutVersion = toLayoutVersionProto( - layoutInfo != null ? layoutInfo.getMetadataLayoutVersion() : 0, - layoutInfo != null ? layoutInfo.getSoftwareLayoutVersion() : 0); + this.lastKnownSoftwareVersion = + HDDSVersionUtils.deserializeHDDSVersionOrLayoutVersion(versionInfo.getSoftwareLayoutVersion()); + this.lastKnownApparentVersion = + HDDSVersionUtils.deserializeHDDSVersionOrLayoutVersion(versionInfo.getMetadataLayoutVersion()); this.storageReports = Collections.emptyList(); this.nodeStatus = nodeStatus; this.metadataStorageReports = Collections.emptyList(); @@ -108,17 +110,18 @@ public void updateLastHeartbeatTime(long milliSecondsSinceEpoch) { } /** - * Updates the last LayoutVersion. + * Updates the last known apparent and software versions for this datanode. */ - public void updateLastKnownLayoutVersion(LayoutVersionProto version) { + public void updateLastKnownVersions(LayoutVersionProto version) { if (version == null) { return; } try { lock.writeLock().lock(); - lastKnownLayoutVersion = toLayoutVersionProto( - version.getMetadataLayoutVersion(), - version.getSoftwareLayoutVersion()); + lastKnownSoftwareVersion = + HDDSVersionUtils.deserializeHDDSVersionOrLayoutVersion(version.getSoftwareLayoutVersion()); + lastKnownApparentVersion = + HDDSVersionUtils.deserializeHDDSVersionOrLayoutVersion(version.getMetadataLayoutVersion()); } finally { lock.writeLock().unlock(); } @@ -138,15 +141,19 @@ public long getLastHeartbeatTime() { } } - /** - * Returns the last known Layout Version . - * - * @return last Layout Version. - */ - public LayoutVersionProto getLastKnownLayoutVersion() { + public ComponentVersion getLastKnownSoftwareVersion() { + try { + lock.readLock().lock(); + return lastKnownSoftwareVersion; + } finally { + lock.readLock().unlock(); + } + } + + public ComponentVersion getLastKnownApparentVersion() { try { lock.readLock().lock(); - return lastKnownLayoutVersion; + return lastKnownApparentVersion; } finally { lock.readLock().unlock(); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java index 69b62dec4d49..cd31eba5c4fd 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java @@ -17,7 +17,7 @@ package org.apache.hadoop.hdds.scm.node; -import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.defaultLayoutVersionProto; +import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.defaultVersionProto; import jakarta.annotation.Nullable; import java.io.Closeable; @@ -27,6 +27,7 @@ import java.util.Map; import java.util.Set; import java.util.function.BiConsumer; +import org.apache.hadoop.hdds.ComponentVersion; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.DatanodeID; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState; @@ -43,7 +44,6 @@ import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineID; import org.apache.hadoop.hdds.server.events.EventHandler; -import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager; import org.apache.hadoop.ozone.protocol.StorageContainerNodeProtocol; import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode; import org.apache.hadoop.ozone.protocol.commands.RegisteredCommand; @@ -91,7 +91,7 @@ default RegisteredCommand register( DatanodeDetails datanodeDetails, NodeReportProto nodeReport, PipelineReportsProto pipelineReportsProto) { return register(datanodeDetails, nodeReport, pipelineReportsProto, - defaultLayoutVersionProto()); + defaultVersionProto()); } /** @@ -173,19 +173,17 @@ default DatanodeFinalizationCounts getDatanodeFinalizationCounts() { totalHealthyNodes++; DatanodeInfo datanodeInfo = getDatanodeInfo(dn); if (datanodeInfo == null) { - LOG.warn("Could not get DatanodeInfo for {}, skipping in " + - "finalization wait.", dn.getHostName()); + LOG.warn("Could not get DatanodeInfo for {}, skip counting for finalization.", dn.getHostName()); continue; } - LayoutVersionProto dnLayout = datanodeInfo.getLastKnownLayoutVersion(); - int dnMlv = dnLayout.getMetadataLayoutVersion(); - int dnSlv = dnLayout.getSoftwareLayoutVersion(); + ComponentVersion dnApparentVersion = datanodeInfo.getLastKnownApparentVersion(); + ComponentVersion dnSoftwareVersion = datanodeInfo.getLastKnownSoftwareVersion(); - if (dnMlv < dnSlv) { + if (!dnApparentVersion.equals(dnSoftwareVersion)) { // Datanode has not yet finalized - LOG.debug("Datanode {} has not yet finalized: MLV={}, SLV={}", - dn.getHostName(), dnMlv, dnSlv); + LOG.debug("Datanode {} has not yet finalized: apparent version={}, software version={}", + dn.getHostName(), dnApparentVersion, dnSoftwareVersion); } else { finalizedNodes++; } @@ -372,8 +370,8 @@ void processNodeReport(DatanodeDetails datanodeDetails, * @param datanodeDetails * @param layoutReport */ - void processLayoutVersionReport(DatanodeDetails datanodeDetails, - LayoutVersionProto layoutReport); + void processVersionReport(DatanodeDetails datanodeDetails, + LayoutVersionProto layoutReport); /** * Get the number of commands of the given type queued on the datanode at the @@ -473,10 +471,6 @@ default Collection getPeerList(DatanodeDetails dn) { return null; } - default HDDSLayoutVersionManager getLayoutVersionManager() { - return null; - } - /** * This API allows removal of only DECOMMISSIONED, IN_MAINTENANCE and DEAD nodes * from NodeManager data structures and cleanup memory. diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java index eb341cb21d00..968eb75513ae 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java @@ -234,7 +234,7 @@ public void addNode(DatanodeDetails datanodeDetails, LayoutVersionProto layoutInfo) throws NodeAlreadyExistsException { nodeStateMap.addNode(newDatanodeInfo(datanodeDetails, layoutInfo)); try { - updateLastKnownLayoutVersion(datanodeDetails, layoutInfo); + updateLastKnownVersionInfo(datanodeDetails, layoutInfo); } catch (NodeNotFoundException ex) { throw new IllegalStateException("Inconsistent NodeStateMap! Datanode " + datanodeDetails.getID() + " was added but not found in map: " + nodeStateMap); @@ -318,11 +318,10 @@ public void updateLastHeartbeatTime(DatanodeDetails datanodeDetails) * * @throws NodeNotFoundException if the node is not present */ - public void updateLastKnownLayoutVersion(DatanodeDetails datanodeDetails, - LayoutVersionProto layoutInfo) - throws NodeNotFoundException { + public void updateLastKnownVersionInfo(DatanodeDetails datanodeDetails, + LayoutVersionProto layoutInfo) throws NodeNotFoundException { nodeStateMap.getNodeInfo(datanodeDetails.getID()) - .updateLastKnownLayoutVersion(layoutInfo); + .updateLastKnownVersions(layoutInfo); } /** @@ -339,7 +338,7 @@ public void updateNode(DatanodeDetails datanodeDetails, final DatanodeInfo oldInfo = nodeStateMap.updateNode(newInfo); LOG.info("Updated datanode {} {} to {} {}", oldInfo, oldInfo.getNodeStatus(), newInfo, newInfo.getNodeStatus()); - updateLastKnownLayoutVersion(datanodeDetails, layoutInfo); + updateLastKnownVersionInfo(datanodeDetails, layoutInfo); } /** diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java index 22fd52835d9e..c9309c6a9952 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java @@ -50,7 +50,7 @@ import java.util.function.Predicate; import java.util.stream.Collectors; import javax.management.ObjectName; -import org.apache.hadoop.hdds.HDDSVersion; +import org.apache.hadoop.hdds.ComponentVersion; import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -84,8 +84,9 @@ import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; import org.apache.hadoop.hdds.scm.pipeline.PipelineNotFoundException; import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; +import org.apache.hadoop.hdds.scm.server.upgrade.ScmVersionManager; import org.apache.hadoop.hdds.server.events.EventPublisher; -import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager; +import org.apache.hadoop.hdds.upgrade.HDDSVersionUtils; import org.apache.hadoop.ipc_.Server; import org.apache.hadoop.metrics2.util.MBeans; import org.apache.hadoop.ozone.OzoneConfigKeys; @@ -135,7 +136,7 @@ public class SCMNodeManager implements NodeManager { private final Map> dnsToDnIdMap = new ConcurrentHashMap<>(); private final int numPipelinesPerMetadataVolume; private final int datanodePipelineLimit; - private final HDDSLayoutVersionManager scmLayoutVersionManager; + private final ScmVersionManager versionManager; private final EventPublisher scmNodeEventPublisher; private final SCMContext scmContext; private final Map scmSlv} as an invalid node. - */ - private static boolean shouldFenceDatanode(int dnSoftwareVersion, int scmSoftwareVersion) { - return dnSoftwareVersion > scmSoftwareVersion && dnSoftwareVersion != HDDSVersion.ZDU.serialize(); - } - - /** - * Constructs SCM machine Manager. + * Constructs SCM machine Manager using {@link ScmVersionManager} (production SCM). */ + @VisibleForTesting public SCMNodeManager( OzoneConfiguration conf, SCMStorageConfig scmStorageConfig, EventPublisher eventPublisher, NetworkTopology networkTopology, SCMContext scmContext, - HDDSLayoutVersionManager layoutVersionManager) { + ScmVersionManager versionManager) { this(conf, scmStorageConfig, eventPublisher, networkTopology, scmContext, - layoutVersionManager, hostname -> null); + versionManager, hostname -> null); } public SCMNodeManager( @@ -187,14 +178,14 @@ public SCMNodeManager( EventPublisher eventPublisher, NetworkTopology networkTopology, SCMContext scmContext, - HDDSLayoutVersionManager layoutVersionManager, + ScmVersionManager versionManager, Function nodeResolver) { this.scmNodeEventPublisher = eventPublisher; this.nodeStateManager = new NodeStateManager(conf, eventPublisher, scmContext); this.version = VersionInfo.getLatestVersion(); this.commandQueue = new CommandQueue(); this.scmStorageConfig = scmStorageConfig; - this.scmLayoutVersionManager = layoutVersionManager; + this.versionManager = versionManager; LOG.info("Entering startup safe mode."); registerMXBean(); this.metrics = SCMNodeMetrics.create(this); @@ -384,10 +375,8 @@ public RegisteredCommand register( PipelineReportsProto pipelineReportsProto) { return register(datanodeDetails, nodeReport, pipelineReportsProto, LayoutVersionProto.newBuilder() - .setMetadataLayoutVersion( - scmLayoutVersionManager.getMetadataLayoutVersion()) - .setSoftwareLayoutVersion( - scmLayoutVersionManager.getSoftwareLayoutVersion()) + .setMetadataLayoutVersion(versionManager.getApparentVersion().serialize()) + .setSoftwareLayoutVersion(versionManager.getSoftwareVersion().serialize()) .build()); } @@ -407,10 +396,8 @@ public RegisteredCommand register( public RegisteredCommand register( DatanodeDetails datanodeDetails, NodeReportProto nodeReport, PipelineReportsProto pipelineReportsProto, - LayoutVersionProto layoutInfo) { - int dnSlvRegister = layoutInfo.getSoftwareLayoutVersion(); - int scmSlvRegister = scmLayoutVersionManager.getSoftwareLayoutVersion(); - if (shouldFenceDatanode(dnSlvRegister, scmSlvRegister)) { + LayoutVersionProto dnVersionInfo) { + if (shouldFenceDatanode(datanodeDetails, dnVersionInfo)) { return RegisteredCommand.newBuilder() .setErrorCode(ErrorCode.errorNodeNotPermitted) .setDatanode(datanodeDetails) @@ -440,7 +427,7 @@ public RegisteredCommand register( if (!isNodeRegistered(datanodeDetails)) { try { clusterMap.add(datanodeDetails); - nodeStateManager.addNode(datanodeDetails, layoutInfo); + nodeStateManager.addNode(datanodeDetails, dnVersionInfo); // Check that datanode in nodeStateManager has topology parent set DatanodeDetails dn = nodeStateManager.getNode(datanodeDetails); Preconditions.checkState(dn.getParent() != null); @@ -466,7 +453,7 @@ public RegisteredCommand register( hostName, ipAddress, dnId)) { LOG.info("Updating datanode from {} to {}", oldNode, datanodeDetails); clusterMap.update(oldNode, datanodeDetails); - nodeStateManager.updateNode(datanodeDetails, layoutInfo); + nodeStateManager.updateNode(datanodeDetails, dnVersionInfo); DatanodeDetails dn = nodeStateManager.getNode(datanodeDetails); Preconditions.checkState(dn.getParent() != null); processNodeReport(datanodeDetails, nodeReport); @@ -476,7 +463,7 @@ public RegisteredCommand register( LOG.info("Update the version for registered datanode {}, " + "oldVersion = {}, newVersion = {}.", datanodeDetails, oldNode.getVersion(), datanodeDetails.getVersion()); - nodeStateManager.updateNode(datanodeDetails, layoutInfo); + nodeStateManager.updateNode(datanodeDetails, dnVersionInfo); } } catch (NodeNotFoundException e) { LOG.error("Cannot find datanode {} from nodeStateManager", @@ -735,11 +722,11 @@ public void processNodeReport(DatanodeDetails datanodeDetails, * Process Layout Version report. * * @param datanodeDetails - * @param layoutVersionReport + * @param versionReport */ @Override - public void processLayoutVersionReport(DatanodeDetails datanodeDetails, - LayoutVersionProto layoutVersionReport) { + public void processVersionReport(DatanodeDetails datanodeDetails, + LayoutVersionProto versionReport) { if (LOG.isDebugEnabled()) { LOG.debug("Processing Layout Version report from [datanode={}]", datanodeDetails.getHostName()); @@ -747,63 +734,65 @@ public void processLayoutVersionReport(DatanodeDetails datanodeDetails, if (LOG.isTraceEnabled()) { LOG.trace("HB is received from [datanode={}]: {}", datanodeDetails.getHostName(), - layoutVersionReport.toString().replaceAll("\n", "\\\\n")); + versionReport.toString().replaceAll("\n", "\\\\n")); } try { - nodeStateManager.updateLastKnownLayoutVersion(datanodeDetails, - layoutVersionReport); + nodeStateManager.updateLastKnownVersionInfo(datanodeDetails, + versionReport); } catch (NodeNotFoundException e) { LOG.error("SCM trying to process Layout Version from an " + "unregistered node {}.", datanodeDetails); return; } - sendFinalizeToDatanodeIfNeeded(datanodeDetails, layoutVersionReport); + sendFinalizeToDatanodeIfNeeded(datanodeDetails, versionReport); } protected void sendFinalizeToDatanodeIfNeeded(DatanodeDetails datanodeDetails, - LayoutVersionProto layoutVersionReport) { - // Software layout version is hardcoded to the SCM. - int scmSlv = scmLayoutVersionManager.getSoftwareLayoutVersion(); - int dnSlv = layoutVersionReport.getSoftwareLayoutVersion(); - int dnMlv = layoutVersionReport.getMetadataLayoutVersion(); - - // A datanode with a larger software layout version is from a future - // version of ozone. It should not have been added to the cluster. - // TODO HDDS-15129 REMOVE WHEN SCM USES new versioning framework. - // For now, do not treat datanodes with ZDU future software version as invalid. - if (shouldFenceDatanode(dnSlv, scmSlv)) { - LOG.error("Invalid data node in the cluster : {}. " + - "DataNode SoftwareLayoutVersion = {}, SCM " + - "SoftwareLayoutVersion = {}", - datanodeDetails.getHostName(), dnSlv, scmSlv); + LayoutVersionProto versionReport) { + ComponentVersion dnSoftwareVersion = HDDSVersionUtils.deserializeHDDSVersionOrLayoutVersion( + versionReport.getSoftwareLayoutVersion()); + ComponentVersion dnApparentVersion = HDDSVersionUtils.deserializeHDDSVersionOrLayoutVersion( + versionReport.getMetadataLayoutVersion()); + ComponentVersion scmSoftwareVersion = versionManager.getSoftwareVersion(); + ComponentVersion scmApparentVersion = versionManager.getApparentVersion(); + + if (shouldFenceDatanode(datanodeDetails, dnSoftwareVersion, dnApparentVersion)) { + LOG.error("Invalid datanode in the cluster : {}. " + + "Datanode software version = {}, " + + "Datanode apparent version = {}, " + + "SCM software version = {} " + + "SCM apparent version = {}", + datanodeDetails.getHostName(), dnSoftwareVersion, dnApparentVersion, + scmSoftwareVersion, scmApparentVersion); return; } - if (!scmContext.isLeader() || scmLayoutVersionManager.needsFinalization()) { + if (!scmContext.isLeader() || versionManager.needsFinalization()) { return; } - int scmMlv = scmLayoutVersionManager.getMetadataLayoutVersion(); - if (dnMlv == scmMlv) { - // datanode is already finalized, so there is nothing to do + if (versionManager.getApparentVersion().equals(dnApparentVersion)) { + // If SCM and DN apparent version match, then the datanode is already finalized. + LOG.debug("Skip sending finalize command to datanode {} because its apparent version matches SCM's apparent " + + "version {}", datanodeDetails, versionManager.getApparentVersion()); return; } // Because the finalizationManager / versionManager says finalization is not needed it means any DN reporting a // metadata layout version less than the SCM's metadata layout version can be finalized. - LOG.warn("Data node {} has a MetadataLayoutVersion = {}, SCM MetadataLayoutVersion = {}. Sending finalize", - datanodeDetails.getHostName(), dnMlv, scmMlv); + LOG.info("Sending finalize command to datanode {} with apparent version {} which is less than SCM's finalized " + + "apparent version {}", datanodeDetails, dnApparentVersion, scmApparentVersion); FinalizeVersionCommand finalizeCmd = new FinalizeVersionCommand(true, LayoutVersionProto.newBuilder() - .setSoftwareLayoutVersion(dnSlv) - .setMetadataLayoutVersion(dnSlv).build()); + .setSoftwareLayoutVersion(dnSoftwareVersion.serialize()) + .setMetadataLayoutVersion(dnSoftwareVersion.serialize()).build()); try { finalizeCmd.setTerm(scmContext.getTermOfLeader()); - // Send Finalize command to the data node. Its OK to send Finalize command multiple times. + // Send Finalize command to the data node. It's OK to send Finalize command multiple times. scmNodeEventPublisher.fireEvent(SCMEvents.DATANODE_COMMAND, new CommandForDatanode<>(datanodeDetails, finalizeCmd)); @@ -1987,15 +1976,6 @@ long getSkippedHealthChecks() { return nodeStateManager.getSkippedHealthChecks(); } - /** - * @return HDDSLayoutVersionManager - */ - @VisibleForTesting - @Override - public HDDSLayoutVersionManager getLayoutVersionManager() { - return scmLayoutVersionManager; - } - private ReentrantReadWriteLock.WriteLock writeLock() { return lock.writeLock(); } @@ -2035,4 +2015,42 @@ public void removeNode(DatanodeDetails datanodeDetails) throws NodeNotFoundExcep writeLock().unlock(); } } + + protected boolean shouldFenceDatanode(DatanodeDetails dnDetails, LayoutVersionProto versionReport) { + ComponentVersion dnSoftwareVersion = HDDSVersionUtils.deserializeHDDSVersionOrLayoutVersion( + versionReport.getSoftwareLayoutVersion()); + ComponentVersion dnApparentVersion = HDDSVersionUtils.deserializeHDDSVersionOrLayoutVersion( + versionReport.getMetadataLayoutVersion()); + return shouldFenceDatanode(dnDetails, dnSoftwareVersion, dnApparentVersion); + } + + /** + * TODO Update this method to fence datanodes based on their software and apparent version and log the results. + * For now, maintain the non-rolling upgrade requirement that DN and SCM must have the same software version. + * Datanodes still cannot have a higher apparent version than SCM. + */ + private boolean shouldFenceDatanode(DatanodeDetails dnDetails, ComponentVersion softwareVersion, + ComponentVersion apparentVersion) { + // Check datanode software version against SCM. + if (!versionManager.getSoftwareVersion().equals(softwareVersion)) { + // TODO Once SCM implementation for ZDU is complete, Datanodes with lower software versions will be allowed as + // long as SCM is pre-finalized. + LOG.error("Datanode {} with software version {} which does not match SCM software version {} will not be " + + "allowed to join the cluster. This requirement will be lifted when ZDU is complete.", + dnDetails, softwareVersion, versionManager.getSoftwareVersion()); + return true; + } + + // Check datanode apparent version against SCM. + if (!versionManager.isAllowed(apparentVersion)) { + // Datanodes can never have a higher apparent version than SCM. + LOG.error("Datanode {} with apparent version {} which is larger than SCM's apparent version {} will not be " + + "allowed to join the cluster.", dnDetails, apparentVersion, versionManager.getApparentVersion()); + return true; + } + + // Datanodes with lower apparent version than SCM are allowed in the cluster but will be instructed to finalize + // if SCM has finalized. + return false; + } } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocolServerSideTranslatorPB.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocolServerSideTranslatorPB.java index f9fa80bf42d5..007376670ba4 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocolServerSideTranslatorPB.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocolServerSideTranslatorPB.java @@ -129,9 +129,7 @@ private SCMBlockLocationResponse processMessage( try { switch (request.getCmdType()) { case AllocateScmBlock: - if (scm.getLayoutVersionManager().needsFinalization() && - !scm.getLayoutVersionManager() - .isAllowed(HDDSLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT) + if (!scm.getVersionManager().isAllowed(HDDSLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT) ) { if (request.getAllocateScmBlockRequest().hasEcReplicationConfig()) { throw new SCMException("Cluster is not finalized yet, it is" diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java index 2a160b07e406..49507152dd5f 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java @@ -497,8 +497,7 @@ public ScmContainerLocationResponse processRequest( request.getScmCloseContainerRequest())) .build(); case AllocatePipeline: - if (scm.getLayoutVersionManager().needsFinalization() && - !scm.getLayoutVersionManager().isAllowed( + if (!scm.getVersionManager().isAllowed( HDDSLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT) ) { if (request.getPipelineRequest().getReplicationType() == diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java index 806e0fed9a10..2d4c52b630fe 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java @@ -29,6 +29,8 @@ import static org.apache.hadoop.hdds.server.ServerUtils.getRemoteUserName; import static org.apache.hadoop.hdds.server.ServerUtils.updateRPCListenAddress; import static org.apache.hadoop.hdds.utils.HddsServerUtil.getRemoteUser; +import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.FINALIZATION_DONE_MSG; +import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.FINALIZATION_REQUIRED_MSG; import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.Status.ALREADY_FINALIZED; import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.Status.STARTING_FINALIZATION; @@ -1152,7 +1154,7 @@ public ReplicationManagerReport getReplicationManagerReport() { @Deprecated public StatusAndMessages finalizeScmUpgrade(String upgradeClientID) throws IOException { - if (scm.getLayoutVersionManager().getUpgradeState() == ALREADY_FINALIZED) { + if (!scm.getVersionManager().needsFinalization()) { return new StatusAndMessages(ALREADY_FINALIZED, Collections.emptyList()); } finalizeUpgrade(); @@ -1187,12 +1189,13 @@ public StatusAndMessages queryUpgradeFinalizationProgress( auditMap.put("readonly", String.valueOf(readonly)); try { - // check admin authorization - if (!readonly) { - getScm().checkAdminAccess(getRemoteUser(), true); + getScm().checkAdminAccess(getRemoteUser(), true); + StatusAndMessages result; + if (scm.getVersionManager().needsFinalization()) { + result = FINALIZATION_REQUIRED_MSG; + } else { + result = FINALIZATION_DONE_MSG; } - StatusAndMessages result = scm.getFinalizationManager() - .queryUpgradeFinalizationProgress(upgradeClientID, force, readonly); AUDIT.logReadSuccess(buildAuditMessageForSuccess( SCMAction.QUERY_UPGRADE_FINALIZATION_PROGRESS, auditMap)); return result; @@ -1208,7 +1211,7 @@ public HddsProtos.UpgradeStatus queryUpgradeStatus() throws IOException { try { getScm().checkAdminAccess(getRemoteUser(), true); - boolean scmFinalized = !scm.getLayoutVersionManager().needsFinalization(); + boolean scmFinalized = !scm.getVersionManager().needsFinalization(); NodeManager.DatanodeFinalizationCounts datanodeFinalizationCounts = scm.getScmNodeManager().getDatanodeFinalizationCounts(); int finalizedDatanodes = datanodeFinalizationCounts.getNumFinalizedDatanodes(); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMConfigurator.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMConfigurator.java index bc542c6722c2..a823c41f11fd 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMConfigurator.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMConfigurator.java @@ -28,10 +28,8 @@ import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; import org.apache.hadoop.hdds.scm.pipeline.WritableContainerFactory; import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager; -import org.apache.hadoop.hdds.scm.server.upgrade.SCMUpgradeFinalizationContext; import org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateServer; import org.apache.hadoop.ozone.lease.LeaseManager; -import org.apache.hadoop.ozone.upgrade.UpgradeFinalizationExecutor; /** * This class acts as an SCM builder Class. This class is important for us @@ -74,8 +72,6 @@ public final class SCMConfigurator { private SCMHAManager scmHAManager; private SCMContext scmContext; private WritableContainerFactory writableContainerFactory; - private UpgradeFinalizationExecutor - finalizationExecutor; private LeaseManager leaseManager; /** @@ -185,15 +181,6 @@ public void setWritableContainerFactory( this.writableContainerFactory = writableContainerFactory; } - /** - * Allows user to set the executor for upgrade finalization. - * @param executor - Finalization executor to use. - */ - public void setUpgradeFinalizationExecutor( - UpgradeFinalizationExecutor executor) { - this.finalizationExecutor = executor; - } - /** * Allows user to specify a custom version lease manager. * @param leaseManager - lease Manager. @@ -298,15 +285,6 @@ public WritableContainerFactory getWritableContainerFactory() { return writableContainerFactory; } - /** - * Get the upgrade finalization executor. - * @return UpgradeFinalizationExecutor. - */ - public UpgradeFinalizationExecutor - getUpgradeFinalizationExecutor() { - return finalizationExecutor; - } - /** * Get the lease manager. * @return LeaseManager diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java index 1641a5353a34..efb7a5b81166 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java @@ -25,7 +25,7 @@ import static org.apache.hadoop.hdds.scm.events.SCMEvents.PIPELINE_ACTIONS; import static org.apache.hadoop.hdds.scm.events.SCMEvents.PIPELINE_REPORT; import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.INITIAL_VERSION; -import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.toLayoutVersionProto; +import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.toVersionProto; import com.google.protobuf.Message; import java.util.List; @@ -97,18 +97,17 @@ public List> dispatch(SCMHeartbeatRequestProto heartbeat) { } else { - LayoutVersionProto layoutVersion = null; + LayoutVersionProto versionInfo = null; if (!heartbeat.hasDataNodeLayoutVersion()) { // Backward compatibility to make sure old Datanodes can still talk to // SCM. - layoutVersion = toLayoutVersionProto(INITIAL_VERSION.layoutVersion(), - INITIAL_VERSION.layoutVersion()); + versionInfo = toVersionProto(INITIAL_VERSION, INITIAL_VERSION); } else { - layoutVersion = heartbeat.getDataNodeLayoutVersion(); + versionInfo = heartbeat.getDataNodeLayoutVersion(); } LOG.debug("Processing DataNode Layout Report."); - nodeManager.processLayoutVersionReport(datanodeDetails, layoutVersion); + nodeManager.processVersionReport(datanodeDetails, versionInfo); CommandQueueReportProto commandQueueReport = null; if (heartbeat.hasCommandQueueReport()) { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMStorageConfig.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMStorageConfig.java index d41537fed0fb..75d2c557b64e 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMStorageConfig.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMStorageConfig.java @@ -17,7 +17,6 @@ package org.apache.hadoop.hdds.scm.server; -import static org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager.maxLayoutVersion; import static org.apache.hadoop.ozone.OzoneConsts.PRIMARY_SCM_NODE_ID; import static org.apache.hadoop.ozone.OzoneConsts.SCM_CERT_SERIAL_ID; import static org.apache.hadoop.ozone.OzoneConsts.SCM_HA; @@ -28,10 +27,10 @@ import java.io.IOException; import java.util.Properties; import java.util.UUID; +import org.apache.hadoop.hdds.HDDSVersion; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType; import org.apache.hadoop.hdds.server.ServerUtils; -import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager; import org.apache.hadoop.ozone.common.Storage; /** @@ -49,12 +48,12 @@ public class SCMStorageConfig extends Storage { public SCMStorageConfig(OzoneConfiguration conf) throws IOException { super(NodeType.SCM, ServerUtils.getScmDbDir(conf), STORAGE_DIR, getInitApparentVersion(conf, TESTING_INIT_LAYOUT_VERSION_KEY, - HDDSLayoutVersionManager::maxLayoutVersion)); + HDDSVersion.SOFTWARE_VERSION::serialize)); } public SCMStorageConfig(NodeType type, File root, String sdName) throws IOException { - super(type, root, sdName, maxLayoutVersion()); + super(type, root, sdName, HDDSVersion.SOFTWARE_VERSION.serialize()); } public void setScmId(String scmId) throws IOException { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index 45faab72f01b..393bed6b5be4 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -147,7 +147,7 @@ import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.IncrementalContainerReportFromDatanode; import org.apache.hadoop.hdds.scm.server.upgrade.FinalizationManager; import org.apache.hadoop.hdds.scm.server.upgrade.FinalizationManagerImpl; -import org.apache.hadoop.hdds.scm.server.upgrade.SCMUpgradeFinalizationContext; +import org.apache.hadoop.hdds.scm.server.upgrade.ScmVersionManager; import org.apache.hadoop.hdds.security.SecurityConfig; import org.apache.hadoop.hdds.security.symmetric.SecretKeyManager; import org.apache.hadoop.hdds.security.token.ContainerTokenGenerator; @@ -169,8 +169,6 @@ import org.apache.hadoop.hdds.server.events.FixedThreadPoolWithAffinityExecutor; import org.apache.hadoop.hdds.server.http.RatisDropwizardExports; import org.apache.hadoop.hdds.tracing.TracingConfig; -import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager; -import org.apache.hadoop.hdds.upgrade.ScmUpgradeActionProvider; import org.apache.hadoop.hdds.utils.HAUtils; import org.apache.hadoop.hdds.utils.HddsServerUtil; import org.apache.hadoop.hdds.utils.HddsVersionInfo; @@ -189,8 +187,6 @@ import org.apache.hadoop.ozone.common.Storage.StorageState; import org.apache.hadoop.ozone.lease.LeaseManager; import org.apache.hadoop.ozone.lease.LeaseManagerNotRunningException; -import org.apache.hadoop.ozone.upgrade.DefaultUpgradeFinalizationExecutor; -import org.apache.hadoop.ozone.upgrade.UpgradeFinalizationExecutor; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; @@ -249,7 +245,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl private NodeDecommissionManager scmDecommissionManager; private WritableContainerFactory writableContainerFactory; private FinalizationManager finalizationManager; - private HDDSLayoutVersionManager scmLayoutVersionManager; + private ScmVersionManager versionManager; private LeaseManager leaseManager; private SCMMetadataStore scmMetadataStore; @@ -702,22 +698,11 @@ private void initializeSystemManagers(OzoneConfiguration conf, leaseManager = new LeaseManager<>(threadNamePrefix, timeDuration); } - scmLayoutVersionManager = new HDDSLayoutVersionManager( - scmStorageConfig.getApparentVersion(), new ScmUpgradeActionProvider(), null); - - UpgradeFinalizationExecutor - finalizationExecutor; - if (configurator.getUpgradeFinalizationExecutor() != null) { - finalizationExecutor = configurator.getUpgradeFinalizationExecutor(); - } else { - finalizationExecutor = new DefaultUpgradeFinalizationExecutor<>(); - } + versionManager = new ScmVersionManager(scmStorageConfig, this); finalizationManager = new FinalizationManagerImpl.Builder() - .setLayoutVersionManager(scmLayoutVersionManager) - .setStorage(scmStorageConfig) .setHAManager(scmHAManager) .setFinalizationStore(scmMetadataStore.getMetaTable()) - .setFinalizationExecutor(finalizationExecutor) + .setVersionManager(versionManager) .build(); // inline upgrade for SequenceIdGenerator @@ -753,7 +738,7 @@ private void initializeSystemManagers(OzoneConfiguration conf, scmNodeManager = configurator.getScmNodeManager(); } else { scmNodeManager = new SCMNodeManager(conf, scmStorageConfig, eventQueue, - clusterMap, scmContext, scmLayoutVersionManager, + clusterMap, scmContext, versionManager, this::resolveNodeLocation); } @@ -784,8 +769,6 @@ private void initializeSystemManagers(OzoneConfiguration conf, ); } - finalizationManager.buildUpgradeContext(scmNodeManager, scmContext); - ReplicationManager.ReplicationManagerConfiguration rmConf = conf.getObject(ReplicationManager.ReplicationManagerConfiguration.class); containerReplicaPendingOps = @@ -1695,8 +1678,8 @@ public void stop() { LOG.error("Storage Container Manager HTTP server stop failed.", ex); } - LOG.info("Stopping SCM LayoutVersionManager Service."); - scmLayoutVersionManager.close(); + LOG.info("Stopping SCM version manager metrics."); + versionManager.close(); if (getSecurityProtocolServer() != null) { getSecurityProtocolServer().stop(); @@ -2109,14 +2092,21 @@ public String getClusterId() { return getScmStorageConfig().getClusterID(); } - public HDDSLayoutVersionManager getLayoutVersionManager() { - return scmLayoutVersionManager; - } - + /** + * @return The {@link FinalizationManager} which can be used to finalize an SCM quorum through Ratis. + */ public FinalizationManager getFinalizationManager() { return finalizationManager; } + /** + * @return The {@link ScmVersionManager} which can be used to check this SCM's local apparent and software versions. + * To finalize all SCM's via Ratis, use {@link FinalizationManager} instead. + */ + public ScmVersionManager getVersionManager() { + return versionManager; + } + /** * Return the node Id of this SCM. * @return node Id. diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationManager.java index 3bdfa2dbd3e3..f7e610636125 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationManager.java @@ -17,36 +17,15 @@ package org.apache.hadoop.hdds.scm.server.upgrade; -import com.google.common.annotations.VisibleForTesting; import java.io.IOException; -import org.apache.hadoop.hdds.scm.ha.SCMContext; -import org.apache.hadoop.hdds.scm.node.NodeManager; -import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager; import org.apache.hadoop.hdds.utils.db.Table; -import org.apache.hadoop.ozone.upgrade.BasicUpgradeFinalizer; -import org.apache.hadoop.ozone.upgrade.UpgradeFinalization; /** * Class to initiate SCM finalization and query its progress. */ public interface FinalizationManager { - UpgradeFinalization.StatusAndMessages finalizeUpgrade(String upgradeClientID) - throws IOException; - void finalizeUpgrade() throws IOException; - UpgradeFinalization.StatusAndMessages queryUpgradeFinalizationProgress( - String upgradeClientID, boolean takeover, boolean readonly - ) throws IOException; - - @VisibleForTesting - BasicUpgradeFinalizer - getUpgradeFinalizer(); - - void buildUpgradeContext(NodeManager nodeManager, - SCMContext scmContext); - void reinitialize(Table finalizationStore) throws IOException; - } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationManagerImpl.java index ccff51018980..f02a8231d546 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationManagerImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationManagerImpl.java @@ -19,27 +19,14 @@ import com.google.common.annotations.VisibleForTesting; import java.io.IOException; -import java.util.Collections; import java.util.Objects; -import org.apache.hadoop.hdds.scm.ha.SCMContext; import org.apache.hadoop.hdds.scm.ha.SCMHAManager; -import org.apache.hadoop.hdds.scm.node.NodeManager; -import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; -import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager; import org.apache.hadoop.hdds.utils.db.Table; -import org.apache.hadoop.ozone.upgrade.BasicUpgradeFinalizer; -import org.apache.hadoop.ozone.upgrade.DefaultUpgradeFinalizationExecutor; -import org.apache.hadoop.ozone.upgrade.UpgradeFinalization; -import org.apache.hadoop.ozone.upgrade.UpgradeFinalizationExecutor; /** * Class to initiate SCM finalization and query its progress. */ public class FinalizationManagerImpl implements FinalizationManager { - - private SCMUpgradeFinalizer upgradeFinalizer; - private SCMUpgradeFinalizationContext context; - private SCMStorageConfig storage; private final FinalizationStateManager finalizationStateManager; /** @@ -48,74 +35,25 @@ public class FinalizationManagerImpl implements FinalizationManager { @VisibleForTesting protected FinalizationManagerImpl(Builder builder, FinalizationStateManager stateManager) throws IOException { - initCommonFields(builder); this.finalizationStateManager = stateManager; - } private FinalizationManagerImpl(Builder builder) throws IOException { - initCommonFields(builder); this.finalizationStateManager = new FinalizationStateManagerImpl.Builder() - .setUpgradeFinalizer(this.upgradeFinalizer) .setFinalizationStore(builder.finalizationStore) .setTransactionBuffer(builder.scmHAManager.getDBTransactionBuffer()) .setRatisServer(builder.scmHAManager.getRatisServer()) + .setVersionManager(builder.versionManager) .build(); } - private void initCommonFields(Builder builder) { - this.storage = builder.storage; - this.upgradeFinalizer = new SCMUpgradeFinalizer(builder.versionManager, builder.executor); - } - - @Override - public void buildUpgradeContext(NodeManager nodeManager, - SCMContext scmContext) { - this.context = new SCMUpgradeFinalizationContext.Builder() - .setStorage(this.storage) - .setFinalizationStateManager(finalizationStateManager) - .setNodeManager(nodeManager) - .setSCMContext(scmContext) - .build(); - - finalizationStateManager.setUpgradeContext(this.context); - } - - @Override - public UpgradeFinalization.StatusAndMessages finalizeUpgrade( - String upgradeClientID) - throws IOException { - Objects.requireNonNull(context, "Cannot finalize upgrade without " + - "first building the upgrade context."); - return upgradeFinalizer.finalize(upgradeClientID, context); - } - @Override public void finalizeUpgrade() throws IOException { - Objects.requireNonNull(context, "Cannot finalize upgrade without first building the upgrade context."); - upgradeFinalizer.finalize(context); - } - - @Override - public UpgradeFinalization.StatusAndMessages queryUpgradeFinalizationProgress( - String upgradeClientID, boolean takeover, boolean readonly - ) throws IOException { - if (readonly) { - return new UpgradeFinalization.StatusAndMessages( - upgradeFinalizer.getStatus(), Collections.emptyList()); - } - return upgradeFinalizer.reportStatus(upgradeClientID, takeover); - } - - @Override - public BasicUpgradeFinalizer getUpgradeFinalizer() { - return upgradeFinalizer; + finalizationStateManager.finalizeUpgrade(); } @Override - public void reinitialize(Table finalizationStore) - throws IOException { + public void reinitialize(Table finalizationStore) throws IOException { finalizationStateManager.reinitialize(finalizationStore); } @@ -123,24 +61,15 @@ public void reinitialize(Table finalizationStore) * Builds a {@link FinalizationManagerImpl}. */ public static class Builder { - private HDDSLayoutVersionManager versionManager; - private SCMStorageConfig storage; - private SCMHAManager scmHAManager; private Table finalizationStore; - private UpgradeFinalizationExecutor executor; + private SCMHAManager scmHAManager; + private ScmVersionManager versionManager; public Builder() { - executor = new DefaultUpgradeFinalizationExecutor<>(); - } - - public Builder setLayoutVersionManager( - HDDSLayoutVersionManager layoutVersionManager) { - this.versionManager = layoutVersionManager; - return this; } - public Builder setStorage(SCMStorageConfig storage) { - this.storage = storage; + public Builder setVersionManager(ScmVersionManager versionManager) { + this.versionManager = versionManager; return this; } @@ -155,18 +84,10 @@ public Builder setFinalizationStore( return this; } - public Builder setFinalizationExecutor( - UpgradeFinalizationExecutor finalizationExecutor) { - this.executor = finalizationExecutor; - return this; - } - public FinalizationManagerImpl build() throws IOException { - Objects.requireNonNull(versionManager, "versionManager == null"); - Objects.requireNonNull(storage, "storage == null"); Objects.requireNonNull(scmHAManager, "scmHAManager == null"); Objects.requireNonNull(finalizationStore, "finalizationStore == null"); - Objects.requireNonNull(executor, "executor == null"); + Objects.requireNonNull(versionManager, "versionManager == null"); return new FinalizationManagerImpl(this); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManager.java index ed4a07924c6a..0b8d550ab342 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManager.java @@ -28,10 +28,19 @@ */ public interface FinalizationStateManager extends SCMHandler { + // TODO this will need a parameter for peer version info to validate. @Replicate - void finalizeLayoutFeatures(Integer toLayoutVersion) - throws IOException; + void finalizeUpgrade() throws IOException; + /** + * Legacy layout-feature finalization API. Retained until obsolete finalizer classes are removed. + */ + @Replicate + void finalizeLayoutFeatures(Integer toLayoutVersion) throws IOException; + + /** + * Legacy finalization context. Retained until obsolete finalizer classes are removed. + */ void setUpgradeContext(SCMUpgradeFinalizationContext context); /** diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManagerImpl.java index 16f6cb3b336a..491e32c9582a 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManagerImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationStateManagerImpl.java @@ -21,12 +21,8 @@ import java.util.Objects; import org.apache.hadoop.hdds.scm.ha.SCMRatisServer; import org.apache.hadoop.hdds.scm.metadata.DBTransactionBuffer; -import org.apache.hadoop.hdds.scm.metadata.Replicate; -import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature; -import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.upgrade.LayoutFeature; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,132 +36,65 @@ public class FinalizationStateManagerImpl implements FinalizationStateManager { private Table finalizationStore; private final DBTransactionBuffer transactionBuffer; - private final HDDSLayoutVersionManager versionManager; - // Ensures that we are not in the process of updating checkpoint state as - // we read it to determine the current checkpoint. - private SCMUpgradeFinalizationContext upgradeContext; - private final SCMUpgradeFinalizer upgradeFinalizer; + private final ScmVersionManager versionManager; - protected FinalizationStateManagerImpl(Builder builder) throws IOException { + protected FinalizationStateManagerImpl(Builder builder) { this.finalizationStore = builder.finalizationStore; this.transactionBuffer = builder.transactionBuffer; - this.upgradeFinalizer = builder.upgradeFinalizer; - this.versionManager = this.upgradeFinalizer.getVersionManager(); + this.versionManager = builder.versionManager; } @Override - public void setUpgradeContext(SCMUpgradeFinalizationContext context) { - this.upgradeContext = context; + public void finalizeUpgrade() throws IOException { + versionManager.finalizeUpgrade(); + transactionBuffer.addToBuffer(finalizationStore, + OzoneConsts.APPARENT_VERSION_KEY, String.valueOf(versionManager.getApparentVersion().serialize())); } @Override - public synchronized void finalizeLayoutFeatures(Integer toVersion) throws IOException { - for (LayoutFeature feature : versionManager.unfinalizedFeatures()) { - finalizeLayoutFeatureLocal((HDDSLayoutFeature) feature); - } + public void setUpgradeContext(SCMUpgradeFinalizationContext context) { + // Retained for compile compatibility with SCMUpgradeFinalizer until it is removed in a follow-up PR. } - /** - * A version of finalizeLayoutFeature without the {@link Replicate} - * annotation that can be called by followers to finalize from a snapshot. - */ - private void finalizeLayoutFeatureLocal(HDDSLayoutFeature layoutFeature) - throws IOException { - // The VERSION file is the source of truth for the current layout - // version. This is updated in the replicated finalization steps. - // Layout version will be written to the DB as well so followers can - // finalize from a snapshot. - if (versionManager.getMetadataLayoutVersion() >= layoutFeature.layoutVersion()) { - LOG.warn("Attempting to finalize layout feature for layout version {}, but " + - "current metadata layout version is {}. Skipping finalization for this layout version.", - layoutFeature.layoutVersion(), versionManager.getMetadataLayoutVersion()); - } else { - upgradeFinalizer.replicatedFinalizationSteps(layoutFeature, upgradeContext); - } - transactionBuffer.addToBuffer(finalizationStore, - OzoneConsts.APPARENT_VERSION_KEY, String.valueOf(layoutFeature.layoutVersion())); + @Override + public void finalizeLayoutFeatures(Integer toLayoutVersion) throws IOException { + throw new UnsupportedOperationException( + "Layout feature finalization was removed; use finalizeUpgrade() instead."); } /** - * Called on snapshot installation. + * Called on snapshot installation, which is coordinated by Ratis. */ @Override - public synchronized void reinitialize(Table newFinalizationStore) - throws IOException { + public void reinitialize(Table newFinalizationStore) throws IOException { try { this.finalizationStore = newFinalizationStore; - - int dbLayoutVersion = getDBLayoutVersion(); - int currentLayoutVersion = versionManager.getMetadataLayoutVersion(); - if (currentLayoutVersion < dbLayoutVersion) { - // Snapshot contained a higher metadata layout version. Finalize this - // follower SCM as a result. - LOG.info("New SCM snapshot received with metadata layout version {}, " + - "which is higher than this SCM's metadata layout version {}." + - "Attempting to finalize current SCM to that version.", - dbLayoutVersion, currentLayoutVersion); - // Since the SCM is finalizing from a snapshot, it is a follower, and - // does not need to run the leader only finalization driving actions - // that the UpgradeFinalizationExecutor contains. Just run the - // upgrade actions for the layout features, set the finalization - // checkpoint, and increase the version in the VERSION file. - finalizeLayoutFeatures(dbLayoutVersion); - } + versionManager.finalizeFromSnapshotIfRequired(finalizationStore); } catch (Exception ex) { LOG.error("Failed to reinitialize finalization state", ex); throw new IOException(ex); } } - /** - * Gets the metadata layout version from the SCM RocksDB. This is used for - * Ratis snapshot based finalization in a slow follower. In all other - * cases, the VERSION file should be the source of truth. - * - * MLV was not stored in RocksDB until SCM HA supported snapshot based - * finalization, which was after a few HDDS layout features - * were introduced. If the SCM has not finalized since this code - * was added, the layout version will not be there. Defer to the MLV in the - * VERSION file in this case, since finalization is not ongoing. The key will - * be added once finalization is started with this software version. - */ - private int getDBLayoutVersion() throws IOException { - String dbLayoutVersion = finalizationStore.get( - OzoneConsts.APPARENT_VERSION_KEY); - if (dbLayoutVersion == null) { - return versionManager.getMetadataLayoutVersion(); - } else { - try { - return Integer.parseInt(dbLayoutVersion); - } catch (NumberFormatException ex) { - String msg = String.format( - "Failed to read layout version from SCM DB. Found string %s", - dbLayoutVersion); - LOG.error(msg, ex); - throw new IOException(msg, ex); - } - } - } - /** * Builds a {@link FinalizationManagerImpl}. */ public static class Builder { private Table finalizationStore; private DBTransactionBuffer transactionBuffer; - private SCMRatisServer scmRatisServer; - private SCMUpgradeFinalizer upgradeFinalizer; + private SCMRatisServer ratisServer; + private ScmVersionManager versionManager; public Builder() { } - public Builder setUpgradeFinalizer(final SCMUpgradeFinalizer finalizer) { - upgradeFinalizer = finalizer; + public Builder setRatisServer(SCMRatisServer ratisServer) { + this.ratisServer = ratisServer; return this; } - public Builder setRatisServer(final SCMRatisServer ratisServer) { - scmRatisServer = ratisServer; + public Builder setVersionManager(ScmVersionManager versionManager) { + this.versionManager = versionManager; return this; } @@ -183,9 +112,9 @@ public Builder setTransactionBuffer(DBTransactionBuffer transactionBuffer) { public FinalizationStateManager build() throws IOException { Objects.requireNonNull(finalizationStore, "finalizationStore == null"); Objects.requireNonNull(transactionBuffer, "transactionBuffer == null"); - Objects.requireNonNull(upgradeFinalizer, "upgradeFinalizer == null"); + Objects.requireNonNull(versionManager, "versionManager == null"); - return scmRatisServer.getProxyHandler(FinalizationStateManager.class, new FinalizationStateManagerImpl(this)); + return ratisServer.getProxyHandler(FinalizationStateManager.class, new FinalizationStateManagerImpl(this)); } } } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/ScmOnFinalizeActionForDatanodeSchemaV2.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/ScmOnFinalizeActionForDatanodeSchemaV2.java index bd62e87def75..d6b3cef1ce3d 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/ScmOnFinalizeActionForDatanodeSchemaV2.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/ScmOnFinalizeActionForDatanodeSchemaV2.java @@ -19,6 +19,7 @@ import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.DATANODE_SCHEMA_V2; +import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; import org.apache.hadoop.hdds.upgrade.ScmUpgradeAction; import org.apache.hadoop.ozone.upgrade.UpgradeActionScm; import org.slf4j.Logger; @@ -28,13 +29,11 @@ * SCM Upgrade Action for the very first Upgrade Version. */ @UpgradeActionScm(feature = DATANODE_SCHEMA_V2) -public class ScmOnFinalizeActionForDatanodeSchemaV2 implements - ScmUpgradeAction { - private static final Logger LOG = - LoggerFactory.getLogger(ScmOnFinalizeActionForDatanodeSchemaV2.class); +public class ScmOnFinalizeActionForDatanodeSchemaV2 implements ScmUpgradeAction { + private static final Logger LOG = LoggerFactory.getLogger(ScmOnFinalizeActionForDatanodeSchemaV2.class); @Override - public void execute(SCMUpgradeFinalizationContext context) throws Exception { + public void execute(OzoneStorageContainerManager context) throws Exception { LOG.info("Executing SCM On Finalize action for layout feature {}", DATANODE_SCHEMA_V2); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/ScmVersionManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/ScmVersionManager.java new file mode 100644 index 000000000000..7d28a3328500 --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/ScmVersionManager.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.server.upgrade; + +import com.google.common.annotations.VisibleForTesting; +import java.io.IOException; +import java.util.Map; +import org.apache.hadoop.hdds.ComponentVersion; +import org.apache.hadoop.hdds.HDDSVersion; +import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; +import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; +import org.apache.hadoop.hdds.upgrade.HDDSVersionUtils; +import org.apache.hadoop.hdds.upgrade.ScmUpgradeAction; +import org.apache.hadoop.hdds.upgrade.ScmUpgradeActionProvider; +import org.apache.hadoop.ozone.upgrade.ComponentUpgradeActionProvider; +import org.apache.hadoop.ozone.upgrade.RatisBasedVersionManager; +import org.apache.hadoop.ozone.upgrade.UpgradeException; + +/** + * SCM-specific version manager that wires upgrade actions internally. + */ +public class ScmVersionManager extends RatisBasedVersionManager { + + private final Map upgradeActions; + private final OzoneStorageContainerManager upgradeActionArg; + + public ScmVersionManager(SCMStorageConfig storage, OzoneStorageContainerManager upgradeActionArg) throws IOException { + this(storage, upgradeActionArg, new ScmUpgradeActionProvider()); + } + + @VisibleForTesting + public ScmVersionManager(SCMStorageConfig storage, + OzoneStorageContainerManager upgradeActionArg, + ComponentUpgradeActionProvider upgradeActionProvider) + throws IOException { + super(storage, HDDSVersionUtils.deserializedPersistedApparentVersion(storage.getApparentVersion()), + HDDSVersion.SOFTWARE_VERSION); + this.upgradeActionArg = upgradeActionArg; + upgradeActions = upgradeActionProvider.load(); + } + + @VisibleForTesting + public Map getUpgradeActionsForTesting() { + return upgradeActions; + } + + @Override + protected void runUpgradeAction(ComponentVersion version) throws UpgradeException { + ScmUpgradeAction action = upgradeActions.get(version); + if (action == null) { + return; + } + try { + action.execute(upgradeActionArg); + } catch (Exception e) { + logAndThrow(e, "SCM upgrade action for version " + version + " failed.", + UpgradeException.ResultCodes.FINALIZE_UPGRADE_ACTION_FAILED); + } + } + + @Override + protected ComponentVersion computeApparentVersion(int serializedVersion) throws IOException { + return HDDSVersionUtils.deserializedPersistedApparentVersion(serializedVersion); + } +} diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/upgrade/ScmUpgradeAction.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/upgrade/ScmUpgradeAction.java index 25fa5d3e9dbc..47cc02ed7bab 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/upgrade/ScmUpgradeAction.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/upgrade/ScmUpgradeAction.java @@ -17,12 +17,12 @@ package org.apache.hadoop.hdds.upgrade; -import org.apache.hadoop.hdds.scm.server.upgrade.SCMUpgradeFinalizationContext; +import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; import org.apache.hadoop.ozone.upgrade.UpgradeAction; /** * Storage Container Manager Upgrade Action interface. An upgrade action is an operation that * needs to be executed during finalization. */ -public interface ScmUpgradeAction extends UpgradeAction { +public interface ScmUpgradeAction extends UpgradeAction { } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java index 57d38ece3dd6..afc564e6edde 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java @@ -268,7 +268,7 @@ public List getNodes( List healthyNodesWithInfo = new ArrayList<>(); for (DatanodeDetails dd : healthyNodes) { DatanodeInfo di = new DatanodeInfo(dd, NodeStatus.inServiceHealthy(), - UpgradeUtils.defaultLayoutVersionProto(), HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT); + UpgradeUtils.defaultVersionProto(), HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT); long capacity = nodeMetricMap.get(dd).getCapacity().get(); long used = nodeMetricMap.get(dd).getScmUsed().get(); @@ -347,7 +347,7 @@ public List getAllNodes() { nodeStatus = NodeStatus.inServiceDead(); } DatanodeInfo di = new DatanodeInfo(entry.getKey(), nodeStatus, - UpgradeUtils.defaultLayoutVersionProto(), HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT); + UpgradeUtils.defaultVersionProto(), HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT); long capacity = entry.getValue().getCapacity().get(); long used = entry.getValue().getScmUsed().get(); @@ -436,7 +436,7 @@ public DatanodeInfo getDatanodeInfo(DatanodeDetails dd) { } DatanodeInfo di = new DatanodeInfo(dd, NodeStatus.inServiceHealthy(), - UpgradeUtils.defaultLayoutVersionProto(), HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT); + UpgradeUtils.defaultVersionProto(), HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT); long capacity = nodeMetricMap.get(dd).getCapacity().get(); long used = nodeMetricMap.get(dd).getScmUsed().get(); long remaining = nodeMetricMap.get(dd).getRemaining().get(); @@ -626,8 +626,8 @@ public void processNodeReport(DatanodeDetails dnUuid, * @param layoutReport */ @Override - public void processLayoutVersionReport(DatanodeDetails dnUuid, - LayoutVersionProto layoutReport) { + public void processVersionReport(DatanodeDetails dnUuid, + LayoutVersionProto layoutReport) { // do nothing } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java index f2da8fd2878b..7d386913c422 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java @@ -47,6 +47,7 @@ import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineID; import org.apache.hadoop.hdds.server.events.EventPublisher; +import org.apache.hadoop.ozone.container.upgrade.UpgradeUtils; import org.apache.hadoop.ozone.protocol.VersionResponse; import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode; import org.apache.hadoop.ozone.protocol.commands.RegisteredCommand; @@ -67,7 +68,8 @@ public class SimpleMockNodeManager implements NodeManager { public void register(DatanodeDetails dd, NodeStatus status) { dd.setPersistedOpState(status.getOperationalState()); dd.setPersistedOpStateExpiryEpochSec(status.getOpStateExpiryEpochSeconds()); - nodeMap.put(dd.getID(), new DatanodeInfo(dd, status, null, HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT)); + nodeMap.put(dd.getID(), new DatanodeInfo(dd, status, UpgradeUtils.defaultVersionProto(), + HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT)); } public void setNodeStatus(DatanodeDetails dd, NodeStatus status) { @@ -301,8 +303,8 @@ public void processNodeReport(DatanodeDetails datanodeDetails, } @Override - public void processLayoutVersionReport(DatanodeDetails datanodeDetails, - LayoutVersionProto layoutReport) { + public void processVersionReport(DatanodeDetails datanodeDetails, + LayoutVersionProto layoutReport) { } /** diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestIncrementalContainerReportHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestIncrementalContainerReportHandler.java index 8e87cc5d88d9..1e2db41971e3 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestIncrementalContainerReportHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestIncrementalContainerReportHandler.java @@ -26,7 +26,7 @@ import static org.apache.hadoop.hdds.scm.container.TestContainerReportHandler.createMatchingDataChecksumForReplica; import static org.apache.hadoop.hdds.scm.container.TestContainerReportHandler.createUniqueDataChecksumForReplica; import static org.apache.hadoop.hdds.scm.container.TestContainerReportHandler.getContainerReportsProto; -import static org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager.maxLayoutVersion; +import static org.apache.hadoop.hdds.scm.upgrade.ScmUpgradeTestUtils.mockVersionManager; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -81,9 +81,9 @@ import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.ContainerReportFromDatanode; import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.IncrementalContainerReportFromDatanode; import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; +import org.apache.hadoop.hdds.scm.server.upgrade.ScmVersionManager; import org.apache.hadoop.hdds.server.events.EventPublisher; import org.apache.hadoop.hdds.server.events.EventQueue; -import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager; import org.apache.hadoop.hdds.utils.db.DBStore; import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; import org.apache.hadoop.ozone.common.statemachine.InvalidStateTransitionException; @@ -119,9 +119,7 @@ public void setup() throws IOException, InvalidStateTransitionException, NetworkTopology clusterMap = new NetworkTopologyImpl(conf); EventQueue eventQueue = new EventQueue(); SCMStorageConfig storageConfig = new SCMStorageConfig(conf); - HDDSLayoutVersionManager versionManager = mock(HDDSLayoutVersionManager.class); - when(versionManager.getMetadataLayoutVersion()).thenReturn(maxLayoutVersion()); - when(versionManager.getSoftwareLayoutVersion()).thenReturn(maxLayoutVersion()); + ScmVersionManager versionManager = mockVersionManager(); this.nodeManager = new SCMNodeManager(conf, storageConfig, eventQueue, clusterMap, scmContext, versionManager); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java index ed4e96b8de56..863da78884d1 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java @@ -99,7 +99,7 @@ public void testRackAwarePolicy() throws IOException { .createDatanodeDetails(hostname + i, rack + (i / 5)); DatanodeInfo datanodeInfo = new DatanodeInfo( datanodeDetails, NodeStatus.inServiceHealthy(), - UpgradeUtils.defaultLayoutVersionProto(), HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT); + UpgradeUtils.defaultVersionProto(), HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT); StorageReportProto storage1 = HddsTestUtils.createStorageReport( datanodeInfo.getID(), "/data1-" + datanodeInfo.getID(), diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementCapacity.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementCapacity.java index 1fb3f53504d3..44d37f359c04 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementCapacity.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementCapacity.java @@ -64,7 +64,7 @@ public void chooseDatanodes() throws SCMException { DatanodeInfo datanodeInfo = new DatanodeInfo( MockDatanodeDetails.randomDatanodeDetails(), NodeStatus.inServiceHealthy(), - UpgradeUtils.defaultLayoutVersionProto(), + UpgradeUtils.defaultVersionProto(), HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT); StorageReportProto storage1 = HddsTestUtils.createStorageReport( diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java index beb6ad82877c..549a3b7ad35e 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java @@ -112,7 +112,7 @@ private void setup(int datanodeCount) { cluster.add(datanodeDetails); DatanodeInfo datanodeInfo = new DatanodeInfo( datanodeDetails, NodeStatus.inServiceHealthy(), - UpgradeUtils.defaultLayoutVersionProto(), + UpgradeUtils.defaultVersionProto(), HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT); StorageReportProto storage1 = HddsTestUtils.createStorageReport( @@ -456,7 +456,7 @@ public void testDatanodeWithDefaultNetworkLocation(int datanodeCount) hostname + i, null); DatanodeInfo dnInfo = new DatanodeInfo( dn, NodeStatus.inServiceHealthy(), - UpgradeUtils.defaultLayoutVersionProto(), + UpgradeUtils.defaultVersionProto(), HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT); StorageReportProto storage1 = HddsTestUtils.createStorageReport( diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackScatter.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackScatter.java index e015b93c1e31..6403fc7c56b3 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackScatter.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackScatter.java @@ -180,7 +180,7 @@ private void setupDatanode(DatanodeDetails datanodeDetails) { cluster.add(datanodeDetails); DatanodeInfo datanodeInfo = new DatanodeInfo( datanodeDetails, NodeStatus.inServiceHealthy(), - UpgradeUtils.defaultLayoutVersionProto(), + UpgradeUtils.defaultVersionProto(), HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT); StorageReportProto storage1 = HddsTestUtils.createStorageReport( @@ -491,7 +491,7 @@ public void testDatanodeWithDefaultNetworkLocation(int datanodeCount) hostname + i, null); DatanodeInfo dnInfo = new DatanodeInfo( dn, NodeStatus.inServiceHealthy(), - UpgradeUtils.defaultLayoutVersionProto(), + UpgradeUtils.defaultVersionProto(), HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT); StorageReportProto storage1 = HddsTestUtils.createStorageReport( diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRandom.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRandom.java index 47602a385fd6..ed1b349d5ecb 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRandom.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRandom.java @@ -61,7 +61,7 @@ public void chooseDatanodes() throws SCMException { DatanodeInfo datanodeInfo = new DatanodeInfo( MockDatanodeDetails.randomDatanodeDetails(), NodeStatus.inServiceHealthy(), - UpgradeUtils.defaultLayoutVersionProto(), + UpgradeUtils.defaultVersionProto(), HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT); StorageReportProto storage1 = HddsTestUtils.createStorageReport( @@ -168,7 +168,7 @@ public void testIsValidNode() throws SCMException { DatanodeInfo datanodeInfo = new DatanodeInfo( MockDatanodeDetails.randomDatanodeDetails(), NodeStatus.inServiceHealthy(), - UpgradeUtils.defaultLayoutVersionProto(), + UpgradeUtils.defaultVersionProto(), HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT); StorageReportProto storage1 = HddsTestUtils.createStorageReport( diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java index 4dbe79fc1351..5dacbd30c1ab 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java @@ -22,7 +22,7 @@ import static org.apache.hadoop.hdds.scm.net.NetConstants.LEAF_SCHEMA; import static org.apache.hadoop.hdds.scm.net.NetConstants.RACK_SCHEMA; import static org.apache.hadoop.hdds.scm.net.NetConstants.ROOT_SCHEMA; -import static org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager.maxLayoutVersion; +import static org.apache.hadoop.hdds.scm.upgrade.ScmUpgradeTestUtils.mockVersionManager; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.mockito.ArgumentMatchers.any; @@ -71,8 +71,8 @@ import org.apache.hadoop.hdds.scm.pipeline.MockPipelineManager; import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; +import org.apache.hadoop.hdds.scm.server.upgrade.ScmVersionManager; import org.apache.hadoop.hdds.server.events.EventQueue; -import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager; import org.apache.hadoop.hdds.utils.db.DBStore; import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; import org.apache.hadoop.ozone.OzoneConsts; @@ -143,17 +143,14 @@ SCMNodeManager createNodeManager(OzoneConfiguration config) { SCMStorageConfig storageConfig = mock(SCMStorageConfig.class); when(storageConfig.getClusterID()).thenReturn("cluster1"); - HDDSLayoutVersionManager versionManager = mock(HDDSLayoutVersionManager.class); - when(versionManager.getMetadataLayoutVersion()).thenReturn(maxLayoutVersion()); - when(versionManager.getSoftwareLayoutVersion()).thenReturn(maxLayoutVersion()); + ScmVersionManager versionManager = mockVersionManager(); NodeSchema[] schemas = new NodeSchema[] {ROOT_SCHEMA, RACK_SCHEMA, LEAF_SCHEMA}; NodeSchemaManager.getInstance().init(schemas, true); NetworkTopology networkTopology = new NetworkTopologyImpl(NodeSchemaManager.getInstance()); - SCMNodeManager scmNodeManager = new SCMNodeManager(config, storageConfig, + return new SCMNodeManager(config, storageConfig, eventQueue, networkTopology, SCMContext.emptyContext(), versionManager); - return scmNodeManager; } ContainerManager createContainerManager() diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeReportHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeReportHandler.java index ffeec5debe43..e99ebdf48f1c 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeReportHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeReportHandler.java @@ -17,7 +17,7 @@ package org.apache.hadoop.hdds.scm.node; -import static org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager.maxLayoutVersion; +import static org.apache.hadoop.hdds.scm.upgrade.ScmUpgradeTestUtils.mockVersionManager; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.mockito.Mockito.mock; @@ -40,10 +40,10 @@ import org.apache.hadoop.hdds.scm.net.NetworkTopologyImpl; import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.NodeReportFromDatanode; import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; +import org.apache.hadoop.hdds.scm.server.upgrade.ScmVersionManager; import org.apache.hadoop.hdds.server.events.Event; import org.apache.hadoop.hdds.server.events.EventPublisher; import org.apache.hadoop.hdds.server.events.EventQueue; -import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -71,9 +71,7 @@ public void resetEventCollector() throws IOException { when(storageConfig.getClusterID()).thenReturn("cluster1"); NetworkTopology clusterMap = new NetworkTopologyImpl(conf); - HDDSLayoutVersionManager versionManager = mock(HDDSLayoutVersionManager.class); - when(versionManager.getMetadataLayoutVersion()).thenReturn(maxLayoutVersion()); - when(versionManager.getSoftwareLayoutVersion()).thenReturn(maxLayoutVersion()); + ScmVersionManager versionManager = mockVersionManager(); nodeManager = new SCMNodeManager(conf, storageConfig, new EventQueue(), clusterMap, SCMContext.emptyContext(), versionManager); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeStateManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeStateManager.java index db0425fed745..de49a6bc7680 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeStateManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeStateManager.java @@ -17,7 +17,7 @@ package org.apache.hadoop.hdds.scm.node; -import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.defaultLayoutVersionProto; +import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.defaultVersionProto; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; @@ -40,6 +40,7 @@ import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; import org.apache.hadoop.hdds.server.events.Event; import org.apache.hadoop.hdds.server.events.EventPublisher; +import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature; import org.apache.hadoop.hdds.utils.HddsServerUtil; import org.apache.hadoop.ozone.container.upgrade.UpgradeUtils; import org.apache.hadoop.util.Time; @@ -86,7 +87,7 @@ public void testNodeCanBeAddedAndRetrieved() throws NodeAlreadyExistsException, NodeNotFoundException { // Create a datanode, then add and retrieve it DatanodeDetails dn = generateDatanode(); - nsm.addNode(dn, UpgradeUtils.defaultLayoutVersionProto()); + nsm.addNode(dn, UpgradeUtils.defaultVersionProto()); assertEquals(dn.getUuid(), nsm.getNode(dn).getUuid()); // Now get the status of the newly added node and it should be // IN_SERVICE and HEALTHY @@ -98,9 +99,9 @@ public void testNodeCanBeAddedAndRetrieved() public void testGetAllNodesReturnsCorrectly() throws NodeAlreadyExistsException { DatanodeDetails dn = generateDatanode(); - nsm.addNode(dn, UpgradeUtils.defaultLayoutVersionProto()); + nsm.addNode(dn, UpgradeUtils.defaultVersionProto()); dn = generateDatanode(); - nsm.addNode(dn, UpgradeUtils.defaultLayoutVersionProto()); + nsm.addNode(dn, UpgradeUtils.defaultVersionProto()); assertEquals(2, nsm.getAllNodes().size()); assertEquals(2, nsm.getTotalNodeCount()); } @@ -109,7 +110,7 @@ public void testGetAllNodesReturnsCorrectly() public void testGetNodeCountReturnsCorrectly() throws NodeAlreadyExistsException { DatanodeDetails dn = generateDatanode(); - nsm.addNode(dn, UpgradeUtils.defaultLayoutVersionProto()); + nsm.addNode(dn, UpgradeUtils.defaultVersionProto()); assertEquals(1, nsm.getNodes(NodeStatus.inServiceHealthy()).size()); assertEquals(0, nsm.getNodes(NodeStatus.inServiceStale()).size()); } @@ -117,7 +118,7 @@ public void testGetNodeCountReturnsCorrectly() @Test public void testGetNodeCount() throws NodeAlreadyExistsException { DatanodeDetails dn = generateDatanode(); - nsm.addNode(dn, UpgradeUtils.defaultLayoutVersionProto()); + nsm.addNode(dn, UpgradeUtils.defaultVersionProto()); assertEquals(1, nsm.getNodeCount(NodeStatus.inServiceHealthy())); assertEquals(0, nsm.getNodeCount(NodeStatus.inServiceStale())); } @@ -132,15 +133,15 @@ public void testNodesMarkedDeadAndStale() long deadLimit = HddsServerUtil.getDeadNodeInterval(conf) + 1000; DatanodeDetails staleDn = generateDatanode(); - nsm.addNode(staleDn, defaultLayoutVersionProto()); + nsm.addNode(staleDn, defaultVersionProto()); nsm.getNode(staleDn).updateLastHeartbeatTime(now - staleLimit); DatanodeDetails deadDn = generateDatanode(); - nsm.addNode(deadDn, defaultLayoutVersionProto()); + nsm.addNode(deadDn, defaultVersionProto()); nsm.getNode(deadDn).updateLastHeartbeatTime(now - deadLimit); DatanodeDetails healthyDn = generateDatanode(); - nsm.addNode(healthyDn, defaultLayoutVersionProto()); + nsm.addNode(healthyDn, defaultVersionProto()); nsm.getNode(healthyDn).updateLastHeartbeatTime(); nsm.checkNodesHealth(); @@ -165,7 +166,7 @@ public void testNodeCanTransitionThroughHealthStatesAndFiresEvents() long deadLimit = HddsServerUtil.getDeadNodeInterval(conf) + 1000; DatanodeDetails dn = generateDatanode(); - nsm.addNode(dn, defaultLayoutVersionProto()); + nsm.addNode(dn, defaultVersionProto()); DatanodeInfo dni = nsm.getNode(dn); dni.updateLastHeartbeatTime(); @@ -209,7 +210,7 @@ public void testNodeCanTransitionThroughHealthStatesAndFiresEvents() public void testNodeOpStateCanBeSet() throws NodeAlreadyExistsException, NodeNotFoundException { DatanodeDetails dn = generateDatanode(); - nsm.addNode(dn, UpgradeUtils.defaultLayoutVersionProto()); + nsm.addNode(dn, UpgradeUtils.defaultVersionProto()); nsm.setNodeOperationalState(dn, HddsProtos.NodeOperationalState.DECOMMISSIONED); @@ -223,7 +224,7 @@ public void testNodeOpStateCanBeSet() public void testContainerCanBeAddedAndRemovedFromDN() throws NodeAlreadyExistsException, NodeNotFoundException { DatanodeDetails dn = generateDatanode(); - nsm.addNode(dn, UpgradeUtils.defaultLayoutVersionProto()); + nsm.addNode(dn, UpgradeUtils.defaultVersionProto()); nsm.addContainer(dn.getID(), ContainerID.valueOf(1)); nsm.addContainer(dn.getID(), ContainerID.valueOf(2)); @@ -244,7 +245,7 @@ public void testContainerCanBeAddedAndRemovedFromDN() public void testHealthEventsFiredWhenOpStateChanged() throws NodeAlreadyExistsException, NodeNotFoundException { DatanodeDetails dn = generateDatanode(); - nsm.addNode(dn, UpgradeUtils.defaultLayoutVersionProto()); + nsm.addNode(dn, UpgradeUtils.defaultVersionProto()); // First set the node to decommissioned, then run through all op states in // order and ensure the unhealthy_to_healthy event gets fired @@ -297,7 +298,7 @@ public void testUpdateNode() throws NodeAlreadyExistsException, String hostName = "test-host"; StorageContainerDatanodeProtocolProtos.LayoutVersionProto layoutVersionProto = - UpgradeUtils.toLayoutVersionProto(1, 2); + UpgradeUtils.toVersionProto(HDDSLayoutFeature.INITIAL_VERSION, HDDSLayoutFeature.INITIAL_VERSION); DatanodeDetails dn = DatanodeDetails.newBuilder() .setUuid(dnUuid) .setIpAddress(ipAddress) @@ -309,7 +310,7 @@ public void testUpdateNode() throws NodeAlreadyExistsException, String newIpAddress = "2.3.4.5"; String newHostName = "new-host"; StorageContainerDatanodeProtocolProtos.LayoutVersionProto - newLayoutVersionProto = UpgradeUtils.defaultLayoutVersionProto(); + newLayoutVersionProto = UpgradeUtils.defaultVersionProto(); DatanodeDetails newDn = DatanodeDetails.newBuilder() .setUuid(dnUuid) .setIpAddress(newIpAddress) diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestPendingContainerTracker.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestPendingContainerTracker.java index c747dc7d60ae..a9df35f89002 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestPendingContainerTracker.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestPendingContainerTracker.java @@ -28,6 +28,7 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageReportProto; import org.apache.hadoop.hdds.scm.HddsTestUtils; import org.apache.hadoop.hdds.scm.container.ContainerID; +import org.apache.hadoop.ozone.container.upgrade.UpgradeUtils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; @@ -58,8 +59,8 @@ public void setUp() throws IOException { datanodes = new ArrayList<>(NUM_DATANODES); for (int i = 0; i < NUM_DATANODES; i++) { datanodes.add(new DatanodeInfo( - MockDatanodeDetails.randomLocalDatanodeDetails(), NodeStatus.inServiceHealthy(), null, - HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT)); + MockDatanodeDetails.randomLocalDatanodeDetails(), NodeStatus.inServiceHealthy(), + UpgradeUtils.defaultVersionProto(), HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT)); } containers = new ArrayList<>(NUM_CONTAINERS); @@ -126,8 +127,8 @@ public void testRemovePendingAllocation() { public void testTwoWindowRollAgesOutContainerAfterTwoIntervals() throws InterruptedException { long rollMs = 200L; DatanodeInfo shortDn = new DatanodeInfo( - MockDatanodeDetails.randomLocalDatanodeDetails(), NodeStatus.inServiceHealthy(), null, - rollMs); + MockDatanodeDetails.randomLocalDatanodeDetails(), NodeStatus.inServiceHealthy(), + UpgradeUtils.defaultVersionProto(), rollMs); PendingContainerTracker shortRollTracker = new PendingContainerTracker(MAX_CONTAINER_SIZE, rollMs, null); @@ -162,8 +163,8 @@ public void testRemoveNonExistentContainer() { @Test public void testUnknownDatanodeHasZeroPendingCount() { DatanodeInfo unknownDN = new DatanodeInfo( - MockDatanodeDetails.randomDatanodeDetails(), NodeStatus.inServiceHealthy(), null, - HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT); + MockDatanodeDetails.randomDatanodeDetails(), NodeStatus.inServiceHealthy(), + UpgradeUtils.defaultVersionProto(), HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT); assertEquals(0, unknownDN.getPendingContainerAllocations().getCount()); } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java index 543b97530cd0..eb5ffda37ac2 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java @@ -38,7 +38,9 @@ import static org.apache.hadoop.hdds.scm.events.SCMEvents.DATANODE_COMMAND; import static org.apache.hadoop.hdds.scm.events.SCMEvents.DATANODE_COMMAND_COUNT_UPDATED; import static org.apache.hadoop.hdds.scm.events.SCMEvents.NEW_NODE; -import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.toLayoutVersionProto; +import static org.apache.hadoop.hdds.scm.upgrade.ScmUpgradeTestUtils.mockVersionManager; +import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.defaultVersionProto; +import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.toVersionProto; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; @@ -67,6 +69,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.hdds.HDDSVersion; import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -97,9 +100,10 @@ import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.NodeReportFromDatanode; import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; +import org.apache.hadoop.hdds.scm.server.upgrade.ScmVersionManager; import org.apache.hadoop.hdds.server.events.EventPublisher; import org.apache.hadoop.hdds.server.events.EventQueue; -import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager; +import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature; import org.apache.hadoop.ozone.container.upgrade.UpgradeUtils; import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand; import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode; @@ -137,19 +141,21 @@ public class TestSCMNodeManager { private StorageContainerManager scm; private SCMContext scmContext; - private static final int MAX_SOFTWARE_VERSION = HDDSLayoutVersionManager.maxLayoutVersion(); - private static final LayoutVersionProto LARGER_SOFTWARE_PROTO = - toLayoutVersionProto(MAX_SOFTWARE_VERSION, MAX_SOFTWARE_VERSION + 1); + private static final LayoutVersionProto LARGER_SOFTWARE_PROTO = LayoutVersionProto.newBuilder() + .setMetadataLayoutVersion(HDDSVersion.SOFTWARE_VERSION.serialize()) + .setSoftwareLayoutVersion(HDDSVersion.SOFTWARE_VERSION.serialize() + 1) + .build(); private static final LayoutVersionProto SMALLER_APPARENT_VERSION_PROTO = - toLayoutVersionProto(MAX_SOFTWARE_VERSION - 1, MAX_SOFTWARE_VERSION); - // In a real cluster, startup is disallowed if MLV is larger than SLV, so - // increase both numbers to test smaller SLV or larger MLV. + toVersionProto(HDDSLayoutFeature.SCM_HA, HDDSVersion.SOFTWARE_VERSION); + // In a real cluster, startup is disallowed if apparent version is larger than software version, so + // increase both numbers to test smaller software version or larger apparent version. private static final LayoutVersionProto SMALLER_ALL_VERSIONS_PROTO = - toLayoutVersionProto(MAX_SOFTWARE_VERSION - 1, MAX_SOFTWARE_VERSION - 1); - private static final LayoutVersionProto LARGER_ALL_VERSIONS_PROTO = - toLayoutVersionProto(MAX_SOFTWARE_VERSION + 1, MAX_SOFTWARE_VERSION + 1); - private static final LayoutVersionProto MATCHING_VERSION_PROTO = - toLayoutVersionProto(MAX_SOFTWARE_VERSION, MAX_SOFTWARE_VERSION); + toVersionProto(HDDSLayoutFeature.SCM_HA, HDDSLayoutFeature.SCM_HA); + private static final LayoutVersionProto LARGER_ALL_VERSIONS_PROTO = LayoutVersionProto.newBuilder() + .setMetadataLayoutVersion(HDDSVersion.SOFTWARE_VERSION.serialize() + 1) + .setSoftwareLayoutVersion(HDDSVersion.SOFTWARE_VERSION.serialize() + 1) + .build(); + private static final LayoutVersionProto MATCHING_VERSION_PROTO = defaultVersionProto(); @BeforeEach public void setup() { @@ -271,7 +277,7 @@ public void testGetLastHeartbeatTimeDiff() throws Exception { */ private DatanodeDetails registerWithCapacity(SCMNodeManager nodeManager) { return registerWithCapacity(nodeManager, - UpgradeUtils.defaultLayoutVersionProto(), success); + UpgradeUtils.defaultVersionProto(), success); } /** @@ -733,10 +739,10 @@ void testScmHandleJvmPause() throws Exception { } @Test - public void testProcessLayoutVersion() throws IOException { - testProcessLayoutVersionLowerMlv(true); - testProcessLayoutVersionLowerMlv(false); - testProcessLayoutVersionReportHigherMlv(); + public void testProcessVersionReports() throws IOException { + testProcessVersionReportLowerApparentVersion(true); + testProcessVersionReportLowerApparentVersion(false); + testProcessVersionReportHigherApparentVersion(); } @Test @@ -749,27 +755,26 @@ public void testDatanodeFinalizedCounterTracksVersionReports() .getNumFinalizedDatanodes(), "Initial datanode should be counted as finalized"); - int softwareVersion = - nodeManager.getLayoutVersionManager().getSoftwareLayoutVersion(); - int metadataVersion = - nodeManager.getLayoutVersionManager().getMetadataLayoutVersion(); - nodeManager.processLayoutVersionReport(node, + // Report a pre-finalized datanode. + int softwareVersion = HDDSVersion.SOFTWARE_VERSION.serialize(); + nodeManager.processVersionReport(node, LayoutVersionProto.newBuilder() - .setMetadataLayoutVersion(metadataVersion - 1) + .setMetadataLayoutVersion(HDDSLayoutFeature.INITIAL_VERSION.serialize()) .setSoftwareLayoutVersion(softwareVersion) .build()); assertEquals(0, nodeManager.getDatanodeFinalizationCounts() .getNumFinalizedDatanodes(), - "Lower metadata layout version should decrement finalized count"); + "Lower apparent version should decrement finalized count"); - nodeManager.processLayoutVersionReport(node, + // Report a finalized datanode. + nodeManager.processVersionReport(node, LayoutVersionProto.newBuilder() - .setMetadataLayoutVersion(metadataVersion) + .setMetadataLayoutVersion(softwareVersion) .setSoftwareLayoutVersion(softwareVersion) .build()); assertEquals(1, nodeManager.getDatanodeFinalizationCounts() .getNumFinalizedDatanodes(), - "Restored metadata layout version should restore finalized count"); + "Restored apparent version should restore finalized count"); } } @@ -880,7 +885,7 @@ public void testDatanodeFinalizedCounterIncludesAllHealthyOpStates( } // Currently invoked by testProcessLayoutVersion. - public void testProcessLayoutVersionReportHigherMlv() + public void testProcessVersionReportHigherApparentVersion() throws IOException { final int healthCheckInterval = 200; // milliseconds final int heartbeatInterval = 1; // seconds @@ -894,73 +899,61 @@ public void testProcessLayoutVersionReportHigherMlv() SCMStorageConfig scmStorageConfig = mock(SCMStorageConfig.class); when(scmStorageConfig.getClusterID()).thenReturn("xyz111"); EventPublisher eventPublisher = mock(EventPublisher.class); - HDDSLayoutVersionManager lvm = - new HDDSLayoutVersionManager(scmStorageConfig.getApparentVersion(), null, null); + ScmVersionManager versionManager = mockVersionManager(); + SCMContext nodeManagerContext = SCMContext.emptyContext(); SCMNodeManager nodeManager = new SCMNodeManager(conf, scmStorageConfig, eventPublisher, new NetworkTopologyImpl(conf), - nodeManagerContext, lvm); + nodeManagerContext, versionManager); - // Regardless of SCM's finalization checkpoint, datanodes with higher MLV - // than SCM should not be found in the cluster. + // Datanodes should never have higher apparent version than SCM. DatanodeDetails node1 = HddsTestUtils.createRandomDatanodeAndRegister(nodeManager); LogCapturer logCapturer = LogCapturer.captureLogs(SCMNodeManager.class); - int scmMlv = - nodeManager.getLayoutVersionManager().getMetadataLayoutVersion(); - int scmSlv = - nodeManager.getLayoutVersionManager().getSoftwareLayoutVersion(); - nodeManager.processLayoutVersionReport(node1, - LayoutVersionProto.newBuilder() - .setMetadataLayoutVersion(scmMlv + 1) - .setSoftwareLayoutVersion(scmSlv + 1) - .build()); - assertThat(logCapturer.getOutput()) - .contains("Invalid data node in the cluster"); + nodeManager.processVersionReport(node1, LARGER_ALL_VERSIONS_PROTO); + assertThat(logCapturer.getOutput()).contains("will not be allowed to join the cluster"); nodeManager.close(); } // Currently invoked by testProcessLayoutVersion. - public void testProcessLayoutVersionLowerMlv(boolean mvlLessThanSlv) throws IOException { + public void testProcessVersionReportLowerApparentVersion(boolean withScmFinalized) { OzoneConfiguration conf = new OzoneConfiguration(); SCMStorageConfig scmStorageConfig = mock(SCMStorageConfig.class); when(scmStorageConfig.getClusterID()).thenReturn("xyz111"); EventPublisher eventPublisher = mock(EventPublisher.class); - int currentVersion = HDDSLayoutVersionManager.maxLayoutVersion(); - if (mvlLessThanSlv) { - currentVersion -= 1; + + ScmVersionManager versionManager; + if (withScmFinalized) { + versionManager = mockVersionManager(); + } else { + // Use an apparent version for SCM that is in between SCM's software version and the datanode's apparent version. + versionManager = mockVersionManager(HDDSLayoutFeature.SCM_HA); } - HDDSLayoutVersionManager lvm = new HDDSLayoutVersionManager(currentVersion, null, null); SCMContext nodeManagerContext = SCMContext.emptyContext(); SCMNodeManager nodeManager = new SCMNodeManager(conf, scmStorageConfig, eventPublisher, new NetworkTopologyImpl(conf), - nodeManagerContext, lvm); + nodeManagerContext, versionManager); DatanodeDetails node1 = HddsTestUtils.createRandomDatanodeAndRegister(nodeManager); verify(eventPublisher, times(1)).fireEvent(NEW_NODE, node1); - int scmMlv = - nodeManager.getLayoutVersionManager().getMetadataLayoutVersion(); - nodeManager.processLayoutVersionReport(node1, + nodeManager.processVersionReport(node1, LayoutVersionProto.newBuilder() - .setMetadataLayoutVersion(scmMlv - 1) - .setSoftwareLayoutVersion(scmMlv) + .setMetadataLayoutVersion(HDDSLayoutFeature.INITIAL_VERSION.serialize()) + .setSoftwareLayoutVersion(versionManager.getSoftwareVersion().serialize()) .build()); ArgumentCaptor captor = ArgumentCaptor.forClass(CommandForDatanode.class); - if (!lvm.needsFinalization()) { - // If the mlv equals slv checkpoint passed, datanodes with older mlvs - // should be instructed to finalize. + // SCM will only tell datanodes to finalize after it has finalized. + if (withScmFinalized) { verify(eventPublisher, times(1)) .fireEvent(eq(DATANODE_COMMAND), captor.capture()); assertEquals(captor.getValue().getDatanodeId(), node1.getID()); - assertEquals(captor.getValue().getCommand().getType(), - finalizeNewLayoutVersionCommand); + assertEquals(finalizeNewLayoutVersionCommand, + captor.getValue().getCommand().getType()); } else { - // SCM has not finished finalizing its mlv, so datanodes with older - // mlvs should not be instructed to finalize yet. verify(eventPublisher, times(0)) .fireEvent(eq(DATANODE_COMMAND), captor.capture()); } @@ -973,12 +966,10 @@ public void testProcessCommandQueueReport() SCMStorageConfig scmStorageConfig = mock(SCMStorageConfig.class); when(scmStorageConfig.getClusterID()).thenReturn("xyz111"); EventPublisher eventPublisher = mock(EventPublisher.class); - HDDSLayoutVersionManager lvm = - new HDDSLayoutVersionManager(scmStorageConfig.getApparentVersion(), null, null); createNodeManager(getConf()); SCMNodeManager nodeManager = new SCMNodeManager(conf, scmStorageConfig, eventPublisher, new NetworkTopologyImpl(conf), - scmContext, lvm); + scmContext, mockVersionManager()); DatanodeDetails node1 = HddsTestUtils.createRandomDatanodeAndRegister(nodeManager); @@ -2166,11 +2157,10 @@ public void testNodeOperationalStateChange( SCMStorageConfig scmStorageConfig = mock(SCMStorageConfig.class); when(scmStorageConfig.getClusterID()).thenReturn("xyz111"); EventPublisher eventPublisher = mock(EventPublisher.class); - HDDSLayoutVersionManager lvm = new HDDSLayoutVersionManager(scmStorageConfig.getApparentVersion(), null, null); createNodeManager(getConf()); SCMNodeManager nodeManager = new SCMNodeManager(conf, scmStorageConfig, eventPublisher, new NetworkTopologyImpl(conf), - scmContext, lvm); + scmContext, mockVersionManager()); DatanodeDetails datanode = MockDatanodeDetails.randomDatanodeDetails(); datanode.setPersistedOpState(oldState); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeMetrics.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeMetrics.java index 5a5a9c594a73..bfce2f290973 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeMetrics.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeMetrics.java @@ -18,13 +18,11 @@ package org.apache.hadoop.hdds.scm.node; import static java.lang.Thread.sleep; -import static org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager.maxLayoutVersion; +import static org.apache.hadoop.hdds.scm.upgrade.ScmUpgradeTestUtils.mockVersionManager; import static org.apache.ozone.test.MetricsAsserts.assertGauge; import static org.apache.ozone.test.MetricsAsserts.getLongCounter; import static org.apache.ozone.test.MetricsAsserts.getMetrics; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; import java.io.File; import java.io.IOException; @@ -40,8 +38,8 @@ import org.apache.hadoop.hdds.scm.ha.SCMContext; import org.apache.hadoop.hdds.scm.net.NetworkTopologyImpl; import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; +import org.apache.hadoop.hdds.scm.server.upgrade.ScmVersionManager; import org.apache.hadoop.hdds.server.events.EventQueue; -import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -63,9 +61,7 @@ public static void setup() throws Exception { EventQueue publisher = new EventQueue(); SCMStorageConfig config = new SCMStorageConfig(NodeType.DATANODE, new File("/tmp"), "storage"); - HDDSLayoutVersionManager versionManager = mock(HDDSLayoutVersionManager.class); - when(versionManager.getMetadataLayoutVersion()).thenReturn(maxLayoutVersion()); - when(versionManager.getSoftwareLayoutVersion()).thenReturn(maxLayoutVersion()); + ScmVersionManager versionManager = mockVersionManager(); nodeManager = new SCMNodeManager(source, config, publisher, new NetworkTopologyImpl(source), SCMContext.emptyContext(), versionManager); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/states/TestNodeStateMap.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/states/TestNodeStateMap.java index e9f13ee82c8c..d0b9c53d0e75 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/states/TestNodeStateMap.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/states/TestNodeStateMap.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.node.DatanodeInfo; import org.apache.hadoop.hdds.scm.node.NodeStatus; +import org.apache.hadoop.ozone.container.upgrade.UpgradeUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -53,7 +54,8 @@ void addNode(NodeStatus status) throws NodeAlreadyExistsException { } void addNode(DatanodeDetails datanode, NodeStatus status) throws NodeAlreadyExistsException { - map.addNode(new DatanodeInfo(datanode, status, null, HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT)); + map.addNode(new DatanodeInfo(datanode, status, UpgradeUtils.defaultVersionProto(), + HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT)); } @BeforeEach diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementFactory.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementFactory.java index 84afb74f0a5a..3739581a825a 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementFactory.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementFactory.java @@ -103,7 +103,7 @@ private void setupRacks(int datanodeCount, int nodesPerRack, cluster.add(datanodeDetails); DatanodeInfo datanodeInfo = new DatanodeInfo( datanodeDetails, NodeStatus.inServiceHealthy(), - UpgradeUtils.defaultLayoutVersionProto(), + UpgradeUtils.defaultVersionProto(), HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT); StorageContainerDatanodeProtocolProtos.StorageReportProto storage1 = diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/upgrade/TestScmVersionManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/upgrade/TestScmVersionManager.java new file mode 100644 index 000000000000..c112217f80e7 --- /dev/null +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/upgrade/TestScmVersionManager.java @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.server.upgrade; + +import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.DATANODE_SCHEMA_V2; +import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT; +import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.INITIAL_VERSION; +import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION; +import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.WITNESSED_CONTAINER_DB_PROTO_VALUE; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Stream; +import org.apache.hadoop.hdds.ComponentVersion; +import org.apache.hadoop.hdds.HDDSVersion; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.scm.ScmConfigKeys; +import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; +import org.apache.hadoop.hdds.scm.server.StorageContainerManager; +import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature; +import org.apache.hadoop.hdds.upgrade.ScmUpgradeAction; +import org.apache.hadoop.hdds.upgrade.ScmUpgradeActionProvider; +import org.apache.hadoop.ozone.upgrade.AbstractComponentVersionManagerTest; +import org.apache.hadoop.ozone.upgrade.ComponentUpgradeActionProvider; +import org.apache.hadoop.ozone.upgrade.ComponentVersionManager; +import org.apache.hadoop.ozone.upgrade.UpgradeException; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.provider.Arguments; + +/** + * Tests for {@link ScmVersionManager} using on-disk {@link SCMStorageConfig} + * under a JUnit temp directory. + */ +class TestScmVersionManager extends AbstractComponentVersionManagerTest { + + private OzoneConfiguration conf; + + @TempDir + private Path tempDir; + + private static final List ALL_VERSIONS; + + static { + ALL_VERSIONS = new ArrayList<>(Arrays.asList(HDDSLayoutFeature.values())); + for (HDDSVersion version : HDDSVersion.values()) { + // Add all defined versions after and including ZDU to get the complete version list. + if (HDDSVersion.ZDU.isSupportedBy(version) && version != HDDSVersion.UNKNOWN_VERSION) { + ALL_VERSIONS.add(version); + } + } + } + + @BeforeEach + public void init() { + conf = new OzoneConfiguration(); + } + + public static Stream preFinalizedVersionArgs() { + return ALL_VERSIONS.stream() + .limit(ALL_VERSIONS.size() - 1) + .map(org.junit.jupiter.params.provider.Arguments::of); + } + + @Override + protected ComponentVersionManager createManager(int serializedApparentVersion) throws IOException { + return createManager(serializedApparentVersion, HashMap::new); + } + + private ScmVersionManager createManager(int serializedApparentVersion, + ComponentUpgradeActionProvider actions) throws IOException { + SCMStorageConfig storage = newScmStorage(serializedApparentVersion); + StorageContainerManager context = mock(StorageContainerManager.class); + return new ScmVersionManager(storage, context, actions); + } + + private SCMStorageConfig newScmStorage(int apparentVersion) throws IOException { + Path storageRoot = Files.createTempDirectory(tempDir, "scm-version-manager-"); + conf.set(ScmConfigKeys.OZONE_SCM_DB_DIRS, storageRoot.toString()); + SCMStorageConfig storage = new SCMStorageConfig(conf); + storage.setScmId("test-scm"); + storage.setApparentVersion(apparentVersion); + storage.initialize(); + return storage; + } + + @Override + protected List allVersionsInOrder() { + return ALL_VERSIONS; + } + + @Override + protected ComponentVersion expectedSoftwareVersion() { + return HDDSVersion.SOFTWARE_VERSION; + } + + @Override + @Test + public void testClasspathScanDiscoversUpgradeActions() throws Exception { + try (ScmVersionManager versionManager = createManager(INITIAL_VERSION.serialize(), + new ScmUpgradeActionProvider())) { + assertTrue(versionManager.needsFinalization()); + ScmUpgradeAction upgradeAction = versionManager.getUpgradeActionsForTesting().get(DATANODE_SCHEMA_V2); + assertInstanceOf(ScmOnFinalizeActionForDatanodeSchemaV2.class, upgradeAction); + } + + try (ScmVersionManager versionManager = createManager(HDDSVersion.SOFTWARE_VERSION.serialize(), + new ScmUpgradeActionProvider())) { + assertFalse(versionManager.needsFinalization()); + ScmUpgradeAction upgradeAction = versionManager.getUpgradeActionsForTesting().get(DATANODE_SCHEMA_V2); + assertInstanceOf(ScmOnFinalizeActionForDatanodeSchemaV2.class, upgradeAction); + } + } + + @Override + @Test + public void testFinalizeRunsSuppliedUpgradeAction() throws Exception { + ScmUpgradeAction mockECAction = mock(ScmUpgradeAction.class); + ScmUpgradeAction mockZDUAction = mock(ScmUpgradeAction.class); + + ComponentUpgradeActionProvider provider = () -> { + Map m = new HashMap<>(); + m.put(ERASURE_CODED_STORAGE_SUPPORT, mockECAction); + m.put(HDDSVersion.ZDU, mockZDUAction); + return m; + }; + + try (ScmVersionManager versionManager = createManager(ERASURE_CODED_STORAGE_SUPPORT.serialize(), provider)) { + versionManager.finalizeUpgrade(); + assertEquals(HDDSVersion.SOFTWARE_VERSION, versionManager.getApparentVersion()); + + // Apparent version is already EC; finalization runs actions for later versions only, not for EC itself. + verify(mockECAction, never()).execute(any()); + verify(mockZDUAction, atLeastOnce()).execute(any()); + assertScmApparentVersionOnDisk(conf, HDDSVersion.SOFTWARE_VERSION.serialize()); + } + } + + @Override + @Test + public void testUpgradeActionFailureAbortsFinalize() throws Exception { + ComponentUpgradeActionProvider provider = () -> { + Map m = new HashMap<>(); + m.put(STORAGE_SPACE_DISTRIBUTION, o -> { + throw new IOException("expected test failure"); + }); + return m; + }; + + try (ScmVersionManager versionManager = + createManager(WITNESSED_CONTAINER_DB_PROTO_VALUE.serialize(), provider)) { + UpgradeException thrown = assertThrows(UpgradeException.class, versionManager::finalizeUpgrade); + assertEquals(UpgradeException.ResultCodes.FINALIZE_UPGRADE_ACTION_FAILED, thrown.getResult()); + // WITNESSED_CONTAINER_DB_PROTO_VALUE is the version before STORAGE_SPACE_DISTRIBUTION, which has failed. + assertEquals(WITNESSED_CONTAINER_DB_PROTO_VALUE, versionManager.getApparentVersion()); + assertScmApparentVersionOnDisk(conf, WITNESSED_CONTAINER_DB_PROTO_VALUE.serialize()); + } + } + + @Override + @Test + public void testPersistFailureRollsBack() throws Exception { + SCMStorageConfig storage = mock(SCMStorageConfig.class); + AtomicInteger persistedApparentVersion = new AtomicInteger(INITIAL_VERSION.serialize()); + when(storage.getApparentVersion()).thenAnswer(invocation -> persistedApparentVersion.get()); + doAnswer(invocation -> { + persistedApparentVersion.set(invocation.getArgument(0)); + return null; + }).when(storage).setApparentVersion(anyInt()); + doThrow(new IOException("persist failed")).when(storage).persistCurrentState(); + + StorageContainerManager context = mock(StorageContainerManager.class); + try (ScmVersionManager versionManager = new ScmVersionManager(storage, context, HashMap::new)) { + assertEquals(INITIAL_VERSION, versionManager.getApparentVersion()); + UpgradeException thrown = assertThrows(UpgradeException.class, versionManager::finalizeUpgrade); + assertEquals(UpgradeException.ResultCodes.APPARENT_VERSION_UPDATE_FAILED, thrown.getResult()); + assertEquals(INITIAL_VERSION, versionManager.getApparentVersion()); + assertEquals(INITIAL_VERSION.serialize(), storage.getApparentVersion()); + } + } + + private static void assertScmApparentVersionOnDisk(OzoneConfiguration conf, int expected) + throws IOException { + SCMStorageConfig reloaded = new SCMStorageConfig(conf); + assertEquals(expected, reloaded.getApparentVersion()); + } +} diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/ScmUpgradeTestUtils.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/ScmUpgradeTestUtils.java new file mode 100644 index 000000000000..3d97c639d68d --- /dev/null +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/ScmUpgradeTestUtils.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.upgrade; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import org.apache.hadoop.hdds.ComponentVersion; +import org.apache.hadoop.hdds.HDDSVersion; +import org.apache.hadoop.hdds.scm.server.upgrade.ScmVersionManager; + +/** + * Utility class to help test SCM upgrade scenarios. + */ +public final class ScmUpgradeTestUtils { + + private ScmUpgradeTestUtils() { + // Utility class. + } + + /** + * Constructs a mock ScmVersionManager for an SCM which may be pre-finalized. + */ + public static ScmVersionManager mockVersionManager(ComponentVersion apparentVersion) { + ScmVersionManager manager = mock(ScmVersionManager.class); + when(manager.getApparentVersion()).thenReturn(apparentVersion); + when(manager.getSoftwareVersion()).thenReturn(HDDSVersion.SOFTWARE_VERSION); + when(manager.isAllowed(any(ComponentVersion.class))) + .thenAnswer(v -> v.getArgument(0, ComponentVersion.class).isSupportedBy(apparentVersion)); + when(manager.needsFinalization()).thenReturn(!HDDSVersion.SOFTWARE_VERSION.equals(apparentVersion)); + return manager; + } + + /** + * Constructs a mock ScmVersionManager for a finalized SCM. + */ + public static ScmVersionManager mockVersionManager() { + return mockVersionManager(HDDSVersion.SOFTWARE_VERSION); + } +} diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/TestScmStartupInvalidApparentVersion.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/TestScmStartupInvalidApparentVersion.java index cb78e35968be..0f678bdf5d75 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/TestScmStartupInvalidApparentVersion.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/TestScmStartupInvalidApparentVersion.java @@ -28,21 +28,20 @@ import java.io.IOException; import java.nio.file.Path; import java.util.Properties; +import org.apache.hadoop.hdds.HDDSVersion; import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature; -import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager; import org.apache.hadoop.ozone.upgrade.UpgradeTestUtils; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; /** - * Ensures SCM does not start when the VERSION file records an apparent version (still persisted as the layout-version - * integer) that is not supported by the layout version manager for this build - * ({@link HDDSLayoutVersionManager} / {@link HDDSLayoutFeature}). + * Ensures SCM does not start when the VERSION file records an apparent version + * that is larger than the software version. */ public class TestScmStartupInvalidApparentVersion { @@ -74,18 +73,16 @@ private void assertStartupFailsWithComponentVersionMessage(int serializedApparen conf.set(ScmConfigKeys.OZONE_SCM_DB_DIRS, folder.toAbsolutePath().toString()); conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, folder.toAbsolutePath().toString()); - int softwareLayoutVersion = HDDSLayoutVersionManager.maxLayoutVersion(); - Properties properties = new Properties(); properties.setProperty(SCM_ID, "scm"); properties.setProperty(SCM_HA, "true"); UpgradeTestUtils.createVersionFile(scmSubdir, HddsProtos.NodeType.SCM, serializedApparentVersion, properties); - // TODO update this message when SCM migrated to using HDDSVersionManager. - String expectedMessage = String.format( - "Cannot initialize VersionManager. Metadata layout version (%s) > software layout version (%s)", - serializedApparentVersion, softwareLayoutVersion); + String expectedMessage = + "Initialization failed. Disk contains unknown apparent version " + serializedApparentVersion + + " for software version " + HDDSVersion.SOFTWARE_VERSION + ". Make sure this component was not" + + " downgraded after finalization"; IOException ioException = assertThrows(IOException.class, () -> new StorageContainerManager(conf)); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java index c9a66f7c56c0..feb244edb56a 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java @@ -19,7 +19,7 @@ import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails; import static org.apache.hadoop.ozone.container.common.ContainerTestUtils.createEndpoint; -import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.defaultLayoutVersionProto; +import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.defaultVersionProto; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertInstanceOf; @@ -398,7 +398,7 @@ public void testRegister() throws Exception { nodeToRegister.getID()))), HddsTestUtils.getRandomContainerReports(10), HddsTestUtils.getRandomPipelineReports(), - defaultLayoutVersionProto()); + defaultVersionProto()); assertNotNull(responseProto); assertEquals(nodeToRegister.getUuidString(), responseProto.getDatanodeUUID()); assertNotNull(responseProto.getClusterID()); diff --git a/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/datanode/schemaupgrade/UpgradeContainerSchema.java b/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/datanode/schemaupgrade/UpgradeContainerSchema.java index 7add67421fae..ffd9dfb8fa45 100644 --- a/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/datanode/schemaupgrade/UpgradeContainerSchema.java +++ b/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/datanode/schemaupgrade/UpgradeContainerSchema.java @@ -34,7 +34,7 @@ import java.util.Objects; import java.util.concurrent.CompletableFuture; import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hdds.HDDSVersion; import org.apache.hadoop.hdds.StringUtils; import org.apache.hadoop.hdds.cli.HddsVersionProvider; import org.apache.hadoop.hdds.conf.ConfigurationSource; @@ -50,6 +50,7 @@ import org.apache.hadoop.io.nativeio.NativeIO; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.common.Storage; +import org.apache.hadoop.ozone.container.common.DatanodeStorage; import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils; import org.apache.hadoop.ozone.container.common.impl.ContainerData; import org.apache.hadoop.ozone.container.common.impl.ContainerDataYaml; @@ -63,6 +64,7 @@ import org.apache.hadoop.ozone.container.metadata.DatanodeSchemaThreeDBDefinition; import org.apache.hadoop.ozone.container.metadata.DatanodeStore; import org.apache.hadoop.ozone.container.metadata.DatanodeStoreSchemaThreeImpl; +import org.apache.hadoop.ozone.container.upgrade.DatanodeVersionManager; import org.apache.hadoop.ozone.repair.RepairTool; import org.apache.hadoop.util.Time; import picocli.CommandLine; @@ -131,22 +133,21 @@ public void execute() throws Exception { DatanodeDetails dnDetail = UpgradeUtils.getDatanodeDetails(configuration); - Pair layoutFeature = - UpgradeUtils.getLayoutFeature(dnDetail, configuration); - final HDDSLayoutFeature softwareLayoutFeature = layoutFeature.getLeft(); - final HDDSLayoutFeature metadataLayoutFeature = layoutFeature.getRight(); - final int needLayoutVersion = - HDDSLayoutFeature.DATANODE_SCHEMA_V3.layoutVersion(); - - if (metadataLayoutFeature.layoutVersion() < needLayoutVersion || - softwareLayoutFeature.layoutVersion() < needLayoutVersion) { - fatal( - "Please upgrade your software version, no less than %s," + - " current metadata layout version is %s," + - " software layout version is %s", - HDDSLayoutFeature.DATANODE_SCHEMA_V3.toString(), - metadataLayoutFeature.toString(), softwareLayoutFeature.toString()); - return; + DatanodeStorage storage = new DatanodeStorage(configuration, dnDetail.getUuidString()); + try (DatanodeVersionManager versionManager = new DatanodeVersionManager(storage, null)) { + // Ensure repair tool is not run in a newer version that supports schema V3 while the datanode that will read the + // containers does not. + if (!HDDSLayoutFeature.DATANODE_SCHEMA_V3.isSupportedBy(HDDSVersion.SOFTWARE_VERSION)) { + fatal("Please upgrade your software version to at least %s, current software version is %s", + HDDSLayoutFeature.DATANODE_SCHEMA_V3, HDDSVersion.SOFTWARE_VERSION); + return; + } + + + if (!versionManager.isAllowed(HDDSLayoutFeature.DATANODE_SCHEMA_V3)) { + fatal("Please finalize the cluster to enable support for Datanode container schema V3"); + return; + } } if (!Strings.isNullOrEmpty(volume)) { diff --git a/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/datanode/schemaupgrade/UpgradeUtils.java b/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/datanode/schemaupgrade/UpgradeUtils.java index 164f91b6a327..5c559c52f285 100644 --- a/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/datanode/schemaupgrade/UpgradeUtils.java +++ b/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/datanode/schemaupgrade/UpgradeUtils.java @@ -29,13 +29,9 @@ import java.util.List; import java.util.Objects; import java.util.Set; -import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature; -import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager; import org.apache.hadoop.hdds.utils.HddsServerUtil; -import org.apache.hadoop.ozone.container.common.DatanodeStorage; import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils; import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil; import org.apache.hadoop.ozone.container.common.volume.HddsVolume; @@ -93,28 +89,6 @@ public static boolean createFile(File file) throws IOException { return file.exists(); } - public static Pair getLayoutFeature( - DatanodeDetails dnDetail, OzoneConfiguration conf) throws IOException { - DatanodeStorage layoutStorage = - new DatanodeStorage(conf, dnDetail.getUuidString()); - HDDSLayoutVersionManager layoutVersionManager = - new HDDSLayoutVersionManager(layoutStorage.getApparentVersion(), null, null); - - final int metadataLayoutVersion = - layoutVersionManager.getMetadataLayoutVersion(); - final HDDSLayoutFeature metadataLayoutFeature = - (HDDSLayoutFeature) layoutVersionManager.getFeature( - metadataLayoutVersion); - - final int softwareLayoutVersion = - layoutVersionManager.getSoftwareLayoutVersion(); - final HDDSLayoutFeature softwareLayoutFeature = - (HDDSLayoutFeature) layoutVersionManager.getFeature( - softwareLayoutVersion); - - return Pair.of(softwareLayoutFeature, metadataLayoutFeature); - } - public static List getAllVolume(DatanodeDetails detail, OzoneConfiguration configuration) throws IOException { final MutableVolumeSet dataVolumeSet = getHddsVolumes(configuration, StorageVolume.VolumeType.DATA_VOLUME, diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 3421ca9ec000..5c3a3328782d 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -361,7 +361,7 @@ public static OzoneManagerVersion getOmVersion(ServiceInfoEx info) { static boolean validateOmVersion(OzoneManagerVersion minimumVersion, List serviceInfoList) { - if (minimumVersion == OzoneManagerVersion.FUTURE_VERSION) { + if (minimumVersion == OzoneManagerVersion.UNKNOWN_VERSION) { // A FUTURE_VERSION should not be expected ever. throw new IllegalArgumentException("Configuration error, expected " + "OzoneManager version config evaluates to a future version."); diff --git a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/rpc/TestRpcClient.java b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/rpc/TestRpcClient.java index b6b501f67741..c6d996971113 100644 --- a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/rpc/TestRpcClient.java +++ b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/rpc/TestRpcClient.java @@ -59,13 +59,13 @@ private enum ValidateOmVersionTestCases { NULL_EXPECTED_ONE_CURRENT_ONE_FUTURE_OM( null, OzoneManagerVersion.SOFTWARE_VERSION, - OzoneManagerVersion.FUTURE_VERSION, + OzoneManagerVersion.UNKNOWN_VERSION, true ), NULL_EXPECTED_TWO_FUTURE_OM( null, - OzoneManagerVersion.FUTURE_VERSION, - OzoneManagerVersion.FUTURE_VERSION, + OzoneManagerVersion.UNKNOWN_VERSION, + OzoneManagerVersion.UNKNOWN_VERSION, true ), @@ -86,7 +86,7 @@ private enum ValidateOmVersionTestCases { true), DEFAULT_EXPECTED_ONE_FUTURE_OM( OzoneManagerVersion.DEFAULT_VERSION, - OzoneManagerVersion.FUTURE_VERSION, + OzoneManagerVersion.UNKNOWN_VERSION, null, true), DEFAULT_EXPECTED_TWO_DEFAULT_OM( @@ -101,8 +101,8 @@ private enum ValidateOmVersionTestCases { true), DEFAULT_EXPECTED_TWO_FUTURE_OM( OzoneManagerVersion.DEFAULT_VERSION, - OzoneManagerVersion.FUTURE_VERSION, - OzoneManagerVersion.FUTURE_VERSION, + OzoneManagerVersion.UNKNOWN_VERSION, + OzoneManagerVersion.UNKNOWN_VERSION, true), DEFAULT_EXPECTED_ONE_DEFAULT_ONE_CURRENT_OM( OzoneManagerVersion.DEFAULT_VERSION, @@ -112,12 +112,12 @@ private enum ValidateOmVersionTestCases { DEFAULT_EXPECTED_ONE_DEFAULT_ONE_FUTURE_OM( OzoneManagerVersion.DEFAULT_VERSION, OzoneManagerVersion.DEFAULT_VERSION, - OzoneManagerVersion.FUTURE_VERSION, + OzoneManagerVersion.UNKNOWN_VERSION, true), DEFAULT_EXPECTED_ONE_CURRENT_ONE_FUTURE_OM( OzoneManagerVersion.DEFAULT_VERSION, OzoneManagerVersion.SOFTWARE_VERSION, - OzoneManagerVersion.FUTURE_VERSION, + OzoneManagerVersion.UNKNOWN_VERSION, true), CURRENT_EXPECTED_NO_OM( @@ -137,7 +137,7 @@ private enum ValidateOmVersionTestCases { true), CURRENT_EXPECTED_ONE_FUTURE_OM( OzoneManagerVersion.SOFTWARE_VERSION, - OzoneManagerVersion.FUTURE_VERSION, + OzoneManagerVersion.UNKNOWN_VERSION, null, true), CURRENT_EXPECTED_TWO_DEFAULT_OM( @@ -152,8 +152,8 @@ private enum ValidateOmVersionTestCases { true), CURRENT_EXPECTED_TWO_FUTURE_OM( OzoneManagerVersion.SOFTWARE_VERSION, - OzoneManagerVersion.FUTURE_VERSION, - OzoneManagerVersion.FUTURE_VERSION, + OzoneManagerVersion.UNKNOWN_VERSION, + OzoneManagerVersion.UNKNOWN_VERSION, true), CURRENT_EXPECTED_ONE_DEFAULT_ONE_CURRENT_OM( OzoneManagerVersion.SOFTWARE_VERSION, @@ -163,12 +163,12 @@ private enum ValidateOmVersionTestCases { CURRENT_EXPECTED_ONE_DEFAULT_ONE_FUTURE_OM( OzoneManagerVersion.SOFTWARE_VERSION, OzoneManagerVersion.DEFAULT_VERSION, - OzoneManagerVersion.FUTURE_VERSION, + OzoneManagerVersion.UNKNOWN_VERSION, false), CURRENT_EXPECTED_ONE_CURRENT_ONE_FUTURE_OM( OzoneManagerVersion.SOFTWARE_VERSION, OzoneManagerVersion.SOFTWARE_VERSION, - OzoneManagerVersion.FUTURE_VERSION, + OzoneManagerVersion.UNKNOWN_VERSION, true); private final OzoneManagerVersion expectedVersion; @@ -213,6 +213,6 @@ public void testValidateOmVersion(ValidateOmVersionTestCases testCase) { public void testFutureVersionShouldNotBeAnExpectedVersion() { assertThrows( IllegalArgumentException.class, - () -> validateOmVersion(OzoneManagerVersion.FUTURE_VERSION, null)); + () -> validateOmVersion(OzoneManagerVersion.UNKNOWN_VERSION, null)); } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestDNDataDistributionFinalization.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestDNDataDistributionFinalization.java index 6ab15c96aeeb..5f038f6b7c41 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestDNDataDistributionFinalization.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestDNDataDistributionFinalization.java @@ -21,6 +21,7 @@ import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -31,7 +32,6 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.scm.ScmConfig; import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol; -import org.apache.hadoop.hdds.scm.server.SCMConfigurator; import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; import org.apache.hadoop.ozone.HddsDatanodeService; import org.apache.hadoop.ozone.MiniOzoneCluster; @@ -75,10 +75,7 @@ public void cleanup() { public void init(OzoneConfiguration conf) throws Exception { - SCMConfigurator configurator = new SCMConfigurator(); - configurator.setUpgradeFinalizationExecutor(null); - - conf.setInt(SCMStorageConfig.TESTING_INIT_LAYOUT_VERSION_KEY, HDDSLayoutFeature.HBASE_SUPPORT.layoutVersion()); + conf.setInt(SCMStorageConfig.TESTING_INIT_LAYOUT_VERSION_KEY, HDDSLayoutFeature.HBASE_SUPPORT.serialize()); conf.setTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 100, TimeUnit.MILLISECONDS); conf.setTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 100, @@ -101,7 +98,6 @@ public void init(OzoneConfiguration conf) throws Exception { .setSCMServiceId("scmservice") .setOMServiceId("omServiceId") .setNumOfOzoneManagers(1) - .setSCMConfigurator(configurator) .setNumDatanodes(NUM_DATANODES) .setDatanodeFactory(UniformDatanodesFactory.newBuilder() .setLayoutVersion(HDDSLayoutFeature.INITIAL_VERSION.layoutVersion()) @@ -110,8 +106,8 @@ public void init(OzoneConfiguration conf) throws Exception { scmClient = cluster.getStorageContainerLocationClient(); cluster.waitForClusterToBeReady(); - assertEquals(HDDSLayoutFeature.HBASE_SUPPORT.layoutVersion(), - cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion()); + assertEquals(HDDSLayoutFeature.HBASE_SUPPORT, + cluster.getStorageContainerManager().getVersionManager().getApparentVersion()); // Create Volume and Bucket try (OzoneClient ozoneClient = OzoneClientFactory.getRpcClient(conf)) { @@ -136,8 +132,8 @@ public void testDataDistributionUpgradeScenario() throws Exception { init(new OzoneConfiguration()); // Verify initial state - STORAGE_SPACE_DISTRIBUTION should not be finalized yet - assertEquals(HDDSLayoutFeature.HBASE_SUPPORT.layoutVersion(), - cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion()); + assertEquals(HDDSLayoutFeature.HBASE_SUPPORT, + cluster.getStorageContainerManager().getVersionManager().getApparentVersion()); // Create some data and delete operations to trigger pending deletion logic String keyName1 = "testKey1"; @@ -163,8 +159,7 @@ public void testDataDistributionUpgradeScenario() throws Exception { TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient); // Verify finalization completed - assertEquals(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION.layoutVersion(), - cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion()); + assertFalse(cluster.getStorageContainerManager().getVersionManager().needsFinalization()); // Create more data and deletions to test post-finalization behavior String keyName3 = "testKey3"; @@ -198,8 +193,7 @@ public void testMissingPendingDeleteMetadataRecalculation() throws Exception { scmClient.finalizeUpgrade(); TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient); - assertEquals(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION.layoutVersion(), - cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion()); + assertFalse(cluster.getStorageContainerManager().getVersionManager().needsFinalization()); // Verify the system can handle scenarios where pendingDeleteBlockCount // might be missing and needs recalculation diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestHddsUpgradeUtils.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestHddsUpgradeUtils.java index bc9a71b72acb..ae7e091658ca 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestHddsUpgradeUtils.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestHddsUpgradeUtils.java @@ -21,13 +21,16 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.fail; +import java.util.Collection; import java.util.List; import java.util.concurrent.TimeoutException; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; +import org.apache.hadoop.hdds.utils.db.CodecException; +import org.apache.hadoop.hdds.utils.db.RocksDatabaseException; import org.apache.hadoop.ozone.HddsDatanodeService; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine; import org.apache.hadoop.ozone.container.upgrade.DatanodeVersionManager; import org.apache.ozone.test.GenericTestUtils; @@ -52,21 +55,6 @@ public static void waitForFinalizationFromClient(StorageContainerLocationProtoco }); } - /* - * Helper function to test Pre-Upgrade conditions on the SCM - */ - public static void testPreUpgradeConditionsSCM( - List scms) { - for (StorageContainerManager scm : scms) { - assertEquals(HDDSLayoutFeature.INITIAL_VERSION.layoutVersion(), - scm.getLayoutVersionManager().getMetadataLayoutVersion()); - for (ContainerInfo ci : scm.getContainerManager() - .getContainers()) { - assertEquals(HddsProtos.LifeCycleState.OPEN, ci.getState()); - } - } - } - /* * Helper function to test Post-Upgrade conditions on the SCM */ @@ -81,10 +69,9 @@ public static void testPostUpgradeConditionsSCM( public static void testPostUpgradeConditionsSCM(StorageContainerManager scm, int numContainers) { - HDDSLayoutVersionManager scmVersionManager = scm.getLayoutVersionManager(); - assertEquals(scmVersionManager.getSoftwareLayoutVersion(), - scmVersionManager.getMetadataLayoutVersion()); - assertThat(scmVersionManager.getMetadataLayoutVersion()).isGreaterThanOrEqualTo(1); + assertEquals(scm.getVersionManager().getSoftwareVersion(), + scm.getVersionManager().getApparentVersion()); + assertThat(scm.getVersionManager().getApparentVersion().serialize()).isGreaterThanOrEqualTo(1); int countContainers = scm.getContainerManager().getContainers().size(); assertThat(countContainers).isGreaterThanOrEqualTo(numContainers); @@ -122,4 +109,40 @@ public static void testPostUpgradeConditionsDataNodes( } assertThat(countContainers).isGreaterThanOrEqualTo(numContainers); } + + public static void waitForScmsToFinalize(Collection scms) + throws Exception { + for (StorageContainerManager scm: scms) { + // SCM will flush entries to the DB async, the state is kept in memory and the ratis logs. + // In the common case, the apparent version will not appear in the DB at the time of finalization since the logs + // Will not have been flushed. + waitForScmToFinalize(scm, false); + } + } + + public static void waitForScmToFinalize(StorageContainerManager scm, boolean waitForDBKeyFlush) + throws Exception { + GenericTestUtils.waitFor(() -> isScmFinalized(scm, waitForDBKeyFlush), 2_000, 60_000); + } + + private static boolean isScmFinalized(StorageContainerManager scm, boolean waitForDBKeyFlush) { + boolean exitedSafemode = !scm.isInSafeMode(); + boolean isFinalized = !scm.getVersionManager().needsFinalization(); + boolean dbKeyFlushed = false; + + try { + dbKeyFlushed = scm.getScmMetadataStore().getMetaTable().get(OzoneConsts.APPARENT_VERSION_KEY) != null; + } catch (RocksDatabaseException | CodecException e) { + throw new RuntimeException(e); + } + + LOG.info("Waiting for SCM {} (leader? {}) to finalize.\n" + + "Exited safemode? {}\n" + + "version manager finalized? {}\n" + + "DB key flushed? {}\n" + + "Requiring DB key to flush? {}", + scm.getSCMNodeId(), scm.checkLeader(), exitedSafemode, isFinalized, dbKeyFlushed, waitForDBKeyFlush); + + return exitedSafemode && isFinalized && (!waitForDBKeyFlush || dbKeyFlushed); + } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmDataDistributionFinalization.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmDataDistributionFinalization.java index b3230ee026b0..3339e3c23871 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmDataDistributionFinalization.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmDataDistributionFinalization.java @@ -28,6 +28,7 @@ import static org.apache.hadoop.hdds.client.ReplicationType.RATIS; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL; import static org.apache.hadoop.hdds.scm.block.SCMDeletedBlockTransactionStatusManager.EMPTY_SUMMARY; +import static org.apache.hadoop.hdds.upgrade.TestHddsUpgradeUtils.waitForScmsToFinalize; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL; import static org.apache.hadoop.ozone.common.BlockGroup.SIZE_NOT_AVAILABLE; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -38,7 +39,6 @@ import java.io.IOException; import java.time.Duration; import java.util.ArrayList; -import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -59,7 +59,6 @@ import org.apache.hadoop.hdds.scm.server.SCMConfigurator; import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; -import org.apache.hadoop.hdds.scm.server.upgrade.SCMUpgradeFinalizationContext; import org.apache.hadoop.hdds.utils.db.CodecException; import org.apache.hadoop.hdds.utils.db.RocksDatabaseException; import org.apache.hadoop.hdds.utils.db.Table; @@ -76,21 +75,15 @@ import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.common.DeletedBlock; import org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration; -import org.apache.hadoop.ozone.upgrade.UpgradeFinalizationExecutor; import org.apache.ozone.test.GenericTestUtils; import org.apache.ozone.test.tag.Flaky; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Tests upgrade finalization failure scenarios and corner cases specific to SCM data distribution feature. */ public class TestScmDataDistributionFinalization { - private static final Logger LOG = - LoggerFactory.getLogger(TestScmDataDistributionFinalization.class); - private StorageContainerLocationProtocol scmClient; private MiniOzoneHAClusterImpl cluster; private static final int NUM_DATANODES = 3; @@ -101,11 +94,9 @@ public class TestScmDataDistributionFinalization { private static final long BLOCK_SIZE = 1024 * 1024; // 1 MB private static final long BLOCKS_PER_TX = 5; // 1 MB - public void init(OzoneConfiguration conf, - UpgradeFinalizationExecutor executor, boolean doFinalize) throws Exception { + public void init(OzoneConfiguration conf) throws Exception { SCMConfigurator configurator = new SCMConfigurator(); - configurator.setUpgradeFinalizationExecutor(executor); conf.setInt(SCMStorageConfig.TESTING_INIT_LAYOUT_VERSION_KEY, HDDSLayoutFeature.HBASE_SUPPORT.layoutVersion()); conf.setTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 100, TimeUnit.MILLISECONDS); @@ -142,8 +133,8 @@ public void init(OzoneConfiguration conf, scmClient = cluster.getStorageContainerLocationClient(); cluster.waitForClusterToBeReady(); - assertEquals(HDDSLayoutFeature.HBASE_SUPPORT.layoutVersion(), - cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion()); + assertEquals(HDDSLayoutFeature.HBASE_SUPPORT, + cluster.getStorageContainerManager().getVersionManager().getApparentVersion()); // Create Volume and Bucket try (OzoneClient ozoneClient = OzoneClientFactory.getRpcClient(conf)) { @@ -169,15 +160,13 @@ public void shutdown() { @Test @Flaky("HDDS-14050") public void testFinalizationEmptyClusterDataDistribution() throws Exception { - init(new OzoneConfiguration(), null, true); + init(new OzoneConfiguration()); assertEquals(EMPTY_SUMMARY, cluster.getStorageContainerLocationClient().getDeletedBlockSummary()); scmClient.finalizeUpgrade(); TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient); // Make sure old leader has caught up and all SCMs have finalized. waitForScmsToFinalize(cluster.getStorageContainerManagersList()); - assertEquals(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION.layoutVersion(), - cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion()); TestHddsUpgradeUtils.testPostUpgradeConditionsSCM( cluster.getStorageContainerManagersList(), 0); @@ -264,7 +253,7 @@ public void testFinalizationEmptyClusterDataDistribution() throws Exception { */ @Test public void testFinalizationNonEmptyClusterDataDistribution() throws Exception { - init(new OzoneConfiguration(), null, false); + init(new OzoneConfiguration()); // stop SCMBlockDeletingService for (StorageContainerManager scm: cluster.getStorageContainerManagersList()) { scm.getScmBlockManager().getSCMBlockDeletingService().stop(); @@ -281,8 +270,6 @@ public void testFinalizationNonEmptyClusterDataDistribution() throws Exception { TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient); // Make sure old leader has caught up and all SCMs have finalized. waitForScmsToFinalize(cluster.getStorageContainerManagersList()); - assertEquals(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION.layoutVersion(), - cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion()); TestHddsUpgradeUtils.testPostUpgradeConditionsSCM( cluster.getStorageContainerManagersList(), 0); @@ -410,22 +397,6 @@ private long findLastTx() throws RocksDatabaseException, CodecException { return lastTxId; } - private void waitForScmsToFinalize(Collection scms) - throws Exception { - for (StorageContainerManager scm: scms) { - waitForScmToFinalize(scm); - } - } - - private void waitForScmToFinalize(StorageContainerManager scm) - throws Exception { - GenericTestUtils.waitFor(() -> !scm.isInSafeMode(), 500, 5000); - GenericTestUtils.waitFor(() -> { - LOG.info("Waiting for SCM {} (leader? {}) to finalize.", scm.getSCMNodeId(), scm.checkLeader()); - return !scm.getLayoutVersionManager().needsFinalization(); - }, 2_000, 60_000); - } - private void flushDBTransactionBuffer(StorageContainerManager scm) throws IOException { DBTransactionBuffer dbTxBuffer = scm.getScmHAManager().getDBTransactionBuffer(); if (dbTxBuffer instanceof SCMHADBTransactionBuffer) { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmHAFinalization.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmHAFinalization.java index b5b70eda97b6..c024bf3ebb95 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmHAFinalization.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmHAFinalization.java @@ -17,11 +17,13 @@ package org.apache.hadoop.hdds.upgrade; +import static org.apache.hadoop.hdds.upgrade.TestHddsUpgradeUtils.waitForScmToFinalize; +import static org.apache.hadoop.hdds.upgrade.TestHddsUpgradeUtils.waitForScmsToFinalize; import static org.assertj.core.api.Assertions.assertThat; import java.util.ArrayList; -import java.util.Collection; import java.util.List; +import org.apache.hadoop.hdds.HDDSVersion; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.ScmConfigKeys; @@ -30,14 +32,10 @@ import org.apache.hadoop.hdds.scm.server.SCMConfigurator; import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; -import org.apache.hadoop.hdds.scm.server.upgrade.FinalizationStateManagerImpl; -import org.apache.hadoop.hdds.scm.server.upgrade.SCMUpgradeFinalizationContext; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl; import org.apache.hadoop.ozone.UniformDatanodesFactory; -import org.apache.hadoop.ozone.upgrade.DefaultUpgradeFinalizationExecutor; -import org.apache.hadoop.ozone.upgrade.UpgradeFinalizationExecutor; -import org.apache.ozone.test.GenericTestUtils; +import org.apache.hadoop.ozone.upgrade.RatisBasedVersionManager; import org.apache.ozone.test.GenericTestUtils.LogCapturer; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; @@ -57,12 +55,9 @@ public class TestScmHAFinalization { private static final int NUM_DATANODES = 3; private static final int NUM_SCMS = 3; - public void init(OzoneConfiguration conf, - UpgradeFinalizationExecutor executor, - int numInactiveSCMs) throws Exception { + public void init(OzoneConfiguration conf, int numInactiveSCMs) throws Exception { SCMConfigurator configurator = new SCMConfigurator(); - configurator.setUpgradeFinalizationExecutor(executor); conf.setInt(SCMStorageConfig.TESTING_INIT_LAYOUT_VERSION_KEY, HDDSLayoutFeature.INITIAL_VERSION.layoutVersion()); conf.set(ScmConfigKeys.OZONE_SCM_HA_RATIS_SERVER_RPC_FIRST_ELECTION_TIMEOUT, "5s"); @@ -94,7 +89,7 @@ public void shutdown() { @Test public void testFinalization() throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); - init(conf, new DefaultUpgradeFinalizationExecutor<>(), 0); + init(conf, 0); scmClient.finalizeUpgrade(); TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient); // Ensure all SCMs finalize, indicating the message has been propagated across them all @@ -116,9 +111,9 @@ public void testSnapshotFinalization() throws Exception { conf.setLong(ScmConfigKeys.OZONE_SCM_HA_RATIS_SNAPSHOT_THRESHOLD, 5); - init(conf, new DefaultUpgradeFinalizationExecutor<>(), numInactiveSCMs); + init(conf, numInactiveSCMs); - LogCapturer logCapture = LogCapturer.captureLogs(FinalizationStateManagerImpl.class); + LogCapturer logCapture = LogCapturer.captureLogs(RatisBasedVersionManager.class); StorageContainerManager inactiveScm = cluster.getInactiveSCM().next(); LOG.info("Inactive SCM node ID: {}", inactiveScm.getSCMNodeId()); @@ -154,29 +149,16 @@ public void testSnapshotFinalization() throws Exception { } cluster.startInactiveSCM(inactiveScm.getSCMNodeId()); - waitForScmToFinalize(inactiveScm); + LOG.info("Waiting for restarted SCM to finalize"); + // When the leader sends a snapshot to the follower, it should have flushed all entries to the DB, including the + // apparent versin. This means the follower should see it in the DB it receives immediately to trigger finalization. + waitForScmToFinalize(inactiveScm, true); TestHddsUpgradeUtils.testPostUpgradeConditionsSCM( inactiveScm, 0); // Use log to verify a snapshot was installed. - assertThat(logCapture.getOutput()).contains("New SCM snapshot " + - "received with metadata layout version"); - } - - private void waitForScmsToFinalize(Collection scms) - throws Exception { - for (StorageContainerManager scm: scms) { - waitForScmToFinalize(scm); - } - } - - private void waitForScmToFinalize(StorageContainerManager scm) - throws Exception { - GenericTestUtils.waitFor(() -> !scm.isInSafeMode(), 500, 5000); - GenericTestUtils.waitFor(() -> { - LOG.info("Waiting for SCM {} (leader? {}) to finalize.", scm.getSCMNodeId(), scm.checkLeader()); - return !scm.getLayoutVersionManager().needsFinalization(); - }, 2_000, 60_000); + assertThat(logCapture.getOutput()).contains("New snapshot received with higher apparent version " + + HDDSVersion.SOFTWARE_VERSION + ". Attempting to finalize to that version."); } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmStartupInvalidApparentVersion.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmStartupInvalidApparentVersion.java index bd4aad16a882..519421413a0c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmStartupInvalidApparentVersion.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmStartupInvalidApparentVersion.java @@ -67,7 +67,7 @@ private void assertStartupFailsWithComponentVersionMessage(int serializedApparen UpgradeTestUtils.createVersionFile(omSubdir, HddsProtos.NodeType.OM, serializedApparentVersion); - MiniOzoneCluster.Builder clusterBuilder = MiniOzoneCluster.newBuilder(conf); + MiniOzoneCluster.Builder clusterBuilder = MiniOzoneCluster.newBuilder(conf).withoutDatanodes(); String expectedMessage = "Initialization failed. Disk contains unknown apparent version " + serializedApparentVersion diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestBlockDeletionService.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestBlockDeletionService.java index aa14184c2b66..257260f4622e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestBlockDeletionService.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestBlockDeletionService.java @@ -18,7 +18,6 @@ package org.apache.hadoop.ozone.om.service; import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.HBASE_SUPPORT; -import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.params.provider.Arguments.arguments; @@ -98,8 +97,7 @@ public static void init() throws Exception { .build(); cluster.waitForClusterToBeReady(); scmClient = cluster.getStorageContainerLocationClient(); - assertEquals(HBASE_SUPPORT.ordinal(), - cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion()); + assertEquals(HBASE_SUPPORT, cluster.getStorageContainerManager().getVersionManager().getApparentVersion()); metrics = cluster.getStorageContainerManager().getBlockProtocolServer().getMetrics(); OzoneClient ozoneClient = cluster.newClient(); @@ -140,8 +138,6 @@ public void testDeleteKeyQuotaWithUpgrade() throws Exception { // UPGRADE SCM (if specified) scmClient.finalizeUpgrade(); TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient); - assertEquals(STORAGE_SPACE_DISTRIBUTION.ordinal(), - cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion()); // POST-UPGRADE //Step 6: Repeat the same steps in pre-upgrade diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index f89dae3b3c0a..c4d54db6d4c6 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -946,7 +946,7 @@ private void instantiateServices(boolean withNewSnapshot) throws IOException { OmMetadataManagerImpl metadataManagerImpl = new OmMetadataManagerImpl(configuration, this); this.metadataManager = metadataManagerImpl; - versionManager.validateDBVersion(metadataManager); + versionManager.validateDBVersion(metadataManager.getMetaTable()); LOG.info("S3 Multi-Tenancy is {}", isS3MultiTenancyEnabled ? "enabled" : "disabled"); if (isS3MultiTenancyEnabled) { @@ -1033,7 +1033,7 @@ public void close() { updateActiveSnapshotMetrics(); if (withNewSnapshot) { - versionManager.finalizeFromSnapshotIfRequired(metadataManager); + versionManager.finalizeFromSnapshotIfRequired(metadataManager.getMetaTable()); instantiatePrepareStateAfterSnapshot(); } else { // Prepare state depends on the transaction ID of metadataManager after a diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMVersionManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMVersionManager.java index 99bbab840e9a..4d6547792c05 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMVersionManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMVersionManager.java @@ -17,28 +17,21 @@ package org.apache.hadoop.ozone.om.upgrade; -import static org.apache.hadoop.ozone.OzoneConsts.APPARENT_VERSION_KEY; - import com.google.common.annotations.VisibleForTesting; import java.io.IOException; import java.util.Map; import org.apache.hadoop.hdds.ComponentVersion; import org.apache.hadoop.ozone.OzoneManagerVersion; -import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OMStorage; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.upgrade.ComponentUpgradeActionProvider; -import org.apache.hadoop.ozone.upgrade.ComponentVersionManager; +import org.apache.hadoop.ozone.upgrade.RatisBasedVersionManager; import org.apache.hadoop.ozone.upgrade.UpgradeException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Component version manager for Ozone Manager. */ -public class OMVersionManager extends ComponentVersionManager { - - private static final Logger LOG = LoggerFactory.getLogger(OMVersionManager.class); +public class OMVersionManager extends RatisBasedVersionManager { private final Map upgradeActions; @@ -53,33 +46,11 @@ public OMVersionManager(OMStorage storage, OzoneManager upgradeActionArg) throws @VisibleForTesting public OMVersionManager(OMStorage storage, OzoneManager upgradeActionArg, ComponentUpgradeActionProvider upgradeActionProvider) throws IOException { - super(storage, computeApparentVersion(storage.getApparentVersion()), OzoneManagerVersion.SOFTWARE_VERSION); + super(storage, computeApparentVersionInternal(storage.getApparentVersion()), OzoneManagerVersion.SOFTWARE_VERSION); this.upgradeActionArg = upgradeActionArg; upgradeActions = upgradeActionProvider.load(); } - public void validateDBVersion(OMMetadataManager metadataManager) throws IOException { - ComponentVersion dbVersion = getApparentVersionInDB(metadataManager); - ComponentVersion apparentVersion = getApparentVersion(); - - if (!apparentVersion.equals(dbVersion)) { - LOG.info("Version file has different apparent version ({}) than OM DB ({}). That is expected if this " - + "OM has never been finalized to a newer version.", apparentVersion, dbVersion); - } - } - - public void finalizeFromSnapshotIfRequired(OMMetadataManager metadataManager) throws IOException { - ComponentVersion apparentVersionInDB = getApparentVersionInDB(metadataManager); - if (apparentVersionInDB != null && !isAllowed(apparentVersionInDB)) { - LOG.info("New OM snapshot received with higher apparent version {}. " - + "Attempting to finalize current OM to that version.", apparentVersionInDB); - finalizeUpgrade(); - // Update the apparent version in the DB to match the VERSION file. - // When finalization is not done with a snapshot, this DB value is updated by OMFinalizeUpgradeRequest. - metadataManager.getMetaTable().put(APPARENT_VERSION_KEY, String.valueOf(getApparentVersion().serialize())); - } - } - @VisibleForTesting public Map getUpgradeActionsForTesting() { return upgradeActions; @@ -99,9 +70,9 @@ protected void runUpgradeAction(ComponentVersion componentVersion) throws Upgrad } } - private static ComponentVersion getApparentVersionInDB(OMMetadataManager metadataManager) throws IOException { - String apparentVersion = metadataManager.getMetaTable().get(APPARENT_VERSION_KEY); - return (apparentVersion == null) ? null : computeApparentVersion(Integer.parseInt(apparentVersion)); + @Override + protected ComponentVersion computeApparentVersion(int serializedApparentVersion) throws IOException { + return computeApparentVersionInternal(serializedApparentVersion); } /** @@ -113,10 +84,10 @@ private static ComponentVersion getApparentVersionInDB(OMMetadataManager metadat * the gap between the largest {@link OMLayoutFeature} and ZDU are not valid legacy layout values; startup fails with * the persisted integer in the exception message. */ - private static ComponentVersion computeApparentVersion(int serializedApparentVersion) throws IOException { + private static ComponentVersion computeApparentVersionInternal(int serializedApparentVersion) throws IOException { if (serializedApparentVersion >= OzoneManagerVersion.ZDU.serialize()) { OzoneManagerVersion fromOm = OzoneManagerVersion.deserialize(serializedApparentVersion); - if (fromOm != OzoneManagerVersion.FUTURE_VERSION) { + if (fromOm != OzoneManagerVersion.UNKNOWN_VERSION) { return fromOm; } } else { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutFeature.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutFeature.java index 9e98935b5b3d..9534c0ccba19 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutFeature.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutFeature.java @@ -101,7 +101,7 @@ public void testIsSupportedByFeatureBoundary() { public void testAllLayoutFeaturesAreSupportedByFutureVersions() { for (OMLayoutFeature feature : OMLayoutFeature.values()) { assertSupportedBy(feature, OzoneManagerVersion.ZDU); - assertSupportedBy(feature, OzoneManagerVersion.FUTURE_VERSION); + assertSupportedBy(feature, OzoneManagerVersion.UNKNOWN_VERSION); // No ComponentVersion instance represents an arbitrary unknown future version. assertTrue(feature.isSupportedBy(Integer.MAX_VALUE)); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMVersionManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMVersionManager.java index 2565109a73f6..6d232e85be9d 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMVersionManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMVersionManager.java @@ -83,7 +83,7 @@ class TestOMVersionManager extends AbstractComponentVersionManagerTest { ALL_VERSIONS = new ArrayList<>(Arrays.asList(OMLayoutFeature.values())); for (OzoneManagerVersion version : OzoneManagerVersion.values()) { // Add all defined versions after and including ZDU to get the complete version list. - if (ZDU.isSupportedBy(version) && version != OzoneManagerVersion.FUTURE_VERSION) { + if (ZDU.isSupportedBy(version) && version != OzoneManagerVersion.UNKNOWN_VERSION) { ALL_VERSIONS.add(version); } } diff --git a/hadoop-ozone/recon/pom.xml b/hadoop-ozone/recon/pom.xml index ecaf38d8de64..9dacdf1f8ca1 100644 --- a/hadoop-ozone/recon/pom.xml +++ b/hadoop-ozone/recon/pom.xml @@ -296,6 +296,12 @@ test-jar test + + org.apache.ozone + hdds-server-scm + test-jar + test + org.apache.ozone hdds-test-utils diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/DatanodeMetadata.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/DatanodeMetadata.java index 324ce4f38d9a..b8fb46d294de 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/DatanodeMetadata.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/DatanodeMetadata.java @@ -84,9 +84,9 @@ public final class DatanodeMetadata { @JsonInclude(JsonInclude.Include.NON_NULL) private String revision; - @XmlElement(name = "layoutVersion") + @XmlElement(name = "apparentVersion") @JsonInclude(JsonInclude.Include.NON_DEFAULT) - private int layoutVersion; + private int apparentVersion; @XmlElement(name = "networkLocation") @JsonInclude(JsonInclude.Include.NON_NULL) @@ -106,7 +106,7 @@ private DatanodeMetadata(Builder builder) { this.version = builder.version; this.setupTime = builder.setupTime; this.revision = builder.revision; - this.layoutVersion = builder.layoutVersion; + this.apparentVersion = builder.apparentVersion; this.networkLocation = builder.networkLocation; } @@ -162,8 +162,8 @@ public String getRevision() { return revision; } - public int getLayoutVersion() { - return layoutVersion; + public int getApparentVersion() { + return apparentVersion; } public String getNetworkLocation() { @@ -196,7 +196,7 @@ public static final class Builder { private String version; private long setupTime; private String revision; - private int layoutVersion; + private int apparentVersion; private String networkLocation; public Builder() { @@ -260,7 +260,7 @@ public Builder setDatanode(DatanodeInfo datanode) { this.version = datanode.getVersion(); this.revision = datanode.getRevision(); - this.layoutVersion = datanode.getLastKnownLayoutVersion().getMetadataLayoutVersion(); + this.apparentVersion = datanode.getLastKnownApparentVersion().serialize(); this.setupTime = datanode.getSetupTime(); return this; diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconNodeManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconNodeManager.java index e5cdf65c9103..0eacc90936b9 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconNodeManager.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconNodeManager.java @@ -47,12 +47,13 @@ import org.apache.hadoop.hdds.scm.node.SCMNodeManager; import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; +import org.apache.hadoop.hdds.scm.server.upgrade.ScmVersionManager; import org.apache.hadoop.hdds.server.events.EventPublisher; import org.apache.hadoop.hdds.server.events.EventQueue; -import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager; import org.apache.hadoop.hdds.utils.HddsServerUtil; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; +import org.apache.hadoop.ozone.container.upgrade.UpgradeUtils; import org.apache.hadoop.ozone.protocol.VersionResponse; import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode; import org.apache.hadoop.ozone.protocol.commands.RegisteredCommand; @@ -89,9 +90,9 @@ public ReconNodeManager(OzoneConfiguration conf, EventPublisher eventPublisher, NetworkTopology networkTopology, Table nodeDB, - HDDSLayoutVersionManager scmLayoutVersionManager) { + ScmVersionManager versionManager) { super(conf, scmStorageConfig, eventPublisher, networkTopology, - SCMContext.emptyContext(), scmLayoutVersionManager); + SCMContext.emptyContext(), versionManager); final int reconStaleDatanodeMultiplier = 3; this.reconDatanodeOutdatedTime = reconStaleDatanodeMultiplier * HddsServerUtil.getReconHeartbeatInterval(conf); @@ -100,8 +101,8 @@ public ReconNodeManager(OzoneConfiguration conf, public ReconNodeManager(OzoneConfiguration conf, SCMStorageConfig scmStorageConfig, EventQueue eventQueue, NetworkTopology clusterMap, Table table, - HDDSLayoutVersionManager scmLayoutVersionManager, ReconContext reconContext) { - this(conf, scmStorageConfig, eventQueue, clusterMap, table, scmLayoutVersionManager); + ScmVersionManager versionManager, ReconContext reconContext) { + this(conf, scmStorageConfig, eventQueue, clusterMap, table, versionManager); this.reconContext = reconContext; loadExistingNodes(); } @@ -112,13 +113,7 @@ private void loadExistingNodes() { int nodeCount = 0; while (iterator.hasNext()) { DatanodeDetails datanodeDetails = iterator.next().getValue(); - register(datanodeDetails, null, null, - LayoutVersionProto.newBuilder() - .setMetadataLayoutVersion( - HDDSLayoutVersionManager.maxLayoutVersion()) - .setSoftwareLayoutVersion( - HDDSLayoutVersionManager.maxLayoutVersion()) - .build()); + register(datanodeDetails, null, null, UpgradeUtils.defaultVersionProto()); nodeCount++; } LOG.info("Loaded {} nodes from node DB.", nodeCount); @@ -212,7 +207,7 @@ public void refreshAllHealthyDnUsageInfo() { public RegisteredCommand register( DatanodeDetails datanodeDetails, NodeReportProto nodeReport, PipelineReportsProto pipelineReportsProto, - LayoutVersionProto layoutInfo) { + LayoutVersionProto dnVersionInfo) { if (isNodeRegistered(datanodeDetails)) { try { nodeDB.put(datanodeDetails.getID(), datanodeDetails); @@ -223,7 +218,7 @@ public RegisteredCommand register( } try { RegisteredCommand registeredCommand = super.register(datanodeDetails, nodeReport, pipelineReportsProto, - layoutInfo); + dnVersionInfo); reconContext.updateHealthStatus(new AtomicBoolean(true)); reconContext.getErrors().remove(ReconContext.ErrorCode.INVALID_NETWORK_TOPOLOGY); return registeredCommand; @@ -298,31 +293,9 @@ public void removeNode(DatanodeDetails datanodeDetails) throws NodeNotFoundExcep @Override protected void sendFinalizeToDatanodeIfNeeded(DatanodeDetails datanodeDetails, - LayoutVersionProto layoutVersionReport) { - // Recon should do nothing here. - int scmSlv = getLayoutVersionManager().getSoftwareLayoutVersion(); - int scmMlv = getLayoutVersionManager().getMetadataLayoutVersion(); - int dnSlv = layoutVersionReport.getSoftwareLayoutVersion(); - int dnMlv = layoutVersionReport.getMetadataLayoutVersion(); - - if (dnSlv > scmSlv) { - LOG.error("Invalid data node reporting to Recon : {}. " + - "DataNode SoftwareLayoutVersion = {}, Recon/SCM " + - "SoftwareLayoutVersion = {}", - datanodeDetails.getHostName(), dnSlv, scmSlv); - } - - if (scmMlv == scmSlv) { - // Recon metadata is finalised. - if (dnMlv < scmMlv) { - if (LOG.isDebugEnabled()) { - LOG.debug("Data node {} reports a lower MLV than Recon " - + "DataNode MetadataLayoutVersion = {}, Recon/SCM " - + "MetadataLayoutVersion = {}. SCM needs to finalize this DN", - datanodeDetails.getHostName(), dnMlv, scmMlv); - } - } - } - + LayoutVersionProto versionReport) { + // Recon will not send commands to datanodes, but it should still log if a datanode with an invalid version is + // heartbeating. + shouldFenceDatanode(datanodeDetails, versionReport); } } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java index f44d4e57a1a1..466bc50878ab 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java @@ -98,9 +98,9 @@ import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.ContainerReportFromDatanode; import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.IncrementalContainerReportFromDatanode; import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; +import org.apache.hadoop.hdds.scm.server.upgrade.ScmVersionManager; import org.apache.hadoop.hdds.server.events.EventQueue; import org.apache.hadoop.hdds.server.events.FixedThreadPoolWithAffinityExecutor; -import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.hdds.utils.db.DBCheckpoint; import org.apache.hadoop.hdds.utils.db.DBStore; @@ -215,8 +215,8 @@ public ReconStorageContainerManagerFacade(OzoneConfiguration conf, NetworkTopology clusterMap = new NetworkTopologyImpl(conf); this.dbStore = DBStoreBuilder.createDBStore(ozoneConfiguration, ReconSCMDBDefinition.get()); - HDDSLayoutVersionManager scmLayoutVersionManager = - new HDDSLayoutVersionManager(scmStorageConfig.getApparentVersion(), null, null); + // TODO HDDS-15374 Fully switch recon to the new versioning framework. + ScmVersionManager versionManager = new ScmVersionManager(scmStorageConfig, this); this.scmhaManager = SCMHAManagerStub.getInstance( true, new SCMDBTransactionBufferImpl()); this.sequenceIdGen = new SequenceIdGenerator( @@ -225,7 +225,7 @@ public ReconStorageContainerManagerFacade(OzoneConfiguration conf, this.nodeManager = new ReconNodeManager(conf, scmStorageConfig, eventQueue, clusterMap, ReconSCMDBDefinition.NODES.getTable(dbStore), - scmLayoutVersionManager, reconContext); + versionManager, reconContext); SCMContainerPlacementMetrics placementMetrics = SCMContainerPlacementMetrics.create(); PlacementPolicy containerPlacementPolicy = ContainerPlacementPolicyFactory.getPolicy(conf, nodeManager, clusterMap, true, placementMetrics); diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java index 330bd3adbe21..7b28bd793d21 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java @@ -18,7 +18,7 @@ package org.apache.hadoop.ozone.recon.api; import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails; -import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.defaultLayoutVersionProto; +import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.defaultVersionProto; import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getRandomPipeline; import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager; import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.initializeNewOmMetadataManager; @@ -427,7 +427,7 @@ public void setUp() throws Exception { NodeReportProto.newBuilder() .addStorageReport(storageReportProto3) .addStorageReport(storageReportProto4).build(); - LayoutVersionProto layoutInfo = defaultLayoutVersionProto(); + LayoutVersionProto layoutInfo = defaultVersionProto(); DatanodeDetailsProto datanodeDetailsProto3 = DatanodeDetailsProto.newBuilder() @@ -471,12 +471,12 @@ public void setUp() throws Exception { .register(extendedDatanodeDetailsProto2, nodeReportProto2, ContainerReportsProto.newBuilder().build(), PipelineReportsProto.newBuilder().build(), - defaultLayoutVersionProto()); + defaultVersionProto()); reconScm.getDatanodeProtocolServer() .register(extendedDatanodeDetailsProto3, nodeReportProto3, ContainerReportsProto.newBuilder().build(), PipelineReportsProto.newBuilder().build(), - defaultLayoutVersionProto()); + defaultVersionProto()); // Process all events in the event queue reconScm.getEventQueue().processAll(1000); }); @@ -654,7 +654,7 @@ private void testDatanodeResponse(DatanodeMetadata datanodeMetadata) fail(String.format("Datanode %s not registered", hostname)); } - assertEquals(HDDSVersion.SOFTWARE_VERSION.serialize(), datanodeMetadata.getLayoutVersion()); + assertEquals(HDDSVersion.SOFTWARE_VERSION.serialize(), datanodeMetadata.getApparentVersion()); } @Test @@ -1276,7 +1276,7 @@ private void waitAndCheckConditionAfterHeartbeat(Callable check) .setContainerReport(containerReportsProto) .setDatanodeDetails(extendedDatanodeDetailsProto .getDatanodeDetails()) - .setDataNodeLayoutVersion(defaultLayoutVersionProto()) + .setDataNodeLayoutVersion(defaultVersionProto()) .build(); reconScm.getDatanodeProtocolServer().sendHeartbeat(heartbeatRequestProto); LambdaTestUtils.await(30000, 1000, check); diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOpenContainerCount.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOpenContainerCount.java index be2fd9bd8247..fb8f4b66d7b9 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOpenContainerCount.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOpenContainerCount.java @@ -18,7 +18,7 @@ package org.apache.hadoop.ozone.recon.api; import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails; -import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.defaultLayoutVersionProto; +import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.defaultVersionProto; import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getRandomPipeline; import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager; import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.initializeNewOmMetadataManager; @@ -321,7 +321,7 @@ public void setUp() throws Exception { reconScm.getDatanodeProtocolServer() .register(extendedDatanodeDetailsProto, nodeReportProto, containerReportsProto, pipelineReportsProto, - defaultLayoutVersionProto()); + defaultVersionProto()); // Process all events in the event queue reconScm.getEventQueue().processAll(1000); }); @@ -411,7 +411,7 @@ private void updateContainerReport(long containerId) { reconScm.getDatanodeProtocolServer() .register(extendedDatanodeDetailsProto, nodeReportProto, containerReportsProto, pipelineReportsProto, - defaultLayoutVersionProto()); + defaultVersionProto()); // Process all events in the event queue reconScm.getEventQueue().processAll(1000); }); diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestStorageDistributionEndpoint.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestStorageDistributionEndpoint.java index d62aae159139..1b646e2aa20e 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestStorageDistributionEndpoint.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestStorageDistributionEndpoint.java @@ -17,6 +17,7 @@ package org.apache.hadoop.ozone.recon.api; +import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.defaultVersionProto; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -225,7 +226,7 @@ private List mockStorageDistributionData(int numNodes) throws Exception .build(); pendingDeletionMetrics.add(new DatanodePendingDeletionMetrics(hostName, uuid.toString(), PENDING_DELETION_SIZE)); - dataNodes.add(new DatanodeInfo(datanode, NodeStatus.inServiceHealthy(), null, 5 * 60 * 1000)); + dataNodes.add(new DatanodeInfo(datanode, NodeStatus.inServiceHealthy(), defaultVersionProto(), 5 * 60 * 1000)); when(nodeManager.getNodeStat(datanode)) .thenReturn(new SCMNodeMetric(OZONE_CAPACITY, OZONE_USED, OZONE_REMAINING, COMMITTED, MIN_FREE_SPACE, RESERVED)); diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/AbstractReconContainerManagerTest.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/AbstractReconContainerManagerTest.java index 33e20413bfd6..153519ecc33b 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/AbstractReconContainerManagerTest.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/AbstractReconContainerManagerTest.java @@ -21,7 +21,7 @@ import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_NAMES; import static org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition.CONTAINERS; -import static org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager.maxLayoutVersion; +import static org.apache.hadoop.hdds.scm.upgrade.ScmUpgradeTestUtils.mockVersionManager; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS; import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getRandomPipeline; import static org.mockito.Mockito.mock; @@ -52,8 +52,8 @@ import org.apache.hadoop.hdds.scm.node.SCMNodeManager; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; +import org.apache.hadoop.hdds.scm.server.upgrade.ScmVersionManager; import org.apache.hadoop.hdds.server.events.EventQueue; -import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager; import org.apache.hadoop.hdds.utils.db.DBStore; import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; import org.apache.hadoop.hdds.utils.db.Table; @@ -89,13 +89,9 @@ public void setUp(@TempDir File tempDir) throws Exception { SCMStorageConfig scmStorageConfig = new ReconStorageConfig(conf, new ReconUtils()); NetworkTopology clusterMap = new NetworkTopologyImpl(conf); EventQueue eventQueue = new EventQueue(); - HDDSLayoutVersionManager layoutVersionManager = mock(HDDSLayoutVersionManager.class); - when(layoutVersionManager.getSoftwareLayoutVersion()) - .thenReturn(maxLayoutVersion()); - when(layoutVersionManager.getMetadataLayoutVersion()) - .thenReturn(maxLayoutVersion()); + ScmVersionManager versionManager = mockVersionManager(); NodeManager nodeManager = new SCMNodeManager(conf, scmStorageConfig, - eventQueue, clusterMap, scmContext, layoutVersionManager); + eventQueue, clusterMap, scmContext, versionManager); pipelineManager = ReconPipelineManager.newReconPipelineManager( conf, nodeManager, diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconIncrementalContainerReportHandler.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconIncrementalContainerReportHandler.java index 78f9bbceafcd..2ec3c5b1b5da 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconIncrementalContainerReportHandler.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconIncrementalContainerReportHandler.java @@ -19,7 +19,7 @@ import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.OPEN; -import static org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager.maxLayoutVersion; +import static org.apache.hadoop.hdds.scm.upgrade.ScmUpgradeTestUtils.mockVersionManager; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.any; @@ -51,9 +51,9 @@ import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.IncrementalContainerReportFromDatanode; import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; +import org.apache.hadoop.hdds.scm.server.upgrade.ScmVersionManager; import org.apache.hadoop.hdds.server.events.EventPublisher; import org.apache.hadoop.hdds.server.events.EventQueue; -import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager; import org.apache.hadoop.ozone.recon.TestReconUtils; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -93,11 +93,7 @@ public void testProcessICR(@TempDir Path scmPath) NetworkTopology clusterMap = new NetworkTopologyImpl(conf); EventQueue eventQueue = new EventQueue(); SCMStorageConfig storageConfig = new SCMStorageConfig(conf); - HDDSLayoutVersionManager versionManager = mock(HDDSLayoutVersionManager.class); - when(versionManager.getMetadataLayoutVersion()) - .thenReturn(maxLayoutVersion()); - when(versionManager.getSoftwareLayoutVersion()) - .thenReturn(maxLayoutVersion()); + ScmVersionManager versionManager = mockVersionManager(); NodeManager nodeManager = new SCMNodeManager(conf, storageConfig, eventQueue, clusterMap, SCMContext.emptyContext(), versionManager); diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconNodeManager.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconNodeManager.java index adb09e4ad90a..c1315cee1146 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconNodeManager.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconNodeManager.java @@ -35,6 +35,7 @@ import java.io.IOException; import java.nio.file.Path; +import java.util.HashMap; import java.util.List; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; @@ -45,8 +46,9 @@ import org.apache.hadoop.hdds.scm.net.NetworkTopology; import org.apache.hadoop.hdds.scm.net.NetworkTopologyImpl; import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; +import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; +import org.apache.hadoop.hdds.scm.server.upgrade.ScmVersionManager; import org.apache.hadoop.hdds.server.events.EventQueue; -import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager; import org.apache.hadoop.hdds.utils.db.DBStore; import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; import org.apache.hadoop.hdds.utils.db.Table; @@ -71,7 +73,7 @@ public class TestReconNodeManager { private OzoneConfiguration conf; private DBStore store; - private HDDSLayoutVersionManager versionManager; + private ScmVersionManager versionManager; private ReconContext reconContext; @BeforeEach @@ -81,8 +83,8 @@ public void setUp() throws Exception { conf.set(OZONE_SCM_NAMES, "localhost"); ReconUtils reconUtils = new ReconUtils(); ReconStorageConfig reconStorageConfig = new ReconStorageConfig(conf, reconUtils); - versionManager = new HDDSLayoutVersionManager( - reconStorageConfig.getApparentVersion(), null, null); + versionManager = new ScmVersionManager(reconStorageConfig, + mock(OzoneStorageContainerManager.class), HashMap::new); store = DBStoreBuilder.createDBStore(conf, ReconSCMDBDefinition.get()); reconContext = new ReconContext(conf, reconUtils); } diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconPipelineManager.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconPipelineManager.java index f837d1a41fd6..86b0e48417f7 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconPipelineManager.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconPipelineManager.java @@ -19,7 +19,7 @@ import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_NAMES; -import static org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager.maxLayoutVersion; +import static org.apache.hadoop.hdds.scm.upgrade.ScmUpgradeTestUtils.mockVersionManager; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS; import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getRandomPipeline; import static org.assertj.core.api.Assertions.assertThat; @@ -57,8 +57,8 @@ import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager; import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; +import org.apache.hadoop.hdds.scm.server.upgrade.ScmVersionManager; import org.apache.hadoop.hdds.server.events.EventQueue; -import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager; import org.apache.hadoop.hdds.utils.db.DBStore; import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; import org.apache.hadoop.ozone.recon.ReconUtils; @@ -129,11 +129,7 @@ public void testInitialize() throws IOException, TimeoutException { NetworkTopology clusterMap = new NetworkTopologyImpl(conf); EventQueue eventQueue = new EventQueue(); - HDDSLayoutVersionManager versionManager = mock(HDDSLayoutVersionManager.class); - when(versionManager.getMetadataLayoutVersion()) - .thenReturn(maxLayoutVersion()); - when(versionManager.getSoftwareLayoutVersion()) - .thenReturn(maxLayoutVersion()); + ScmVersionManager versionManager = mockVersionManager(); NodeManager nodeManager = new SCMNodeManager(conf, scmStorageConfig, eventQueue, clusterMap, SCMContext.emptyContext(), versionManager); @@ -183,11 +179,7 @@ public void testAddPipeline() throws IOException, TimeoutException { Pipeline pipeline = getRandomPipeline(); NetworkTopology clusterMap = new NetworkTopologyImpl(conf); EventQueue eventQueue = new EventQueue(); - HDDSLayoutVersionManager versionManager = mock(HDDSLayoutVersionManager.class); - when(versionManager.getMetadataLayoutVersion()) - .thenReturn(maxLayoutVersion()); - when(versionManager.getSoftwareLayoutVersion()) - .thenReturn(maxLayoutVersion()); + ScmVersionManager versionManager = mockVersionManager(); NodeManager nodeManager = new SCMNodeManager(conf, scmStorageConfig, eventQueue, clusterMap, SCMContext.emptyContext(), versionManager); @@ -210,9 +202,7 @@ public void testDuplicatePipelineHandling() throws IOException { Pipeline pipeline = getRandomPipeline(); NetworkTopology clusterMap = new NetworkTopologyImpl(conf); EventQueue eventQueue = new EventQueue(); - HDDSLayoutVersionManager versionManager = mock(HDDSLayoutVersionManager.class); - when(versionManager.getMetadataLayoutVersion()).thenReturn(maxLayoutVersion()); - when(versionManager.getSoftwareLayoutVersion()).thenReturn(maxLayoutVersion()); + ScmVersionManager versionManager = mockVersionManager(); NodeManager nodeManager = new SCMNodeManager(conf, scmStorageConfig, eventQueue, clusterMap, SCMContext.emptyContext(), versionManager); diff --git a/hadoop-ozone/vapor/src/main/java/org/apache/hadoop/ozone/freon/SCMThroughputBenchmark.java b/hadoop-ozone/vapor/src/main/java/org/apache/hadoop/ozone/freon/SCMThroughputBenchmark.java index e575576428b3..dfbf0ded77d0 100644 --- a/hadoop-ozone/vapor/src/main/java/org/apache/hadoop/ozone/freon/SCMThroughputBenchmark.java +++ b/hadoop-ozone/vapor/src/main/java/org/apache/hadoop/ozone/freon/SCMThroughputBenchmark.java @@ -799,7 +799,7 @@ public void register() throws IOException { createNodeReport(datanodeDetails.getUuid()), createContainerReport(), createPipelineReport(), - UpgradeUtils.defaultLayoutVersionProto()); + UpgradeUtils.defaultVersionProto()); if (response.hasHostname() && response.hasIpAddress()) { datanodeDetails.setHostName(response.getHostname()); @@ -816,7 +816,7 @@ public void sendHeartbeat() throws IOException, TimeoutException { .newBuilder() .setDatanodeDetails(datanodeDetails.getProtoBufMessage()) .setContainerReport(containerReport) - .setDataNodeLayoutVersion(UpgradeUtils.defaultLayoutVersionProto()) + .setDataNodeLayoutVersion(UpgradeUtils.defaultVersionProto()) .build(); datanodeScmClient.sendHeartbeat(heartbeatRequest); // scm commands are ignored