HDDS-15374. Switch Recon to the new versioning framework

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HDDSVersion.java

@@ -41,8 +41,8 @@ public enum HDDSVersion implements ComponentVersion {
 
   ZDU(100, "Version that supports zero downtime upgrade"),
 
-  FUTURE_VERSION(-1, "Used internally in the client when the server side is "
-      + " newer and an unknown server version has arrived to the client.");
+  UNKNOWN_VERSION(-1, "Used when a version cannot be deserialized to any version recognized by this" +
+      " component, which may indicate it came from a component in a newer version");
 
   //////////////////////////////  //////////////////////////////
 
@@ -66,7 +66,7 @@ public String description() {
   }
 
   /**
-   * @return The next version immediately following this one and excluding FUTURE_VERSION,
+   * @return The next version immediately following this one and excluding {@link #UNKNOWN_VERSION},
    *    or null if there is no such version.
    */
   @Override
@@ -85,11 +85,11 @@ public int serialize() {
 
   /**
    * @param value The serialized version to convert.
-   * @return The version corresponding to this serialized value, or {@link #FUTURE_VERSION} if no matching version is
+   * @return The version corresponding to this serialized value, or {@link #UNKNOWN_VERSION} if no matching version is
    *    found.
    */
   public static HDDSVersion deserialize(int value) {
-    return BY_VALUE.getOrDefault(value, FUTURE_VERSION);
+    return BY_VALUE.getOrDefault(value, UNKNOWN_VERSION);
   }
 
   @Override

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java

@@ -42,7 +42,7 @@ public enum ClientVersion implements ComponentVersion {
       "This client version has support for Object Store and File " +
           "System Optimized Bucket Layouts."),
 
-  FUTURE_VERSION(-1, "Used internally when the server side is older and an"
+  FUTURE_VERSION(-1, "Used internally by the server when the server side is older and an"
       + " unknown client version has arrived from the client.");
 
   private static final SortedMap<Integer, ClientVersion> BY_VALUE =

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java

@@ -64,8 +64,8 @@ public enum OzoneManagerVersion implements ComponentVersion {
 
   ZDU(100, "OzoneManager version that supports zero downtime upgrade"),
 
-  FUTURE_VERSION(-1, "Used internally in the client when the server side is "
-      + " newer and an unknown server version has arrived to the client.");
+  UNKNOWN_VERSION(-1, "Used when a version cannot be deserialized to any version recognized by this" +
+      " component, which may indicate it came from a component in a newer version");
 
   private static final SortedMap<Integer, OzoneManagerVersion> BY_VALUE =
       Arrays.stream(values())
@@ -93,11 +93,11 @@ public int serialize() {
 
   /**
    * @param value The serialized version to convert.
-   * @return The version corresponding to this serialized value, or {@link #FUTURE_VERSION} if no matching version is
+   * @return The version corresponding to this serialized value, or {@link #UNKNOWN_VERSION} if no matching version is
    *    found.
    */
   public static OzoneManagerVersion deserialize(int value) {
-    return BY_VALUE.getOrDefault(value, FUTURE_VERSION);
+    return BY_VALUE.getOrDefault(value, UNKNOWN_VERSION);
   }
 
 

hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/AbstractComponentVersionTest.java

@@ -35,26 +35,23 @@ public abstract class AbstractComponentVersionTest {
 
   protected abstract ComponentVersion getDefaultVersion();
 
-  protected abstract ComponentVersion getFutureVersion();
+  protected abstract ComponentVersion getUnknownVersion();
 
   protected abstract ComponentVersion deserialize(int value);
 
-  // FUTURE_VERSION is the latest
   @Test
-  public void testFutureVersionHasTheHighestOrdinal() {
+  public void testUnknownFutureVersionHasTheHighestOrdinal() {
     ComponentVersion[] values = getValues();
-    ComponentVersion futureValue = getFutureVersion();
+    ComponentVersion futureValue = getUnknownVersion();
     assertEquals(values[values.length - 1], futureValue);
   }
 
-  // FUTURE_VERSION's internal version id is -1
   @Test
   public void testFutureVersionSerializesToMinusOne() {
-    ComponentVersion futureValue = getFutureVersion();
-    assertEquals(-1, futureValue.serialize());
+    ComponentVersion unknownVersion = getUnknownVersion();
+    assertEquals(-1, unknownVersion.serialize());
   }
 
-  // DEFAULT_VERSION's internal version id is 0
   @Test
   public void testDefaultVersionSerializesToZero() {
     ComponentVersion defaultValue = getDefaultVersion();
@@ -75,7 +72,7 @@ public void testSerializedValuesAreMonotonic() {
   @Test
   public void testNextVersionProgression() {
     ComponentVersion[] values = getValues();
-    ComponentVersion futureValue = getFutureVersion();
+    ComponentVersion futureValue = getUnknownVersion();
     int knownVersionCount = values.length - 1;
     for (int i = 0; i < knownVersionCount - 1; i++) {
       assertEquals(values[i + 1], values[i].nextVersion(),
@@ -84,7 +81,7 @@ public void testNextVersionProgression() {
     assertNull(values[knownVersionCount - 1].nextVersion(),
         "Expected latest known version to have no nextVersion");
     assertNull(futureValue.nextVersion(),
-        "Expected FUTURE_VERSION.nextVersion() to return null");
+        "Expected unknown version to have no nextVersion");
   }
 
   @Test
@@ -109,12 +106,13 @@ public void testFutureVersionSupportsAllKnownVersions() {
     ComponentVersion[] values = getValues();
     int unknownFutureVersion = Integer.MAX_VALUE;
     for (ComponentVersion knownVersion : values) {
-      if (knownVersion == getFutureVersion()) {
-        // FUTURE_VERSION with serialized value < 0 is considered larger than any version with a concrete
-        // positive value.
+      if (knownVersion == getUnknownVersion()) {
+        // Two unknown future versions should not support each other.
         assertFalse(knownVersion.isSupportedBy(unknownFutureVersion), knownVersion +
             " should not support unknown future version " + unknownFutureVersion);
       } else {
+        // The unknown future version should deserialize to a negative value, but still be considered larger than all
+        // known versions.
         assertTrue(knownVersion.isSupportedBy(unknownFutureVersion), knownVersion +
             " should support unknown future version " + unknownFutureVersion);
       }
@@ -129,7 +127,7 @@ public void testVersionSerDes() {
   }
 
   @Test
-  public void testDeserializeUnknownReturnsFutureVersion() {
-    assertEquals(getFutureVersion(), deserialize(Integer.MAX_VALUE));
+  public void testDeserializeUnknownVersion() {
+    assertEquals(getUnknownVersion(), deserialize(Integer.MAX_VALUE));
   }
 }

hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestClientVersion.java

@@ -35,7 +35,7 @@ protected ComponentVersion getDefaultVersion() {
   }
 
   @Override
-  protected ComponentVersion getFutureVersion() {
+  protected ComponentVersion getUnknownVersion() {
     return ClientVersion.FUTURE_VERSION;
   }
 

hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHDDSVersion.java

@@ -37,8 +37,8 @@ protected ComponentVersion getDefaultVersion() {
   }
 
   @Override
-  protected ComponentVersion getFutureVersion() {
-    return HDDSVersion.FUTURE_VERSION;
+  protected ComponentVersion getUnknownVersion() {
+    return HDDSVersion.UNKNOWN_VERSION;
   }
 
   @Override

hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestOzoneManagerVersion.java

@@ -38,8 +38,8 @@ protected ComponentVersion getDefaultVersion() {
   }
 
   @Override
-  protected ComponentVersion getFutureVersion() {
-    return OzoneManagerVersion.FUTURE_VERSION;
+  protected ComponentVersion getUnknownVersion() {
+    return OzoneManagerVersion.UNKNOWN_VERSION;
   }
 
   @Override

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DatanodeStorage.java

@@ -17,12 +17,12 @@
 
 package org.apache.hadoop.ozone.container.common;
 
-import static org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager.maxLayoutVersion;
 import static org.apache.hadoop.ozone.OzoneConsts.DATANODE_LAYOUT_VERSION_DIR;
 
 import java.io.File;
 import java.io.IOException;
 import java.util.Properties;
+import org.apache.hadoop.hdds.HDDSVersion;
 import org.apache.hadoop.hdds.conf.ConfigurationSource;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType;
@@ -43,7 +43,7 @@ public class DatanodeStorage extends Storage {
   public DatanodeStorage(ConfigurationSource conf, String dataNodeId)
       throws IOException {
     super(NodeType.DATANODE, ServerUtils.getOzoneMetaDirPath(conf),
-        DATANODE_LAYOUT_VERSION_DIR, dataNodeId, getDefaultLayoutVersion(conf));
+        DATANODE_LAYOUT_VERSION_DIR, dataNodeId, getDefaultApparentVersion(conf));
   }
 
   public DatanodeStorage(OzoneConfiguration conf, String dataNodeId,
@@ -56,7 +56,7 @@ public DatanodeStorage(OzoneConfiguration conf, String dataNodeId,
   public DatanodeStorage(ConfigurationSource conf)
       throws IOException {
     super(NodeType.DATANODE, ServerUtils.getOzoneMetaDirPath(conf),
-        DATANODE_LAYOUT_VERSION_DIR, getDefaultLayoutVersion(conf));
+        DATANODE_LAYOUT_VERSION_DIR, getDefaultApparentVersion(conf));
   }
 
   @Override
@@ -94,15 +94,14 @@ public void setClusterId(String clusterId) throws IOException {
    * @return The layout version that should be used for the datanode if no
    * layout version is found on disk.
    */
-  private static int getDefaultLayoutVersion(ConfigurationSource conf) {
-    int defaultLayoutVersion = maxLayoutVersion();
+  private static int getDefaultApparentVersion(ConfigurationSource conf) {
+    int defaultApparentVersion = HDDSVersion.SOFTWARE_VERSION.serialize();
 
     File dnIdFile = new File(HddsServerUtil.getDatanodeIdFilePath(conf));
     if (dnIdFile.exists()) {
-      defaultLayoutVersion =
-          HDDSLayoutFeature.INITIAL_VERSION.layoutVersion();
+      defaultApparentVersion = HDDSLayoutFeature.INITIAL_VERSION.serialize();
     }
 
-    return defaultLayoutVersion;
+    return defaultApparentVersion;
   }
 }

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java

@@ -21,8 +21,7 @@
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_ACTION_MAX_LIMIT_DEFAULT;
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PIPELINE_ACTION_MAX_LIMIT;
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PIPELINE_ACTION_MAX_LIMIT_DEFAULT;
-import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type.finalizeNewLayoutVersionCommand;
-import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.toLayoutVersionProto;
+import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.toVersionProto;
 
 import com.google.common.base.Preconditions;
 import com.google.protobuf.Descriptors;
@@ -129,13 +128,13 @@ public EndpointStateMachine.EndPointStates call() throws Exception {
     try {
       Preconditions.checkState(this.datanodeDetailsProto != null);
 
-      LayoutVersionProto layoutinfo = toLayoutVersionProto(
-          versionManager.getApparentVersion().serialize(),
-          versionManager.getSoftwareVersion().serialize());
+      LayoutVersionProto versionInfo = toVersionProto(
+          versionManager.getApparentVersion(),
+          versionManager.getSoftwareVersion());
 
       requestBuilder = SCMHeartbeatRequestProto.newBuilder()
           .setDatanodeDetails(datanodeDetailsProto)
-          .setDataNodeLayoutVersion(layoutinfo);
+          .setDataNodeLayoutVersion(versionInfo);
       addReports(requestBuilder);
       addContainerActions(requestBuilder);
       addPipelineActions(requestBuilder);

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DatanodeVersionManager.java

@@ -21,19 +21,22 @@
 import java.io.IOException;
 import java.util.Map;
 import org.apache.hadoop.hdds.ComponentVersion;
+import org.apache.hadoop.hdds.HDDSVersion;
 import org.apache.hadoop.hdds.upgrade.DatanodeUpgradeAction;
 import org.apache.hadoop.hdds.upgrade.DatanodeUpgradeActionProvider;
-import org.apache.hadoop.hdds.upgrade.HDDSVersionManager;
+import org.apache.hadoop.hdds.upgrade.HDDSVersionUtils;
 import org.apache.hadoop.ozone.container.common.DatanodeStorage;
 import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
 import org.apache.hadoop.ozone.upgrade.ComponentUpgradeActionProvider;
+import org.apache.hadoop.ozone.upgrade.ComponentVersionManager;
 import org.apache.hadoop.ozone.upgrade.UpgradeException;
 
 /**
  * Datanode-specific version manager that wires upgrade actions internally.
  */
-public class DatanodeVersionManager extends HDDSVersionManager {
+public class DatanodeVersionManager extends ComponentVersionManager {
 
+  private final DatanodeStorage storage;
   private final Map<ComponentVersion, DatanodeUpgradeAction> upgradeActions;
   private final DatanodeStateMachine upgradeActionArg;
 
@@ -44,11 +47,24 @@ public DatanodeVersionManager(DatanodeStorage storage, DatanodeStateMachine upgr
   @VisibleForTesting
   public DatanodeVersionManager(DatanodeStorage storage, DatanodeStateMachine upgradeActionArg,
       ComponentUpgradeActionProvider<DatanodeUpgradeAction> upgradeActionProvider) throws IOException {
-    super(storage);
+    super(HDDSVersionUtils.deserializedPersistedApparentVersion(storage.getApparentVersion()),
+        HDDSVersion.SOFTWARE_VERSION);
+    this.storage = storage;
     this.upgradeActionArg = upgradeActionArg;
     upgradeActions = upgradeActionProvider.load();
   }
 
+  @Override
+  protected void persistApparentVersion(ComponentVersion newVersion) throws IOException {
+    storage.setApparentVersion(newVersion.serialize());
+    storage.persistCurrentState();
+  }
+
+  @Override
+  public int getPersistedApparentVersion() {
+    return storage.getApparentVersion();
+  }
+
   @VisibleForTesting
   public Map<ComponentVersion, DatanodeUpgradeAction> getUpgradeActionsForTesting() {
     return upgradeActions;

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/UpgradeUtils.java

@@ -17,6 +17,7 @@
 
 package org.apache.hadoop.ozone.container.upgrade;
 
+import org.apache.hadoop.hdds.ComponentVersion;
 import org.apache.hadoop.hdds.HDDSVersion;
 import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto;
 
@@ -28,16 +29,17 @@ public final class UpgradeUtils {
   private UpgradeUtils() {
   }
 
-  public static LayoutVersionProto defaultLayoutVersionProto() {
+  public static LayoutVersionProto defaultVersionProto() {
     int softwareVersion = HDDSVersion.SOFTWARE_VERSION.serialize();
     return LayoutVersionProto.newBuilder()
         .setMetadataLayoutVersion(softwareVersion)
         .setSoftwareLayoutVersion(softwareVersion).build();
   }
 
-  public static LayoutVersionProto toLayoutVersionProto(int mLv, int sLv) {
+  public static LayoutVersionProto toVersionProto(ComponentVersion apparentVersion, ComponentVersion softwareVersion) {
     return LayoutVersionProto.newBuilder()
-        .setMetadataLayoutVersion(mLv)
-        .setSoftwareLayoutVersion(sLv).build();
+        .setMetadataLayoutVersion(apparentVersion.serialize())
+        .setSoftwareLayoutVersion(softwareVersion.serialize())
+        .build();
   }
 }

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java

@@ -18,7 +18,7 @@
 package org.apache.hadoop.ozone.protocolPB;
 
 import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.INITIAL_VERSION;
-import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.toLayoutVersionProto;
+import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.toVersionProto;
 
 import com.google.protobuf.RpcController;
 import com.google.protobuf.ServiceException;
@@ -71,17 +71,16 @@ public SCMRegisteredResponseProto register(
         .getContainerReport();
     NodeReportProto dnNodeReport = request.getNodeReport();
     PipelineReportsProto pipelineReport = request.getPipelineReports();
-    LayoutVersionProto layoutInfo = null;
+    LayoutVersionProto versionInfo = null;
     if (request.hasDataNodeLayoutVersion()) {
-      layoutInfo = request.getDataNodeLayoutVersion();
+      versionInfo = request.getDataNodeLayoutVersion();
     } else {
       // Backward compatibility to make sure old Datanodes can still talk to
       // SCM.
-      layoutInfo = toLayoutVersionProto(INITIAL_VERSION.layoutVersion(),
-          INITIAL_VERSION.layoutVersion());
+      versionInfo = toVersionProto(INITIAL_VERSION, INITIAL_VERSION);
     }
     return impl.register(request.getExtendedDatanodeDetails(), dnNodeReport,
-        containerRequestProto, pipelineReport, layoutInfo);
+        containerRequestProto, pipelineReport, versionInfo);
   }
 
   @Override

hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java

@@ -231,7 +231,7 @@ public void testDatanodeStateContext() throws IOException,
 
       DatanodeStorage layoutStorage = new DatanodeStorage(conf,
           UUID.randomUUID().toString(),
-          HDDSLayoutFeature.DATANODE_SCHEMA_V3.layoutVersion());
+          HDDSLayoutFeature.DATANODE_SCHEMA_V3.serialize());
       layoutStorage.initialize();
 
       // This execute will invoke getVersion calls against all SCM endpoints