HDDS-15063. Switch OM to the new Versioning Framework

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizer.java → hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentUpgradeActionProvider.java

@@ -15,28 +15,24 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.ozone.om.upgrade;
+package org.apache.hadoop.ozone.upgrade;
 
-import org.apache.hadoop.ozone.om.OzoneManager;
-import org.apache.hadoop.ozone.upgrade.BasicUpgradeFinalizer;
-import org.apache.hadoop.ozone.upgrade.LayoutFeature;
-import org.apache.hadoop.ozone.upgrade.UpgradeException;
+import java.util.Map;
+import org.apache.hadoop.hdds.ComponentVersion;
 
 /**
- * UpgradeFinalizer implementation for the Ozone Manager service.
+ * Supplies upgrade actions keyed by {@link ComponentVersion}. Implementations typically perform classpath scanning or
+ * return a fixed map for tests. The component version manager decides when each action is invoked.
+ *
+ * @param <A> concrete upgrade action type (for example OM-specific or HDDS-specific)
  */
-public class OMUpgradeFinalizer extends BasicUpgradeFinalizer<OzoneManager,
-    OMLayoutVersionManager> {
-
-  public OMUpgradeFinalizer(OMLayoutVersionManager versionManager) {
-    super(versionManager);
-  }
+@FunctionalInterface
+public interface ComponentUpgradeActionProvider<A> {
 
-  @Override
-  public void finalizeLayoutFeature(LayoutFeature layoutFeature,
-      OzoneManager om) throws UpgradeException {
-    super.finalizeLayoutFeature(layoutFeature,
-        layoutFeature.action(),
-        om.getOmStorage());
-  }
+  /**
+   * Returns all upgrade actions from this provider, keyed by component version.
+   * <p>
+   * Implementations must return a newly allocated map on each call; the caller may retain and use it directly.
+   */
+  Map<ComponentVersion, A> load();
 }

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/UpgradeException.java

@@ -106,10 +106,8 @@ public String toString() {
   public enum ResultCodes {
     OK,
     INVALID_REQUEST,
-    UPDATE_LAYOUT_VERSION_FAILED,
-    LAYOUT_FEATURE_FINALIZATION_FAILED,
-    PREFINALIZE_ACTION_VALIDATION_FAILED,
-    FIRST_UPGRADE_START_ACTION_FAILED,
-    PREFINALIZE_VALIDATION_FAILED;
+    APPARENT_VERSION_UPDATE_FAILED,
+    UPGRADE_FINALIZATION_FAILED,
+    FINALIZE_UPGRADE_ACTION_FAILED,
   }
 }

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/UpgradeFinalization.java

@@ -49,12 +49,19 @@ public final class UpgradeFinalization {
   );
 
   /**
-   * Default message to provide when the service is in ALREADY_FINALIZED state.
+   * Returned from finalize upgrade commands when finalization is not required.
    */
   public static final StatusAndMessages FINALIZED_MSG = new StatusAndMessages(
       Status.ALREADY_FINALIZED, Collections.emptyList()
   );
 
+  /**
+   * Returned from progress/status queries when finalization is not required.
+   */
+  public static final StatusAndMessages FINALIZATION_DONE_MSG = new StatusAndMessages(
+      Status.FINALIZATION_DONE, Collections.emptyList()
+  );
+
   /**
    * Represents the current state in which the service is with regards to
    * finalization after an upgrade.

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/DatanodeIdYaml.java

@@ -261,7 +261,7 @@ private static DatanodeDetailsYaml getDatanodeDetailsYaml(
           HDDSLayoutFeature layoutFeature
               = f.getAnnotation(BelongsToHDDSLayoutVersion.class).value();
           if (layoutFeature.layoutVersion() >
-              datanodeLayoutStorage.getLayoutVersion()) {
+              datanodeLayoutStorage.getApparentVersion()) {
             continue;
           }
         }

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java

@@ -168,7 +168,7 @@ public DatanodeStateMachine(HddsDatanodeService hddsDatanodeService,
         datanodeDetails.getUuidString());
 
     layoutVersionManager = new HDDSLayoutVersionManager(
-        layoutStorage.getLayoutVersion());
+        layoutStorage.getApparentVersion());
     upgradeFinalizer = new DataNodeUpgradeFinalizer(layoutVersionManager);
     VersionedDatanodeFeatures.initialize(layoutVersionManager);
 

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DataNodeUpgradeFinalizer.java

@@ -47,7 +47,7 @@ public void finalizeLayoutFeature(LayoutFeature layoutFeature,
       String msg = String.format("Failed to finalize datanode layout feature " +
           "%s. It is not an HDDS Layout Feature.", layoutFeature);
       throw new UpgradeException(msg,
-          UpgradeException.ResultCodes.LAYOUT_FEATURE_FINALIZATION_FAILED);
+          UpgradeException.ResultCodes.UPGRADE_FINALIZATION_FAILED);
     }
   }
 }

hadoop-hdds/docs/content/design/upgrade-dev-primer.md

@@ -36,13 +36,11 @@ Downgrade to a lower version is allowed from the pre-finalized state. This invol
     org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature
 Class to add  a new layout feature being brought in. Layout version is typically 1 + last layout feature in that catalog. 
 
-## LayoutVersionManager
-    org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager
-    org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager
-Every component carries an instance of this interface, which provides APIs to get runtime layout version, and if a feature is allowed based on that or not.
+## Version management (OM vs HDDS)
 
-The LayoutVersionManager interface carries an API that can be used to check if a feature is allowed in the current layout version.
-     org.apache.hadoop.ozone.upgrade.LayoutVersionManager#isAllowed(org.apache.hadoop.ozone.upgrade.LayoutFeature)
+**Ozone Manager** uses [`org.apache.hadoop.ozone.om.upgrade.OMVersionManager`](https://github.com/apache/ozone/blob/master/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMVersionManager.java) ([`org.apache.hadoop.ozone.upgrade.ComponentVersionManager`](https://github.com/apache/ozone/blob/master/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentVersionManager.java)), with upgrade actions discovered via [`org.apache.hadoop.ozone.om.upgrade.OMUpgradeActionProvider`](https://github.com/apache/ozone/blob/master/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeActionProvider.java). It exposes apparent/software `ComponentVersion` and `isAllowed(ComponentVersion)` for gating.
+
+**SCM / DataNode** continue to use [`org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager`](https://github.com/apache/ozone/blob/master/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutVersionManager.java) ([`org.apache.hadoop.ozone.upgrade.AbstractLayoutVersionManager`](https://github.com/apache/ozone/blob/master/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/AbstractLayoutVersionManager.java)), which provides metadata/software layout integers and `isAllowed(LayoutFeature)`.
 
 ## @DisallowedUntilLayoutVersion Annotation
 Method level annotation used to "disallow" an API if current layout version does not include the associated layout feature. Currently it is added only to the OM module, but can easily be moved down to a common module based on need on the HDDS layer.

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSVersionManager.java

@@ -20,29 +20,45 @@
 import java.io.IOException;
 import org.apache.hadoop.hdds.ComponentVersion;
 import org.apache.hadoop.hdds.HDDSVersion;
+import org.apache.hadoop.ozone.common.Storage;
 import org.apache.hadoop.ozone.upgrade.ComponentVersionManager;
+import org.apache.hadoop.ozone.upgrade.UpgradeException;
 
 /**
  * Component version manager for HDDS (Datanodes and SCM).
  */
 public class HDDSVersionManager extends ComponentVersionManager {
-  public HDDSVersionManager(int serializedApparentVersion) throws IOException {
-    super(computeApparentVersion(serializedApparentVersion), HDDSVersion.SOFTWARE_VERSION);
+  public HDDSVersionManager(Storage storage) throws IOException {
+    super(storage, computeApparentVersion(storage.getApparentVersion()), HDDSVersion.SOFTWARE_VERSION);
   }
 
   /**
-   * If the apparent version stored on the disk is >= 100, it indicates the component has been finalized for the
-   * ZDU feature, and the apparent version corresponds to a version in {@link HDDSVersion}.
-   * If the apparent version stored on the disk is < 100, it indicates the component is not yet finalized for the
-   * ZDU feature, and the apparent version corresponds to a version in {@link HDDSLayoutFeature}.
+   * If the apparent version stored on the disk is &gt;= {@link HDDSVersion#ZDU} serialized, the apparent version is
+   * resolved via {@link HDDSVersion#deserialize(int)}. Values with no matching {@link HDDSVersion} fail startup with
+   * the persisted integer in the exception message.
+   * If the value is below that threshold, the apparent version is resolved as a {@link HDDSLayoutFeature}. Integers in
+   * the gap between the largest {@link HDDSLayoutFeature} and ZDU are not valid legacy layout values; startup fails
+   * with the persisted integer in the exception message.
    */
-  private static ComponentVersion computeApparentVersion(int serializedApparentVersion) {
-    if (serializedApparentVersion < HDDSVersion.ZDU.serialize()) {
-      return HDDSLayoutFeature.deserialize(serializedApparentVersion);
+  private static ComponentVersion computeApparentVersion(int serializedApparentVersion) throws IOException {
+    if (serializedApparentVersion >= HDDSVersion.ZDU.serialize()) {
+      HDDSVersion fromHdds = HDDSVersion.deserialize(serializedApparentVersion);
+      if (fromHdds != HDDSVersion.FUTURE_VERSION) {
+        return fromHdds;
+      }
     } else {
-      return HDDSVersion.deserialize(serializedApparentVersion);
+      ComponentVersion fromLayout = HDDSLayoutFeature.deserialize(serializedApparentVersion);
+      if (fromLayout != null) {
+        return fromLayout;
+      }
     }
+    throw new IOException("Initialization failed. Disk contains unknown apparent version " + serializedApparentVersion +
+        " for software version " + HDDSVersion.SOFTWARE_VERSION + ". Make sure this component was not downgraded" +
+        " after finalization");
   }
 
-  // TODO HDDS-14826: Register upgrade actions based on annotations
+  @Override
+  protected void runUpgradeAction(ComponentVersion componentVersion) throws UpgradeException {
+    // TODO HDDS-14826: Register upgrade actions based on annotations
+  }
 }

hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/common/Storage.java

@@ -124,20 +124,12 @@ public void setClusterId(String clusterId) throws IOException {
     }
   }
 
-  public int getLayoutVersion() {
-    return storageInfo.getLayoutVersion();
+  public int getApparentVersion() {
+    return storageInfo.getApparentVersion();
   }
 
-  public void setLayoutVersion(int version) {
-    storageInfo.setLayoutVersion(version);
-  }
-
-  public void setFirstUpgradeActionLayoutVersion(int version) {
-    storageInfo.setFirstUpgradeActionLayoutVersion(version);
-  }
-
-  public int getFirstUpgradeActionLayoutVersion() {
-    return storageInfo.getFirstUpgradeActionLayoutVersion();
+  public void setApparentVersion(int version) {
+    storageInfo.setApparentVersion(version);
   }
 
   /**
@@ -288,8 +280,7 @@ public void persistCurrentState() throws IOException {
     storageInfo.writeTo(getVersionFile());
   }
 
-  protected static int getInitLayoutVersion(OzoneConfiguration conf,
-      String configKey,
+  protected static int getInitApparentVersion(OzoneConfiguration conf, String configKey,
       IntSupplier defaultLvSupplier) {
     int lV = conf.getInt(configKey, OZONE_INIT_DEFAULT_LAYOUT_VERSION_DEFAULT);
     if (lV == OZONE_INIT_DEFAULT_LAYOUT_VERSION_DEFAULT) {

hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/common/StorageInfo.java

@@ -61,9 +61,6 @@ public class StorageInfo {
    */
   private static final String LAYOUT_VERSION = "layoutVersion";
 
-  private static final String FIRST_UPGRADE_ACTION_LAYOUT_VERSION =
-      "firstUpgradeActionLayoutVersion";
-
   private static final int INVALID_LAYOUT_VERSION = -1;
 
   /**
@@ -93,7 +90,7 @@ public StorageInfo(NodeType type, File propertiesFile)
     verifyNodeType(type);
     verifyClusterId();
     verifyCreationTime();
-    verifyLayoutVersion();
+    verifyApparentVersion();
   }
 
   public NodeType getNodeType() {
@@ -112,15 +109,15 @@ public Long  getCreationTime() {
     return null;
   }
 
-  public int getLayoutVersion() {
+  public int getApparentVersion() {
     String layout = properties.getProperty(LAYOUT_VERSION);
     if (layout != null) {
       return Integer.parseInt(layout);
     }
     return 0;
   }
 
-  private void verifyLayoutVersion() {
+  private void verifyApparentVersion() {
     String layout = getProperty(LAYOUT_VERSION);
     if (layout == null) {
       LOG.warn("Found " + STORAGE_FILE_VERSION + " file without any layout " +
@@ -129,20 +126,6 @@ private void verifyLayoutVersion() {
     }
   }
 
-  public int getFirstUpgradeActionLayoutVersion() {
-    String upgradingTo =
-        properties.getProperty(FIRST_UPGRADE_ACTION_LAYOUT_VERSION);
-    if (upgradingTo != null) {
-      return Integer.parseInt(upgradingTo);
-    }
-    return INVALID_LAYOUT_VERSION;
-  }
-
-  public void setFirstUpgradeActionLayoutVersion(int layoutVersion) {
-    properties.setProperty(
-        FIRST_UPGRADE_ACTION_LAYOUT_VERSION, Integer.toString(layoutVersion));
-  }
-
   public String getProperty(String key) {
     return properties.getProperty(key);
   }
@@ -159,7 +142,7 @@ public void setClusterId(String clusterId) {
     properties.setProperty(CLUSTER_ID, clusterId);
   }
 
-  public void setLayoutVersion(int version) {
+  public void setApparentVersion(int version) {
     properties.setProperty(LAYOUT_VERSION, Integer.toString(version));
   }
 

hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/AbstractLayoutVersionManager.java

@@ -38,7 +38,7 @@
  */
 @SuppressWarnings("visibilitymodifier")
 public abstract class AbstractLayoutVersionManager<T extends LayoutFeature>
-    implements LayoutVersionManager, LayoutVersionManagerMXBean {
+    implements LayoutVersionManagerMXBean {
 
   private static final Logger LOG =
       LoggerFactory.getLogger(AbstractLayoutVersionManager.class);
@@ -164,7 +164,6 @@ public int getSoftwareLayoutVersion() {
     return softwareLayoutVersion;
   }
 
-  @Override
   public boolean needsFinalization() {
     lock.readLock().lock();
     try {
@@ -174,7 +173,6 @@ public boolean needsFinalization() {
     }
   }
 
-  @Override
   public boolean isAllowed(LayoutFeature layoutFeature) {
     lock.readLock().lock();
     try {
@@ -184,12 +182,10 @@ public boolean isAllowed(LayoutFeature layoutFeature) {
     }
   }
 
-  @Override
   public LayoutFeature getFeature(int layoutVersion) {
     return features.get(layoutVersion);
   }
 
-  @Override
   public Iterable<LayoutFeature> unfinalizedFeatures() {
     lock.readLock().lock();
     try {
@@ -201,7 +197,6 @@ public Iterable<LayoutFeature> unfinalizedFeatures() {
     }
   }
 
-  @Override
   public void close() {
     if (mBean != null) {
       MBeans.unregister(mBean);

hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/BasicUpgradeFinalizer.java

@@ -17,9 +17,9 @@
 
 package org.apache.hadoop.ozone.upgrade;
 
+import static org.apache.hadoop.ozone.upgrade.UpgradeException.ResultCodes.APPARENT_VERSION_UPDATE_FAILED;
 import static org.apache.hadoop.ozone.upgrade.UpgradeException.ResultCodes.INVALID_REQUEST;
-import static org.apache.hadoop.ozone.upgrade.UpgradeException.ResultCodes.LAYOUT_FEATURE_FINALIZATION_FAILED;
-import static org.apache.hadoop.ozone.upgrade.UpgradeException.ResultCodes.UPDATE_LAYOUT_VERSION_FAILED;
+import static org.apache.hadoop.ozone.upgrade.UpgradeException.ResultCodes.UPGRADE_FINALIZATION_FAILED;
 import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.FINALIZATION_IN_PROGRESS_MSG;
 import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.FINALIZATION_REQUIRED_MSG;
 import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.FINALIZED_MSG;
@@ -291,11 +291,11 @@ protected void updateLayoutVersionInVersionFile(LayoutFeature feature,
   }
 
   private int currentStoredLayoutVersion(Storage config) {
-    return config.getLayoutVersion();
+    return config.getApparentVersion();
   }
 
   private void updateStorageLayoutVersion(int version, Storage config) {
-    config.setLayoutVersion(version);
+    config.setApparentVersion(version);
   }
 
   private void persistStorage(Storage config) throws IOException {
@@ -326,13 +326,13 @@ protected void logAndEmit(String msg) {
   protected void logFinalizationFailureAndThrow(Exception e, String feature)
       throws UpgradeException {
     String msg = "Error during finalization of " + feature + ".";
-    logAndThrow(e, msg, LAYOUT_FEATURE_FINALIZATION_FAILED);
+    logAndThrow(e, msg, UPGRADE_FINALIZATION_FAILED);
   }
 
   private void logLayoutVersionUpdateFailureAndThrow(IOException e)
       throws UpgradeException {
     String msg = "Updating the LayoutVersion in the VERSION file failed.";
-    logAndThrow(e, msg, UPDATE_LAYOUT_VERSION_FAILED);
+    logAndThrow(e, msg, APPARENT_VERSION_UPDATE_FAILED);
   }
 
   private void logAndThrow(Exception e, String msg, ResultCodes resultCode)