HDDS-6760. [SCM HA finalization] Support for happy path finalization

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java

@@ -465,6 +465,11 @@ private OzoneConsts() {
 
   // Layout Version written into Meta Table ONLY during finalization.
   public static final String LAYOUT_VERSION_KEY = "#LAYOUTVERSION";
+  // Key written to Meta Table to indicate a component undergoing finalization.
+  // Currently this is only used on SCM, but may be useful on OM if/when
+  // finalizing one layout feature per Ratis request is implemented in
+  // HDDS-4286.
+  public static final String FINALIZING_KEY = "#FINALIZING";
 
   // Kerberos constants
   public static final String KERBEROS_CONFIG_VALUE = "kerberos";

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/AbstractLayoutVersionManager.java

@@ -49,12 +49,12 @@ public abstract class AbstractLayoutVersionManager<T extends LayoutFeature>
   private static final Logger LOG =
       LoggerFactory.getLogger(AbstractLayoutVersionManager.class);
 
-  private int metadataLayoutVersion; // MLV.
-  private int softwareLayoutVersion; // SLV.
+  private volatile int metadataLayoutVersion; // MLV.
+  private volatile int softwareLayoutVersion; // SLV.
   @VisibleForTesting
-  protected final TreeMap<Integer, T> features = new TreeMap<>();
+  protected final TreeMap<Integer, LayoutFeature> features = new TreeMap<>();
   @VisibleForTesting
-  protected final Map<String, T> featureMap = new HashMap<>();
+  protected final Map<String, LayoutFeature> featureMap = new HashMap<>();
   private volatile Status currentUpgradeState = FINALIZATION_REQUIRED;
   // Allows querying upgrade state while an upgrade is in progress.
   // Note that MLV may have been incremented during the upgrade
@@ -123,9 +123,13 @@ public void finalized(T layoutFeature) {
     try {
       if (layoutFeature.layoutVersion() == metadataLayoutVersion + 1) {
         metadataLayoutVersion = layoutFeature.layoutVersion();
+        LOG.info("Layout feature {} has been finalized.", layoutFeature);
+        if (!needsFinalization()) {
+          completeFinalization();
+        }
       } else {
         String msgStart = "";
-        if (layoutFeature.layoutVersion() < metadataLayoutVersion) {
+        if (layoutFeature.layoutVersion() <= metadataLayoutVersion) {
           msgStart = "Finalize attempt on a layoutFeature which has already "
               + "been finalized.";
         } else {
@@ -135,18 +139,20 @@ public void finalized(T layoutFeature) {
         }
 
         throw new IllegalArgumentException(
-            msgStart + "Software Layout version: " + softwareLayoutVersion
+            msgStart + " Software layout version: " + softwareLayoutVersion
+                + " Metadata layout version: " + metadataLayoutVersion
                 + " Feature Layout version: " + layoutFeature.layoutVersion());
       }
     } finally {
       lock.writeLock().unlock();
     }
   }
 
-  public void completeFinalization() {
+  private void completeFinalization() {
     lock.writeLock().lock();
     try {
       currentUpgradeState = FINALIZATION_DONE;
+      LOG.info("Finalization is complete.");
     } finally {
       lock.writeLock().unlock();
     }
@@ -203,12 +209,17 @@ public boolean isAllowed(String featureName) {
   }
 
   @Override
-  public T getFeature(String name) {
+  public LayoutFeature getFeature(String name) {
     return featureMap.get(name);
   }
 
   @Override
-  public Iterable<T> unfinalizedFeatures() {
+  public LayoutFeature getFeature(int layoutVersion) {
+    return features.get(layoutVersion);
+  }
+
+  @Override
+  public Iterable<LayoutFeature> unfinalizedFeatures() {
     lock.readLock().lock();
     try {
       return new ArrayList<>(features

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/BasicUpgradeFinalizer.java

@@ -18,7 +18,6 @@
 
 package org.apache.hadoop.ozone.upgrade;
 
-import static org.apache.hadoop.ozone.upgrade.LayoutFeature.UpgradeActionType.ON_FINALIZE;
 import static org.apache.hadoop.ozone.upgrade.LayoutFeature.UpgradeActionType.ON_FIRST_UPGRADE_START;
 import static org.apache.hadoop.ozone.upgrade.LayoutFeature.UpgradeActionType.VALIDATE_IN_PREFINALIZE;
 import static org.apache.hadoop.ozone.upgrade.UpgradeException.ResultCodes.FIRST_UPGRADE_START_ACTION_FAILED;
@@ -36,6 +35,8 @@
 import java.util.Optional;
 import java.util.Queue;
 import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
 import java.util.function.Function;
 import java.util.concurrent.TimeUnit;
 
@@ -52,27 +53,58 @@
 public abstract class BasicUpgradeFinalizer
     <T, V extends AbstractLayoutVersionManager> implements UpgradeFinalizer<T> {
 
-  private V versionManager;
+  private final V versionManager;
   private String clientID;
   private T component;
-  private DefaultUpgradeFinalizationExecutor<T> finalizationExecutor;
+  private UpgradeFinalizationExecutor<T> finalizationExecutor;
+  // Ensures that there is only one finalization thread running at a time.
+  private final Lock finalizationLock;
 
-  private Queue<String> msgs = new ConcurrentLinkedQueue<>();
+  private final Queue<String> msgs = new ConcurrentLinkedQueue<>();
   private boolean isDone = false;
 
   public BasicUpgradeFinalizer(V versionManager) {
+    this(versionManager, new DefaultUpgradeFinalizationExecutor<>());
+  }
+
+  public BasicUpgradeFinalizer(V versionManager,
+                               UpgradeFinalizationExecutor<T> executor) {
     this.versionManager = versionManager;
-    this.finalizationExecutor = new DefaultUpgradeFinalizationExecutor<>();
+    this.finalizationExecutor = executor;
+    this.finalizationLock = new ReentrantLock();
   }
 
   public StatusAndMessages finalize(String upgradeClientID, T service)
       throws IOException {
+    // In some components, finalization can be driven asynchronously by a
+    // thread, not a single serialized Ratis request.
+    // A second request could closely follow the first before it
+    // sets the finalization status to FINALIZATION_IN_PROGRESS.
+    // Therefore, a lock is used to make sure only one finalization thread is
+    // running at a time.
     StatusAndMessages response = initFinalize(upgradeClientID, service);
-    if (response.status() != FINALIZATION_REQUIRED) {
-      return response;
+    if (finalizationLock.tryLock()) {
+      try {
+        // Even if the status indicates finalization completed, the component
+        // may not have finished all its specific steps if finalization was
+        // interrupted, so we should re-invoke them here.
+        if (response.status() == FINALIZATION_REQUIRED ||
+            !componentFinishedFinalizationSteps(service)) {
+          finalizationExecutor.execute(service, this);
+          response = STARTING_MSG;
+        }
+      } finally {
+        finalizationLock.unlock();
+      }
+    } else {
+      // We could not acquire the lock, so either finalization is ongoing, or
+      // it already finished but we received multiple requests to
+      // run it at the same time.
+      if (!isFinalized(response.status())) {
+        response = FINALIZATION_IN_PROGRESS_MSG;
+      }
     }
-    finalizationExecutor.execute(service, this);
-    return STARTING_MSG;
+    return response;
   }
 
   public synchronized StatusAndMessages reportStatus(
@@ -102,8 +134,6 @@ protected void postFinalizeUpgrade(T service) throws IOException {
     // No Op by default.
   }
 
-  public abstract void finalizeUpgrade(T service) throws UpgradeException;
-
   @Override
   public void finalizeAndWaitForCompletion(
       String upgradeClientID, T service, long maxTimeToWaitInSeconds)
@@ -139,10 +169,12 @@ public void finalizeAndWaitForCompletion(
     }
   }
 
+  @VisibleForTesting
   public boolean isFinalizationDone() {
     return isDone;
   }
 
+  @VisibleForTesting
   public void markFinalizationDone() {
     isDone = true;
   }
@@ -194,19 +226,24 @@ private static boolean isFinalized(Status status) {
         || status.equals(FINALIZATION_DONE);
   }
 
-  protected void finalizeUpgrade(Function<LayoutFeature,
-      Function<UpgradeActionType, Optional<? extends UpgradeAction>>>
-      aFunction, Storage storage) throws UpgradeException {
-    for (Object obj : versionManager.unfinalizedFeatures()) {
-      LayoutFeature lf = (LayoutFeature) obj;
-      Function<UpgradeActionType, Optional<? extends UpgradeAction>> function =
-          aFunction.apply(lf);
-      Optional<? extends UpgradeAction> action = function.apply(ON_FINALIZE);
-      runFinalizationAction(lf, action);
-      updateLayoutVersionInVersionFile(lf, storage);
-      versionManager.finalized(lf);
-    }
-    versionManager.completeFinalization();
+  /**
+   * Child classes that have additional finalization steps can override this
+   * method to check component specific state to determine whether
+   * finalization still needs to be run.
+   */
+  protected boolean componentFinishedFinalizationSteps(T service) {
+    return true;
+  }
+
+  public abstract void finalizeLayoutFeature(LayoutFeature lf, T context)
+      throws UpgradeException;
+
+  protected void finalizeLayoutFeature(LayoutFeature lf, Optional<?
+      extends UpgradeAction> action, Storage storage)
+      throws UpgradeException {
+    runFinalizationAction(lf, action);
+    updateLayoutVersionInVersionFile(lf, storage);
+    versionManager.finalized(lf);
   }
 
   protected void runFinalizationAction(LayoutFeature feature,
@@ -215,6 +252,7 @@ protected void runFinalizationAction(LayoutFeature feature,
     if (!action.isPresent()) {
       emitNOOPMsg(feature.name());
     } else {
+      LOG.info("Running finalization actions for layout feature: {}", feature);
       try {
         action.get().execute(component);
       } catch (Exception e) {
@@ -362,7 +400,7 @@ private void logAndThrow(Exception e, String msg, ResultCodes resultCode)
   }
 
   @VisibleForTesting
-  public void setFinalizationExecutor(DefaultUpgradeFinalizationExecutor
+  public void setFinalizationExecutor(DefaultUpgradeFinalizationExecutor<T>
                                             executor) {
     finalizationExecutor = executor;
   }

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/DefaultUpgradeFinalizationExecutor.java

@@ -31,14 +31,15 @@
  * Unit/Integration tests can override this to provide error injected version
  * of this class.
  */
-public class DefaultUpgradeFinalizationExecutor<T> {
+public class DefaultUpgradeFinalizationExecutor<T>
+    implements UpgradeFinalizationExecutor<T> {
   static final Logger LOG =
       LoggerFactory.getLogger(DefaultUpgradeFinalizationExecutor.class);
 
   public DefaultUpgradeFinalizationExecutor() {
   }
 
-  public void execute(T component, BasicUpgradeFinalizer finalizer)
+  public void execute(T component, BasicUpgradeFinalizer<T, ?> finalizer)
       throws IOException {
     try {
       finalizer.emitStartingMsg();
@@ -47,7 +48,8 @@ public void execute(T component, BasicUpgradeFinalizer finalizer)
 
       finalizer.preFinalizeUpgrade(component);
 
-      finalizer.finalizeUpgrade(component);
+      finalizeFeatures(component, finalizer,
+          finalizer.getVersionManager().unfinalizedFeatures());
 
       finalizer.postFinalizeUpgrade(component);
 
@@ -63,4 +65,12 @@ public void execute(T component, BasicUpgradeFinalizer finalizer)
       finalizer.markFinalizationDone();
     }
   }
+
+  protected void finalizeFeatures(T component,
+      BasicUpgradeFinalizer<T, ?> finalizer,  Iterable<LayoutFeature> lfs)
+      throws UpgradeException {
+    for (LayoutFeature lf: lfs) {
+      finalizer.finalizeLayoutFeature(lf, component);
+    }
+  }
 }

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutVersionManager.java

@@ -62,6 +62,13 @@ public interface LayoutVersionManager {
    */
   LayoutFeature getFeature(String name);
 
+  /**
+   * Get Feature given its layout version.
+   * @param layoutVersion Version number of the feature.
+   * @return LayoutFeature instance.
+   */
+  LayoutFeature getFeature(int layoutVersion);
+
   Iterable<? extends LayoutFeature> unfinalizedFeatures();
 
   /**

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/UpgradeFinalizationExecutor.java

@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.upgrade;
+
+import java.io.IOException;
+
+/**
+ * An upgrade finalization executor runs the finalization methods of an
+ * {@link UpgradeFinalizer}, providing them the state they need to operate
+ * and optionally injecting actions in between.
+ * @param <T> The component or context that the {@link UpgradeFinalizer}
+ *    needs to run.
+ */
+public interface UpgradeFinalizationExecutor<T> {
+  void execute(T component, BasicUpgradeFinalizer<T, ?> finalizer)
+      throws IOException;
+}

hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/upgrade/InjectedUpgradeFinalizationExecutor.java

@@ -68,7 +68,7 @@ static class UpgradeTestInjectionAbort extends Exception {
   }
 
   @Override
-  public void execute(T component, BasicUpgradeFinalizer finalizer)
+  public void execute(T component, BasicUpgradeFinalizer<T, ?> finalizer)
       throws IOException {
     try {
       injectTestFunctionAtThisPoint(BEFORE_PRE_FINALIZE_UPGRADE);
@@ -79,7 +79,8 @@ public void execute(T component, BasicUpgradeFinalizer finalizer)
       finalizer.preFinalizeUpgrade(component);
       injectTestFunctionAtThisPoint(AFTER_PRE_FINALIZE_UPGRADE);
 
-      finalizer.finalizeUpgrade(component);
+      super.finalizeFeatures(component, finalizer,
+          finalizer.getVersionManager().unfinalizedFeatures());
       injectTestFunctionAtThisPoint(AFTER_COMPLETE_FINALIZATION);
 
       finalizer.postFinalizeUpgrade(component);

hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/upgrade/TestBasicUpgradeFinalizer.java

@@ -60,7 +60,14 @@ public void testFinalizerPhasesAreInvokedInOrder() throws IOException {
     assertEquals(FINALIZATION_DONE, res.status());
 
     inOrder.verify(finalizer).preFinalizeUpgrade(ArgumentMatchers.eq(mockObj));
-    inOrder.verify(finalizer).finalizeUpgrade(ArgumentMatchers.eq(mockObj));
+    inOrder.verify(finalizer).finalizeLayoutFeature(
+        ArgumentMatchers.eq(
+            TestUpgradeFinalizerActions.MockLayoutFeature.VERSION_2),
+        ArgumentMatchers.eq(mockObj));
+    inOrder.verify(finalizer).finalizeLayoutFeature(
+        ArgumentMatchers.eq(
+            TestUpgradeFinalizerActions.MockLayoutFeature.VERSION_3),
+        ArgumentMatchers.eq(mockObj));
     inOrder.verify(finalizer).postFinalizeUpgrade(ArgumentMatchers.eq(mockObj));
 
     assertTrue(finalizer.isFinalizationDone());
@@ -113,9 +120,23 @@ protected void postFinalizeUpgrade(Object service) {
     }
 
     @Override
-    public void finalizeUpgrade(Object service) {
+    public void finalizeLayoutFeature(LayoutFeature lf, Object service)
+        throws UpgradeException {
+      Storage mockStorage = mock(Storage.class);
+      InOrder inOrder = inOrder(mockStorage);
+
+      super.finalizeLayoutFeature(lf,
+          lf.action(LayoutFeature.UpgradeActionType.ON_FINALIZE), mockStorage);
+
+      inOrder.verify(mockStorage)
+          .setLayoutVersion(ArgumentMatchers.eq(lf.layoutVersion()));
+      try {
+        inOrder.verify(mockStorage).persistCurrentState();
+      } catch (IOException ex) {
+        throw new UpgradeException(ex,
+            UpgradeException.ResultCodes.LAYOUT_FEATURE_FINALIZATION_FAILED);
+      }
       finalizeCalled = true;
-      getVersionManager().completeFinalization();
     }
 
     @Override