Merge pull request idaholab#15614 from lindsayad/bug_multilevel

Add auto_advance parameter to PicardSolve

framework/doc/content/source/multiapps/TransientMultiApp.md

@@ -14,6 +14,45 @@ sub-apps is utilized. The ability to do perform sub-cycling, which allows the su
 to perform multiple time steps per execution may be enabled using the
 [!param](/MultiApps/TransientMultiApp/sub_cycling) parameter.
 
+## Time state of TransientMultiApps
+
+`TransientMultiApps` are "auto-advanced" by default whenever we are not doing
+Picard iterations between the master and sub-application. This means that the
+`Transient::endStep` and `Transient::postStep` methods of the sub-applications
+executioner are called, regardless of whether the sub-application solve fails or
+not. The `endStep` method increments the time and also performs
+`EXEC_TIMESTEP_END` output. When sub-applications are auto-advanced, their
+`endStep` call happens before the master application's `endStep` call. This has
+the important benefit that when master application output occurs, the
+sub-application's and master application's time states are the same, which
+enables MOOSE restart/recovery capability.
+
+## Handing sub-application solve failures
+
+As noted above, the default behavior when running `TransientMultiApps` is that
+their time state is incremented, e.g. they are "auto-advanced", regardless of
+whether their solve is actually successful. This is undesirable behavior, but we
+believe that the syncing of master and sub-application states, under normal
+operation, to enable correct [checkpoint](/Checkpoint.md) output is a good
+trade. Given the constraints of the elected design, there are still multiple ways to turn a failed
+sub-application solve from a warning into an exception that will force corrective
+behavior in either the sub- or master-application:
+
+1. The user can set `auto_advance = false` in the `Executioner` block of the
+   master application . This will cause the master application to immediately cut
+   its time-step when the sub-application fails. **However**, setting this
+   parameter to `false` also eliminates the possibility of doing restart/recover
+   because the master and sub will be out of sync if/when checkpoint output occurs.
+2. The user can set `catch_up = true` in the `TransientMultiApp` block. This
+   will cause the sub-application to try and catch up to the master application
+   after a sub-app failed solve. If catch-up is unsuccessful, then MOOSE
+   registers this as a true failure of the solve, and the master dt will *then*
+   get cut. This option has the advantage of keeping the master and sub
+   transient states in sync, enabling accurate restart/recover data.
+
+In general, if the user wants sub-application failed solves to be treated as
+exceptions, we recommend that option 2 over option 1.
+
 ## Example Input File Syntax
 
 The following input file shows the creation of a TransientMultiApp object with the time step

framework/doc/content/source/postprocessors/TimePostprocessor.md

@@ -0,0 +1,9 @@
+# TimePostprocessor
+
+!syntax description /Postprocessors/TimePostprocessor
+
+!syntax parameters /Postprocessors/TimePostprocessor
+
+!syntax inputs /Postprocessors/TimePostprocessor
+
+!syntax children /Postprocessors/TimePostprocessor

framework/include/executioners/PicardSolve.h

@@ -76,6 +76,11 @@ class PicardSolve : public SolveObject
     _picard_self_relaxed_variables = vars;
   }
 
+  /**
+   * Whether sub-applications are automatically advanced no matter what happens during their solves
+   */
+  bool autoAdvance() const;
+
 protected:
   /**
    * Perform one Picard iteration or a full solve.
@@ -162,4 +167,12 @@ class PicardSolve : public SolveObject
   Real _previous_entering_time;
 
   const std::string _solve_message;
+
+  /// Whether the user has set the auto_advance parameter for handling advancement of
+  /// sub-applications in multi-app contexts
+  const bool _auto_advance_set_by_user;
+
+  /// The value of auto_advance set by the user for handling advancement of sub-applications in
+  /// multi-app contexts
+  const bool _auto_advance_user_value;
 };

framework/include/multiapps/MultiApp.h

@@ -117,9 +117,11 @@ class MultiApp : public MooseObject,
    * Calls multi-apps executioners' endStep and postStep methods which creates output and advances
    * time (not the time step; see incrementTStep()) among other things. This method is only called
    * for Picard calculations because for loosely coupled calculations the executioners' endStep and
-   * postStep methods are called from solveStep().
+   * postStep methods are called from solveStep(). This may be called with the optional flag \p
+   * recurse_through_multiapp_levels which may be useful if this method is being called for the
+   * *final* time of program execution
    */
-  virtual void finishStep() {}
+  virtual void finishStep(bool /*recurse_through_multiapp_levels*/ = false) {}
 
   /**
    * Save off the state of every Sub App

framework/include/multiapps/TransientMultiApp.h

@@ -41,7 +41,7 @@ class TransientMultiApp : public MultiApp
 
   virtual void incrementTStep(Real target_time) override;
 
-  virtual void finishStep() override;
+  virtual void finishStep(bool recurse_through_multiapp_levels = false) override;
 
   virtual bool needsRestoration() override;
 

framework/include/postprocessors/TimePostprocessor.h

@@ -0,0 +1,31 @@
+//* This file is part of the MOOSE framework
+//* https://www.mooseframework.org
+//*
+//* All rights reserved, see COPYRIGHT for full restrictions
+//* https://github.com/idaholab/moose/blob/master/COPYRIGHT
+//*
+//* Licensed under LGPL 2.1, please see LICENSE for details
+//* https://www.gnu.org/licenses/lgpl-2.1.html
+
+#pragma once
+
+#include "GeneralPostprocessor.h"
+
+/**
+ * Postprocessor that returns the current time
+ */
+class TimePostprocessor : public GeneralPostprocessor
+{
+public:
+  static InputParameters validParams();
+
+  TimePostprocessor(const InputParameters & parameters);
+
+  void initialize() override {}
+  void execute() override {}
+
+  Real getValue() override;
+
+protected:
+  const FEProblemBase & _feproblem;
+};

framework/include/problems/FEProblemBase.h

@@ -1076,9 +1076,10 @@ class FEProblemBase : public SubProblem, public Restartable
   }
 
   /**
-   * Finish the MultiApp time step (endStep, postStep) associated with the ExecFlagType
+   * Finish the MultiApp time step (endStep, postStep) associated with the ExecFlagType. Optionally
+   * recurse through all multi-app levels
    */
-  void finishMultiAppStep(ExecFlagType type);
+  void finishMultiAppStep(ExecFlagType type, bool recurse_through_multiapp_levels = false);
 
   /**
    * Backup the MultiApps associated with the ExecFlagType

framework/src/executioners/PicardSolve.C

@@ -99,6 +99,9 @@ PicardSolve::validParams()
   params.addParam<bool>("update_xfem_at_timestep_begin",
                         false,
                         "Should XFEM update the mesh at the beginning of the timestep");
+  params.addParam<bool>("auto_advance",
+                        "Whether to automatically advance sub-applications regardless of whether "
+                        "their solve converges.");
 
   return params;
 }
@@ -130,7 +133,9 @@ PicardSolve::PicardSolve(Executioner * ex)
     _xfem_update_count(0),
     _xfem_repeat_step(false),
     _previous_entering_time(_problem.time() - 1),
-    _solve_message(_problem.shouldSolve() ? "Solve Converged!" : "Solve Skipped!")
+    _solve_message(_problem.shouldSolve() ? "Solve Converged!" : "Solve Skipped!"),
+    _auto_advance_set_by_user(isParamValid("auto_advance")),
+    _auto_advance_user_value(_auto_advance_set_by_user ? getParam<bool>("auto_advance") : true)
 {
   if (_relax_factor != 1.0)
     // Store a copy of the previous solution here
@@ -375,6 +380,20 @@ PicardSolve::solve()
   return converged;
 }
 
+bool
+PicardSolve::autoAdvance() const
+{
+  bool auto_advance = !(_has_picard_its && _problem.isTransient());
+
+  if (dynamic_cast<EigenExecutionerBase *>(&_executioner) && _has_picard_its)
+    auto_advance = true;
+
+  if (_auto_advance_set_by_user)
+    auto_advance = _auto_advance_user_value;
+
+  return auto_advance;
+}
+
 bool
 PicardSolve::solveStep(Real begin_norm_old,
                        Real & begin_norm,
@@ -383,10 +402,7 @@ PicardSolve::solveStep(Real begin_norm_old,
                        bool relax,
                        const std::set<dof_id_type> & relaxed_dofs)
 {
-  bool auto_advance = !(_has_picard_its && _problem.isTransient());
-
-  if (dynamic_cast<EigenExecutionerBase *>(&_executioner) && _has_picard_its)
-    auto_advance = true;
+  bool auto_advance = autoAdvance();
 
   _executioner.preSolve();
 

framework/src/executioners/Transient.C

@@ -314,10 +314,18 @@ Transient::execute()
   if (lastSolveConverged())
   {
     _t_step++;
-    if (_picard_solve.hasPicardIteration())
+
+    /*
+     * Call the multi-app executioners endStep and
+     * postStep methods when doing Picard or when not automatically advancing sub-applications for
+     * some other reason. We do not perform these calls for loose-coupling/auto-advancement
+     * problems because Transient::endStep and Transient::postStep get called from
+     * TransientMultiApp::solveStep in that case.
+     */
+    if (!_picard_solve.autoAdvance())
     {
-      _problem.finishMultiAppStep(EXEC_TIMESTEP_BEGIN);
-      _problem.finishMultiAppStep(EXEC_TIMESTEP_END);
+      _problem.finishMultiAppStep(EXEC_TIMESTEP_BEGIN, /*recurse_through_multiapp_levels=*/true);
+      _problem.finishMultiAppStep(EXEC_TIMESTEP_END, /*recurse_through_multiapp_levels=*/true);
     }
   }
 
@@ -365,11 +373,12 @@ Transient::incrementStepOrReject()
 
       /*
        * Call the multi-app executioners endStep and
-       * postStep methods when doing Picard. We do not perform these calls for
-       * loose coupling because Transient::endStep and Transient::postStep get
-       * called from TransientMultiApp::solveStep in that case.
+       * postStep methods when doing Picard or when not automatically advancing sub-applications for
+       * some other reason. We do not perform these calls for loose-coupling/auto-advancement
+       * problems because Transient::endStep and Transient::postStep get called from
+       * TransientMultiApp::solveStep in that case.
        */
-      if (_picard_solve.hasPicardIteration())
+      if (!_picard_solve.autoAdvance())
       {
         _problem.finishMultiAppStep(EXEC_TIMESTEP_BEGIN);
         _problem.finishMultiAppStep(EXEC_TIMESTEP_END);

framework/src/multiapps/TransientMultiApp.C

@@ -535,7 +535,7 @@ TransientMultiApp::incrementTStep(Real target_time)
 }
 
 void
-TransientMultiApp::finishStep()
+TransientMultiApp::finishStep(bool recurse_through_multiapp_levels)
 {
   if (!_sub_cycling)
   {
@@ -544,6 +544,13 @@ TransientMultiApp::finishStep()
       Transient * ex = _transient_executioners[i];
       ex->endStep();
       ex->postStep();
+      if (recurse_through_multiapp_levels)
+      {
+        ex->feProblem().finishMultiAppStep(EXEC_TIMESTEP_BEGIN,
+                                           /*recurse_through_multiapp_levels=*/true);
+        ex->feProblem().finishMultiAppStep(EXEC_TIMESTEP_END,
+                                           /*recurse_through_multiapp_levels=*/true);
+      }
     }
   }
 }
@@ -585,7 +592,8 @@ TransientMultiApp::computeDT()
   return smallest_dt;
 }
 
-void TransientMultiApp::resetApp(
+void
+TransientMultiApp::resetApp(
     unsigned int global_app,
     Real /*time*/) // FIXME: Note that we are passing in time but also grabbing it below
 {
@@ -611,7 +619,8 @@ void TransientMultiApp::resetApp(
   }
 }
 
-void TransientMultiApp::setupApp(unsigned int i, Real /*time*/) // FIXME: Should we be passing time?
+void
+TransientMultiApp::setupApp(unsigned int i, Real /*time*/) // FIXME: Should we be passing time?
 {
   auto & app = _apps[i];
   Transient * ex = dynamic_cast<Transient *>(app->getExecutioner());

framework/src/postprocessors/TimePostprocessor.C

@@ -0,0 +1,32 @@
+//* This file is part of the MOOSE framework
+//* https://www.mooseframework.org
+//*
+//* All rights reserved, see COPYRIGHT for full restrictions
+//* https://github.com/idaholab/moose/blob/master/COPYRIGHT
+//*
+//* Licensed under LGPL 2.1, please see LICENSE for details
+//* https://www.gnu.org/licenses/lgpl-2.1.html
+
+#include "TimePostprocessor.h"
+#include "FEProblem.h"
+
+registerMooseObject("MooseApp", TimePostprocessor);
+
+InputParameters
+TimePostprocessor::validParams()
+{
+  InputParameters params = GeneralPostprocessor::validParams();
+  params.addClassDescription("Reports the current time");
+  return params;
+}
+
+TimePostprocessor::TimePostprocessor(const InputParameters & parameters)
+  : GeneralPostprocessor(parameters), _feproblem(dynamic_cast<FEProblemBase &>(_subproblem))
+{
+}
+
+Real
+TimePostprocessor::getValue()
+{
+  return _feproblem.time();
+}

framework/src/problems/FEProblemBase.C

@@ -4300,7 +4300,7 @@ FEProblemBase::incrementMultiAppTStep(ExecFlagType type)
 }
 
 void
-FEProblemBase::finishMultiAppStep(ExecFlagType type)
+FEProblemBase::finishMultiAppStep(ExecFlagType type, bool recurse_through_multiapp_levels)
 {
   const auto & multi_apps = _multi_apps[type].getActiveObjects();
 
@@ -4310,7 +4310,7 @@ FEProblemBase::finishMultiAppStep(ExecFlagType type)
              << std::endl;
 
     for (const auto & multi_app : multi_apps)
-      multi_app->finishStep();
+      multi_app->finishStep(recurse_through_multiapp_levels);
 
     MooseUtils::parallelBarrierNotify(_communicator, _parallel_barrier_messaging);