MultiApp restart works closes idaholab#5695

framework/include/base/MooseApp.h

@@ -154,7 +154,7 @@ class MooseApp :
    *
    * @param time The start time for the simulation.
    */
-  void setStartTime(const Real time) { _start_time_set = true; _start_time = time; }
+  void setStartTime(const Real time);
 
   /**
    * @return Whether or not a start time has been programmatically set using setStartTime()
@@ -391,8 +391,11 @@ class MooseApp :
 
   /**
    * Restore a Backup.  This sets the App's state.
+   *
+   * @param backup The Backup holding the data for the app
+   * @param for_restart Whether this restoration is explicitly for the first restoration of restart data
    */
-  void restore(MooseSharedPointer<Backup> backup);
+  void restore(MooseSharedPointer<Backup> backup, bool for_restart = false);
 
   /**
    * Returns a string to be printed at the beginning of a simulation
@@ -426,7 +429,24 @@ class MooseApp :
    */
   void executeMeshModifiers();
 
-protected:
+  ///@{
+  /**
+   * Sets the restart/recover flags
+   * @param state The state to set the flag to
+   */
+  void setRestart(const bool & value);
+  void setRecover(const bool & value);
+  ///@}
+
+  /**
+   * Whether or not this MooseApp has cached a Backup to use for restart / recovery
+   */
+  bool hasCachedBackup() { return _cached_backup.get(); }
+
+  /**
+   * Restore from a cached backup
+   */
+  void restoreCachedBackup();
 
   /**
    * Helper method for dynamic loading of objects
@@ -579,14 +599,8 @@ class MooseApp :
   /// Holds the mesh modifiers until they have completed, then this structure is cleared
   std::map<std::string, MooseSharedPointer<MeshModifier> > _mesh_modifiers;
 
-  ///@{
-  /**
-   * Sets the restart/recover flags
-   * @param value The state to set the flag to
-   */
-  void setRestart(const bool & value){ _restart = value; }
-  void setRecover(const bool & value){ _recover = value; }
-  ///@}
+  /// Cache for a Backup to use for restart / recovery
+  MooseSharedPointer<Backup> _cached_backup;
 
   // Allow FEProblem to set the recover/restart state, so make it a friend
   friend class FEProblem;

framework/include/restart/Restartable.h

@@ -205,6 +205,7 @@ class Restartable
   friend class FEProblem;
   friend class Transient;
   friend class TableOutput;
+  friend class TransientMultiApp;
 };
 
 template<typename T>

framework/include/restart/RestartableDataIO.h

@@ -112,7 +112,7 @@ class RestartableDataIO
   /**
    * Restore a Backup for the current system.
    */
-  void restoreBackup(MooseSharedPointer<Backup> backup);
+  void restoreBackup(MooseSharedPointer<Backup> backup, bool for_restart = false);
 
 private:
   /**

framework/src/base/FEProblem.C

@@ -500,6 +500,51 @@ void FEProblem::initialSetup()
        ++it)
     it->second->updateSeeds(EXEC_INITIAL);
 
+  if (!_app.isRecovering())
+  {
+    Moose::setup_perf_log.push("Initial computeUserObjects()","Setup");
+
+    computeUserObjects(EXEC_INITIAL, UserObjectWarehouse::PRE_AUX);
+
+    _aux.compute(EXEC_INITIAL);
+
+    if (_use_legacy_uo_initialization)
+    {
+      _aux.compute(EXEC_TIMESTEP_BEGIN);
+      computeUserObjects(EXEC_TIMESTEP_END);
+    }
+
+    // The only user objects that should be computed here are the initial UOs
+    computeUserObjects(EXEC_INITIAL, UserObjectWarehouse::POST_AUX);
+
+    if (_use_legacy_uo_initialization)
+    {
+      computeUserObjects(EXEC_TIMESTEP_BEGIN);
+      computeUserObjects(EXEC_LINEAR);
+    }
+    Moose::setup_perf_log.pop("Initial computeUserObjects()","Setup");
+  }
+
+  // Here we will initialize the stateful properties once more since they may have been updated
+  // during initialSetup by calls to computeProperties.
+  if (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties())
+  {
+    ConstElemRange & elem_range = *_mesh.getActiveLocalElementRange();
+    ComputeMaterialsObjectThread cmt(*this, _nl, _material_data, _bnd_material_data, _neighbor_material_data,
+                                     _material_props, _bnd_material_props, _materials, _assembly);
+    Threads::parallel_reduce(elem_range, cmt);
+  }
+
+  if (_app.isRestarting() || _app.isRecovering())
+  {
+    if (_app.hasCachedBackup()) // This happens when this app is a sub-app and has been given a Backup
+      _app.restoreCachedBackup();
+    else
+      _resurrector->restartRestartableData();
+  }
+
+  // HUGE NOTE: MultiApp initialSetup() MUST... I repeat MUST be _after_ restartable data has been restored
+
   // Call initialSetup on the MultiApps
   _multi_apps(EXEC_LINEAR)[0].initialSetup();
   _multi_apps(EXEC_NONLINEAR)[0].initialSetup();
@@ -539,43 +584,8 @@ void FEProblem::initialSetup()
     Moose::setup_perf_log.push("Initial execMultiApps()","Setup");
     execMultiApps(EXEC_INITIAL);
     Moose::setup_perf_log.pop("Initial execMultiApps()","Setup");
-
-    Moose::setup_perf_log.push("Initial computeUserObjects()","Setup");
-
-    computeUserObjects(EXEC_INITIAL, UserObjectWarehouse::PRE_AUX);
-
-    _aux.compute(EXEC_INITIAL);
-
-    if (_use_legacy_uo_initialization)
-    {
-      _aux.compute(EXEC_TIMESTEP_BEGIN);
-      computeUserObjects(EXEC_TIMESTEP_END);
-    }
-
-    // The only user objects that should be computed here are the initial UOs
-    computeUserObjects(EXEC_INITIAL, UserObjectWarehouse::POST_AUX);
-
-    if (_use_legacy_uo_initialization)
-    {
-      computeUserObjects(EXEC_TIMESTEP_BEGIN);
-      computeUserObjects(EXEC_LINEAR);
-    }
-    Moose::setup_perf_log.pop("Initial computeUserObjects()","Setup");
-  }
-
-  // Here we will initialize the stateful properties once more since they may have been updated
-  // during initialSetup by calls to computeProperties.
-  if (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties())
-  {
-    ConstElemRange & elem_range = *_mesh.getActiveLocalElementRange();
-    ComputeMaterialsObjectThread cmt(*this, _nl, _material_data, _bnd_material_data, _neighbor_material_data,
-                                     _material_props, _bnd_material_props, _materials, _assembly);
-    Threads::parallel_reduce(elem_range, cmt);
   }
 
-  if (_app.isUltimateMaster() && (_app.isRestarting() || _app.isRecovering()))
-    _resurrector->restartRestartableData();
-
   // Scalar variables need to reinited for the initial conditions to be available for output
   for (unsigned int tid = 0; tid < n_threads; tid++)
     reinitScalars(tid);

framework/src/base/MooseApp.C

@@ -428,14 +428,12 @@ MooseApp::executeExecutioner()
 bool
 MooseApp::isRecovering() const
 {
-  // We never "recover" a sub-app... they just get their state "pushed" to them from  above
   return _recover;
 }
 
 bool
 MooseApp::isRestarting() const
 {
-  // We never "restart" a sub-app... they just get their state "pushed" to them from  above
   return _restart;
 }
 
@@ -514,13 +512,21 @@ MooseApp::backup()
 }
 
 void
-MooseApp::restore(MooseSharedPointer<Backup> backup)
+MooseApp::restore(MooseSharedPointer<Backup> backup, bool for_restart)
 {
+  // This means that a Backup is coming through to use for restart / recovery
+  // We should just cache it for now
+  if (!_executioner)
+  {
+    _cached_backup = backup;
+    return;
+  }
+
   FEProblem & fe_problem = static_cast<FEProblem &>(_executioner->problem());
 
   RestartableDataIO rdio(fe_problem);
 
-  rdio.restoreBackup(backup);
+  rdio.restoreBackup(backup, for_restart);
 }
 
 void
@@ -572,6 +578,13 @@ MooseApp::setOutputPosition(Point p)
     _executioner->parentOutputPositionChanged();
 }
 
+void
+MooseApp::setStartTime(const Real time)
+{
+  _start_time_set = true;
+  _start_time = time;
+}
+
 std::string
 MooseApp::getFileName(bool stripLeadingPath) const
 {
@@ -911,3 +924,30 @@ MooseApp::executeMeshModifiers()
   // Clear the modifiers, they are not used again during the simulation
   _mesh_modifiers.clear();
 }
+
+void
+MooseApp::setRestart(const bool & value)
+{
+  _restart = value;
+
+  MooseSharedPointer<FEProblem> fe_problem = _action_warehouse.problem();
+}
+
+void
+MooseApp::setRecover(const bool & value)
+{
+  _recover = value;
+}
+
+
+void
+MooseApp::restoreCachedBackup()
+{
+  if (!_cached_backup.get())
+    mooseError("No cached Backup to restore!");
+
+  restore(_cached_backup, isRestarting());
+
+  // Release our hold on this Backup
+  _cached_backup.reset();
+}

framework/src/executioners/Transient.C

@@ -93,11 +93,11 @@ Transient::Transient(const InputParameters & parameters) :
     _time_old(_problem.timeOld()),
     _dt(_problem.dt()),
     _dt_old(_problem.dtOld()),
-    _unconstrained_dt(declareRestartableData<Real>("unconstrained_dt", -1)),
-    _at_sync_point(declareRestartableData<bool>("at_sync_point", false)),
+    _unconstrained_dt(declareRecoverableData<Real>("unconstrained_dt", -1)),
+    _at_sync_point(declareRecoverableData<bool>("at_sync_point", false)),
     _first(declareRecoverableData<bool>("first", true)),
-    _multiapps_converged(declareRestartableData<bool>("multiapps_converged", true)),
-    _last_solve_converged(declareRestartableData<bool>("last_solve_converged", true)),
+    _multiapps_converged(declareRecoverableData<bool>("multiapps_converged", true)),
+    _last_solve_converged(declareRecoverableData<bool>("last_solve_converged", true)),
     _end_time(getParam<Real>("end_time")),
     _dtmin(getParam<Real>("dtmin")),
     _dtmax(getParam<Real>("dtmax")),
@@ -107,20 +107,20 @@ Transient::Transient(const InputParameters & parameters) :
     _trans_ss_check(getParam<bool>("trans_ss_check")),
     _ss_check_tol(getParam<Real>("ss_check_tol")),
     _ss_tmin(getParam<Real>("ss_tmin")),
-    _old_time_solution_norm(declareRestartableData<Real>("old_time_solution_norm", 0.0)),
+    _old_time_solution_norm(declareRecoverableData<Real>("old_time_solution_norm", 0.0)),
     _sync_times(_app.getOutputWarehouse().getSyncTimes()),
     _abort(getParam<bool>("abort_on_solve_fail")),
-    _time_interval(declareRestartableData<bool>("time_interval", false)),
+    _time_interval(declareRecoverableData<bool>("time_interval", false)),
     _start_time(getParam<Real>("start_time")),
     _timestep_tolerance(getParam<Real>("timestep_tolerance")),
-    _target_time(declareRestartableData<Real>("target_time", -1)),
+    _target_time(declareRecoverableData<Real>("target_time", -1)),
     _use_multiapp_dt(getParam<bool>("use_multiapp_dt")),
-    _picard_it(declareRestartableData<int>("picard_it", 0)),
+    _picard_it(declareRecoverableData<int>("picard_it", 0)),
     _picard_max_its(getParam<unsigned int>("picard_max_its")),
-    _picard_converged(declareRestartableData<bool>("picard_converged", false)),
-    _picard_initial_norm(declareRestartableData<Real>("picard_initial_norm", 0.0)),
-    _picard_timestep_begin_norm(declareRestartableData<Real>("picard_timestep_begin_norm", 0.0)),
-    _picard_timestep_end_norm(declareRestartableData<Real>("picard_timestep_end_norm", 0.0)),
+    _picard_converged(declareRecoverableData<bool>("picard_converged", false)),
+    _picard_initial_norm(declareRecoverableData<Real>("picard_initial_norm", 0.0)),
+    _picard_timestep_begin_norm(declareRecoverableData<Real>("picard_timestep_begin_norm", 0.0)),
+    _picard_timestep_end_norm(declareRecoverableData<Real>("picard_timestep_end_norm", 0.0)),
     _picard_rel_tol(getParam<Real>("picard_rel_tol")),
     _picard_abs_tol(getParam<Real>("picard_abs_tol")),
     _verbose(getParam<bool>("verbose"))
@@ -133,7 +133,7 @@ Transient::Transient(const InputParameters & parameters) :
   // Either a start_time has been forced on us, or we want to tell the App about what our start time is (in case anyone else is interested.
   if (_app.hasStartTime())
     _start_time = _app.getStartTime();
-  else
+  else if (parameters.paramSetByUser("start_time"))
     _app.setStartTime(_start_time);
 
   _time = _time_old = _start_time;

framework/src/multiapps/MultiApp.C

@@ -281,6 +281,12 @@ MultiApp::backup()
 void
 MultiApp::restore()
 {
+  // Must be restarting / recovering so hold off on restoring
+  // Instead - the restore will happen in createApp()
+  // Note that _backups was already populated by dataLoad()
+  if (_apps.empty())
+    return;
+
   for (unsigned int i=0; i<_my_num_apps; i++)
     _apps[i]->restore(_backups[i]);
 }
@@ -481,6 +487,15 @@ MultiApp::createApp(unsigned int i, Real start_time)
   app->setInputFileName(input_file);
   app->setOutputFileBase(output_base.str());
   app->setOutputFileNumbers(_app.getOutputWarehouse().getFileNumbers());
+  app->setRestart(_app.isRestarting());
+  app->setRecover(_app.isRecovering());
+
+  // This means we have a backup of this app that we need to give to it
+  // Note: This won't do the restoration immediately.  The Backup
+  // will be cached by the MooseApp object so that it can be used
+  // during FEProblem::initialSetup() during runInputFile()
+  if (_app.isRestarting() || _app.isRecovering())
+    app->restore(_backups[i]);
 
   if (getParam<bool>("output_in_position"))
     app->setOutputPosition(_app.getOutputPosition() + _positions[_first_local_app + i]);

framework/src/multiapps/TransientMultiApp.C

@@ -64,7 +64,7 @@ TransientMultiApp::TransientMultiApp(const InputParameters & parameters):
     _failures(0),
     _catch_up(getParam<bool>("catch_up")),
     _max_catch_up_steps(getParam<Real>("max_catch_up_steps")),
-    _first(declareRestartableData<bool>("first", true)),
+    _first(declareRecoverableData<bool>("first", true)),
     _auto_advance(false),
     _print_sub_cycles(getParam<bool>("print_sub_cycles"))
 {
@@ -198,6 +198,9 @@ TransientMultiApp::solveStep(Real dt, Real target_time, bool auto_advance)
 
         bool at_steady = false;
 
+        if (_first && !_app.isRecovering())
+          problem->advanceState();
+
         // Now do all of the solves we need
         while (true)
         {
@@ -299,6 +302,10 @@ TransientMultiApp::solveStep(Real dt, Real target_time, bool auto_advance)
       else
       {
         _console << "Solving Normal Step!" << std::endl;
+
+        if (_first)
+          problem->advanceState();
+
         if (auto_advance)
           if (_first != true)
             ex->incrementStepOrReject();

framework/src/restart/RestartableDataIO.C

@@ -308,7 +308,7 @@ RestartableDataIO::createBackup()
 }
 
 void
-RestartableDataIO::restoreBackup(MooseSharedPointer<Backup> backup)
+RestartableDataIO::restoreBackup(MooseSharedPointer<Backup> backup, bool for_restart)
 {
   unsigned int n_threads = libMesh::n_threads();
 
@@ -336,6 +336,11 @@ RestartableDataIO::restoreBackup(MooseSharedPointer<Backup> backup)
     backup->_restartable_data[tid]->read((char *)&this_n_procs, sizeof(this_n_procs));
     backup->_restartable_data[tid]->read((char *)&this_n_threads, sizeof(this_n_threads));
 
-    deserializeRestartableData(restartable_datas[tid], *backup->_restartable_data[tid], std::set<std::string>());
+    std::set<std::string> & recoverable_data = _fe_problem.getMooseApp().getRecoverableData();
+
+    if (for_restart) // When doing restart - make sure we don't read data that is only for recovery...
+      deserializeRestartableData(restartable_datas[tid], *backup->_restartable_data[tid], recoverable_data);
+    else
+      deserializeRestartableData(restartable_datas[tid], *backup->_restartable_data[tid], std::set<std::string>());
   }
 }