Skip to content

Commit

Permalink
Switch stateful restart logic to load later on init
Browse files Browse the repository at this point in the history 
refs idaholab#25840
  • Loading branch information
@loganharbour
loganharbour committed Feb 20, 2024
1 parent 7f44416 commit 03e250d
Show file tree
Hide file tree
Showing 3 changed files with 168 additions and 50 deletions.
64 changes: 51 additions & 13 deletions framework/include/materials/MaterialPropertyStorage.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -321,12 +321,24 @@ class MaterialPropertyStorage
MaterialData & getMaterialData(const THREAD_ID tid) { return _material_data[tid]; }

/**
* Clears the materials that are marked as restored.
* Sets the loading of stateful material properties to recover
*
* Materials that have been restored do not have initStatefulProperties() called
* on them.
* This enforces the requirement of one-to-one stateful material properties,
* disabling advanced restart of stateful properties
*/
void clearRestoredMaterials() { _restored_materials.clear(); }
void setRecovering() { _recovering = true; }

/**
* Sets the loading of stateful material properties in place
*
* On init, this cannot be set because we must first call initProps()
* to properly initialize the dynamic types within _storage. After
* the first sweep through with initProps(), we can then load the stateful
* props directly in place into _storage
*
* Also clears _restartable_map, as it should no longer be needed
*/
void setRestartInPlace();

protected:
/// The actual storage
Expand All @@ -338,22 +350,20 @@ class MaterialPropertyStorage
/// the vector of stateful property ids (the vector index is the map to stateful prop_id)
std::vector<unsigned int> _stateful_prop_id_to_prop_id;

/// The materials that have been restored and should not have initStatefulProperties() called on
std::set<const MaterialBase *> _restored_materials;

void sizeProps(MaterialProperties & mp, unsigned int size);

private:
/// Initializes hashmap entries for element and side to proper qpoint and
/// property count sizes.
void initProps(const THREAD_ID tid, const Elem * elem, unsigned int side, unsigned int n_qpoints);
std::vector<MaterialProperties *>
initProps(const THREAD_ID tid, const Elem * elem, unsigned int side, unsigned int n_qpoints);

/// Initializes just one hashmap's entries
void initProps(const THREAD_ID tid,
const unsigned int state,
const Elem * elem,
unsigned int side,
unsigned int n_qpoints);
MaterialProperties & initProps(const THREAD_ID tid,
const unsigned int state,
const Elem * elem,
unsigned int side,
unsigned int n_qpoints);

///@{
/**
Expand Down Expand Up @@ -397,6 +407,34 @@ class MaterialPropertyStorage
/// The threaded material data
std::vector<MaterialData> _material_data;

/**
* Helper struct for all of the data needed to load stateful property data for a
* [state, elem, side, stateful_id] combination later in time in initStatefulProps()
*
* This is needed beacuse restartable data is loaded in FEProblemBase::initialSetup()
* so early. At the point it is loaded, we have yet to call initProps() on any properties
* to setup the entries in _storage. As this data is dynamically typed, there's no way
* we know its type this early on
*/
struct RestartableEntry
{
unsigned int stateful_id;
unsigned int state;
std::size_t num_q_points;
std::stringstream data;
};

typedef std::unordered_map<std::pair<const Elem *, unsigned int>, std::vector<RestartableEntry>>
RestartableMapType;

/// The restartable data to be loaded in initStatefulProps() later; see RestartableEntry
RestartableMapType _restartable_map;

/// Whether or not we want to restart stateful properties in place; see RestartableEntry
bool _restart_in_place;
/// Whether or not we're recovering; enforces a one-to-one mapping of stateful properties
bool _recovering;

// Need to be able to eraseProperty from here
friend class ProjectMaterialProperties;

Expand Down
133 changes: 102 additions & 31 deletions framework/src/materials/MaterialPropertyStorage.C
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,11 @@
#include <optional>

MaterialPropertyStorage::MaterialPropertyStorage(MaterialPropertyRegistry & registry)
: _max_state(0), _spin_mtx(libMesh::Threads::spin_mtx), _registry(registry)
: _max_state(0),
_spin_mtx(libMesh::Threads::spin_mtx),
_registry(registry),
_restart_in_place(false),
_recovering(false)
{
_material_data.reserve(libMesh::n_threads());
for (const auto tid : make_range(libMesh::n_threads()))
Expand Down Expand Up @@ -80,6 +84,13 @@ MaterialPropertyStorage::eraseProperty(const Elem * elem)
setProps(state).erase(elem);
}

void
MaterialPropertyStorage::setRestartInPlace()
{
_restart_in_place = true;
_restartable_map.clear();
}

void
MaterialPropertyStorage::updateStatefulPropsForPRefinement(
const processor_id_type libmesh_dbg_var(pid),
Expand Down Expand Up @@ -262,21 +273,33 @@ MaterialPropertyStorage::initStatefulProps(const THREAD_ID tid,
// NOTE: since materials are storing their computed properties in MaterialData class, we need to
// juggle the memory between MaterialData and MaterialProperyStorage classes

initProps(tid, &elem, side, n_qpoints);
auto props = initProps(tid, &elem, side, n_qpoints);

// copy from storage to material data
swap(tid, elem, side);
// run custom init on properties
// we don't call this for properties that we've already restored because that
// would overwrite what we've loaded
for (const auto & mat : mats)
if (!_restored_materials.count(mat.get()))
mat->initStatefulProperties(n_qpoints);
mat->initStatefulProperties(n_qpoints);

swapBack(tid, elem, side);

if (!hasStatefulProperties())
{
mooseAssert(_restartable_map.empty(), "Should be empty");
return;
}

// If we have stateful restart data to load in that we couldn't previously (because we haven't
// called initProps() to init the dynamic data), do it now
std::set<unsigned int> restarted_stateful_props;
if (auto it = _restartable_map.find(std::make_pair(&elem, side)); it != _restartable_map.end())
for (auto & entry : it->second)
{
if (n_qpoints != entry.num_q_points)
mooseError("size mismatch");
dataLoad(entry.data, (*props[entry.state])[entry.stateful_id], nullptr);
restarted_stateful_props.insert(entry.stateful_id);
}

// This second call to initProps covers cases where code in
// "init[Qp]StatefulProperties" may have called a get/declare for a stateful
Expand All @@ -285,17 +308,14 @@ MaterialPropertyStorage::initStatefulProps(const THREAD_ID tid,
// getMaterialProperty[Old/Older] can potentially trigger a material to
// become stateful that previously wasn't. This needs to go after the
// swapBack.
initProps(tid, &elem, side, n_qpoints);
props = initProps(tid, &elem, side, n_qpoints);

// Copy to older states as needed
const auto & current_props = props(&elem, side, 0);
// Copy to older states as needed, only for non-restarted data
for (const auto state : statefulIndexRange())
{
auto & to_props = setProps(&elem, side, state);
for (const auto i : index_range(_stateful_prop_id_to_prop_id))
for (const auto qp : make_range(n_qpoints))
to_props[i].qpCopy(qp, current_props[i], qp);
}
for (const auto stateful_id : index_range(_stateful_prop_id_to_prop_id))
if (!restarted_stateful_props.count(stateful_id))
for (const auto qp : make_range(n_qpoints))
(*props[state])[stateful_id].qpCopy(qp, (*props[0])[stateful_id], qp);
}

void
Expand Down Expand Up @@ -413,17 +433,19 @@ MaterialPropertyStorage::addProperty(const std::string & prop_name,
return prop_id;
}

void
std::vector<MaterialProperties *>
MaterialPropertyStorage::initProps(const THREAD_ID tid,
const Elem * elem,
unsigned int side,
unsigned int n_qpoints)
{
std::vector<MaterialProperties *> props(numStates());
for (const auto state : stateIndexRange())
this->initProps(tid, state, elem, side, n_qpoints);
props[state] = &this->initProps(tid, state, elem, side, n_qpoints);
return props;
}

void
MaterialProperties &
MaterialPropertyStorage::initProps(const THREAD_ID tid,
const unsigned int state,
const Elem * elem,
Expand Down Expand Up @@ -451,6 +473,8 @@ MaterialPropertyStorage::initProps(const THREAD_ID tid,
mat_props.setPointer(i, material_data.props(0)[prop_id].clone(n_qpoints), {});
mooseAssert(mat_props[i].id() == prop_id, "Inconsistent id");
}

return mat_props;
}

void
Expand Down Expand Up @@ -497,8 +521,16 @@ dataStore(std::ostream & stream, MaterialPropertyStorage & storage, void * conte
n_q_points = entry.size();
dataStore(stream, n_q_points, nullptr);

// Here we actually store a stringstream of the data instead of the data directly, because
// upon load we don't know if we can load it immediately into place or if we need to cache
// it to load later. We also store it as skippable so that we can support not loading a
// property if it no longer exists in restart
for (auto & entry : props)
dataStoreSkippable(stream, entry, nullptr);
{
std::stringstream out;
dataStore(out, entry, nullptr);
dataStoreSkippable(stream, out, nullptr);
}
}
}
}
Expand All @@ -507,7 +539,7 @@ dataStore(std::ostream & stream, MaterialPropertyStorage & storage, void * conte
void
dataLoad(std::istream & stream, MaterialPropertyStorage & storage, void * context)
{
storage._restored_materials.clear();
storage._restartable_map.clear();

const auto & registry = storage.getMaterialPropertyRegistry();

Expand Down Expand Up @@ -582,10 +614,18 @@ dataLoad(std::istream & stream, MaterialPropertyStorage & storage, void * contex
error << "\nCurrent stateful properties:\n";
for (const auto & prop : props)
error << " - " << prop << "\n";
error << "\nThis is not supported in advanced restart.";
mooseError(error.str());
}
}
// We're recovering and we haven't found a stateful material object. We require a
// one-to-one stateful mapping when recovering
else if (storage._recovering)
{
mooseError("The ",
object_string(object),
" was stored in restart but no longer exists. This is not supported when "
"recovering stateful material properties.");
}
// We have not found a material object that was stored with the same name
// with stateful material properties. Here, we enforce that no other new
// stateful material properties are declared in a new material with the
Expand All @@ -601,7 +641,7 @@ dataLoad(std::istream & stream, MaterialPropertyStorage & storage, void * contex
object_string(find->second),
" but is now declared in ",
object_string(object),
".\n\nThis is not supported in advanced restart due to ambiguity.\n\n");
".");
}
}
}
Expand Down Expand Up @@ -634,9 +674,8 @@ dataLoad(std::istream & stream, MaterialPropertyStorage & storage, void * contex
if (to_record.stateful())
{
if (from_record.type != to_record.type)
mooseError("The type for the restarted stateful material property '",
name,
"' in does not match");
mooseError(
"The type for the restarted stateful material property '", name, "' does not match");

// I'm not sure if we need to enforce this one, but I don't want to think
// about it deeply so we'll just make it an error until someone complains
Expand All @@ -659,9 +698,8 @@ dataLoad(std::istream & stream, MaterialPropertyStorage & storage, void * contex
std::distance(storage._stateful_prop_id_to_prop_id.begin(), find_prop_id);
to_stateful_ids[from_stateful_id] = to_stateful_id;

const auto material_ptr = std::get_if<const MaterialBase *>(&to_record.declarer);
mooseAssert(material_ptr && *material_ptr, "Should have a declarer");
storage._restored_materials.insert(*material_ptr);
if (storage._recovering)
mooseAssert(from_stateful_id == to_stateful_id, "Does not have direct mapping");
}
}
}
Expand Down Expand Up @@ -692,12 +730,45 @@ dataLoad(std::istream & stream, MaterialPropertyStorage & storage, void * contex
std::size_t num_q_points;
dataLoad(stream, num_q_points, nullptr);

storage.initProps(0, state, elem, side, num_q_points);
auto & props = storage.setProps(elem, side, state);
// Avoid multiple map lookups for entries pertaining to [elem, side]
MaterialPropertyStorage::RestartableMapType::mapped_type * restart_entry = nullptr;
MaterialProperties * in_place_entry = nullptr;

// Load each stateful property
for (const auto from_stateful_id : make_range(num_props))
// We have a property to load into
if (const auto to_stateful_id_ptr = to_stateful_ids[from_stateful_id])
dataLoadSkippable(stream, props[*to_stateful_id_ptr], nullptr);
{
const auto to_stateful_id = *to_stateful_id_ptr;

// Load the binary data, which lets us choose in a moment whether
// or not to load it in place or to cache it to load later
std::stringstream data;
dataLoadSkippable(stream, data, nullptr);

// Load the data directly into _storage
if (storage._restart_in_place)
{
if (!in_place_entry)
in_place_entry = &storage.setProps(elem, side, state);

dataLoad(data, (*in_place_entry)[to_stateful_id], nullptr);
}
// Properties aren't initialized, so load the data into
// _restartable_map to be loaded later in initStatefulProps()
else
{
if (!restart_entry)
restart_entry = &storage._restartable_map[std::make_pair(elem, side)];

auto & entry = restart_entry->emplace_back();
entry.stateful_id = to_stateful_id;
entry.state = state;
entry.num_q_points = num_q_points;
entry.data = std::move(data);
}
}
// We do not have a property to load into, so skip
else
dataLoadSkip(stream);
}
Expand Down
21 changes: 15 additions & 6 deletions framework/src/problems/FEProblemBase.C
View file
Original file line number Diff line number Diff line change
Expand Up @@ -786,6 +786,11 @@ FEProblemBase::initialSetup()
// Only load all of the vectors if we're recovering
_req.set().setLoadAllVectors(_app.isRecovering());

// This forces stateful material property loading to be an exact one-to-one match
if (_app.isRecovering())
for (auto props : {&_material_props, &_bnd_material_props, &_neighbor_material_props})
props->setRecovering();

TIME_SECTION("restore", 3, "Restoring from backup");

// We could have a cached backup when this app is a sub-app and has been given a Backup
Expand Down Expand Up @@ -934,7 +939,6 @@ FEProblemBase::initialSetup()
}
}

if (!_has_initialized_stateful)
{
TIME_SECTION("computingInitialStatefulProps", 3, "Computing Initial Material Values");

Expand All @@ -943,6 +947,16 @@ FEProblemBase::initialSetup()
if (_material_props.hasStatefulProperties() || _bnd_material_props.hasStatefulProperties() ||
_neighbor_material_props.hasStatefulProperties())
_has_initialized_stateful = true;

// setRestartInPlace() is set because the property maps have now been setup and we can
// dataLoad() them directly in place
// setRecovering() is set because from now on we require a one-to-one mapping of
// stateful properties because we shouldn't be declaring any more
for (auto props : {&_material_props, &_bnd_material_props, &_neighbor_material_props})
{
props->setRestartInPlace();
props->setRecovering();
}
}
}

Expand Down Expand Up @@ -7373,11 +7387,6 @@ FEProblemBase::initElementStatefulProps(const ConstElemRange & elem_range, const
Threads::parallel_reduce(elem_range, cmt);
else
cmt(elem_range, true);

// Now that we've initialized stateful props, we no longer want to skip initialization of
// the materials have been skipped in restart
for (auto props : {&_material_props, &_bnd_material_props, &_neighbor_material_props})
props->clearRestoredMaterials();
}

void
Expand Down

0 comments on commit 03e250d

Please sign in to comment.