Add reduction action for AMR diagnostics

src/Executables/Examples/CMakeLists.txt

@@ -6,3 +6,4 @@ add_subdirectory(HelloWorldNoCharm)
 # [add_subdirectory]
 add_subdirectory(ParallelTutorial)
 # [add_subdirectory]
+add_subdirectory(RandomAmr)

src/Executables/Examples/RandomAmr/CMakeLists.txt

@@ -0,0 +1,35 @@
+# Distributed under the MIT License.
+# See LICENSE.txt for details.
+
+set(LIBS_TO_LINK
+  Amr
+  AmrCriteria
+  DomainCreators
+  Informer
+  Options
+  Utilities
+  )
+
+add_spectre_parallel_executable(
+  RandomAmr1D
+  RandomAmr
+  Executables/Examples/RandomAmr
+  RandomAmrMetavars<1>
+  "${LIBS_TO_LINK}"
+  )
+
+add_spectre_parallel_executable(
+  RandomAmr2D
+  RandomAmr
+  Executables/Examples/RandomAmr
+  RandomAmrMetavars<2>
+  "${LIBS_TO_LINK}"
+  )
+
+add_spectre_parallel_executable(
+  RandomAmr3D
+  RandomAmr
+  Executables/Examples/RandomAmr
+  RandomAmrMetavars<3>
+  "${LIBS_TO_LINK}"
+  )

src/Executables/Examples/RandomAmr/InitializeDomain.hpp

@@ -0,0 +1,66 @@
+// Distributed under the MIT License.
+// See LICENSE.txt for details.
+
+#pragma once
+
+#include <array>
+#include <cstddef>
+#include <vector>
+
+#include "Domain/Block.hpp"
+#include "Domain/CreateInitialElement.hpp"
+#include "Domain/Domain.hpp"
+#include "Domain/Structure/CreateInitialMesh.hpp"
+#include "Domain/Structure/Element.hpp"
+#include "Domain/Structure/ElementId.hpp"
+#include "Domain/Tags.hpp"
+#include "Evolution/DiscontinuousGalerkin/Initialization/QuadratureTag.hpp"
+#include "Utilities/Gsl.hpp"
+#include "Utilities/TMPL.hpp"
+
+namespace amr::Initialization {
+
+/// \ingroup InitializationGroup
+/// \brief Initialize items related to the structure of a Domain
+///
+/// \see InitializeItems
+///
+/// \note This only initializes the Element and the Mesh on each element of
+/// an array component.  This is all that is needed about the Domain in order to
+/// test the mechanics of adaptive mesh refinement.
+template <size_t Dim>
+struct Domain {
+  using const_global_cache_tags = tmpl::list<::domain::Tags::Domain<Dim>>;
+  using mutable_global_cache_tags = tmpl::list<>;
+  using simple_tags_from_options =
+      tmpl::list<::domain::Tags::InitialExtents<Dim>,
+                 ::domain::Tags::InitialRefinementLevels<Dim>,
+                 evolution::dg::Tags::Quadrature>;
+
+  using argument_tags =
+      tmpl::append<const_global_cache_tags, simple_tags_from_options,
+                   tmpl::list<::Parallel::Tags::ArrayIndex>>;
+
+  using return_tags =
+      tmpl::list<::domain::Tags::Mesh<Dim>, ::domain::Tags::Element<Dim>>;
+
+  using simple_tags = return_tags;
+  using compute_tags = tmpl::list<>;
+
+  /// Given the items fetched from a DataBox by the argument_tags, mutate
+  /// the items in the DataBox corresponding to return_tags
+  static void apply(
+      const gsl::not_null<Mesh<Dim>*> mesh,
+      const gsl::not_null<Element<Dim>*> element, const ::Domain<Dim>& domain,
+      const std::vector<std::array<size_t, Dim>>& initial_extents,
+      const std::vector<std::array<size_t, Dim>>& initial_refinement,
+      const Spectral::Quadrature& quadrature,
+      const ElementId<Dim>& element_id) {
+    const auto& my_block = domain.blocks()[element_id.block_id()];
+    *mesh = ::domain::Initialization::create_initial_mesh(
+        initial_extents, element_id, quadrature);
+    *element = ::domain::Initialization::create_initial_element(
+        element_id, my_block, initial_refinement);
+  }
+};
+}  // namespace amr::Initialization

src/Executables/Examples/RandomAmr/RandomAmr.hpp

@@ -0,0 +1,92 @@
+// Distributed under the MIT License.
+// See LICENSE.txt for details.
+
+#pragma once
+
+#include <cstddef>
+#include <deque>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "Domain/Creators/DomainCreator.hpp"
+#include "Domain/Creators/Factory1D.hpp"
+#include "Domain/Creators/Factory2D.hpp"
+#include "Domain/Creators/Factory3D.hpp"
+#include "Domain/Creators/RegisterDerivedWithCharm.hpp"
+#include "Evolution/DiscontinuousGalerkin/DgElementArray.hpp"
+#include "Executables/Examples/RandomAmr/InitializeDomain.hpp"
+#include "Options/Options.hpp"
+#include "Options/Protocols/FactoryCreation.hpp"
+#include "Parallel/GlobalCache.hpp"
+#include "Parallel/InitializationFunctions.hpp"
+#include "Parallel/Phase.hpp"
+#include "Parallel/PhaseDependentActionList.hpp"
+#include "Parallel/RegisterDerivedClassesWithCharm.hpp"
+#include "ParallelAlgorithms/Actions/InitializeItems.hpp"
+#include "ParallelAlgorithms/Actions/TerminatePhase.hpp"
+#include "ParallelAlgorithms/Amr/Actions/Component.hpp"
+#include "ParallelAlgorithms/Amr/Actions/Initialize.hpp"
+#include "ParallelAlgorithms/Amr/Actions/SendAmrDiagnostics.hpp"
+#include "Utilities/ErrorHandling/FloatingPointExceptions.hpp"
+#include "Utilities/ProtocolHelpers.hpp"
+#include "Utilities/TMPL.hpp"
+
+namespace {
+struct DummySystem {};
+}  // namespace
+
+/// \page RandomAmrExecutablePage RandomAmr Executable
+/// The RandomAmr executable is being used to develop the mechanics of
+/// adaptive mesh refinement.
+///
+/// See RandomAmrMetavars for a description of the metavariables of this
+/// executable.
+
+/// \brief The metavariables for the RandomAmr executable
+template <size_t Dim>
+struct RandomAmrMetavars {
+  static constexpr size_t volume_dim = Dim;
+  using system = DummySystem;
+
+  static constexpr Options::String help{
+      "Test anisotropic refinement by randomly refining a grid.\n"};
+
+  struct factory_creation
+      : tt::ConformsTo<Options::protocols::FactoryCreation> {
+    using factory_classes = tmpl::map<
+        tmpl::pair<DomainCreator<volume_dim>, domain_creators<volume_dim>>>;
+  };
+
+  using const_global_cache_tags = tmpl::list<>;
+
+  static constexpr auto default_phase_order =
+      std::array{Parallel::Phase::Initialization, Parallel::Phase::CheckDomain,
+                 Parallel::Phase::Exit};
+
+  using amr_element_component = DgElementArray<
+      RandomAmrMetavars,
+      tmpl::list<
+          Parallel::PhaseActions<
+              Parallel::Phase::Initialization,
+              tmpl::list<Initialization::Actions::InitializeItems<
+                             amr::Initialization::Domain<volume_dim>,
+                             amr::Initialization::Initialize<volume_dim>>,
+                         Parallel::Actions::TerminatePhase>>,
+          Parallel::PhaseActions<
+              Parallel::Phase::CheckDomain,
+              tmpl::list<::amr::Actions::SendAmrDiagnostics,
+                         Parallel::Actions::TerminatePhase>>>>;
+
+  using component_list =
+      tmpl::list<amr::Component<RandomAmrMetavars>, amr_element_component>;
+
+  // NOLINTNEXTLINE(google-runtime-references)
+  void pup(PUP::er& /*p*/) {}
+};
+
+static const std::vector<void (*)()> charm_init_node_funcs{
+    &setup_error_handling, &domain::creators::register_derived_with_charm,
+    &Parallel::register_factory_classes_with_charm<metavariables>};
+static const std::vector<void (*)()> charm_init_proc_funcs{
+    &enable_floating_point_exceptions};

src/Executables/Examples/RandomAmr/RandomAmrFwd.hpp

@@ -0,0 +1,11 @@
+// Distributed under the MIT License.
+// See LICENSE.txt for details.
+
+#pragma once
+
+#include <cstddef>
+
+/// \cond
+template <size_t Dim>
+struct RandomAmrMetavars;
+/// \endcond

src/ParallelAlgorithms/Amr/Actions/CMakeLists.txt

@@ -10,5 +10,7 @@ spectre_target_headers(
   EvaluateRefinementCriteria.hpp
   Initialization.hpp
   Initialize.hpp
+  RunAmrDiagnostics.hpp
+  SendAmrDiagnostics.hpp
   UpdateAmrDecision.hpp
   )

src/ParallelAlgorithms/Amr/Actions/Initialize.hpp

@@ -5,7 +5,6 @@
 
 #include <array>
 #include <cstddef>
-#include <unordered_map>
 
 #include "Domain/Amr/Flag.hpp"
 #include "Domain/Amr/Tags/Flags.hpp"
@@ -14,37 +13,29 @@
 #include "Utilities/MakeArray.hpp"
 #include "Utilities/TMPL.hpp"
 
-/// \cond
-template <size_t VolumeDim>
-class ElementId;
-/// \endcond
-
 namespace amr::Initialization {
-/*!
- * \brief %Mutator meant to be used with
- *  `Initialization::Actions::AddSimpleTags`to initialize flags used for
- * adaptive mesh refinement
- *
- * DataBox:
- * - Adds:
- *   - `amr::Tags::Flags<Dim>`
- *   - `amr::Tags::NeighborFlags<Dim>`
- * - Removes: nothing
- * - Modifies: nothing
- *
- */
+/// \ingroup InitializationGroup
+/// \brief Initialize items related to adaptive mesh refinement
+///
+/// \see InitializeItems
 template <size_t Dim>
 struct Initialize {
-  using return_tags =
-      tmpl::list<amr::Tags::Flags<Dim>, amr::Tags::NeighborFlags<Dim>>;
+  using const_global_cache_tags = tmpl::list<>;
+  using mutable_global_cache_tags = tmpl::list<>;
+  using simple_tags_from_options = tmpl::list<>;
+
   using argument_tags = tmpl::list<>;
+  using return_tags = tmpl::list<amr::Tags::Flags<Dim>>;
+  using simple_tags =
+      tmpl::push_back<return_tags, amr::Tags::NeighborFlags<Dim>>;
+
+  using compute_tags = tmpl::list<>;
 
+  /// Given the items fetched from a DataBox by the argument_tags, mutate
+  /// the items in the DataBox corresponding to return_tags
   static void apply(
-      const gsl::not_null<std::array<amr::Flag, Dim>*> amr_flags,
-      const gsl::not_null<std::unordered_map<
-          ElementId<Dim>, std::array<amr::Flag, Dim>>*> /*neighbor_flags*/) {
+      const gsl::not_null<std::array<amr::Flag, Dim>*> amr_flags) {
     *amr_flags = make_array<Dim>(amr::Flag::Undefined);
-    // default initialization of NeighborFlags is okay
   }
 };
 }  // namespace amr::Initialization

src/ParallelAlgorithms/Amr/Actions/RunAmrDiagnostics.hpp

@@ -0,0 +1,71 @@
+// Distributed under the MIT License.
+// See LICENSE.txt for details.
+
+#pragma once
+
+#include <boost/rational.hpp>
+#include <cstddef>
+#include <vector>
+
+#include "DataStructures/DataBox/DataBox.hpp"
+#include "Domain/Domain.hpp"
+#include "Domain/Tags.hpp"
+#include "Parallel/GlobalCache.hpp"
+#include "Parallel/Printf.hpp"
+#include "Parallel/Reduction.hpp"
+#include "Utilities/MakeString.hpp"
+#include "Utilities/StdHelpers.hpp"
+#include "Utilities/System/Abort.hpp"
+
+namespace amr::Actions {
+
+/// \brief Use the AMR diagnostics gathered from all of the Element%s
+///
+/// Checks the following:
+/// - That the fraction of Block volume (in the logical coordinate frame)
+/// covered by all Element%s is equal to the number of Block%s in the Domain
+///
+/// Prints the following (as integers):
+/// - The number of elements
+/// - The number of grid points
+/// - The average refinement level by logical dimension (i.e. not by the
+///   physical dimensions)
+/// - The average number of grid points by logical dimension
+struct RunAmrDiagnostics {
+  template <typename ParallelComponent, typename DbTagList,
+            typename Metavariables, typename ArrayIndex>
+  static void apply(db::DataBox<DbTagList>& box,
+                    const Parallel::GlobalCache<Metavariables>& /*cache*/,
+                    const ArrayIndex& /*array_index*/,
+                    const boost::rational<size_t>& volume,
+                    const size_t number_of_elements,
+                    const size_t number_of_grid_points,
+                    std::vector<size_t> refinement_levels_by_dim,
+                    std::vector<size_t> extents_by_dim) {
+    constexpr size_t volume_dim = Metavariables::volume_dim;
+    const boost::rational<size_t> number_of_blocks{
+        db::get<::domain::Tags::Domain<Metavariables::volume_dim>>(box)
+            .blocks()
+            .size()};
+    if (number_of_blocks != volume) {
+      sys::abort(MakeString{} << "Check Domain failed!  Expected volume "
+                              << number_of_blocks << ", not " << volume
+                              << "\n");
+    }
+    for (size_t d = 0; d < volume_dim; ++d) {
+      extents_by_dim[d] /= number_of_elements;
+      refinement_levels_by_dim[d] /= number_of_elements;
+    }
+    const std::string string_gcc_needs_to_use_in_order_for_printf_to_compile =
+        MakeString{} << "Average refinement levels: "
+                     << refinement_levels_by_dim
+                     << "\nAverage grid points: " << extents_by_dim << "\n";
+    Parallel::printf(
+        "Number of elements: %zu\n"
+        "Number of grid points: %zu\n"
+        "%s\n",
+        number_of_elements, number_of_grid_points,
+        string_gcc_needs_to_use_in_order_for_printf_to_compile);
+  }
+};
+}  // namespace amr::Actions