Skip to content

Commit

Permalink
Add copy_to_device
Browse files Browse the repository at this point in the history
  • Loading branch information
@masterleinad
masterleinad committed May 14, 2024
1 parent 432c229 commit 9e520b7
Show file tree
Hide file tree
Showing 3 changed files with 60 additions and 47 deletions.
6 changes: 3 additions & 3 deletions application/adamantine.hh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -606,7 +606,7 @@ void refine_mesh(
const double refinement_beam_cutoff =
refinement_database.get<double>("beam_cutoff", 1.0e-15);

adamantine::HeatSources<dealii::MemorySpace::Host, dim> host_heat_sources = heat_sources.copy_to_host();
adamantine::HeatSources<dealii::MemorySpace::Host, dim> host_heat_sources = heat_sources.copy_to(dealii::MemorySpace::Host{});

for (unsigned int i = 0; i < n_refinements; ++i)
{
Expand Down Expand Up @@ -935,7 +935,7 @@ run(MPI_Comm const &communicator, boost::property_tree::ptree const &database,
mechanical_physics, displacement, material_properties, timers);
++n_time_step;

adamantine::HeatSources<dealii::MemorySpace::Host, dim> host_heat_sources = heat_sources.copy_to_host();
adamantine::HeatSources<dealii::MemorySpace::Host, dim> host_heat_sources = heat_sources.copy_to(dealii::MemorySpace::Host{});

// Create the bounding boxes used for material deposition
auto [material_deposition_boxes, deposition_times, deposition_cos,
Expand Down Expand Up @@ -1719,7 +1719,7 @@ run_ensemble(MPI_Comm const &global_communicator,
refine_mesh(thermal_physics_ensemble[member],
*material_properties_ensemble[member],
solution_augmented_ensemble[member].block(base_state),
heat_sources_ensemble[member].copy_to_host(), time, next_refinement_time,
heat_sources_ensemble[member].copy_to(dealii::MemorySpace::Host{}), time, next_refinement_time,
time_steps_refinement, refinement_database);
solution_augmented_ensemble[member].collect_sizes();
}
Expand Down
90 changes: 50 additions & 40 deletions source/HeatSources.hh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,11 @@ public:
HeatSources(boost::property_tree::ptree const &source_database);

/**
* Return a copy of this instance in host memory space.
* Return a copy of this instance in the target memory space.
*/
HeatSources<dealii::MemorySpace::Host, dim> copy_to_host() const;
template <typename TargetMemorySpace>
HeatSources<TargetMemorySpace, dim>
copy_to(TargetMemorySpace target_memory_space) const;

/**
* Set the time variable.
Expand Down Expand Up @@ -75,6 +77,7 @@ public:

private:
friend class HeatSources<dealii::MemorySpace::Default, dim>;
friend class HeatSources<dealii::MemorySpace::Host, dim>;

/**
* Private constructor used by copy_to_host.
Expand Down Expand Up @@ -339,88 +342,95 @@ void HeatSources<MemorySpace, dim>::set_beam_properties(
}

template <typename MemorySpace, int dim>
HeatSources<dealii::MemorySpace::Host, dim>
HeatSources<MemorySpace, dim>::copy_to_host() const
template <typename TargetMemorySpace>
HeatSources<TargetMemorySpace, dim> HeatSources<MemorySpace, dim>::copy_to(
TargetMemorySpace /*target_memory_space*/) const
{
if constexpr (std::is_same_v<MemorySpace, dealii::MemorySpace::Host>)
if constexpr (std::is_same_v<MemorySpace, TargetMemorySpace>)
return *this;
else
{
Kokkos::View<ElectronBeamHeatSource<dim, dealii::MemorySpace::Host> *,
Kokkos::HostSpace>
host_electron_beam_heat_sources(
Kokkos::View<ElectronBeamHeatSource<dim, TargetMemorySpace> *,
typename TargetMemorySpace::kokkos_space>
target_electron_beam_heat_sources(
Kokkos::view_alloc(Kokkos::WithoutInitializing,
"electron_beam_heat_sources"),
_electron_beam_heat_sources.size());
std::vector<Kokkos::View<ScanPathSegment *, Kokkos::HostSpace>>
host_electron_beam_scan_path_segments(
std::vector<Kokkos::View<ScanPathSegment *,
typename TargetMemorySpace::kokkos_space>>
target_electron_beam_scan_path_segments(
_electron_beam_scan_path_segments.size());
{
for (unsigned int i = 0; i < _electron_beam_scan_path_segments.size();
++i)
host_electron_beam_scan_path_segments[i] =
target_electron_beam_scan_path_segments[i] =
Kokkos::create_mirror_view_and_copy(
Kokkos::HostSpace{}, _electron_beam_scan_path_segments[i]);
auto host_copy_electron_beam_heat_sources =
auto target_copy_electron_beam_heat_sources =
Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{},
_electron_beam_heat_sources);
std::vector<BeamHeatSourceProperties> electron_beam_beams(
_electron_beam_heat_sources.size());
for (unsigned int i = 0; i < _electron_beam_heat_sources.size(); ++i)
electron_beam_beams[i] =
host_copy_electron_beam_heat_sources(i).get_beam_properties();
std::vector<ElectronBeamHeatSource<dim, dealii::MemorySpace::Host>>
target_copy_electron_beam_heat_sources(i).get_beam_properties();
std::vector<ElectronBeamHeatSource<dim, TargetMemorySpace>>
electron_beam_heat_source_vector;
for (unsigned int i = 0; i < _electron_beam_heat_sources.size(); ++i)
electron_beam_heat_source_vector.emplace_back(
electron_beam_beams[i],
ScanPath<dealii::MemorySpace::Host>(
host_electron_beam_scan_path_segments[i]));
ScanPath<TargetMemorySpace>(
target_electron_beam_scan_path_segments[i]));
Kokkos::deep_copy(
host_electron_beam_heat_sources,
Kokkos::View<ElectronBeamHeatSource<dim, dealii::MemorySpace::Host> *,
target_electron_beam_heat_sources,
Kokkos::View<ElectronBeamHeatSource<dim, TargetMemorySpace> *,
Kokkos::HostSpace>(
electron_beam_heat_source_vector.data(),
electron_beam_heat_source_vector.size()));
}

Kokkos::View<GoldakHeatSource<dim, dealii::MemorySpace::Host> *,
Kokkos::HostSpace>
host_goldak_heat_sources(Kokkos::view_alloc(Kokkos::WithoutInitializing,
"goldak_heat_sources"),
_goldak_heat_sources.size());
std::vector<Kokkos::View<ScanPathSegment *, Kokkos::HostSpace>>
host_goldak_scan_path_segments(_goldak_scan_path_segments.size());
Kokkos::View<GoldakHeatSource<dim, TargetMemorySpace> *,
typename TargetMemorySpace::kokkos_space>
target_goldak_heat_sources(
Kokkos::view_alloc(Kokkos::WithoutInitializing,
"goldak_heat_sources"),
_goldak_heat_sources.size());
std::vector<Kokkos::View<ScanPathSegment *,
typename TargetMemorySpace::kokkos_space>>
target_goldak_scan_path_segments(_goldak_scan_path_segments.size());
{
for (unsigned int i = 0; i < _goldak_scan_path_segments.size(); ++i)
host_goldak_scan_path_segments[i] = Kokkos::create_mirror_view_and_copy(
Kokkos::HostSpace{}, _goldak_scan_path_segments[i]);
auto host_copy_goldak_heat_sources = Kokkos::create_mirror_view_and_copy(
Kokkos::HostSpace{}, _goldak_heat_sources);
target_goldak_scan_path_segments[i] =
Kokkos::create_mirror_view_and_copy(
typename TargetMemorySpace::kokkos_space{},
_goldak_scan_path_segments[i]);
auto target_copy_goldak_heat_sources =
Kokkos::create_mirror_view_and_copy(
typename TargetMemorySpace::kokkos_space{}, _goldak_heat_sources);
std::vector<BeamHeatSourceProperties> goldak_beams(
_goldak_heat_sources.size());
for (unsigned int i = 0; i < _goldak_heat_sources.size(); ++i)
goldak_beams[i] =
host_copy_goldak_heat_sources(i).get_beam_properties();
std::vector<GoldakHeatSource<dim, dealii::MemorySpace::Host>>
target_copy_goldak_heat_sources(i).get_beam_properties();
std::vector<GoldakHeatSource<dim, TargetMemorySpace>>
goldak_heat_source_vector;
for (unsigned int i = 0; i < _goldak_heat_sources.size(); ++i)
goldak_heat_source_vector.emplace_back(
goldak_beams[i], ScanPath<dealii::MemorySpace::Host>(
host_goldak_scan_path_segments[i]));
goldak_beams[i],
ScanPath<TargetMemorySpace>(target_goldak_scan_path_segments[i]));
Kokkos::deep_copy(
host_goldak_heat_sources,
Kokkos::View<GoldakHeatSource<dim, dealii::MemorySpace::Host> *,
target_goldak_heat_sources,
Kokkos::View<GoldakHeatSource<dim, TargetMemorySpace> *,
Kokkos::HostSpace>(goldak_heat_source_vector.data(),
goldak_heat_source_vector.size()));
}

auto host_cube_heat_sources = Kokkos::create_mirror_view_and_copy(
Kokkos::HostSpace{}, _cube_heat_sources);
auto target_cube_heat_sources = Kokkos::create_mirror_view_and_copy(
typename TargetMemorySpace::kokkos_space{}, _cube_heat_sources);

return {host_electron_beam_heat_sources, host_cube_heat_sources,
host_goldak_heat_sources, host_electron_beam_scan_path_segments,
host_goldak_scan_path_segments};
return {target_electron_beam_heat_sources, target_cube_heat_sources,
target_goldak_heat_sources, target_electron_beam_scan_path_segments,
target_goldak_scan_path_segments};
}
}

Expand Down
11 changes: 7 additions & 4 deletions source/ThermalPhysics.templates.hh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ evaluate_thermal_physics_impl(

// Compute the source term.
// TODO do this on the GPU
auto host_heat_sources = heat_sources.copy_to_host();
auto host_heat_sources = heat_sources.copy_to(dealii::MemorySpace::Host{});
host_heat_sources.update_time(t);
dealii::LA::distributed::Vector<double, dealii::MemorySpace::Host> source(
y.get_partitioner());
Expand Down Expand Up @@ -466,7 +466,7 @@ ThermalPhysics<dim, p_order, fe_degree, MaterialStates, MemorySpaceType,
cell->set_active_fe_index(1);
}

auto host_heat_sources = _heat_sources.copy_to_host();
auto host_heat_sources = _heat_sources.copy_to(dealii::MemorySpace::Host{});
_current_source_height = host_heat_sources.get_current_height(0.0);
}

Expand Down Expand Up @@ -809,7 +809,9 @@ void ThermalPhysics<
{
// Update the heat source from heat_source_database to reflect changes during
// the simulation (i.e. due to data assimilation)
_heat_sources.set_beam_properties(heat_source_database);
auto host_heat_sources = _heat_sources.copy_to(dealii::MemorySpace::Host{});
host_heat_sources.set_beam_properties(heat_source_database);
_heat_sources = host_heat_sources.copy_to(MemorySpaceType{});
}

template <int dim, int p_order, int fe_degree, typename MaterialStates,
Expand All @@ -821,7 +823,8 @@ double ThermalPhysics<dim, p_order, fe_degree, MaterialStates, MemorySpaceType,
dealii::LA::distributed::Vector<double, MemorySpaceType> &solution,
std::vector<Timer> &timers)
{
_current_source_height = _heat_sources.copy_to_host().get_current_height(t);
_current_source_height =
_heat_sources.copy_to(dealii::MemorySpace::Host{}).get_current_height(t);

auto eval = [&](double const t, LA_Vector const &y)
{ return evaluate_thermal_physics(t, y, timers); };
Expand Down

0 comments on commit 9e520b7

Please sign in to comment.