Skip to content

Commit

Permalink
Replace MemoryBlock and MemoryBlockView with Kokkos::View
Browse files Browse the repository at this point in the history
  • Loading branch information
@Rombur
Rombur committed Jan 24, 2024
1 parent 04cf703 commit b79b41f
Show file tree
Hide file tree
Showing 13 changed files with 407 additions and 1,139 deletions.
44 changes: 13 additions & 31 deletions application/adamantine.hh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
#include <Geometry.hh>
#include <MaterialProperty.hh>
#include <MechanicalPhysics.hh>
#include <MemoryBlock.hh>
#include <PointCloud.hh>
#include <PostProcessor.hh>
#include <RayTracing.hh>
Expand Down Expand Up @@ -135,11 +134,8 @@ void output_pvtu(
#endif
timers[adamantine::output].start();
auto state = material_properties.get_state();
adamantine::MemoryBlock<double, dealii::MemorySpace::Host> state_host(
state.extent(0), state.extent(1));
adamantine::MemoryBlockView<double, dealii::MemorySpace::Host>
state_host_view(state_host);
adamantine::deep_copy(state_host_view, state);
auto state_host = Kokkos::create_mirror_view_and_copy(
dealii::MemorySpace::Host::kokkos_space, state);
if (thermal_physics)
{
dealii::LinearAlgebra::distributed::Vector<double,
Expand All @@ -152,14 +148,14 @@ void output_pvtu(
mechanical_physics->get_affine_constraints().distribute(displacement);
post_processor.write_output(
n_time_step, time, temperature_host, displacement,
mechanical_physics->get_stress_tensor(), state_host_view,
mechanical_physics->get_stress_tensor(), state_host,
material_properties.get_dofs_map(),
material_properties.get_dof_handler());
}
else
{
post_processor.write_thermal_output(
n_time_step, time, temperature_host, state_host_view,
n_time_step, time, temperature_host, state_host,
material_properties.get_dofs_map(),
material_properties.get_dof_handler());
}
Expand All @@ -169,7 +165,7 @@ void output_pvtu(
mechanical_physics->get_affine_constraints().distribute(displacement);
post_processor.write_mechanical_output(
n_time_step, time, displacement,
mechanical_physics->get_stress_tensor(), state_host_view,
mechanical_physics->get_stress_tensor(), state_host,
material_properties.get_dofs_map(),
material_properties.get_dof_handler());
}
Expand Down Expand Up @@ -389,18 +385,8 @@ void refine_and_transfer(
std::vector<double> dummy_cell_data(n_material_states + direction_data_size +
phase_history_data_size,
std::numeric_limits<double>::infinity());
adamantine::MemoryBlockView<double, MemorySpaceType> material_state_view =
material_properties.get_state();
adamantine::MemoryBlock<double, dealii::MemorySpace::Host>
material_state_host(material_state_view.extent(0),
material_state_view.extent(1));
typename decltype(material_state_view)::memory_space memspace;
adamantine::deep_copy(material_state_host.data(), dealii::MemorySpace::Host{},
material_state_view.data(), memspace,
material_state_view.size());

adamantine::MemoryBlockView<double, dealii::MemorySpace::Host>
state_host_view(material_state_host);
auto state_host = Kokkos::create_mirror_view_and_copy(
Kokkos::HostSpace{}, material_properties.get_state());
unsigned int cell_id = 0;
unsigned int activated_cell_id = 0;
for (auto const &cell : dof_handler.active_cell_iterators())
Expand All @@ -410,7 +396,7 @@ void refine_and_transfer(
std::vector<double> cell_data(n_material_states + direction_data_size +
phase_history_data_size);
for (unsigned int i = 0; i < n_material_states; ++i)
cell_data[i] = state_host_view(i, cell_id);
cell_data[i] = state_host(i, cell_id);
if (cell->active_fe_index() == 0)
{
cell_data[n_material_states] =
Expand Down Expand Up @@ -508,10 +494,8 @@ void refine_and_transfer(
std::vector<double>(n_material_states + direction_data_size +
phase_history_data_size));
cell_data_trans.unpack(transferred_data);
material_state_view = material_properties.get_state();
material_state_host.reinit(material_state_view.extent(0),
material_state_view.extent(1));
state_host_view.reinit(material_state_host);
auto state = material_properties.get_state();
state_host = Kokkos::create_mirror_view(state);
unsigned int total_cell_id = 0;
cell_id = 0;
std::vector<double> transferred_cos;
Expand All @@ -523,7 +507,7 @@ void refine_and_transfer(
{
for (unsigned int i = 0; i < n_material_states; ++i)
{
state_host_view(i, cell_id) = transferred_data[total_cell_id][i];
state_host(i, cell_id) = transferred_data[total_cell_id][i];
}
if (cell->active_fe_index() == 0)
{
Expand Down Expand Up @@ -552,9 +536,7 @@ void refine_and_transfer(
thermal_physics->set_has_melted_vector(has_melted);

// Copy the data back to material_property
adamantine::deep_copy(material_state_view.data(), memspace,
material_state_host.data(), dealii::MemorySpace::Host{},
material_state_view.size());
Kokkos::deep_copy(state, state_host);

// Update the material states in the ThermalOperator
thermal_physics->get_state_from_material_properties();
Expand All @@ -569,7 +551,7 @@ void refine_and_transfer(
double material_ratio = 0.;
for (unsigned int i = 0; i < n_material_states; ++i)
{
material_ratio += state_host_view(i, cell_id);
material_ratio += state_host(i, cell_id);
}
ASSERT(std::abs(material_ratio - 1.) < 1e-14, "Material is lost.");
++cell_id;
Expand Down
2 changes: 0 additions & 2 deletions source/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,6 @@ set(Adamantine_HEADERS
${CMAKE_CURRENT_SOURCE_DIR}/MaterialProperty.templates.hh
${CMAKE_CURRENT_SOURCE_DIR}/MechanicalOperator.hh
${CMAKE_CURRENT_SOURCE_DIR}/MechanicalPhysics.hh
${CMAKE_CURRENT_SOURCE_DIR}/MemoryBlock.hh
${CMAKE_CURRENT_SOURCE_DIR}/MemoryBlockView.hh
${CMAKE_CURRENT_SOURCE_DIR}/NewtonSolver.hh
${CMAKE_CURRENT_SOURCE_DIR}/Operator.hh
${CMAKE_CURRENT_SOURCE_DIR}/PointCloud.hh
Expand Down
162 changes: 86 additions & 76 deletions source/MaterialProperty.hh
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
/* Copyright (c) 2016 - 2022, the adamantine authors.
/* Copyright (c) 2016 - 2023, the adamantine authors.
*
* This file is subject to the Modified BSD License and may not be distributed
* without copyright and license information. Please refer to the file LICENSE
Expand All @@ -8,8 +8,6 @@
#ifndef MATERIAL_PROPERTY_HH
#define MATERIAL_PROPERTY_HH

#include <MemoryBlock.hh>
#include <MemoryBlockView.hh>
#include <types.hh>
#include <utils.hh>

Expand Down Expand Up @@ -39,6 +37,17 @@ template <int dim, typename MemorySpaceType>
class MaterialProperty
{
public:
/**
* Order of the polynomial used to describe the material properties.
*/
static unsigned int constexpr polynomial_order = 4;

/**
* Size of the table, i.e. number of temperature/property pairs, used to
* describe the material properties.
*/
static unsigned int constexpr table_size = 4;

/**
* Constructor.
*/
Expand Down Expand Up @@ -85,24 +94,29 @@ public:
StateProperty prop) const;

/**
* Return a MemoryBlockView of the properties of the material that are
* independent of the state of the material.
* Return the properties of the material that are independent of the state of
* the material.
*/
MemoryBlockView<double, MemorySpaceType> get_properties();
Kokkos::View<double *[g_n_properties], typename MemorySpaceType::kokkos_space>
get_properties();

/**
* Return a MemoryBlockView of the properties of the material that are
* dependent of the state of the material and which have been set using
* tables.
* Return the properties of the material that are dependent of the state of
* the material and which have been set using tables.
*/
MemoryBlockView<double, MemorySpaceType> get_state_property_tables();
Kokkos::View<double *[g_n_material_states][g_n_thermal_state_properties]
[table_size][2],
typename MemorySpaceType::kokkos_space>
get_state_property_tables();

/**
* Return a MemoryBlockView of the properties of the material that are
* dependent of the state of the material and which have beese set using
* polynomials.
* Return the properties of the material that are dependent of the state of
* the material and which have beese set using polynomials.
*/
MemoryBlockView<double, MemorySpaceType> get_state_property_polynomials();
Kokkos::View<double *[g_n_material_states][g_n_thermal_state_properties]
[polynomial_order + 1],
typename MemorySpaceType::kokkos_space>
get_state_property_polynomials();

/**
* Reinitialize the DoFHandler associated with MaterialProperty and resize the
Expand Down Expand Up @@ -153,7 +167,8 @@ public:
* correspond to a cell in the mesh and has a value between 0 and 1. The sum
* of the states for a given cell is equal to 1.
*/
MemoryBlockView<double, MemorySpaceType> get_state() const;
Kokkos::View<double **, typename MemorySpaceType::kokkos_space>
get_state() const;

/**
* Get the ratio of a given MaterialState for a given cell. The sum
Expand Down Expand Up @@ -184,8 +199,10 @@ public:
* Set the ratio of the material states from ThermalOperatorDevice.
*/
void set_state_device(
MemoryBlock<double, MemorySpaceType> const &liquid_ratio,
MemoryBlock<double, MemorySpaceType> const &powder_ratio,
Kokkos::View<double *, typename MemorySpaceType::kokkos_space>
liquid_ratio,
Kokkos::View<double *, typename MemorySpaceType::kokkos_space>
powder_ratio,
std::map<typename dealii::DoFHandler<dim>::cell_iterator,
std::vector<unsigned int>> const &_cell_it_to_mf_pos,
dealii::DoFHandler<dim> const &dof_handler);
Expand All @@ -209,22 +226,11 @@ public:
* Compute a property from a table given the temperature.
*/
static KOKKOS_FUNCTION double compute_property_from_table(
MemoryBlockView<double, MemorySpaceType> const
&state_property_tables_view,
Kokkos::View<double ****[2], typename MemorySpaceType::kokkos_space>
state_property_tables,
unsigned int const material_id, unsigned int const material_state,
unsigned int const property, double const temperature);

/**
* Order of the polynomial used to describe the material properties.
*/
static unsigned int constexpr polynomial_order = 4;

/**
* Size of the table, i.e. number of temperature/property pairs, used to
* describe the material properties.
*/
static unsigned int constexpr table_size = 4;

private:
/**
* Fill the _properties map.
Expand Down Expand Up @@ -257,52 +263,57 @@ private:
*/
bool _use_table;
/**
* MemoryBlock that stores the thermal material properties which have been set
* using tables.
* Thermal material properties which have been set using tables.
*/
MemoryBlock<double, MemorySpaceType> _state_property_tables;
Kokkos::View<double *[g_n_material_states][g_n_thermal_state_properties]
[table_size][2],
typename MemorySpaceType::kokkos_space>
_state_property_tables;
/**
* MemoryBlock that stores the thermal material properties which have been set
* Thermal material properties which have been set
* using polynomials.
*/
MemoryBlock<double, MemorySpaceType> _state_property_polynomials;
Kokkos::View<double *[g_n_material_states][g_n_thermal_state_properties]
[polynomial_order + 1],
typename MemorySpaceType::kokkos_space>
_state_property_polynomials;
/**
* MemoryBlock that stores the properties of the material that are independent
* of the state of the material.
* Properties of the material that are independent of the state of the
* material.
*/
MemoryBlock<double, MemorySpaceType> _properties;
Kokkos::View<double *[g_n_properties], typename MemorySpaceType::kokkos_space>
_properties;
/**
* MemoryBlockView associated with _properties.
* Ratio of each in MaterarialState in each cell.
*/
MemoryBlockView<double, MemorySpaceType> _properties_view;
Kokkos::View<double **, typename MemorySpaceType::kokkos_space> _state;
/**
* MemoryBlock that stores the ratio of each in MaterarialState in each cell.
* Thermal properties of the material that are dependent of the state of the
* material.
*/
MemoryBlock<double, MemorySpaceType> _state;
Kokkos::View<double **, typename MemorySpaceType::kokkos_space>
_property_values;
/**
* MemoryBlock that stores the thermal properties of the material that are
* dependent of the state of the material.
*/
MemoryBlock<double, MemorySpaceType> _property_values;
/**
* MemoryBlock that stores the mechanical properties which have been set using
* tables.
* Mechanical properties which have been set using tables.
*/
// We cannot put the mechanical properties with the thermal properties because
// the mechanical properties can only exist on the host while the thermal ones
// can be on the host or the device.
MemoryBlock<double, dealii::MemorySpace::Host>
Kokkos::View<double *[g_n_mechanical_state_properties][table_size][2],
typename dealii::MemorySpace::Host::kokkos_space>
_mechanical_properties_tables_host;
/**
* MemoryBlock that stores the mechanical properties which have been set using
* polynomials.
* Mechanical properties which have been set using polynomials.
*/
MemoryBlock<double, dealii::MemorySpace::Host>
Kokkos::View<double *[g_n_mechanical_state_properties][polynomial_order + 1],
typename dealii::MemorySpace::Host::kokkos_space>
_mechanical_properties_polynomials_host;
/**
* MemoryBlock that stores the mechanical properties.
* Temperature independent mechanical properties.
*/
MemoryBlock<double, dealii::MemorySpace::Host> _mechanical_properties_host;
Kokkos::View<double *[g_n_mechanical_state_properties],
typename dealii::MemorySpace::Host::kokkos_space>
_mechanical_properties_host;
/**
* Discontinuous piecewise constant finite element.
*/
Expand All @@ -321,18 +332,13 @@ template <int dim, typename MemorySpaceType>
inline double MaterialProperty<dim, MemorySpaceType>::get(
dealii::types::material_id material_id, Property property) const
{
// This function works only on the host because the MemoryBlockView needs to
// be created on the host. Using MemoryBlock directly doesn't work because the
// _data ptr is on the host.
return _properties_view(material_id, static_cast<unsigned int>(property));
return _properties(material_id, static_cast<unsigned int>(property));
}

template <int dim, typename MemorySpaceType>
inline MemoryBlockView<double, MemorySpaceType>
MaterialProperty<dim, MemorySpaceType>::get_properties()
{
return _properties_view;
}
inline Kokkos::View<double *[g_n_properties],
typename MemorySpaceType::kokkos_space>
MaterialProperty<dim, MemorySpaceType>::get_properties() { return _properties; }

template <int dim, typename MemorySpaceType>
inline bool MaterialProperty<dim, MemorySpaceType>::properties_use_table() const
Expand All @@ -341,24 +347,28 @@ inline bool MaterialProperty<dim, MemorySpaceType>::properties_use_table() const
}

template <int dim, typename MemorySpaceType>
inline MemoryBlockView<double, MemorySpaceType>
inline Kokkos::View<double *[g_n_material_states]
[g_n_thermal_state_properties][MaterialProperty<
dim, MemorySpaceType>::table_size][2],
typename MemorySpaceType::kokkos_space>
MaterialProperty<dim, MemorySpaceType>::get_state_property_tables()
{
return MemoryBlockView<double, MemorySpaceType>(_state_property_tables);
}
{ return _state_property_tables; }

template <int dim, typename MemorySpaceType>
inline MemoryBlockView<double, MemorySpaceType>
MaterialProperty<dim, MemorySpaceType>::get_state_property_polynomials()
{
return MemoryBlockView<double, MemorySpaceType>(_state_property_polynomials);
}
inline Kokkos::View<
double *[g_n_material_states]
[g_n_thermal_state_properties]
[MaterialProperty<dim, MemorySpaceType>::polynomial_order + 1],
typename MemorySpaceType::kokkos_space> MaterialProperty<dim,
MemorySpaceType>::
get_state_property_polynomials() { return _state_property_polynomials; }

template <int dim, typename MemorySpaceType>
inline MemoryBlockView<double, MemorySpaceType>
MaterialProperty<dim, MemorySpaceType>::get_state() const
inline Kokkos::
View<double **, typename MemorySpaceType::kokkos_space> MaterialProperty<
dim, MemorySpaceType>::get_state() const
{
return MemoryBlockView<double, MemorySpaceType>(_state);
return _state;
}

template <int dim, typename MemorySpaceType>
Expand Down

0 comments on commit b79b41f

Please sign in to comment.