Add Timers for FSI (#1221)

* Add Timers for FSI

* Style

include/aero/AeroContainer.h

@@ -55,6 +55,8 @@ class AeroContainer
   const stk::mesh::PartVector fsi_parts();
   const stk::mesh::PartVector fsi_bndry_parts();
   const std::vector<std::string> fsi_bndry_part_names();
+  double openfast_accumulated_time();
+  double nalu_fsi_accumulated_time();
 
 private:
   bool has_actuators() { return actuatorModel_.is_active(); }

include/aero/fsi/OpenfastFSI.h

@@ -47,6 +47,9 @@ class OpenfastFSI
     std::array<double, 3> axis, double omega, double curTime);
   void end_openfast();
 
+  double total_openfastfsi_execution_time() { return openFastTimer_.second; }
+  double total_nalu_fsi_execution_time() { return naluTimer_.second; }
+
 private:
   OpenfastFSI() = delete;
   OpenfastFSI(const OpenfastFSI&) = delete;
@@ -58,6 +61,8 @@ class OpenfastFSI
   void compute_mapping();
 
   void send_loads(const double curTime);
+  void timer_start(std::pair<double, double>& timer);
+  void timer_stop(std::pair<double, double>& timer);
 
   std::shared_ptr<stk::mesh::BulkData> bulk_;
 
@@ -79,6 +84,11 @@ class OpenfastFSI
 
   double dt_{-1.0}; // Store nalu-wind step
 
+  std::pair<double, double> openFastTimer_{
+    0.0, 0.0}; // store time taken in openfast calls
+  std::pair<double, double> naluTimer_{
+    0.0, 0.0}; // store time taken in openfast calls
+
   int writeFreq_{
     30}; // Frequency to write line loads and deflections to netcdf file
 

src/Realm.C

@@ -4038,6 +4038,40 @@ Realm::dump_simulation_time()
       << std::endl;
   }
 
+  if (aeroModels_->has_fsi()) {
+    double naluFsiTimer = aeroModels_->nalu_fsi_accumulated_time();
+    double openFastFsiTimer = aeroModels_->openfast_accumulated_time();
+    // nalu fsi calculations
+    double g_totalNalu = 0.0, g_minNalu = 0.0, g_maxNalu = 0.0;
+    stk::all_reduce_min(
+      NaluEnv::self().parallel_comm(), &naluFsiTimer, &g_minNalu, 1);
+    stk::all_reduce_max(
+      NaluEnv::self().parallel_comm(), &naluFsiTimer, &g_maxNalu, 1);
+    stk::all_reduce_sum(
+      NaluEnv::self().parallel_comm(), &naluFsiTimer, &g_totalNalu, 1);
+
+    NaluEnv::self().naluOutputP0()
+      << "Timing for FSI Computations :    " << std::endl;
+    NaluEnv::self().naluOutputP0()
+      << "        Nalu-Wind::computations --  "
+      << " \tavg: " << g_totalNalu / double(nprocs) << " \tmin: " << g_minNalu
+      << " \tmax: " << g_maxNalu << std::endl;
+
+    // openfast calculations (excluding data fetch operations)
+    double g_totalFast = 0.0, g_minFast = 0.0, g_maxFast = 0.0;
+    stk::all_reduce_min(
+      NaluEnv::self().parallel_comm(), &openFastFsiTimer, &g_minFast, 1);
+    stk::all_reduce_max(
+      NaluEnv::self().parallel_comm(), &openFastFsiTimer, &g_maxFast, 1);
+    stk::all_reduce_sum(
+      NaluEnv::self().parallel_comm(), &openFastFsiTimer, &g_totalFast, 1);
+
+    NaluEnv::self().naluOutputP0()
+      << "        OpenFAST::computations --  "
+      << " \tavg: " << g_totalFast / double(nprocs) << " \tmin: " << g_minFast
+      << " \tmax: " << g_maxFast << std::endl;
+  }
+
   // consolidated sort
   if (solutionOptions_->useConsolidatedSolverAlg_) {
     double g_totalSort = 0.0, g_minSort = 0.0, g_maxSort = 0.0;

src/aero/AeroContainer.C

@@ -213,5 +213,29 @@ AeroContainer::fsi_bndry_part_names()
   return bndry_part_names;
 }
 
+double
+AeroContainer::openfast_accumulated_time()
+{
+#ifdef NALU_USES_OPENFAST_FSI
+  if (has_fsi())
+    return fsiContainer_->total_openfastfsi_execution_time();
+  else
+    return -1.0;
+#endif
+  return -1.0;
+}
+
+double
+AeroContainer::nalu_fsi_accumulated_time()
+{
+#ifdef NALU_USES_OPENFAST_FSI
+  if (has_fsi())
+    return fsiContainer_->total_nalu_fsi_execution_time();
+  else
+    return -1.0;
+#endif
+  return -1.0;
+}
+
 } // namespace nalu
 } // namespace sierra

src/aero/fsi/OpenfastFSI.C

@@ -382,21 +382,27 @@ OpenfastFSI::compute_mapping()
 void
 OpenfastFSI::predict_struct_states()
 {
+  timer_start(openFastTimer_);
   FAST.predict_states();
+  timer_stop(openFastTimer_);
 }
 
 void
 OpenfastFSI::predict_struct_timestep(const double curTime)
 {
   send_loads(curTime);
+  timer_start(openFastTimer_);
   FAST.update_states_driver_time_step();
+  timer_stop(openFastTimer_);
 }
 
 void
 OpenfastFSI::advance_struct_timestep(const double curTime)
 {
 
+  timer_start(openFastTimer_);
   FAST.advance_to_next_driver_time_step();
+  timer_stop(openFastTimer_);
 
   tStep_ += 1;
 
@@ -561,11 +567,13 @@ OpenfastFSI::compute_div_mesh_velocity()
 {
 
   int nTurbinesGlob = FAST.get_nTurbinesGlob();
+  timer_start(naluTimer_);
   for (int i = 0; i < nTurbinesGlob; i++) {
     if (fsiTurbineData_[i] != NULL) // This may not be a turbine intended for
                                     // blade-resolved simulation
       fsiTurbineData_[i]->compute_div_mesh_velocity();
   }
+  timer_stop(naluTimer_);
 }
 
 void
@@ -584,6 +592,7 @@ void
 OpenfastFSI::map_displacements(double current_time, bool updateCurCoor)
 {
 
+  timer_start(naluTimer_);
   get_displacements(current_time);
 
   stk::mesh::Selector sel;
@@ -627,12 +636,13 @@ OpenfastFSI::map_displacements(double current_time, bool updateCurCoor)
     curCoords->modify_on_host();
     curCoords->sync_to_device();
   }
+  timer_stop(naluTimer_);
 }
 
 void
 OpenfastFSI::map_loads(const int tStep, const double curTime)
 {
-
+  timer_start(naluTimer_);
   int nTurbinesGlob = FAST.get_nTurbinesGlob();
   for (int i = 0; i < nTurbinesGlob; i++) {
     if (fsiTurbineData_[i] != nullptr) { // This may not be a turbine intended
@@ -661,6 +671,20 @@ OpenfastFSI::map_loads(const int tStep, const double curTime)
       fsiTurbineData_[i]->write_nc_def_loads(tStep, curTime);
     }
   }
+  timer_stop(naluTimer_);
+}
+
+void
+OpenfastFSI::timer_start(std::pair<double, double>& timer)
+{
+  timer.first = NaluEnv::self().nalu_time();
+}
+
+void
+OpenfastFSI::timer_stop(std::pair<double, double>& timer)
+{
+  timer.first = NaluEnv::self().nalu_time() - timer.first;
+  timer.second += timer.first;
 }
 
 } // namespace nalu