GPU make edge scal 3d (GPU hydro PR 7/n)

Exec/Make.Maestro

@@ -261,6 +261,14 @@ $(objEXETempDir)/AMReX_buildInfo.o: .FORCE
 
 include $(AMREX_HOME)/Tools/GNUMake/Make.rules
 
+ifeq ($(USE_CUDA),TRUE)
+ifeq ($(USE_GPU_PRAGMA), TRUE)
+
+include $(TOP)/Exec/Make.cuda_rules
+
+endif
+endif
+
 clean::
 
 	$(SILENT) $(RM) extern.F90 extern.f90

Exec/Make.cuda_rules

@@ -0,0 +1,72 @@
+
+# Specialize rules for files that need register capping for CUDA.
+# This corresponds to what we do to AMReX_filcc_mod.F90 in AMReX.
+
+$(objEXETempDir)/MaestroHydro.o: MaestroHydro.cpp
+	@echo Compiling $(<F)
+	@if [ !-d $(objExeTempDir) ]; then mkdir -p $(objEXETempDir); fi
+ifeq ($(cap_registers),1)
+	$(SILENT) $(CCACHE) $(CXX) $(patsubst -maxrregcount=$(CUDA_MAXREGCOUNT), -maxrregcount=128, $(CXXFLAGS)) $(CPPFLAGS) $(includes) -c $(srcTempDir)/$(<F) -o $(objEXETempDir)/MaestroHydro.o
+else
+	$(SILENT) $(CCACHE) $(CXX) $(CXXFLAGS) $(CPPFLAGS) $(includes) -c $(srcTempDir)/$(<F) -o $(objEXETempDir)/MaestroHydro.o
+endif
+
+$(objEXETempDir)/ppm.o: ppm.F90
+	@echo Compiling $(<F) ...
+	@if [ ! -d $(objEXETempDir) ]; then mkdir -p $(objEXETempDir); fi
+	@if [ ! -d $(srcTempDir) ]; then mkdir -p $(srcTempDir); fi
+	@$(SHELL) -ec 'cp $< $(srcTempDir)'
+	$(AMREX_HOME)/Tools/F_scripts/gpu_fortran.py --fortran "$(srcTempDir)/$(<F)"
+ifeq ($(cap_registers),1)
+	$(SILENT) $(F90CACHE) $(F90) $(patsubst $(cap_register_flag)$(CUDA_MAXREGCOUNT), $(cap_register_flag)128, $(F90FLAGS)) $(FMODULES) -DBL_LANG_FORT -DAMREX_LANG_FORT $(FCPPFLAGS) $(fincludes) -c $(srcTempDir)/$(<F) -o $(objEXETempDir)/ppm.o
+else
+	$(SILENT) $(F90CACHE) $(F90) $(F90FLAGS) $(FMODULES) -DBL_LANG_FORT -DAMREX_LANG_FORT $(FCPPFLAGS) $(fincludes) -c $(srcTempDir)/$(<F) -o $(objEXETempDir)/ppm.o
+endif
+
+$(objEXETempDir)/slope.o: slope.F90
+	@echo Compiling $(<F) ...
+	@if [ ! -d $(objEXETempDir) ]; then mkdir -p $(objEXETempDir); fi
+	@if [ ! -d $(srcTempDir) ]; then mkdir -p $(srcTempDir); fi
+	@$(SHELL) -ec 'cp $< $(srcTempDir)'
+	$(AMREX_HOME)/Tools/F_scripts/gpu_fortran.py --fortran "$(srcTempDir)/$(<F)"
+ifeq ($(cap_registers),1)
+	$(SILENT) $(F90CACHE) $(F90) $(patsubst $(cap_register_flag)$(CUDA_MAXREGCOUNT), $(cap_register_flag)128, $(F90FLAGS)) $(FMODULES) -DBL_LANG_FORT -DAMREX_LANG_FORT $(FCPPFLAGS) $(fincludes) -c $(srcTempDir)/$(<F) -o $(objEXETempDir)/slope.o
+else
+	$(SILENT) $(F90CACHE) $(F90) $(F90FLAGS) $(FMODULES) -DBL_LANG_FORT -DAMREX_LANG_FORT $(FCPPFLAGS) $(fincludes) -c $(srcTempDir)/$(<F) -o $(objEXETempDir)/slope.o
+endif
+
+$(objEXETempDir)/velpred.o: velpred.F90
+	@echo Compiling $(<F) ...
+	@if [ ! -d $(objEXETempDir) ]; then mkdir -p $(objEXETempDir); fi
+	@if [ ! -d $(srcTempDir) ]; then mkdir -p $(srcTempDir); fi
+	@$(SHELL) -ec 'cp $< $(srcTempDir)'
+	$(AMREX_HOME)/Tools/F_scripts/gpu_fortran.py --fortran "$(srcTempDir)/$(<F)"
+ifeq ($(cap_registers),1)
+	$(SILENT) $(F90CACHE) $(F90) $(patsubst $(cap_register_flag)$(CUDA_MAXREGCOUNT), $(cap_register_flag)128, $(F90FLAGS)) $(FMODULES) -DBL_LANG_FORT -DAMREX_LANG_FORT $(FCPPFLAGS) $(fincludes) -c $(srcTempDir)/$(<F) -o $(objEXETempDir)/velpred.o
+else
+	$(SILENT) $(F90CACHE) $(F90) $(F90FLAGS) $(FMODULES) -DBL_LANG_FORT -DAMREX_LANG_FORT $(FCPPFLAGS) $(fincludes) -c $(srcTempDir)/$(<F) -o $(objEXETempDir)/velpred.o
+endif
+
+$(objEXETempDir)/mkutrans.o: mkutrans.F90
+	@echo Compiling $(<F) ...
+	@if [ ! -d $(objEXETempDir) ]; then mkdir -p $(objEXETempDir); fi
+	@if [ ! -d $(srcTempDir) ]; then mkdir -p $(srcTempDir); fi
+	@$(SHELL) -ec 'cp $< $(srcTempDir)'
+	$(AMREX_HOME)/Tools/F_scripts/gpu_fortran.py --fortran "$(srcTempDir)/$(<F)"
+ifeq ($(cap_registers),1)
+	$(SILENT) $(F90CACHE) $(F90) $(patsubst $(cap_register_flag)$(CUDA_MAXREGCOUNT), $(cap_register_flag)128, $(F90FLAGS)) $(FMODULES) -DBL_LANG_FORT -DAMREX_LANG_FORT $(FCPPFLAGS) $(fincludes) -c $(srcTempDir)/$(<F) -o $(objEXETempDir)/mkutrans.o
+else
+	$(SILENT) $(F90CACHE) $(F90) $(F90FLAGS) $(FMODULES) -DBL_LANG_FORT -DAMREX_LANG_FORT $(FCPPFLAGS) $(fincludes) -c $(srcTempDir)/$(<F) -o $(objEXETempDir)/mkutrans.o
+endif
+
+$(objEXETempDir)/make_edge_scal.o: make_edge_scal.F90
+	@echo Compiling $(<F) ...
+	@if [ ! -d $(objEXETempDir) ]; then mkdir -p $(objEXETempDir); fi
+	@if [ ! -d $(srcTempDir) ]; then mkdir -p $(srcTempDir); fi
+	@$(SHELL) -ec 'cp $< $(srcTempDir)'
+	$(AMREX_HOME)/Tools/F_scripts/gpu_fortran.py --fortran "$(srcTempDir)/$(<F)"
+ifeq ($(cap_registers),1)
+	$(SILENT) $(F90CACHE) $(F90) $(patsubst $(cap_register_flag)$(CUDA_MAXREGCOUNT), $(cap_register_flag)128, $(F90FLAGS)) $(FMODULES) -DBL_LANG_FORT -DAMREX_LANG_FORT $(FCPPFLAGS) $(fincludes) -c $(srcTempDir)/$(<F) -o $(objEXETempDir)/make_edge_scal.o
+else
+	$(SILENT) $(F90CACHE) $(F90) $(F90FLAGS) $(FMODULES) -DBL_LANG_FORT -DAMREX_LANG_FORT $(FCPPFLAGS) $(fincludes) -c $(srcTempDir)/$(<F) -o $(objEXETempDir)/make_edge_scal.o
+endif

Source/Maestro.H

@@ -15,7 +15,6 @@
 #include <AMReX_PlotFileUtil.H>
 
 #include <PhysBCFunctMaestro.H>
-#include <Maestro_F.H>
 
 #ifdef AMREX_USE_CUDA
 // #include <AMReX_CudaAllocators.H>
@@ -26,8 +25,10 @@
 /// this will be stored in CUDA managed memory.
 #ifdef AMREX_USE_CUDA
 typedef amrex::Gpu::ManagedVector<amrex::Real> RealVector;
+typedef amrex::Gpu::ManagedVector<int> IntVector;
 #else
 typedef amrex::Vector< amrex::Real > RealVector;
+typedef amrex::Vector< int > IntVector;
 #endif
 
 class Maestro
@@ -1246,9 +1247,9 @@ private:
     amrex::Vector<amrex::MultiFab> normal;
     amrex::Vector<amrex::MultiFab> cell_cc_to_r;
 
-    /// Stores domain boundary conditions.
+    /// stores domain boundary conditions.
     /// These muse be vectors (rather than arrays) so we can ParmParse them
-    amrex::Vector<int> phys_bc;
+    IntVector phys_bc;
 
     /// Boundary condition objects needed for FillPatch routines.
     /// This is essentially an array (over components)
@@ -1301,12 +1302,12 @@ private:
     int nr_irreg;
 
     // these provide information about the multilevel base state configuration
-    amrex::Vector<int> numdisjointchunks;
-    amrex::Vector<int> r_start_coord;
-    amrex::Vector<int> r_end_coord;
+    IntVector numdisjointchunks;
+    IntVector r_start_coord;
+    IntVector r_end_coord;
 
     /// array of tagged boxes (planar)
-    amrex::Vector<int> tag_array;
+    IntVector tag_array;
 
     // diag file array buffers
     amrex::Vector<amrex::Real> diagfile1_data;

Source/Maestro.cpp

@@ -1,5 +1,6 @@
 
 #include <Maestro.H>
+#include <Maestro_F.H>
 
 using namespace amrex;
 

Source/MaestroAdvance.cpp

@@ -1,5 +1,6 @@
 
 #include <Maestro.H>
+#include <Maestro_F.H>
 
 using namespace amrex;
 

Source/MaestroAdvanceAvg.cpp

@@ -1,5 +1,6 @@
 
 #include <Maestro.H>
+#include <Maestro_F.H>
 
 using namespace amrex;
 

Source/MaestroAdvanceIrreg.cpp

@@ -1,5 +1,6 @@
 
 #include <Maestro.H>
+#include <Maestro_F.H>
 
 using namespace amrex;