Gpu

.gitignore

@@ -18,13 +18,16 @@ test_parallel_rk2
 test_parallel_rkSplit
 test_rkSplit
 test_weno
+test_hdf5_parallel_rk2
+test_hdf5_rk2
 
 test_*.o
 !test_*.cc
 gtest_main.a
 gtest.a
 
 *.dat
+*.hdf5
 
 Doxumentation/html/*
 
@@ -37,11 +40,21 @@ runMultipleSims.sh
 cleanData.sh
 
 .cache
-
 ProfileParallel.nvvp
 
 slurm*
 
+# Don't track the data files!
 Data
+Tests/TestData
+Examples/Data*
+Scripts/Data*
 
+# Ignore the build directories
 build
+
+# Don't track files from the docker mount
+dockermount
+# Don't track files from the virtual environment or IDE
+venv
+.idea

Examples/KelvinHelmholtz/SingleFluidIdealRandomMPIHDF5/Makefile

@@ -0,0 +1,193 @@
+# Make file for the main function. Builds all modules
+# and links for main.cc
+
+# Compiler
+CC = mpic++
+##CC = g++
+
+USE_MPI = 1
+USE_OMP = 1
+USE_HDF = 1
+
+PROJECT_TYPE = CPU
+
+# Module directory
+MODULE_DIR = ./../../../Project/${PROJECT_TYPE}/Src
+
+# Rootfinder directory
+RTFIND_DIR = ./../../../Project/${PROJECT_TYPE}/CminpackLibrary
+
+# Include directory
+INC_DIR = ./../../../Project/${PROJECT_TYPE}/Include
+
+# Cminpack include directory
+RTFIND_INC_DIR = ./../../../Project/${PROJECT_TYPE}/CminpackLibrary/Include
+
+# Cminpack source directory
+RTFIND_SRC_DIR = ./../../../Project/${PROJECT_TYPE}//CminpackLibrary/Src
+
+ifeq ($(USE_OMP), 1)
+    OMP_FLAGS = -fopenmp
+endif
+
+# C++ compiler flags
+CXXFLAGS = -Wall -std=c++11 -O3 $(OMP_FLAGS) -Wno-unknown-pragmas
+
+# Sources
+SRCS = simulation.cc \
+			 srmhd.cc \
+			 initFunc.cc \
+			 simData.cc \
+			 wenoUpwinds.cc \
+			 weno.cc \
+			 RK2.cc \
+			 rkSplit.cc \
+			 boundaryConds.cc \
+			 fluxVectorSplitting.cc \
+
+SERIAL_SRCS = serialSaveData.cc \
+			  serialEnv.cc
+
+ifeq ($(USE_HDF), 1)
+    SERIAL_SRCS += serialSaveDataHDF5.cc
+endif
+
+PARALLEL_SRCS = parallelSaveData.cc \
+				parallelBoundaryConds.cc \
+				parallelEnv.cc
+
+ifeq ($(USE_HDF), 1)
+    PARALLEL_SRCS += parallelSaveDataHDF5.cc
+endif
+
+ifeq ($(USE_MPI), 1)
+    SRCS += ${PARALLEL_SRCS}
+else
+    SRCS += ${SERIAL_SRCS}
+endif
+
+ifeq ($(USE_HDF), 1)
+    export HDF5_CXX := $(CC)
+    export HDF5_CLINKER := $(CC)
+
+    ifeq ($(USE_OMP), 1)
+		CC = h5pcc.openmpi
+    else
+    	CC = h5c++
+    endif
+endif
+
+# Headers
+HDRS = ${SRCS:.cc=.h} cudaErrorCheck.h
+
+# Objects
+OBJS = ${SRCS:.cc=.o}
+
+# Rootfinder objects
+RTFIND_OBJS = $(RTFIND_SRC_DIR)/dogleg.o \
+							$(RTFIND_SRC_DIR)/dpmpar.o \
+							$(RTFIND_SRC_DIR)/enorm.o \
+							$(RTFIND_SRC_DIR)/fdjac1.o \
+							$(RTFIND_SRC_DIR)/hybrd.o \
+							$(RTFIND_SRC_DIR)/hybrd1.o \
+							$(RTFIND_SRC_DIR)/qform.o \
+							$(RTFIND_SRC_DIR)/qrfac.o \
+							$(RTFIND_SRC_DIR)/r1mpyq.o \
+							$(RTFIND_SRC_DIR)/r1updt.o
+
+# Programmes
+PROGS = ${SRCS:.cc=}
+
+# Main programme executable
+EXEC = main
+
+# Rootfinder executables
+RTFIND = buildRootfinder
+
+
+# Build everything
+build : $(RTFIND) $(EXEC)
+
+# Make and run all tests on simulation scripts and run main
+all : $(RTFIND) $(EXEC)
+	cd $(TEST_DIR) && $(MAKE) test
+	@$(MAKE) run
+
+# Build and run main (does not run any tests if called explicitly)
+run : $(RTFIND) $(EXEC)
+	@echo '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
+	@echo ''
+	@echo ''
+	@echo '######################################################'
+	@echo '#              Executing main programme              #'
+	@echo '######################################################'
+	@./$(EXEC)
+
+clean :
+	rm	-f	$(EXEC)	*.o	*.gch
+
+
+#################
+# Build objects #
+#################
+
+simData.o : $(MODULE_DIR)/simData.cc $(INC_DIR)/simData.h
+	$(CC)	$<	-c	$(CXXFLAGS) -I$(INC_DIR)
+
+initFunc.o : $(MODULE_DIR)/initFunc.cc $(INC_DIR)/initFunc.h	$(INC_DIR)/simData.h
+	$(CC)	$<	-c	$(CXXFLAGS) -I$(INC_DIR)
+
+srmhd.o : $(MODULE_DIR)/srmhd.cc $(INC_DIR)/srmhd.h
+	$(CC)	$<	-c	$(CXXFLAGS) -I$(INC_DIR) -I$(RTFIND_INC_DIR)
+
+simulation.o : $(MODULE_DIR)/simulation.cc	$(INC_DIR)/simulation.h $(INC_DIR)/model.h	$(INC_DIR)/simData.h $(INC_DIR)/saveData.h
+	$(CC)	$<	-c	$(CXXFLAGS) -I$(INC_DIR)
+
+main.o : ./main.cc $(INC_DIR)/simulation.h $(INC_DIR)/model.h $(INC_DIR)/simData.h $(INC_DIR)/initFunc.h
+	$(CC)	$<	-c	$(CXXFLAGS) -I$(INC_DIR) -I$(RTFIND_INC_DIR) $(CXXFLAGS)
+
+wenoUpwinds.o : $(MODULE_DIR)/wenoUpwinds.cc $(INC_DIR)/wenoUpwinds.h
+	$(CC)	$<	-c	$(CXXFLAGS) -I$(INC_DIR)
+
+weno.o : $(MODULE_DIR)/weno.cc $(INC_DIR)/weno.h $(INC_DIR)/wenoUpwinds.h
+	$(CC)	$<	-c	$(CXXFLAGS) -I$(INC_DIR)
+
+RK2.o : $(MODULE_DIR)/RK2.cc $(INC_DIR)/RK2.h
+		$(CC)	$<	-c	$(CXXFLAGS) -I$(INC_DIR)
+
+rkSplit.o : $(MODULE_DIR)/rkSplit.cc $(INC_DIR)/rkSplit.h $(INC_DIR)/RK2.h
+	$(CC)	$<	-c	$(CXXFLAGS) -I$(INC_DIR)
+
+boundaryConds.o : $(MODULE_DIR)/boundaryConds.cc $(INC_DIR)/boundaryConds.h
+	$(CC)	$<	-c	$(CXXFLAGS) -I$(INC_DIR)
+
+parallelBoundaryConds.o : $(MODULE_DIR)/parallelBoundaryConds.cc $(INC_DIR)/parallelBoundaryConds.h
+	$(CC)	$<	-c	$(CXXFLAGS) -I$(INC_DIR)
+
+serialSaveData.o : $(MODULE_DIR)/serialSaveData.cc $(INC_DIR)/serialSaveData.h
+	$(CC)   $<  -c  $(CXXFLAGS) -I$(INC_DIR)
+
+serialSaveDataHDF5.o : $(MODULE_DIR)/serialSaveDataHDF5.cc $(INC_DIR)/serialSaveDataHDF5.h
+	$(CC)   $<  -c  $(CXXFLAGS) -I$(INC_DIR)
+
+parallelSaveData.o : $(MODULE_DIR)/parallelSaveData.cc $(INC_DIR)/parallelSaveData.h
+	$(CC)   $<  -c  $(CXXFLAGS) -I$(INC_DIR)
+
+parallelSaveDataHDF5.o : $(MODULE_DIR)/parallelSaveDataHDF5.cc $(INC_DIR)/parallelSaveDataHDF5.h
+	$(CC)   $<  -c  $(CXXFLAGS) -I$(INC_DIR)
+
+fluxVectorSplitting.o : $(MODULE_DIR)/fluxVectorSplitting.cc $(INC_DIR)/fluxVectorSplitting.h $(INC_DIR)/weno.h
+	$(CC)	$<	-c	$(CXXFLAGS) -I$(INC_DIR)
+
+serialEnv.o : $(MODULE_DIR)/serialEnv.cc $(INC_DIR)/serialEnv.h
+	$(CC)   $<  -c  $(CXXFLAGS) -I$(INC_DIR)
+
+parallelEnv.o : $(MODULE_DIR)/parallelEnv.cc $(INC_DIR)/parallelEnv.h
+	$(CC)   $<  -c  $(CXXFLAGS) -I$(INC_DIR)
+
+# Executable
+main : main.o $(OBJS) $(RTFIND_OBJS)
+	$(CC)	$^ -o $@ $(CXXFLAGS) $(CXXFLAGS)
+
+buildRootfinder:
+	@cd $(RTFIND_DIR) && $(MAKE)	objects

Examples/KelvinHelmholtz/SingleFluidIdealRandomMPIHDF5/README.md

@@ -0,0 +1,56 @@
+Model
+-----
+  - Ideal MHD
+
+
+
+Initial data
+------------
+  - Kelvin-Helmholtz instability with random perturbation
+
+
+
+InteractivePlot help
+--------------------
+  - This is a 2D problem, so we want to plot heatmaps. After running
+  interactivePlot.py, use
+
+      `Plot.plotHeatMaps()`
+
+  to plot the primitive variables as heatmaps.
+
+  - Use
+
+      `help(Plot.plotHeatMaps)`
+
+    for more information about plotting heatmaps.
+
+
+
+Animation help
+--------------
+  - This is a great simulations to generate animations with. Running animation.py
+  as main will save a gif of the density evolution in the Examples/ directory.
+
+
+
+Notes
+-----
+  - As this is a 2D simulation, it will take significantly longer to run. Expect
+  this to take five minutes or so to generate the timeseries data.
+
+  - If you have the time, I recommend running a higher resolution simulation of
+  this. It is aesthetically pleasing!
+
+  - This simulation should be run using `make main` and then `./main seed` where
+  `seed` is some integar seed for the random perturbation on the layer.
+
+
+Parallel version -- current limitations
+---------------------------------------
+  - Save data gathers all state vectors to proc0 so while this version will be faster, it doesn't currently allow a larger problem to be simulated than will fit on one node
+  - nx, ny, nz must currently be even multiple of nxRanks, nyRanks, nzRanks, and these must be specified as args to PlatformEnv manually. 
+  - nxRanks x nyRanks x nzRanks must match the total number of processes launched
+  - Save data does not include boundary cells
+
+

Examples/KelvinHelmholtz/SingleFluidIdealRandomMPIHDF5/main.cc

@@ -0,0 +1,89 @@
+// Serial main
+#include "simData.h"
+#include "simulation.h"
+#include "initFunc.h"
+#include "srmhd.h"
+#include "parallelBoundaryConds.h"
+#include "rkSplit.h"
+#include "fluxVectorSplitting.h"
+#include "parallelSaveDataHDF5.h"
+#include "platformEnv.h"
+#include "weno.h"
+#include <cstring>
+
+using namespace std;
+
+int main(int argc, char *argv[]) {
+
+
+  // Set up domain
+  int Ng(4);
+  int nx(64);
+  int ny(64);
+  int nz(0);
+  double xmin(0.0);
+  double xmax(1.0);
+  double ymin(0.0);
+  double ymax(1.0);
+  double zmin(0.0);
+  double zmax(1.0);
+  double endTime(3.0);
+  double cfl(0.6);
+  double gamma(4.0/3.0);
+  double sigma(10);
+  double cp(1.0);
+  double mu1(-100);
+  double mu2(100);
+  int frameSkip(10);
+  bool output(false);
+  if (argc != 2) throw std::invalid_argument("Expected ./main seed!\n");
+  int seed(atoi(argv[1]));
+
+  double nxRanks(2);
+  double nyRanks(2);
+  double nzRanks(1);
+
+  ParallelEnv env(&argc, &argv, nxRanks, nyRanks, nzRanks);
+
+  Data data(nx, ny, nz, xmin, xmax, ymin, ymax, zmin, zmax, endTime, &env,
+            cfl, Ng, gamma, sigma, cp, mu1, mu2, frameSkip);
+
+  // Choose particulars of simulation
+  SRMHD model(&data);
+
+  Weno3 weno(&data);
+
+  FVS fluxMethod(&data, &weno, &model);
+
+  ParallelPeriodic bcs(&data, &env);
+
+  Simulation sim(&data, &env);
+
+  if (env.rank==0){
+      printf("Seed: %d\n", seed);
+  }
+
+  KHRandomInstabilitySingleFluid init(&data, 1, seed);
+
+  RKSplit timeInt(&data, &model, &bcs, &fluxMethod);
+
+  ParallelSaveDataHDF5 save(&data, &env, "data_parallel", ParallelSaveDataHDF5::OUTPUT_ALL);
+
+  // Now objects have been created, set up the simulation
+  sim.set(&init, &model, &timeInt, &bcs, &fluxMethod, &save);
+
+  // Time execution of programme
+  //double startTime(omp_get_wtime());
+
+  // Run until end time and save results
+  sim.evolve(output);
+
+  //double timeTaken(omp_get_wtime() - startTime);
+
+  save.saveAll();
+  //printf("\nRuntime: %.5fs\nCompleted %d iterations.\n", timeTaken, data.iters);
+  if(env.rank==0) printf("\nCompleted %d iterations.\n", data.iters);
+
+  return 0;
+
+}