Fixed bugs

README.md

@@ -2,7 +2,7 @@
 
 FDPS is a general-purpose, high-performance library for particle simulations.
 
-The current version is 5.0d. The previous versions are [here](https://github.com/FDPS/FDPS/releases).
+The current version is 5.0e. The previous versions are [here](https://github.com/FDPS/FDPS/releases).
 
 We maintain this from subversion-over-github interface.
 

sample/c++/nbody+sph/Makefile.K

@@ -0,0 +1,73 @@
+PS_PATH = ../../../src/
+INC = -I$(PS_PATH)
+
+#CXX = time -p FCCpx
+CXX = time -p mpiFCCpx
+CXXFLAGS = -std=c++11 -Kfast -Ksimd=2 -Krestp=all
+CXXFLAGS += -DPARTICLE_SIMULATOR_THREAD_PARALLEL -Kopenmp
+CXXFLAGS += -DPARTICLE_SIMULATOR_MPI_PARALLEL
+
+# fdps-autotest-set-vars (DO NOT CHANGE THIS LINE)
+
+# Simulation control macros 
+CXXFLAGS += -DENABLE_VARIABLE_SMOOTHING_LENGTH
+CXXFLAGS += -DUSE_ENTROPY
+CXXFLAGS += -DUSE_BALSARA_SWITCH
+#CXXFLAGS += -DUSE_PRESCR_OF_THOMAS_COUCHMAN_1992
+CXXFLAGS += -DISOTHERMAL_EOS
+
+CXXFLAGS += -DINITIAL_CONDITION=0
+#CXXFLAGS += -DINITIAL_CONDITION=1
+#CXXFLAGS += -DINITIAL_CONDITION=2
+#CXXFLAGS += -DINITIAL_CONDITION=3
+
+CXXFLAGS += -DREAD_DATA_WITH_BYTESWAP
+
+#use_phantom_grape_x86 = yes
+
+# fdps-autotest-set-vars (DO NOT CHANGE THIS LINE)
+
+ifeq ($(use_phantom_grape_x86),yes)
+PG_ROOT = $(PS_PATH)/phantom_grape_x86/G5/newton/libpg5
+INC += -I$(PG_ROOT)
+CXXFLAGS += -DENABLE_PHANTOM_GRAPE_X86
+LIBS = -L$(PG_ROOT) -lpg5
+PG_BUILD = cd $(PG_ROOT) && $(MAKE) distclean libpg5.a
+PG_CLEAN = cd $(PG_ROOT) && $(MAKE) distclean
+else
+PG_BUILD =
+PG_CLEAN = 
+endif
+
+# Job submission variables 
+QSUB = pjsub
+JOB_FILE_NAME = job.K.sh
+WORKDIR = /data/ra000008/namekata/sample/c++/nbody+sph
+
+CPPOBJS = $(patsubst %.cpp, %.o, $(wildcard *.cpp))
+CPPHDRS = $(wildcard *.h) $(wildcard *.hpp)
+PROGRAM = nbodysph.out
+
+.PHONY:	clean all
+
+all: $(CPPOBJS) $(CPPHDRS)
+	$(PG_BUILD)
+	$(CXX) $(CXXFLAGS) $(CPPOBJS) -o $(PROGRAM) $(LIBS) $(INC)
+	mkdir -p $(WORKDIR)
+	mv $(PROGRAM) $(WORKDIR)
+	cp $(JOB_FILE_NAME) $(WORKDIR)
+	cp -fR ./magi_data $(WORKDIR)/
+	cd $(WORKDIR); $(QSUB) $(JOB_FILE_NAME) 
+
+%.o: %.cpp $(CPPHDRS)
+	$(CXX) -c $< $(CXXFLAGS) $(INC)
+
+clean:
+	rm -f $(CPPOBJS)
+
+distclean: clean
+	$(PG_CLEAN)
+	rm -f $(PROGRAM)
+	rm -rf result
+
+# fdps-autotest-run (DO NOT CHANGE THIS LINE)

sample/c++/nbody+sph/ic.hpp

@@ -2,6 +2,29 @@
 #include "mathematical_constants.h"
 #include "physical_constants.h"
 
+template <class T>
+T byteswap(const T val) {
+    constexpr int size = sizeof(T);
+    constexpr int block_size = sizeof(uint16_t);
+    if (size > block_size) {
+        assert(size % block_size == 0);
+        constexpr int n_block = size / block_size;
+        std::array<uint16_t *, n_block> block;
+        T T_tmp = val;
+        uint16_t * head = reinterpret_cast<uint16_t *>(&T_tmp);
+        for (int i = 0; i < n_block; i++) block[i] = head + i;
+        for (int i = 0; i < n_block/2; i++) {
+            uint16_t high = *block[i];
+            uint16_t low  = *block[n_block - 1 - i];
+            *block[n_block - 1 - i] = (high >> 8) | (high << 8);
+            *block[i] = (low >> 8) | (low << 8);
+        }
+        return T_tmp;
+    } else {
+        return val;
+    }
+}
+
 /* Class definitions */
 // The following two classes are used in function readTipsyFile,
 // which is used to read particle data created by MAGI.
@@ -39,6 +62,11 @@ void readTipsyFile(std::string& input_file_name,
     input_file.open(input_file_name.c_str(), std::ios::in | std::ios::binary);
     if (input_file) {
         input_file.read((char *)&header, sizeof(MAGI_Tipsy_header));
+#ifdef READ_DATA_WITH_BYTESWAP
+        std::cout << "nbodies = " << header.nbodies << std::endl;
+        header.nbodies = byteswap(header.nbodies);
+        std::cout << "nbodies = " << header.nbodies << std::endl;
+#endif
         ptcl.resize(header.nbodies);
         input_file.read((char *)&ptcl[0], sizeof(MAGI_Tipsy_particle)*header.nbodies);
     }
@@ -52,13 +80,23 @@ void readTipsyFile(std::string& input_file_name,
     // Copy particle data
     psys.setNumberOfParticleLocal(header.nbodies);
     for (PS::S32 i=0; i<header.nbodies; i++) {
+#ifdef READ_DATA_WITH_BYTESWAP
+        psys[i].mass  = byteswap(ptcl[i].mass);
+        psys[i].pos.x = byteswap(ptcl[i].pos[0]);
+        psys[i].pos.y = byteswap(ptcl[i].pos[1]);
+        psys[i].pos.z = byteswap(ptcl[i].pos[2]);
+        psys[i].vel.x = byteswap(ptcl[i].vel[0]);
+        psys[i].vel.y = byteswap(ptcl[i].vel[1]);
+        psys[i].vel.z = byteswap(ptcl[i].vel[2]);
+#else
         psys[i].mass  = ptcl[i].mass;
         psys[i].pos.x = ptcl[i].pos[0];
         psys[i].pos.y = ptcl[i].pos[1];
         psys[i].pos.z = ptcl[i].pos[2];
         psys[i].vel.x = ptcl[i].vel[0];
         psys[i].vel.y = ptcl[i].vel[1];
         psys[i].vel.z = ptcl[i].vel[2];
+#endif
     }
 
 }
@@ -340,7 +378,7 @@ void MakeGlassIC(PS::ParticleSystem<FP_nbody>& psys_nbody,
                  PS::F64 & time_end) {
     // Model parameters
     const PS::S64 N_nbody = 1; // dummy value
-    const PS::S64 N_sph   = std::pow(2,18);
+    const PS::S64 N_sph   = (1<<18); // 2^{18}
     // Initialize pseudorandom number generator
     PS::MTTS mt;
     mt.init_genrand(0);
@@ -379,8 +417,8 @@ void MakeGlassIC(PS::ParticleSystem<FP_nbody>& psys_nbody,
     }
     // Set boundary condition
     bc = PS::BOUNDARY_CONDITION_PERIODIC_XYZ;
-    pos_root_domain.low_  = (PS::F64vec)(-1.0, -1.0, -1.0);
-    pos_root_domain.high_ = (PS::F64vec)( 1.0,  1.0,  1.0);
+    pos_root_domain.low_  = PS::F64vec(-1.0, -1.0, -1.0);
+    pos_root_domain.high_ = PS::F64vec( 1.0,  1.0,  1.0);
     // Set gravitational softening
     eps_grav = 0.01;
     // Set I/O intervals
@@ -431,7 +469,7 @@ void GalaxyIC(PS::ParticleSystem<FP_nbody>& psys_nbody,
         psys_nbody[i].pot  = 0.0;
     }
     // Place SPH particles to form an exponential-disk
-    const PS::S64 N_sph = std::pow(2,18);
+    const PS::S64 N_sph = (1<<18); // 2^{18}
     const PS::F64 M_gas = 1.0e10 * phys_const::Msolar;
     const PS::F64 Rs = 7.0 * phys_const::kpc; // scale radius
     const PS::F64 Rt = 12.5 * phys_const::kpc; // truncation radius

sample/c++/nbody+sph/job.K.sh

@@ -0,0 +1,60 @@
+#!/bin/bash -x
+#PJM --name "nbodysph.cpp"
+#PJM -o "00stdout-%j.log"
+#PJM -j
+#PJM --norestart
+#---- for test ----
+##PJM --rsc-list "node=64"
+##PJM --rsc-list "elapse=00:30:00"
+##PJM --rsc-list "rscgrp=small"
+#---- for product ----
+#PJM --rsc-list "node=512"
+#PJM --rsc-list "elapse=24:00:00"
+#PJM --rsc-list "rscgrp=large"
+#----
+#PJM -m b,e
+#PJM --mail-list daisuke.namekata@riken.jp
+#PJM --stg-transfiles all
+#PJM --stgin "./nbodysph.out ./"
+#PJM --stgin-dir "./magi_data/dat ./magi_data/dat"
+#PJM --stgout-dir "./result ./result"
+#PJM -s
+#PJM --spath "stat-%j.log"
+#
+. /work/system/Env_base
+export OMP_STACKSIZE=512M
+mkdir -p result
+# Perform the job
+#mpiexec -n 64 ./nbodysph.out -Wl,-T
+mpiexec -n 512 ./nbodysph.out -Wl,-T
+
+# [Notes]
+#   1) In the K-computer, 1 node have 1 CPU which has 8 cores.
+#   2) "-j" option merges STDERR to STDOUT.
+#   3) "--norestart" option stops the system restarting this job. 
+#   4) The group of resource is specified by the `--rsc-list "rscgrp="` option.
+#      We have to select a resource group from the following:
+#      -----------------------------------------------------------
+#         Resource group  | Max. # of nodes  | Max. # of cores
+#      -----------------------------------------------------------
+#         small/interact  |   384            |  3072
+#         micro           |   1152           |  9216
+#         large           |   36864          |  294912
+#         huge            |   82944          |  663552
+#      -----------------------------------------------------------
+#      Note that the resource group `micro` must be performed in a dedicated
+#      directory (/scratch/groupname).
+#
+#   5) If "-m" option is given, the system informs the user abount job status
+#      by e-mails. The e-mail address should be specified by "--mail-list"
+#      option. 
+#   6) "-s" option activates the output of statistical information on job. 
+#      We can specify the name of output file by "--spath" option.
+#   7) With "-Wl,-T" option, we can perfrom IO with little endian.
+#   8) The profiling data created by fipp can be converted into 
+#      a text form by the command:
+#      $ fipppx -Icpu,balance,call,hwm -A -d profile_data
+#      On the other hand, that created by fapp can be converted by the command:
+#      $ fapppx -Impi,hwm -ttext -o profile.txt -A -d profile_data
+#
+#