diff --git a/include/t3pio.h b/include/t3pio.h index 7a6d8be..ff38eff 100644 --- a/include/t3pio.h +++ b/include/t3pio.h @@ -27,9 +27,12 @@ typedef struct int stripeSize; /* stripe size in bytes*/ int nWritersPer; /* number of writers per node */ int numNodes; /* number of nodes */ + int S_dne; /* the do not exceed number of stripes */ + int S_auto_max; /* min(s_dne, GOOD_CITZENSHIP_STRIPES) */ + int nStripesT3; /* Number of stripes T3PIO would choose */ + } T3PIO_results_t; -void t3pio_extract_key_values(MPI_Info info, T3PIO_results_t* r); int t3pio_set_info(MPI_Comm comm, MPI_Info info, const char *dir, ...); const char* t3pio_version(); diff --git a/libt3pio/f2c_internal.c b/libt3pio/f2c_internal.c index a4438d0..1467ae8 100644 --- a/libt3pio/f2c_internal.c +++ b/libt3pio/f2c_internal.c @@ -6,42 +6,48 @@ int t3piointernal_(int* f_comm, int* f_info, const char* dir, int* global_sz, int* max_stripes, int* mStripeSz, const char* file, - int* nWriters) + int* nWriters, int* s_dne, int* s_auto_max, int* nStripesT3) { return t3pio_internal(f_comm, f_info, dir, global_sz, max_stripes, mStripeSz, - file, nWriters); + file, nWriters, s_dne, s_auto_max, nStripesT3); } int T3PIOINTERNAL(int* f_comm, int* f_info, const char* dir, int* global_sz, int* max_stripes, int* mStripeSz, const char* file, - int* nWriters) + int* nWriters, int* s_dne, int* s_auto_max, int* nStripesT3) { return t3pio_internal(f_comm, f_info, dir, global_sz, max_stripes, mStripeSz, - file, nWriters); + file, nWriters, s_dne, s_auto_max, nStripesT3); } int t3piointernal(int* f_comm, int* f_info, const char* dir, int* global_sz, int* max_stripes, int* mStripeSz, const char* file, - int* nWriters) + int* nWriters, int* s_dne, int* s_auto_max, int* nStripesT3) { return t3pio_internal(f_comm, f_info, dir, global_sz, max_stripes, mStripeSz, - file, nWriters); + file, nWriters, s_dne, s_auto_max, nStripesT3) } int t3pio_internal(int* f_comm, int* f_info, const char* dir, int* global_sz, int* max_stripes, int* mStripeSz, const char* file, - int* nWriters) + int* nWriters, int* s_dne, int* s_auto_max, int* nStripesT3) { - int ierr; - MPI_Comm comm = MPI_Comm_f2c(*f_comm); - MPI_Info info = MPI_Info_f2c(*f_info); + int ierr; + T3PIO_results_t results; + MPI_Comm comm = MPI_Comm_f2c(*f_comm); + MPI_Info info = MPI_Info_f2c(*f_info); ierr = t3pio_set_info(comm, info, dir, T3PIO_GLOBAL_SIZE, *global_sz, T3PIO_STRIPE_COUNT, *max_stripes, T3PIO_MAX_AGGREGATORS, *nWriters, T3PIO_STRIPE_SIZE_MB, *mStripeSz, - T3PIO_FILE, file); - *f_info = MPI_Info_c2f(info); + T3PIO_FILE, file, + T3PIO_RESULTS, &results); + + *s_dne = results.S_dne; + *s_auto_max = results.S_auto_max; + *nStripesT3 = results.nStripesT3; + *f_info = MPI_Info_c2f(info); return ierr; } diff --git a/libt3pio/pfs.c b/libt3pio/pfs.c index 9a9ef52..0d08a30 100644 --- a/libt3pio/pfs.c +++ b/libt3pio/pfs.c @@ -49,6 +49,9 @@ void t3pio_init(T3Pio_t* t3) t3->globalSz = -1; t3->nodeMem = -1; t3->fn = NULL; + t3->S_dne = -1; + t3->S_auto_max = -1; + t3->nStripesT3 = -1; if ((p=getenv("T3PIO_STRIPE_COUNT")) != NULL) t3->maxStripes = strtol(p, (char **) NULL, 10); diff --git a/libt3pio/set_info.c b/libt3pio/set_info.c index 35a5e95..b629dbb 100644 --- a/libt3pio/set_info.c +++ b/libt3pio/set_info.c @@ -19,7 +19,7 @@ int t3pio_parse_int_arg(int orig, int value) return value; } -void t3pio_extract_key_values(MPI_Info info, T3PIO_results_t* r) +void t3pio_extract_key_values(MPI_Info info, T3Pio_t* t3, T3PIO_results_t* r) { int ierr; if (r) @@ -39,20 +39,21 @@ void t3pio_extract_key_values(MPI_Info info, T3PIO_results_t* r) else if (strcmp("striping_unit", key) == 0) sscanf(value, "%d", &(*r).stripeSize); } } + r->S_dne = t3->S_dne; + r->S_auto_max = t3->S_auto_max; + r->nStripesT3 = t3->nStripesT3; } int t3pio_set_info(MPI_Comm comm, MPI_Info info, const char* path, ...) { - T3Pio_t t3; int argType; int ierr = 0; va_list ap; int nProcs, myProc; char buf[128]; - int S_dne, S_auto_max; int mStripeSz = -1; T3PIO_results_t *results = NULL; @@ -111,12 +112,13 @@ int t3pio_set_info(MPI_Comm comm, MPI_Info info, const char* path, ...) t3.stripeSz = 1024 * 1024; - S_dne = t3pio_maxStripes(comm, myProc, path); + t3.S_dne = t3pio_maxStripes(comm, myProc, path); + t3.S_auto_max = min(t3.S_dne, GOOD_CITZENSHIP_STRIPES); t3pio_numComputerNodes(comm, nProcs, &t3.numNodes); if (getenv("T3PIO_BYPASS")) { if (results) - t3pio_extract_key_values(info, results); + t3pio_extract_key_values(info, &t3, results); return ierr; } @@ -126,19 +128,20 @@ int t3pio_set_info(MPI_Comm comm, MPI_Info info, const char* path, ...) t3.numStripes = t3pio_readStripes(comm, myProc, t3.fn); t3.stripeSz = -1; /* Can not change stripe sz on files to be read */ + t3.nStripesT3 = t3.numStripes; } else if (t3.maxStripes != T3PIO_BYPASS) { - S_auto_max = min(S_dne, GOOD_CITZENSHIP_STRIPES); - if (t3.numNodes >= S_auto_max ) - t3.numStripes = S_auto_max; + if (t3.numNodes >= t3.S_auto_max ) + t3.numStripes = t3.S_auto_max; else { - int k = min(S_auto_max / t3.numNodes, MAX_STRIPES_PER_NODE); + int k = min(t3.S_auto_max / t3.numNodes, MAX_STRIPES_PER_NODE); t3.numStripes = k * t3.numNodes; } + t3.nStripesT3 = t3.numStripes; if (t3.maxStripes > 0) - t3.numStripes = min(S_dne, t3.maxStripes); + t3.numStripes = min(t3.S_dne, t3.maxStripes); } if (t3.maxWriters == T3PIO_BYPASS) @@ -167,7 +170,7 @@ int t3pio_set_info(MPI_Comm comm, MPI_Info info, const char* path, ...) } if (results) - t3pio_extract_key_values(info, results); + t3pio_extract_key_values(info, &t3, results); return ierr; } diff --git a/libt3pio/t3pio.f90 b/libt3pio/t3pio.f90 index 35f5050..20e3c5b 100644 --- a/libt3pio/t3pio.f90 +++ b/libt3pio/t3pio.f90 @@ -58,6 +58,7 @@ subroutine t3pio_set_info(comm, info, dirIn, ierr, global_size, & integer :: nWriters integer :: gblSz, maxStripes, f integer :: t3piointernal, maxStripeSz + integer :: s_dne, s_auto_max, nStripesT3 type(T3PIO_Results_t), optional :: results nWriters = T3PIO_OPTIMAL @@ -80,7 +81,7 @@ subroutine t3pio_set_info(comm, info, dirIn, ierr, global_size, & usrFile = usrFile(1:len-1) // CHAR(0) ierr = t3piointernal(comm, info, dir, gblSz, maxStripes, maxStripeSz, & - usrFile, nWriters) + usrFile, nWriters, s_dne, s_auto_max, nStripesT3) if (present(results)) then call t3pio_extract_key_values(info, results) diff --git a/libt3pio/t3pio_internal.h b/libt3pio/t3pio_internal.h index 8d4d908..86c13f7 100644 --- a/libt3pio/t3pio_internal.h +++ b/libt3pio/t3pio_internal.h @@ -1,6 +1,7 @@ #ifndef T3PIO_INTERNAL_H #define T3PIO_INTERNAL_H +#include "t3pio.h" #include typedef struct { @@ -15,9 +16,13 @@ typedef struct int nodeMem; int stripeSz; int maxWriters; + int S_dne; + int S_auto_max; + int nStripesT3; char* fn; } T3Pio_t; +void t3pio_extract_key_values(MPI_Info info, T3Pio_t *t3, T3PIO_results_t* r); void t3pio_init(T3Pio_t* t3); void t3pio_numComputerNodes(MPI_Comm comm, int nProc, int* numNodes); int t3pio_maxStripes(MPI_Comm comm, int myProc, const char* dir); diff --git a/tools/generateGraphs b/tools/generateGraphs index 0552c50..8529f1a 100755 --- a/tools/generateGraphs +++ b/tools/generateGraphs @@ -22,7 +22,7 @@ Version = "0.0" require("strict") require("pairsByKeys") -require("string_split") +require("string_utils") local Dbg = require("Dbg") local Optiks = require("Optiks") local concatTbl = table.concat diff --git a/unstructTest/main.C b/unstructTest/main.C index 75ed6c8..4f04273 100644 --- a/unstructTest/main.C +++ b/unstructTest/main.C @@ -73,10 +73,13 @@ void outputResults(CmdLineOptions& cmd, ParallelIO& pio) " rate (MB/s): %12.3f\n" " wrtStyle: %12s\n" " xferStyle: %12s\n" + " S_dne: %12d\n" + " S_auto_max: %12d\n" + " nStripesT3: %12d\n" " t3pioV: %12s\n", P.nProcs, cmd.localSz, pio.numvar(), pio.nIOUnits(), pio.aggregators(), pio.nStripes(), pio.stripeSzMB(), fileSz, pio.time(), pio.totalTime(), - pio.rate(), cmd.wrtStyle.c_str(), cmd.xferStyle.c_str(), t3pioV); - + pio.rate(), cmd.wrtStyle.c_str(), cmd.xferStyle.c_str(), pio.dne_stripes(), + pio.auto_max_stripes(), pio.nStripesT3(), t3pioV); } } diff --git a/unstructTest/parallel_io.C b/unstructTest/parallel_io.C index 3d92adf..616fb0e 100644 --- a/unstructTest/parallel_io.C +++ b/unstructTest/parallel_io.C @@ -30,7 +30,8 @@ Var_t varT[] = ParallelIO::ParallelIO() : m_t(0.0), m_rate(0.0), m_totalSz(1.0), m_nStripes(1), - m_nIOUnits(1), m_stripeSz(-1), m_numvar(1), m_aggregators(0) + m_nIOUnits(1), m_stripeSz(-1), m_numvar(1), m_aggregators(0), + m_dne_stripes(-1), m_auto_max_stripes(-1), m_stripesT3(-1) {} @@ -120,7 +121,10 @@ void ParallelIO::h5writer(CmdLineOptions& cmd) T3PIO_STRIPE_SIZE_MB, cmd.stripeSz, T3PIO_MAX_AGGREGATORS, cmd.maxWriters, T3PIO_RESULTS, &results); - m_nIOUnits = results.numIO; + m_nIOUnits = results.numIO; + m_dne_stripes = results.S_dne_stripes; + m_auto_max_stripes = results.S_auto_max_stripes; + m_nStripesT3 = results.nStripesT3; } xfer_mode = (cmd.collective) ? H5FD_MPIO_COLLECTIVE : H5FD_MPIO_INDEPENDENT; diff --git a/unstructTest/parallel_io.h b/unstructTest/parallel_io.h index 9ee3dcc..95c900f 100644 --- a/unstructTest/parallel_io.h +++ b/unstructTest/parallel_io.h @@ -17,17 +17,20 @@ class ParallelIO void h5writer(CmdLineOptions& cmd); void MPIIOwriter(CmdLineOptions& cmd); void add_attribute(hid_t id, const char* descript, const char* value); - double rate() { return m_rate;} - double time() { return m_t;} - double totalTime() { return m_totalTime;} - double totalSz() { return m_totalSz;} - int nStripes() { return m_nStripes;} - int nIOUnits() { return m_nIOUnits;} - int aggregators() { return m_aggregators;} - int numvar() { return m_numvar;} - int stripeSz() { return m_stripeSz;} - int nWriters() { return m_nWriters; } - int stripeSzMB() { return m_stripeSz/(1024*1024);} + double rate() { return m_rate;} + double time() { return m_t;} + double totalTime() { return m_totalTime;} + double totalSz() { return m_totalSz;} + int nStripes() { return m_nStripes;} + int nIOUnits() { return m_nIOUnits;} + int aggregators() { return m_aggregators;} + int numvar() { return m_numvar;} + int stripeSz() { return m_stripeSz;} + int nWriters() { return m_nWriters; } + int stripeSzMB() { return m_stripeSz/(1024*1024);} + int dne_stripes() { return m_dne_stripes; } + int auto_max_stripes() { return m_auto_max_stripes; } + int nStripesT3() { return m_nStripesT3; } private: double m_t; @@ -40,6 +43,9 @@ class ParallelIO int m_numvar; int m_nWriters; int m_aggregators; + int m_dne_stripes; + int m_auto_max_stripes; + int m_nStripesT3; }; #endif // PARALLELIO_H