Merge pull request #4119 from pnorbert/user-options-yaml

Add support for user options in ~/.config/adios2/adios2.yaml

CMakeLists.txt

@@ -183,7 +183,7 @@ adios_option(Profiling  "Enable support for profiling" AUTO)
 adios_option(Endian_Reverse "Enable support for Little/Big Endian Interoperability" AUTO)
 adios_option(Sodium     "Enable support for Sodium for encryption" AUTO)
 adios_option(Catalyst   "Enable support for in situ visualization plugin using ParaView Catalyst" AUTO)
-adios_option(Campaign   "Enable support for Campaigns (requires SQLite3 and ZLIB)" OFF)
+adios_option(Campaign   "Enable support for Campaigns (requires SQLite3 and ZLIB)" AUTO)
 adios_option(AWSSDK     "Enable support for S3 compatible storage using AWS SDK's S3 module" OFF)
 adios_option(Derived_Variable    "Enable support for derived variables" OFF)
 adios_option(PIP        "Enable support for pip packaging" OFF)

source/adios2/common/ADIOSTypes.h

@@ -357,6 +357,33 @@ std::string ToString(const Dims &dims);
 std::string ToString(const Box<Dims> &box);
 std::string ToString(const MemorySpace value);
 
+/** UserOptions holds all user options from ~/.config/adios2/adios2.yaml */
+struct UserOptions
+{
+    struct General
+    {
+        int verbose;
+    };
+
+    struct Campaign
+    {
+        bool active;
+        int verbose;
+        std::string hostname;
+        std::string campaignstorepath;
+        std::string cachepath;
+    };
+
+    struct SST
+    {
+        int verbose;
+    };
+
+    General general;
+    Campaign campaign;
+    SST sst;
+};
+
 /**
  * os << [adios2_type] enables output of adios2 enums/classes directly
  * to output streams (e.g. std::cout), if ToString() can handle [adios2_type].

source/adios2/core/ADIOS.cpp

@@ -19,6 +19,7 @@
 #include "adios2/core/IO.h"
 #include "adios2/helper/adiosCommDummy.h"
 #include "adios2/helper/adiosFunctions.h" //InquireKey, BroadcastFile
+#include "adios2/helper/adiosYAML.h"
 #include "adios2/operator/OperatorFactory.h"
 #include <adios2sys/SystemTools.hxx>
 
@@ -106,6 +107,10 @@ std::mutex PerfStubsMutex;
 static std::atomic_uint adios_refcount(0); // adios objects at the same time
 static std::atomic_uint adios_count(0);    // total adios objects during runtime
 
+/** User defined options from ~/.config/adios2/adios2.yaml if it exists */
+static adios2::UserOptions UserOptions;
+const adios2::UserOptions &ADIOS::GetUserOptions() { return UserOptions; };
+
 ADIOS::ADIOS(const std::string configFile, helper::Comm comm, const std::string hostLanguage)
 : m_HostLanguage(hostLanguage), m_Comm(std::move(comm)), m_ConfigFile(configFile),
   m_CampaignManager(m_Comm)
@@ -124,6 +129,7 @@ ADIOS::ADIOS(const std::string configFile, helper::Comm comm, const std::string
         }
     }
 #endif
+    ProcessUserConfig();
     if (!configFile.empty())
     {
         if (!adios2sys::SystemTools::FileExists(configFile))
@@ -143,8 +149,11 @@ ADIOS::ADIOS(const std::string configFile, helper::Comm comm, const std::string
 #ifdef ADIOS2_HAVE_KOKKOS
     m_GlobalServices.Init_Kokkos_API();
 #endif
-    std::string campaignName = "campaign_" + std::to_string(adios_count);
-    m_CampaignManager.Open(campaignName);
+    if (UserOptions.campaign.active)
+    {
+        std::string campaignName = "campaign_" + std::to_string(adios_count);
+        m_CampaignManager.Open(campaignName, UserOptions);
+    }
 }
 
 ADIOS::ADIOS(const std::string configFile, const std::string hostLanguage)
@@ -166,7 +175,26 @@ ADIOS::~ADIOS()
     {
         m_GlobalServices.Finalize();
     }
-    m_CampaignManager.Close();
+    if (UserOptions.campaign.active)
+    {
+        m_CampaignManager.Close();
+    }
+}
+
+void ADIOS::ProcessUserConfig()
+{
+    // read config parameters from config file
+    std::string homePath;
+#ifdef _WIN32
+    homePath = getenv("HOMEPATH");
+#else
+    homePath = getenv("HOME");
+#endif
+    const std::string cfgFile = homePath + "/.config/adios2/adios2.yaml";
+    if (adios2sys::SystemTools::FileExists(cfgFile))
+    {
+        helper::ParseUserOptionsFile(m_Comm, cfgFile, UserOptions, homePath);
+    }
 }
 
 IO &ADIOS::DeclareIO(const std::string name, const ArrayOrdering ArrayOrder)
@@ -330,7 +358,10 @@ void ADIOS::YAMLInitIO(const std::string &configFileYAML, const std::string &con
 
 void ADIOS::RecordOutputStep(const std::string &name, const size_t step, const double time)
 {
-    m_CampaignManager.Record(name, step, time);
+    if (UserOptions.campaign.active)
+    {
+        m_CampaignManager.Record(name, step, time);
+    }
 }
 
 void ADIOS::Global_init_AWS_API()

source/adios2/core/ADIOS.h

@@ -164,6 +164,9 @@ class ADIOS
     void RecordOutputStep(const std::string &name, const size_t step = UnknownStep,
                           const double time = UnknownTime);
 
+    /** A constant reference to the user options from ~/.config/adios2/adios2.yaml */
+    static const adios2::UserOptions &GetUserOptions();
+
 private:
     /** Communicator given to parallel constructor. */
     helper::Comm m_Comm;
@@ -204,6 +207,8 @@ class ADIOS
     void YAMLInitIO(const std::string &configFileYAML, const std::string &configFileContents,
                     core::IO &io);
 
+    void ProcessUserConfig();
+
 private:
     /* Global services that we want to initialize at most once and shutdown
        automatically when the ADIOS object is destructed. This only works

source/adios2/core/Engine.cpp

@@ -23,7 +23,8 @@ namespace core
 
 Engine::Engine(const std::string engineType, IO &io, const std::string &name, const Mode openMode,
                helper::Comm comm)
-: m_EngineType(engineType), m_IO(io), m_Name(name), m_OpenMode(openMode), m_Comm(std::move(comm))
+: m_EngineType(engineType), m_IO(io), m_Name(name), m_OpenMode(openMode), m_Comm(std::move(comm)),
+  m_UserOptions(io.m_ADIOS.GetUserOptions())
 {
     m_FailVerbose = (m_Comm.Rank() == 0);
 }

source/adios2/core/Engine.h

@@ -505,6 +505,9 @@ class Engine
      *  if no communicator is passed */
     helper::Comm m_Comm;
 
+    /** User options parsed by the ADIOS object, here just for easy reference */
+    const UserOptions &m_UserOptions;
+
     /** keeps track of current advance status */
     StepStatus m_AdvanceStatus = StepStatus::OK;
 

source/adios2/engine/campaign/CampaignData.cpp

@@ -17,7 +17,6 @@
 #include <iostream>
 #include <sys/stat.h>
 #include <sys/types.h>
-#include <unistd.h>
 #include <zlib.h>
 
 namespace adios2
@@ -70,12 +69,13 @@ static int sqlcb_bpdataset(void *p, int argc, char **argv, char **azColName)
 {
     CampaignData *cdp = reinterpret_cast<CampaignData *>(p);
     CampaignBPDataset cds;
-    size_t hostid = helper::StringToSizeT(std::string(argv[0]), "SQL callback convert text to int");
-    size_t dirid = helper::StringToSizeT(std::string(argv[1]), "SQL callback convert text to int");
+    size_t dsid = helper::StringToSizeT(std::string(argv[0]), "SQL callback convert text to int");
+    size_t hostid = helper::StringToSizeT(std::string(argv[1]), "SQL callback convert text to int");
+    size_t dirid = helper::StringToSizeT(std::string(argv[2]), "SQL callback convert text to int");
     cds.hostIdx = hostid - 1; // SQL rows start from 1, vector idx start from 0
     cds.dirIdx = dirid - 1;
-    cds.name = argv[2];
-    cdp->bpdatasets.push_back(cds);
+    cds.name = argv[3];
+    cdp->bpdatasets[dsid] = cds;
     return 0;
 };
 
@@ -84,16 +84,15 @@ static int sqlcb_bpfile(void *p, int argc, char **argv, char **azColName)
     CampaignData *cdp = reinterpret_cast<CampaignData *>(p);
     CampaignBPFile cf;
     size_t dsid = helper::StringToSizeT(std::string(argv[0]), "SQL callback convert text to int");
-    cf.bpDatasetIdx = dsid - 1;
+    cf.bpDatasetIdx = dsid;
     cf.name = std::string(argv[1]);
     int comp = helper::StringTo<int>(std::string(argv[2]), "SQL callback convert text to int");
     cf.compressed = (bool)comp;
     cf.lengthOriginal =
         helper::StringToSizeT(std::string(argv[3]), "SQL callback convert text to int");
     cf.lengthCompressed =
         helper::StringToSizeT(std::string(argv[4]), "SQL callback convert text to int");
-    cf.ctime = static_cast<long>(
-        helper::StringTo<int64_t>(std::string(argv[5]), "SQL callback convert ctime to int"));
+    cf.ctime = helper::StringTo<int64_t>(std::string(argv[5]), "SQL callback convert ctime to int");
 
     CampaignBPDataset &cds = cdp->bpdatasets[cf.bpDatasetIdx];
     cds.files.push_back(cf);
@@ -139,7 +138,7 @@ void ReadCampaignData(sqlite3 *db, CampaignData &cd)
         sqlite3_free(zErrMsg);
     }
 
-    sqlcmd = "SELECT hostid, dirid, name FROM bpdataset";
+    sqlcmd = "SELECT rowid, hostid, dirid, name FROM bpdataset";
     rc = sqlite3_exec(db, sqlcmd.c_str(), sqlcb_bpdataset, &cd, &zErrMsg);
     if (rc != SQLITE_OK)
     {
@@ -233,9 +232,9 @@ int inflateToFile(const unsigned char *source, const size_t blobsize, std::ofstr
     return ret == Z_STREAM_END ? Z_OK : Z_DATA_ERROR;
 }
 
-static long timeToSec(long ct)
+static int64_t timeToSec(int64_t ct)
 {
-    long t;
+    int64_t t;
     if (ct > 99999999999999999)
     {
         /* nanosec to sec */
@@ -258,7 +257,7 @@ static long timeToSec(long ct)
     return t;
 }
 
-static bool isFileNewer(const std::string path, long ctime)
+static bool isFileNewer(const std::string path, int64_t ctime)
 {
     int result;
 #ifdef _WIN32
@@ -273,9 +272,9 @@ static bool isFileNewer(const std::string path, long ctime)
         return false;
     }
 
-    long ct = s.st_ctime;
-    long ctSec = timeToSec(ct);
-    long ctimeSec = timeToSec(ctime);
+    int64_t ct = static_cast<int64_t>(s.st_ctime);
+    int64_t ctSec = timeToSec(ct);
+    int64_t ctimeSec = timeToSec(ctime);
 
     /*std::cout << "   Stat(" << path << "): size = " << s.st_size
               << " ct = " << ctSec << " ctime = " << ctimeSec << "\n";*/
@@ -292,13 +291,13 @@ void SaveToFile(sqlite3 *db, const std::string &path, const CampaignBPFile &bpfi
     int rc;
     char *zErrMsg = 0;
     std::string sqlcmd;
-    std::string id = std::to_string(bpfile.bpDatasetIdx + 1);
+    std::string id = std::to_string(bpfile.bpDatasetIdx);
 
     sqlite3_stmt *statement;
     sqlcmd =
         "SELECT data FROM bpfile WHERE bpdatasetid = " + id + " AND name = '" + bpfile.name + "'";
     // std::cout << "SQL statement: " << sqlcmd << "\n";
-    rc = sqlite3_prepare_v2(db, sqlcmd.c_str(), sqlcmd.size(), &statement, NULL);
+    rc = sqlite3_prepare_v2(db, sqlcmd.c_str(), static_cast<int>(sqlcmd.size()), &statement, NULL);
     if (rc != SQLITE_OK)
     {
         std::cout << "SQL error: " << zErrMsg << std::endl;

source/adios2/engine/campaign/CampaignData.h

@@ -14,8 +14,8 @@
 
 #include <fstream>
 #include <iostream>
+#include <map>
 #include <string>
-#include <unordered_map>
 #include <vector>
 
 #include <sqlite3.h>
@@ -37,11 +37,11 @@ struct CampaignHost
 struct CampaignBPFile
 {
     std::string name;
-    size_t bpDatasetIdx;
+    size_t bpDatasetIdx; // index of parent CampaignBPDataset in the map
     bool compressed;
     size_t lengthOriginal;
     size_t lengthCompressed;
-    long ctime;
+    int64_t ctime;
 };
 
 struct CampaignBPDataset
@@ -56,7 +56,7 @@ struct CampaignData
 {
     std::string version;
     std::vector<CampaignHost> hosts;
-    std::vector<CampaignBPDataset> bpdatasets;
+    std::map<size_t, CampaignBPDataset> bpdatasets;
 };
 
 void ReadCampaignData(sqlite3 *db, CampaignData &cd);

source/adios2/engine/campaign/CampaignManager.cpp

@@ -43,7 +43,7 @@ int CMapToSqlite(const CampaignRecordMap &cmap, const int rank, std::string name
                                              "SQL error on writing records:");
         sqlite3_free(zErrMsg);
     }
-    sqlcmd = "CREATE TABLE if not exists bpfiles (name);";
+    sqlcmd = "CREATE TABLE if not exists bpfiles (name PRIMARY KEY);";
     rc = sqlite3_exec(db, sqlcmd.c_str(), 0, 0, &zErrMsg);
     if (rc != SQLITE_OK)
     {
@@ -56,7 +56,7 @@ int CMapToSqlite(const CampaignRecordMap &cmap, const int rank, std::string name
 
     for (auto &r : cmap)
     {
-        sqlcmd = "INSERT INTO bpfiles (name)\n";
+        sqlcmd = "INSERT OR IGNORE INTO bpfiles (name)\n";
         sqlcmd += "VALUES('" + r.first + "');";
         rc = sqlite3_exec(db, sqlcmd.c_str(), 0, 0, &zErrMsg);
         if (rc != SQLITE_OK)
@@ -74,31 +74,27 @@ int CMapToSqlite(const CampaignRecordMap &cmap, const int rank, std::string name
     return 0;
 }
 
-CampaignManager::CampaignManager(adios2::helper::Comm &comm)
-{
-    m_WriterRank = comm.Rank();
-    if (m_Verbosity == 5)
-    {
-        std::cout << "Campaign Manager " << m_WriterRank << " constructor called" << std::endl;
-    }
-}
+CampaignManager::CampaignManager(adios2::helper::Comm &comm) { m_WriterRank = comm.Rank(); }
 
 CampaignManager::~CampaignManager()
 {
-    if (m_Verbosity == 5)
-    {
-        std::cout << "Campaign Manager " << m_WriterRank << " desctructor called\n";
-    }
     if (m_Opened)
     {
         Close();
     }
 }
 
-void CampaignManager::Open(const std::string &name)
+void CampaignManager::Open(const std::string &name, const UserOptions &options)
 {
-    m_Name = m_CampaignDir + "/" + name + "_" + std::to_string(m_WriterRank);
-    if (m_Verbosity == 5)
+    const UserOptions::Campaign &opts = options.campaign;
+    m_Options.active = opts.active;
+    m_Options.hostname = opts.hostname;
+    m_Options.campaignstorepath = opts.campaignstorepath;
+    m_Options.cachepath = opts.cachepath;
+    m_Options.verbose = opts.verbose;
+
+    m_Name = m_CampaignDir + PathSeparator + name + "_" + std::to_string(m_WriterRank);
+    if (m_Options.verbose > 0)
     {
         std::cout << "Campaign Manager " << m_WriterRank << " Open(" << m_Name << ")\n";
     }
@@ -107,7 +103,7 @@ void CampaignManager::Open(const std::string &name)
 
 void CampaignManager::Record(const std::string &name, const size_t step, const double time)
 {
-    if (m_Verbosity == 5)
+    if (m_Options.verbose > 0)
     {
         std::cout << "Campaign Manager " << m_WriterRank << "   Record name = " << name
                   << " step = " << step << " time = " << time << "\n";