From d5f056ad1f9b5a639039ea5b1eac453b3e18d692 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20B=C3=B6hme?= Date: Fri, 31 May 2019 15:59:00 -0700 Subject: [PATCH] Implement config manager and built-in channel controllers (#174) * ConfigManager: initial implementation * Reorganize built-in controllers * Add test for spot controller * Add Spot v1 controller * Rename ConfigManager.hpp -> ConfigManager.h * Add missing include --- examples/apps/cali-basic-annotations.cpp | 20 ++ include/caliper/ChannelController.h | 3 + include/caliper/ConfigManager.h | 63 +++++ include/caliper/cali-manager.h | 15 ++ manager/include/caliper/ConfigManager.hpp | 49 ++++ manager/include/caliper/cali-manager.h | 15 ++ manager/src/CMakeLists.txt | 33 +++ manager/src/ConfigManager.cpp | 230 ++++++++++++++++++ manager/src/RuntimeProfileController.cpp | 18 ++ manager/src/RuntimeProfileController.hpp | 21 ++ manager/src/SpotController.cpp | 143 +++++++++++ manager/src/SpotController.hpp | 22 ++ manager/src/test/CMakeLists.txt | 9 + manager/src/test/test_configmanager.cpp | 70 ++++++ mpi/mpi-rt/CMakeLists.txt | 3 + mpi/mpi-rt/controllers/CMakeLists.txt | 9 + mpi/mpi-rt/controllers/SpotController.cpp | 162 ++++++++++++ mpi/mpi-rt/controllers/SpotV1Controller.cpp | 65 +++++ mpi/mpi-rt/setup_mpi.cpp | 13 + src/caliper/CMakeLists.txt | 4 + src/caliper/ChannelController.cpp | 6 + src/caliper/ConfigManager.cpp | 224 +++++++++++++++++ src/caliper/controllers/CMakeLists.txt | 10 + .../controllers/EventTraceController.cpp | 58 +++++ .../controllers/RuntimeReportController.cpp | 50 ++++ src/caliper/controllers/controllers.cpp | 16 ++ src/caliper/test/CMakeLists.txt | 1 + src/caliper/test/test_configmanager.cpp | 70 ++++++ test/ci_app_tests/ci_test_mpi_before_cali.cpp | 38 ++- test/ci_app_tests/test_mpi.py | 16 ++ 30 files changed, 1447 insertions(+), 9 deletions(-) create mode 100644 include/caliper/ConfigManager.h create mode 100644 include/caliper/cali-manager.h create mode 100644 manager/include/caliper/ConfigManager.hpp create mode 100644 manager/include/caliper/cali-manager.h create mode 100644 manager/src/CMakeLists.txt create mode 100644 manager/src/ConfigManager.cpp create mode 100644 manager/src/RuntimeProfileController.cpp create mode 100644 manager/src/RuntimeProfileController.hpp create mode 100644 manager/src/SpotController.cpp create mode 100644 manager/src/SpotController.hpp create mode 100644 manager/src/test/CMakeLists.txt create mode 100644 manager/src/test/test_configmanager.cpp create mode 100644 mpi/mpi-rt/controllers/CMakeLists.txt create mode 100644 mpi/mpi-rt/controllers/SpotController.cpp create mode 100644 mpi/mpi-rt/controllers/SpotV1Controller.cpp create mode 100644 src/caliper/ConfigManager.cpp create mode 100644 src/caliper/controllers/CMakeLists.txt create mode 100644 src/caliper/controllers/EventTraceController.cpp create mode 100644 src/caliper/controllers/RuntimeReportController.cpp create mode 100644 src/caliper/controllers/controllers.cpp create mode 100644 src/caliper/test/test_configmanager.cpp diff --git a/examples/apps/cali-basic-annotations.cpp b/examples/apps/cali-basic-annotations.cpp index 358336abf..b564088e3 100644 --- a/examples/apps/cali-basic-annotations.cpp +++ b/examples/apps/cali-basic-annotations.cpp @@ -33,9 +33,26 @@ // A C++ Caliper instrumentation demo #include +#include int main(int argc, char* argv[]) { + cali::ConfigManager mgr; + + mgr.use_mpi(false); + + if (argc > 1) { + mgr.add(argv[1]); + + if (mgr.error()) + std::cerr << "Caliper config error: " << mgr.error_msg() << std::endl; + } + + auto channels = mgr.get_all_channels(); + + for (auto &c : channels) + c->start(); + // Mark begin/end of the current function. // Sets "function=main" in Caliper. CALI_CXX_MARK_FUNCTION; @@ -65,4 +82,7 @@ int main(int argc, char* argv[]) } CALI_CXX_MARK_LOOP_END(mainloop); + + for (auto &c : channels) + c->flush(); } diff --git a/include/caliper/ChannelController.h b/include/caliper/ChannelController.h index ad0fa7ad7..12bfe1fa3 100644 --- a/include/caliper/ChannelController.h +++ b/include/caliper/ChannelController.h @@ -68,6 +68,9 @@ class ChannelController /// \brief Returns true if channel exists and is active, false otherwise. bool is_active() const; + /// \brief Returns the name of the underlying channel + std::string name() const; + /// \brief Flush the underlying %Caliper channel. /// /// Allows derived classes to implement custom %Caliper data processing. diff --git a/include/caliper/ConfigManager.h b/include/caliper/ConfigManager.h new file mode 100644 index 000000000..0f54fcf9d --- /dev/null +++ b/include/caliper/ConfigManager.h @@ -0,0 +1,63 @@ +// Copyright (c) 2019, Lawrence Livermore National Security, LLC. +// See top-level LICENSE file for details. + +/// \file ConfigManager.hpp +/// \brief %Caliper ConfigManager class definition + +#pragma once + +#include +#include +#include +#include + +namespace cali +{ + +class ChannelController; + +class ConfigManager +{ + struct ConfigManagerImpl; + std::shared_ptr mP; + +public: + + typedef std::map argmap_t; + + typedef cali::ChannelController* (*CreateConfigFn)(const argmap_t&, bool); + + struct ConfigInfo { + const char* name; + const char** args; + CreateConfigFn create; + }; + + static void + add_controllers(const ConfigInfo*); + + ConfigManager(); + + explicit ConfigManager(const char* config_string); + + ~ConfigManager(); + + bool add(const char* config_string); + + void use_mpi(bool); + void set_default_parameter(const char* key, const char* value); + + bool error() const; + std::string error_msg() const; + + typedef std::shared_ptr ChannelPtr; + typedef std::vector ChannelList; + + ChannelList + get_all_channels(); + + ChannelPtr + get_channel(const char* name); +}; + +} // namespace cali diff --git a/include/caliper/cali-manager.h b/include/caliper/cali-manager.h new file mode 100644 index 000000000..ac13f7e0e --- /dev/null +++ b/include/caliper/cali-manager.h @@ -0,0 +1,15 @@ +// Copyright (c) 2019, Lawrence Livermore National Security, LLC. +// See top-level LICENSE file for details. + +#pragma once + +#ifdef __cplusplus +#include "caliper/ConfigManager.h" +#include "caliper/ChannelController.h" + +extern "C" { +#endif + +#ifdef __cplusplus +} // extern "C" +#endif diff --git a/manager/include/caliper/ConfigManager.hpp b/manager/include/caliper/ConfigManager.hpp new file mode 100644 index 000000000..aa9b61083 --- /dev/null +++ b/manager/include/caliper/ConfigManager.hpp @@ -0,0 +1,49 @@ +// Copyright (c) 2019, Lawrence Livermore National Security, LLC. +// See top-level LICENSE file for details. + +/// \file ConfigManager.hpp +/// \brief %Caliper ConfigManager class definition + +#pragma once + +#include +#include +#include + +namespace cali +{ + +class ChannelController; + +class ConfigManager +{ + struct ConfigManagerImpl; + std::shared_ptr mP; + +public: + + ConfigManager(); + + explicit ConfigManager(const char* config_string); + + ~ConfigManager(); + + bool add(const char* config_string); + + void use_mpi(bool); + void set_default_parameter(const char* key, const char* value); + + bool error() const; + std::string error_msg() const; + + typedef std::shared_ptr ChannelPtr; + typedef std::vector ChannelList; + + ChannelList + get_all_channels(); + + ChannelPtr + get_channel(const char* name); +}; + +} // namespace cali diff --git a/manager/include/caliper/cali-manager.h b/manager/include/caliper/cali-manager.h new file mode 100644 index 000000000..f1cd5863c --- /dev/null +++ b/manager/include/caliper/cali-manager.h @@ -0,0 +1,15 @@ +// Copyright (c) 2019, Lawrence Livermore National Security, LLC. +// See top-level LICENSE file for details. + +#pragma once + +#ifdef __cplusplus +#include "caliper/ConfigManager.hpp" +#include "caliper/ChannelController.h" + +extern "C" { +#endif + +#ifdef __cplusplus +} // extern "C" +#endif diff --git a/manager/src/CMakeLists.txt b/manager/src/CMakeLists.txt new file mode 100644 index 000000000..891a9149d --- /dev/null +++ b/manager/src/CMakeLists.txt @@ -0,0 +1,33 @@ +include_directories("../include") +include_directories("../../mpi/include") +include_directories(${MPI_CXX_INCLUDE_PATH}) + +if (${CMAKE_CXX_COMPILER_ID} MATCHES Intel) + # CMake does not have proper compiler feature support for Intel compilers :-/ + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99") +else() + set(CMAKE_C_STANDARD_REQUIRED ON) + set(CMAKE_CXX_STANDARD_REQUIRED ON) + + set(CMAKE_CXX_STANDARD 11) + set(CMAKE_C_STANDARD 99) +endif() + +set(CONFIG_MANAGER_SOURCES + ConfigManager.cpp + RuntimeProfileController.cpp + SpotController.cpp) + +add_library(caliper-config-manager + ${CONFIG_MANAGER_SOURCES}) + +target_link_libraries(caliper-config-manager PUBLIC caliper) + +if (CALIPER_HAVE_MPI) + target_link_libraries(caliper-config-manager PUBLIC caliper-mpi) +endif() + +if (BUILD_TESTING) + add_subdirectory(test) +endif() diff --git a/manager/src/ConfigManager.cpp b/manager/src/ConfigManager.cpp new file mode 100644 index 000000000..f5afc5181 --- /dev/null +++ b/manager/src/ConfigManager.cpp @@ -0,0 +1,230 @@ +// Copyright (c) 2019, Lawrence Livermore National Security, LLC. +// See top-level LICENSE file for details. + +#include "caliper/caliper-config.h" + +#include "caliper/ConfigManager.hpp" + +#include "RuntimeProfileController.hpp" +#include "SpotController.hpp" + +#include "../src/common/util/parse_util.h" + +#include + +using namespace cali; + +namespace +{ + +typedef std::map argmap_t; + +typedef cali::ChannelController* (*CreateConfigFn)(const argmap_t&, bool); + +struct ConfigInfo { + const char* name; + const char** args; + CreateConfigFn create; +}; + +cali::ChannelController* +make_runtime_report_controller(const argmap_t& args, bool use_mpi) +{ + auto it = args.find("output"); + std::string output = (it == args.end() ? "" : it->second); + + if (output.empty()) + output = "stderr"; + + return new RuntimeProfileController(use_mpi, output.c_str()); +} + +cali::ChannelController* +make_spot_controller(const argmap_t& args, bool use_mpi) +{ + auto it = args.find("output"); + std::string output = (it == args.end() ? "" : it->second); + + return new SpotController(use_mpi, output.c_str()); +} + +const char* output_args[] = { "output", nullptr }; + +ConfigInfo config_table[] = { + { "runtime_report", output_args, make_runtime_report_controller }, + { "spot", output_args, make_spot_controller }, + + { nullptr, nullptr, nullptr } +}; + +} + +struct ConfigManager::ConfigManagerImpl +{ + ChannelList m_channels; + + bool m_use_mpi; + + bool m_error = false; + std::string m_error_msg = ""; + + argmap_t m_default_parameters; + + void + set_error(const std::string& msg) { + m_error = true; + m_error_msg = msg; + } + + argmap_t + parse_arglist(std::istream& is, const char* argtbl[]) { + argmap_t args = m_default_parameters; + + char c = util::read_char(is); + + if (!is.good()) + return args; + + if (c != '(') { + is.unget(); + return args; + } + + do { + std::string key = util::read_word(is, ",=()\n"); + + const char** arg = argtbl; + + while (*argtbl && key != std::string(*argtbl)) + ++argtbl; + + if (*argtbl == nullptr) { + set_error("Unknown argument: " + key); + args.clear(); + return args; + } + + c = util::read_char(is); + + if (c != '=') { + set_error("Expected '=' after " + key); + args.clear(); + return args; + } + + args[key] = util::read_word(is, ",=()\n"); + c = util::read_char(is); + } while (is.good() && c == ','); + + if (c != ')') { + set_error("Expected ')'"); + is.unget(); + args.clear(); + } + + return args; + } + + bool add(const char* config_string) { + std::istringstream is(config_string); + char c = 0; + + do { + std::string name = util::read_word(is, ",=()\n"); + + const ::ConfigInfo* cfg_p = ::config_table; + + while (cfg_p && cfg_p->name && name != std::string(cfg_p->name)) + ++cfg_p; + + if (!cfg_p || !cfg_p->name) { + set_error("Unknown config: " + name); + return false; + } + + auto args = parse_arglist(is, cfg_p->args); + + if (m_error) + return false; + + m_channels.emplace_back( (*cfg_p->create)(args, m_use_mpi) ); + + c = util::read_char(is); + } while (!m_error && is.good() && c == ','); + + return !m_error; + } + + ConfigManagerImpl() + : m_use_mpi(false) + { +#ifdef CALIPER_HAVE_MPI + m_use_mpi = true; +#endif + } +}; + + +ConfigManager::ConfigManager() + : mP(new ConfigManagerImpl) +{ } + +ConfigManager::ConfigManager(const char* config_string) + : mP(new ConfigManagerImpl) +{ + mP->add(config_string); +} + +ConfigManager::~ConfigManager() +{ + mP.reset(); +} + +bool +ConfigManager::add(const char* config_str) +{ + return mP->add(config_str); +} + +bool +ConfigManager::error() const +{ + return mP->m_error; +} + +std::string +ConfigManager::error_msg() const +{ + return mP->m_error_msg; +} + +void +ConfigManager::use_mpi(bool enable) +{ +#ifndef CALIPER_HAVE_MPI + Log(0).stream() << "ConfigManager: Cannot enable MPI support in non-MPI Caliper build!" << std::endl; +#endif + mP->m_use_mpi = enable; +} + +void +ConfigManager::set_default_parameter(const char* key, const char* value) +{ + mP->m_default_parameters[key] = value; +} + +ConfigManager::ChannelList +ConfigManager::get_all_channels() +{ + return mP->m_channels; +} + +ConfigManager::ChannelPtr +ConfigManager::get_channel(const char* name) +{ + for (ChannelPtr& chn : mP->m_channels) + if (chn->name() == name) + return chn; + + return ChannelPtr(); +} diff --git a/manager/src/RuntimeProfileController.cpp b/manager/src/RuntimeProfileController.cpp new file mode 100644 index 000000000..14117194d --- /dev/null +++ b/manager/src/RuntimeProfileController.cpp @@ -0,0 +1,18 @@ +#include "RuntimeProfileController.hpp" + +using namespace cali; + +RuntimeProfileController::RuntimeProfileController(bool use_mpi, const char* output) + : ChannelController("runtime_report", 0, { + { "CALI_CHANNEL_FLUSH_ON_EXIT", "false" } + }) +{ + if (use_mpi) { + config()["CALI_CONFIG_PROFILE" ] = "mpi-runtime-report"; + config()["CALI_MPIREPORT_FILENAME"] = output; + config()["CALI_MPIREPORT_WRITE_ON_FINALIZE"] = "false"; + } else { + config()["CALI_CONFIG_PROFILE" ] = "runtime-report"; + config()["CALI_REPORT_FILENAME" ] = output; + } +} diff --git a/manager/src/RuntimeProfileController.hpp b/manager/src/RuntimeProfileController.hpp new file mode 100644 index 000000000..0114988b9 --- /dev/null +++ b/manager/src/RuntimeProfileController.hpp @@ -0,0 +1,21 @@ +/// \file RuntimeProfileController.hpp +/// \brief RuntimeProfileController class + +#pragma once + +#include + +namespace cali +{ + +class RuntimeProfileController : public cali::ChannelController +{ +public: + + explicit RuntimeProfileController(bool use_mpi = false, const char* output = "stderr"); + + ~RuntimeProfileController() + { } +}; + +} diff --git a/manager/src/SpotController.cpp b/manager/src/SpotController.cpp new file mode 100644 index 000000000..ba39723b3 --- /dev/null +++ b/manager/src/SpotController.cpp @@ -0,0 +1,143 @@ +#include "caliper/caliper-config.h" + +#include "SpotController.hpp" + +#include +#include + +#include +#include +#include +#include + +#include +#include + +#include + +#include + +#ifdef CALIPER_HAVE_MPI +#include +#include +#endif + +using namespace cali; + +namespace +{ + +std::string +make_filename() +{ + char timestr[16]; + time_t tm = time(NULL); + strftime(timestr, sizeof(timestr), "%y%m%d-%H%M%S", localtime(&tm)); + + return std::string(timestr) + std::to_string(getpid()) + ".cali"; +} + +} + +SpotController::SpotController(bool use_mpi, const char* output) + : ChannelController("spot", 0, { + { "CALI_SERVICES_ENABLE", "aggregate,env,event,timestamp" }, + { "CALI_EVENT_TRIGGER", "function,loop" }, + { "CALI_EVENT_ENABLE_SNAPSHOT_INFO", "false" }, + { "CALI_TIMER_INCLUSIVE_DURATION", "false" }, + { "CALI_TIMER_SNAPSHOT_DURATION", "true" }, + { "CALI_CHANNEL_FLUSH_ON_EXIT", "false" }, + { "CALI_CHANNEL_CONFIG_CHECK", "false" } + }), + m_use_mpi(use_mpi), + m_output(output) +{ +} + +void +SpotController::flush() +{ + Log(1).stream() << "[spot controller]: Flushing Caliper data" << std::endl; + + // --- Setup output reduction aggregator + + QuerySpec output_spec = + CalQLParser("select *," + " min(inclusive#sum#time.duration)" + ",max(inclusive#sum#time.duration)" + ",avg(inclusive#sum#time.duration)" + " format cali").spec(); + + Aggregator output_agg(output_spec); + + CaliperMetadataDB db; + Caliper c; + + // --- Flush Caliper buffers into intermediate aggregator to calculate + // inclusive times + + { + QuerySpec inclusive_spec = + CalQLParser("aggregate" + " inclusive_sum(sum#time.duration)" + " group by prop:nested").spec(); + + Aggregator inclusive_agg(inclusive_spec); + + c.flush(channel(), nullptr, [&db,&inclusive_agg](CaliperMetadataAccessInterface& in_db, + const std::vector& rec){ + EntryList mrec = db.merge_snapshot(in_db, rec); + inclusive_agg.add(db, mrec); + }); + + // write intermediate results into output aggregator + inclusive_agg.flush(db, output_agg); + } + + // --- Calculate min/max/avg times across MPI ranks + + int rank = 0; + +#ifdef CALIPER_HAVE_MPI + if (m_use_mpi) { + Log(2).stream() << "[spot controller]: Performing cross-process aggregation" << std::endl; + + MPI_Comm comm; + MPI_Comm_dup(MPI_COMM_WORLD, &comm); + MPI_Comm_rank(comm, &rank); + + // Do the global cross-process aggregation. + // aggregate_over_mpi() does all the magic. + // Result will be in output_agg on rank 0. + aggregate_over_mpi(db, output_agg, comm); + + MPI_Comm_free(&comm); + } +#endif + + // --- Write output + + if (rank == 0) { + Log(2).stream() << "[spot controller]: Writing output" << std::endl; + + // import globals from Caliper runtime object + db.import_globals(c); + + std::string output = m_output; + + if (m_output.empty()) + output = ::make_filename(); + + OutputStream stream; + stream.set_filename(output.c_str(), c, c.get_globals()); + + FormatProcessor formatter(output_spec, stream); + + output_agg.flush(db, formatter); + formatter.flush(db); + } +} + +SpotController::~SpotController() +{ +} diff --git a/manager/src/SpotController.hpp b/manager/src/SpotController.hpp new file mode 100644 index 000000000..b5ca1db27 --- /dev/null +++ b/manager/src/SpotController.hpp @@ -0,0 +1,22 @@ +#pragma once + +#include + +namespace cali +{ + +class SpotController : public cali::ChannelController +{ + std::string m_output; + bool m_use_mpi; + +public: + + explicit SpotController(bool use_mpi, const char* output); + + ~SpotController(); + + void flush(); +}; + +} diff --git a/manager/src/test/CMakeLists.txt b/manager/src/test/CMakeLists.txt new file mode 100644 index 000000000..b091d7de0 --- /dev/null +++ b/manager/src/test/CMakeLists.txt @@ -0,0 +1,9 @@ +set(CALIPER_MANAGER_TEST_SOURCES + test_configmanager.cpp) + +add_executable(test_caliper-manager + ${CALIPER_MANAGER_TEST_SOURCES}) + +target_link_libraries(test_caliper-manager caliper-config-manager gtest_main) + +add_test(NAME test-caliper-manager COMMAND test_caliper-manager) diff --git a/manager/src/test/test_configmanager.cpp b/manager/src/test/test_configmanager.cpp new file mode 100644 index 000000000..227222482 --- /dev/null +++ b/manager/src/test/test_configmanager.cpp @@ -0,0 +1,70 @@ +#include "caliper/ConfigManager.hpp" + +#include "caliper/ChannelController.h" + +#include + +using namespace cali; + +TEST(ConfigManagerTest, ParseErrors) { + { + cali::ConfigManager mgr; + + EXPECT_FALSE(mgr.add("foo")); + EXPECT_TRUE(mgr.error()); + EXPECT_STREQ(mgr.error_msg().c_str(), "Unknown config: foo"); + } + + { + cali::ConfigManager mgr(" spot(foo = bar)"); + + EXPECT_TRUE(mgr.error()); + EXPECT_STREQ(mgr.error_msg().c_str(), "Unknown argument: foo"); + + auto list = mgr.get_all_channels(); + + EXPECT_EQ(list.size(), 0); + } +} + +TEST(ConfigManagerTest, ParseConfig) { + { + cali::ConfigManager mgr; + + EXPECT_TRUE(mgr.add("runtime_report")); + EXPECT_FALSE(mgr.error()); + + auto list = mgr.get_all_channels(); + + ASSERT_EQ(list.size(), 1); + EXPECT_EQ(std::string("runtime_report"), list.front()->name()); + } + + { + cali::ConfigManager mgr; + + EXPECT_TRUE(mgr.add(" spot, runtime_report ")); + EXPECT_FALSE(mgr.error()); + + auto list = mgr.get_all_channels(); + + ASSERT_EQ(list.size(), 2); + + EXPECT_EQ(std::string("spot"), list[0]->name()); + EXPECT_EQ(std::string("runtime_report"), list[1]->name()); + } + + { + cali::ConfigManager mgr; + + EXPECT_TRUE(mgr.add(" spot ( output = test.cali ), runtime_report(output=stdout) ")); + EXPECT_FALSE(mgr.error()); + + auto list = mgr.get_all_channels(); + + ASSERT_EQ(list.size(), 2); + + EXPECT_EQ(std::string("spot"), list[0]->name()); + EXPECT_EQ(std::string("runtime_report"), list[1]->name()); + } +} diff --git a/mpi/mpi-rt/CMakeLists.txt b/mpi/mpi-rt/CMakeLists.txt index c1192c5bb..02cb599da 100644 --- a/mpi/mpi-rt/CMakeLists.txt +++ b/mpi/mpi-rt/CMakeLists.txt @@ -30,12 +30,15 @@ endmacro() add_subdirectory(services/mpiwrap) add_subdirectory(services/mpireport) +add_subdirectory(controllers) + if (CALIPER_HAVE_MPIT) # add_subdirectory(services/mpit) message(WARNING "MPIT option is currently not supported") endif() add_library(caliper-mpi + $ ${CALIPER_MPI_SOURCES} ${CALIPER_MPI_LIBS}) diff --git a/mpi/mpi-rt/controllers/CMakeLists.txt b/mpi/mpi-rt/controllers/CMakeLists.txt new file mode 100644 index 000000000..5fb93ec10 --- /dev/null +++ b/mpi/mpi-rt/controllers/CMakeLists.txt @@ -0,0 +1,9 @@ +set(CALIPER_MPI_CONTROLLER_SOURCES + SpotController.cpp + SpotV1Controller.cpp) + +add_library(caliper-mpi-controllers OBJECT ${CALIPER_MPI_CONTROLLER_SOURCES}) + +if (${BUILD_SHARED_LIBS}) + set_property(TARGET caliper-mpi-controllers PROPERTY POSITION_INDEPENDENT_CODE TRUE) +endif() diff --git a/mpi/mpi-rt/controllers/SpotController.cpp b/mpi/mpi-rt/controllers/SpotController.cpp new file mode 100644 index 000000000..b0ca4c45b --- /dev/null +++ b/mpi/mpi-rt/controllers/SpotController.cpp @@ -0,0 +1,162 @@ +#include "caliper/caliper-config.h" + +#include +#include +#include + +#include +#include +#include +#include + +#include +#include + +#include + +#include + +#include "caliper/cali-mpi.h" + +#include + +using namespace cali; + +namespace +{ + +std::string +make_filename() +{ + char timestr[16]; + time_t tm = time(NULL); + strftime(timestr, sizeof(timestr), "%y%m%d-%H%M%S", localtime(&tm)); + + return std::string(timestr) + std::to_string(getpid()) + ".cali"; +} + +class SpotController : public cali::ChannelController +{ + std::string m_output; + bool m_use_mpi; + +public: + + void + flush() { + Log(1).stream() << "[spot controller]: Flushing Caliper data" << std::endl; + + // --- Setup output reduction aggregator + + QuerySpec output_spec = + CalQLParser("select *," + " min(inclusive#sum#time.duration)" + ",max(inclusive#sum#time.duration)" + ",avg(inclusive#sum#time.duration)" + " format cali").spec(); + + Aggregator output_agg(output_spec); + + CaliperMetadataDB db; + Caliper c; + + // --- Flush Caliper buffers into intermediate aggregator to calculate + // inclusive times + + { + QuerySpec inclusive_spec = + CalQLParser("aggregate" + " inclusive_sum(sum#time.duration)" + " group by prop:nested").spec(); + + Aggregator inclusive_agg(inclusive_spec); + + c.flush(channel(), nullptr, [&db,&inclusive_agg](CaliperMetadataAccessInterface& in_db, + const std::vector& rec){ + EntryList mrec = db.merge_snapshot(in_db, rec); + inclusive_agg.add(db, mrec); + }); + + // write intermediate results into output aggregator + inclusive_agg.flush(db, output_agg); + } + + // --- Calculate min/max/avg times across MPI ranks + + int rank = 0; + +#ifdef CALIPER_HAVE_MPI + if (m_use_mpi) { + Log(2).stream() << "[spot controller]: Performing cross-process aggregation" << std::endl; + + MPI_Comm comm; + MPI_Comm_dup(MPI_COMM_WORLD, &comm); + MPI_Comm_rank(comm, &rank); + + // Do the global cross-process aggregation. + // aggregate_over_mpi() does all the magic. + // Result will be in output_agg on rank 0. + aggregate_over_mpi(db, output_agg, comm); + + MPI_Comm_free(&comm); + } +#endif + + // --- Write output + + if (rank == 0) { + Log(2).stream() << "[spot controller]: Writing output" << std::endl; + + // import globals from Caliper runtime object + db.import_globals(c); + + std::string output = m_output; + + if (m_output.empty()) + output = ::make_filename(); + + OutputStream stream; + stream.set_filename(output.c_str(), c, c.get_globals()); + + FormatProcessor formatter(output_spec, stream); + + output_agg.flush(db, formatter); + formatter.flush(db); + } + } + + SpotController(bool use_mpi, const char* output) + : ChannelController("spot", 0, { + { "CALI_SERVICES_ENABLE", "aggregate,env,event,timestamp" }, + { "CALI_EVENT_ENABLE_SNAPSHOT_INFO", "false" }, + { "CALI_TIMER_INCLUSIVE_DURATION", "false" }, + { "CALI_TIMER_SNAPSHOT_DURATION", "true" }, + { "CALI_CHANNEL_FLUSH_ON_EXIT", "false" }, + { "CALI_CHANNEL_CONFIG_CHECK", "false" } + }), + m_use_mpi(use_mpi), + m_output(output) + { } + + ~SpotController() + { } +}; + +const char* spot_args[] = { "output", nullptr }; + +cali::ChannelController* +make_spot_controller(const cali::ConfigManager::argmap_t& args, bool use_mpi) { + auto it = args.find("output"); + std::string output = (it == args.end() ? "" : it->second); + + return new SpotController(use_mpi, output.c_str()); +} + +} // namespace [anonymous] + +namespace cali +{ + +ConfigManager::ConfigInfo spot_controller_info { "spot", ::spot_args, ::make_spot_controller }; + +} diff --git a/mpi/mpi-rt/controllers/SpotV1Controller.cpp b/mpi/mpi-rt/controllers/SpotV1Controller.cpp new file mode 100644 index 000000000..ff7925b76 --- /dev/null +++ b/mpi/mpi-rt/controllers/SpotV1Controller.cpp @@ -0,0 +1,65 @@ +// Copyright (c) 2019, Lawrence Livermore National Security, LLC. +// See top-level LICENSE file for details. + +#include "caliper/Caliper.h" +#include "caliper/ChannelController.h" +#include "caliper/ConfigManager.h" + +#include "caliper/cali.h" + +#include + +namespace +{ + +class SpotV1Controller : public cali::ChannelController +{ +public: + + SpotV1Controller(const cali::ConfigManager::argmap_t& args) + : cali::ChannelController("spot-v1", 0, { + { "CALI_SERVICES_ENABLE", "event,aggregate,spot,timestamp" }, + { "CALI_CHANNEL_FLUSH_ON_EXIT", "false" }, + { "CALI_TIMER_SNAPSHOT_DURATION", "false" }, + { "CALI_SPOT_TIME_DIVISOR", "1000" }, + { "CALI_SPOT_Y_AXES", "Milliseconds" } + }) + { + auto it = args.find("config"); + if (it != args.end()) + config()["CALI_SPOT_CONFIG"] = it->second; + + it = args.find("code_version"); + if (it != args.end()) + config()["CALI_SPOT_CODE_VERSION"] = it->second; + + it = args.find("title"); + if (it != args.end()) + config()["CALI_SPOT_TITLE"] = it->second; + } + + void flush() { + int rank = 0; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + + if (rank == 0 && channel()) + cali_channel_flush(channel()->id(), CALI_FLUSH_CLEAR_BUFFERS); + } +}; + +const char* spot_v1_args[] = { "config", "code_version", "title", nullptr }; + +cali::ChannelController* +make_spot_v1_controller(const cali::ConfigManager::argmap_t& args, bool /*use_mpi*/) +{ + return new SpotV1Controller(args); +} + +} // namespace [anonymous] + +namespace cali +{ + +cali::ConfigManager::ConfigInfo spot_v1_controller_info { "spot-v1", ::spot_v1_args, ::make_spot_v1_controller }; + +} diff --git a/mpi/mpi-rt/setup_mpi.cpp b/mpi/mpi-rt/setup_mpi.cpp index 3da05f008..e4cb2cf6c 100644 --- a/mpi/mpi-rt/setup_mpi.cpp +++ b/mpi/mpi-rt/setup_mpi.cpp @@ -36,6 +36,7 @@ #include "caliper/Caliper.h" #include "caliper/CaliperService.h" +#include "caliper/ConfigManager.h" #include "caliper/common/Log.h" @@ -63,6 +64,16 @@ CaliperService cali_mpi_services[] = { { nullptr, nullptr } }; +extern ConfigManager::ConfigInfo spot_controller_info; +extern ConfigManager::ConfigInfo spot_v1_controller_info; + +ConfigManager::ConfigInfo mpi_controllers[] = { + spot_controller_info, + spot_v1_controller_info, + + { nullptr, nullptr, nullptr } +}; + bool is_initialized = false; // Pre-init setup routine that allows us to do some MPI-specific @@ -101,6 +112,8 @@ mpirt_constructor() { Caliper::add_services(cali_mpi_services); Caliper::add_init_hook(setup_mpi); + + ConfigManager::add_controllers(mpi_controllers); is_initialized = true; } diff --git a/src/caliper/CMakeLists.txt b/src/caliper/CMakeLists.txt index b3d96e32b..2bd76366f 100644 --- a/src/caliper/CMakeLists.txt +++ b/src/caliper/CMakeLists.txt @@ -4,6 +4,7 @@ set(CALIPER_SOURCES Blackboard.cpp Caliper.cpp ChannelController.cpp + ConfigManager.cpp SnapshotRecord.cpp MemoryPool.cpp MetadataTree.cpp @@ -11,9 +12,12 @@ set(CALIPER_SOURCES cali.cpp cali_datatracker.cpp config_sanity_check.cpp) + +add_subdirectory(controllers) add_library(caliper $ + $ ${CALIPER_SOURCES} ${CALIPER_SERVICES_LIBS} ${CALIPER_EXTERNAL_SOURCES}) diff --git a/src/caliper/ChannelController.cpp b/src/caliper/ChannelController.cpp index 7c8f2ac2c..92d168d0e 100644 --- a/src/caliper/ChannelController.cpp +++ b/src/caliper/ChannelController.cpp @@ -104,6 +104,12 @@ ChannelController::flush() Caliper().flush_and_write(chn, nullptr); } +std::string +ChannelController::name() const +{ + return mP->name; +} + ChannelController::ChannelController(const char* name, int flags, const config_map_t& cfg) : mP { new ChannelControllerImpl(name, flags, cfg) } { diff --git a/src/caliper/ConfigManager.cpp b/src/caliper/ConfigManager.cpp new file mode 100644 index 000000000..a74a0dcdd --- /dev/null +++ b/src/caliper/ConfigManager.cpp @@ -0,0 +1,224 @@ +// Copyright (c) 2019, Lawrence Livermore National Security, LLC. +// See top-level LICENSE file for details. + +#include "caliper/caliper-config.h" + +#include "caliper/ConfigManager.h" + +#include "caliper/ChannelController.h" + +#include "caliper/common/Log.h" + +#include "../src/common/util/parse_util.h" + +#include + +using namespace cali; + +namespace cali +{ + +// defined in controllers/controllers.cpp +extern ConfigManager::ConfigInfo builtin_controllers_table[]; + +} + +namespace +{ + +struct ConfigInfoList { + const ConfigManager::ConfigInfo* configs; + ConfigInfoList* next; +}; + +ConfigInfoList s_default_configs { cali::builtin_controllers_table, nullptr }; +ConfigInfoList* s_config_list { &s_default_configs }; + +} + +struct ConfigManager::ConfigManagerImpl +{ + ChannelList m_channels; + + bool m_use_mpi; + + bool m_error = false; + std::string m_error_msg = ""; + + argmap_t m_default_parameters; + + void + set_error(const std::string& msg) { + m_error = true; + m_error_msg = msg; + } + + argmap_t + parse_arglist(std::istream& is, const char* argtbl[]) { + argmap_t args = m_default_parameters; + + char c = util::read_char(is); + + if (!is.good()) + return args; + + if (c != '(') { + is.unget(); + return args; + } + + do { + std::string key = util::read_word(is, ",=()\n"); + + const char** arg = argtbl; + + while (*argtbl && key != std::string(*argtbl)) + ++argtbl; + + if (*argtbl == nullptr) { + set_error("Unknown argument: " + key); + args.clear(); + return args; + } + + c = util::read_char(is); + + if (c != '=') { + set_error("Expected '=' after " + key); + args.clear(); + return args; + } + + args[key] = util::read_word(is, ",=()\n"); + c = util::read_char(is); + } while (is.good() && c == ','); + + if (c != ')') { + set_error("Expected ')'"); + is.unget(); + args.clear(); + } + + return args; + } + + bool add(const char* config_string) { + std::istringstream is(config_string); + char c = 0; + + do { + std::string name = util::read_word(is, ",=()\n"); + + const ::ConfigInfoList* lst_p = ::s_config_list; + const ConfigInfo* cfg_p = nullptr; + + while (lst_p) { + cfg_p = lst_p->configs; + + while (cfg_p && cfg_p->name && name != std::string(cfg_p->name)) + ++cfg_p; + + if (cfg_p && cfg_p->name) + break; + + lst_p = lst_p->next; + } + + if (!cfg_p || !cfg_p->name) { + set_error("Unknown config: " + name); + return false; + } + + auto args = parse_arglist(is, cfg_p->args); + + if (m_error) + return false; + + m_channels.emplace_back( (*cfg_p->create)(args, m_use_mpi) ); + + c = util::read_char(is); + } while (!m_error && is.good() && c == ','); + + return !m_error; + } + + ConfigManagerImpl() + : m_use_mpi(false) + { +#ifdef CALIPER_HAVE_MPI + m_use_mpi = true; +#endif + } +}; + + +ConfigManager::ConfigManager() + : mP(new ConfigManagerImpl) +{ } + +ConfigManager::ConfigManager(const char* config_string) + : mP(new ConfigManagerImpl) +{ + mP->add(config_string); +} + +ConfigManager::~ConfigManager() +{ + mP.reset(); +} + +bool +ConfigManager::add(const char* config_str) +{ + return mP->add(config_str); +} + +bool +ConfigManager::error() const +{ + return mP->m_error; +} + +std::string +ConfigManager::error_msg() const +{ + return mP->m_error_msg; +} + +void +ConfigManager::use_mpi(bool enable) +{ +#ifndef CALIPER_HAVE_MPI + Log(0).stream() << "ConfigManager: Cannot enable MPI support in non-MPI Caliper build!" << std::endl; +#endif + mP->m_use_mpi = enable; +} + +void +ConfigManager::set_default_parameter(const char* key, const char* value) +{ + mP->m_default_parameters[key] = value; +} + +ConfigManager::ChannelList +ConfigManager::get_all_channels() +{ + return mP->m_channels; +} + +ConfigManager::ChannelPtr +ConfigManager::get_channel(const char* name) +{ + for (ChannelPtr& chn : mP->m_channels) + if (chn->name() == name) + return chn; + + return ChannelPtr(); +} + +void +ConfigManager::add_controllers(const ConfigManager::ConfigInfo* ctrlrs) +{ + ::ConfigInfoList* elem = new ConfigInfoList { ctrlrs, ::s_config_list }; + s_config_list = elem; +} diff --git a/src/caliper/controllers/CMakeLists.txt b/src/caliper/controllers/CMakeLists.txt new file mode 100644 index 000000000..80b5ccbf0 --- /dev/null +++ b/src/caliper/controllers/CMakeLists.txt @@ -0,0 +1,10 @@ +set(CALIPER_CONTROLLERS_SOURCES + EventTraceController.cpp + RuntimeReportController.cpp + controllers.cpp) + +add_library(caliper-controllers OBJECT ${CALIPER_CONTROLLERS_SOURCES}) + +if (BUILD_SHARED_LIBS) + set_property(TARGET caliper-controllers PROPERTY POSITION_INDEPENDENT_CODE TRUE) +endif() diff --git a/src/caliper/controllers/EventTraceController.cpp b/src/caliper/controllers/EventTraceController.cpp new file mode 100644 index 000000000..47fa7a5e2 --- /dev/null +++ b/src/caliper/controllers/EventTraceController.cpp @@ -0,0 +1,58 @@ +#include "caliper/ChannelController.h" +#include "caliper/ConfigManager.h" + +using namespace cali; + +namespace +{ + +class EventTraceController : public cali::ChannelController +{ +public: + + EventTraceController(const std::string& output) + : ChannelController("event-trace", 0, { + { "CALI_CHANNEL_FLUSH_ON_EXIT", "false" }, + { "CALI_CONFIG_PROFILE", "serial-trace" } + }) + { + std::string tmp = output; + + if (!tmp.empty()) { + if (tmp != "stderr" && tmp != "stdout") { + auto pos = tmp.find_last_of('.'); + + if (pos == std::string::npos || tmp.substr(pos) != ".cali") + tmp.append(".cali"); + } + + config()["CALI_RECORDER_FILENAME"] = tmp; + } + } +}; + +const char* event_trace_args[] = { "output", nullptr }; + +static cali::ChannelController* +make_event_trace_controller(const cali::ConfigManager::argmap_t& args, bool /*use_mpi*/ ) +{ + std::string output; + auto it = args.find("output"); + + if (it != args.end()) + output = it->second; + + return new EventTraceController(output); +} + +} // namespace [anonymous] + +namespace cali +{ + +ConfigManager::ConfigInfo event_trace_controller_info +{ + "event-trace", ::event_trace_args, ::make_event_trace_controller +}; + +} diff --git a/src/caliper/controllers/RuntimeReportController.cpp b/src/caliper/controllers/RuntimeReportController.cpp new file mode 100644 index 000000000..05a9ec5e7 --- /dev/null +++ b/src/caliper/controllers/RuntimeReportController.cpp @@ -0,0 +1,50 @@ +#include "caliper/ChannelController.h" +#include "caliper/ConfigManager.h" + +using namespace cali; + +namespace +{ + +class RuntimeReportController : public cali::ChannelController +{ +public: + + RuntimeReportController(bool use_mpi, const char* output) + : ChannelController("runtime-report", 0, { + { "CALI_CHANNEL_FLUSH_ON_EXIT", "false" } + }) + { + if (use_mpi) { + config()["CALI_CONFIG_PROFILE" ] = "mpi-runtime-report"; + config()["CALI_MPIREPORT_FILENAME"] = output; + config()["CALI_MPIREPORT_WRITE_ON_FINALIZE"] = "false"; + } else { + config()["CALI_CONFIG_PROFILE" ] = "runtime-report"; + config()["CALI_REPORT_FILENAME" ] = output; + } + } +}; + +const char* runtime_report_args[] = { "output", nullptr }; + +static cali::ChannelController* +make_runtime_report_controller(const cali::ConfigManager::argmap_t& args, bool use_mpi) +{ + auto it = args.find("output"); + std::string output = (it == args.end() ? "" : "stderr"); + + return new RuntimeReportController(use_mpi, output.c_str()); +} + +} // namespace [anonymous] + +namespace cali +{ + +ConfigManager::ConfigInfo runtime_report_controller_info +{ + "runtime-report", ::runtime_report_args, ::make_runtime_report_controller +}; + +} diff --git a/src/caliper/controllers/controllers.cpp b/src/caliper/controllers/controllers.cpp new file mode 100644 index 000000000..d3921eebd --- /dev/null +++ b/src/caliper/controllers/controllers.cpp @@ -0,0 +1,16 @@ +#include "caliper/ConfigManager.h" + +namespace cali +{ + +extern ConfigManager::ConfigInfo event_trace_controller_info; +extern ConfigManager::ConfigInfo runtime_report_controller_info; + +ConfigManager::ConfigInfo builtin_controllers_table[] = { + event_trace_controller_info, + runtime_report_controller_info, + + { nullptr, nullptr, nullptr } +}; + +} diff --git a/src/caliper/test/CMakeLists.txt b/src/caliper/test/CMakeLists.txt index 4fc366a5d..fe7c6d987 100644 --- a/src/caliper/test/CMakeLists.txt +++ b/src/caliper/test/CMakeLists.txt @@ -3,6 +3,7 @@ set(CALIPER_TEST_SOURCES test_blackboard.cpp test_channel_api.cpp test_channel_controller.cpp + test_configmanager.cpp test_metadatatree.cpp test_c_snapshot.cpp) diff --git a/src/caliper/test/test_configmanager.cpp b/src/caliper/test/test_configmanager.cpp new file mode 100644 index 000000000..0d7f9e6ad --- /dev/null +++ b/src/caliper/test/test_configmanager.cpp @@ -0,0 +1,70 @@ +#include "caliper/ConfigManager.h" + +#include "caliper/ChannelController.h" + +#include + +using namespace cali; + +TEST(ConfigManagerTest, ParseErrors) { + { + cali::ConfigManager mgr; + + EXPECT_FALSE(mgr.add("foo")); + EXPECT_TRUE(mgr.error()); + EXPECT_STREQ(mgr.error_msg().c_str(), "Unknown config: foo"); + } + + { + cali::ConfigManager mgr(" event-trace(foo = bar)"); + + EXPECT_TRUE(mgr.error()); + EXPECT_STREQ(mgr.error_msg().c_str(), "Unknown argument: foo"); + + auto list = mgr.get_all_channels(); + + EXPECT_EQ(list.size(), 0); + } +} + +TEST(ConfigManagerTest, ParseConfig) { + { + cali::ConfigManager mgr; + + EXPECT_TRUE(mgr.add("runtime-report")); + EXPECT_FALSE(mgr.error()); + + auto list = mgr.get_all_channels(); + + ASSERT_EQ(list.size(), 1); + EXPECT_EQ(std::string("runtime-report"), list.front()->name()); + } + + { + cali::ConfigManager mgr; + + EXPECT_TRUE(mgr.add(" event-trace, runtime-report ")); + EXPECT_FALSE(mgr.error()); + + auto list = mgr.get_all_channels(); + + ASSERT_EQ(list.size(), 2); + + EXPECT_EQ(std::string("event-trace"), list[0]->name()); + EXPECT_EQ(std::string("runtime-report"), list[1]->name()); + } + + { + cali::ConfigManager mgr; + + EXPECT_TRUE(mgr.add(" event-trace ( output = test.cali ), runtime-report(output=stdout) ")); + EXPECT_FALSE(mgr.error()); + + auto list = mgr.get_all_channels(); + + ASSERT_EQ(list.size(), 2); + + EXPECT_EQ(std::string("event-trace"), list[0]->name()); + EXPECT_EQ(std::string("runtime-report"), list[1]->name()); + } +} diff --git a/test/ci_app_tests/ci_test_mpi_before_cali.cpp b/test/ci_app_tests/ci_test_mpi_before_cali.cpp index 7157b85cd..385cd1669 100644 --- a/test/ci_app_tests/ci_test_mpi_before_cali.cpp +++ b/test/ci_app_tests/ci_test_mpi_before_cali.cpp @@ -2,6 +2,7 @@ #include #include +#include #include @@ -11,21 +12,40 @@ int main(int argc, char* argv[]) MPI_Init(&argc, &argv); - CALI_CXX_MARK_FUNCTION; + cali::ConfigManager mgr; - // some MPI functions to test the wrapper blacklist/whitelist + mgr.use_mpi(true); + + if (argc > 1) + mgr.add(argv[1]); + if (mgr.error()) + MPI_Abort(MPI_COMM_WORLD, -1); + + auto list = mgr.get_all_channels(); + + for (auto channel : list) + channel->start(); + + { + CALI_CXX_MARK_FUNCTION; - MPI_Barrier(MPI_COMM_WORLD); + // some MPI functions to test the wrapper blacklist/whitelist - int val = 42; + MPI_Barrier(MPI_COMM_WORLD); + + int val = 42; - MPI_Bcast(&val, 1, MPI_INT, 0, MPI_COMM_WORLD); + MPI_Bcast(&val, 1, MPI_INT, 0, MPI_COMM_WORLD); - int in = val, out; + int in = val, out; - MPI_Reduce(&in, &out, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); + MPI_Reduce(&in, &out, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); - MPI_Barrier(MPI_COMM_WORLD); - + MPI_Barrier(MPI_COMM_WORLD); + } + + for (auto channel : list) + channel->flush(); + MPI_Finalize(); } diff --git a/test/ci_app_tests/test_mpi.py b/test/ci_app_tests/test_mpi.py index c01628fea..6215c4c89 100644 --- a/test/ci_app_tests/test_mpi.py +++ b/test/ci_app_tests/test_mpi.py @@ -166,5 +166,21 @@ def test_mpi_msg_trace(self): snapshots, { 'function', 'mpi.function', 'mpi.coll.type', 'mpi.call.id' })) + def test_spot_controller(self): + target_cmd = [ './ci_test_mpi_before_cali', 'spot(output=stdout)' ] + query_cmd = [ '../../src/tools/cali-query/cali-query', '-e' ] + + caliper_config = { + 'PATH' : '/usr/bin', # for ssh/rsh + 'CALI_LOG_VERBOSITY' : '0', + } + + query_output = cat.run_test_with_query(target_cmd, query_cmd, caliper_config) + snapshots = cat.get_snapshots_from_text(query_output) + + self.assertTrue(cat.has_snapshot_with_keys( + snapshots, { 'function', 'avg#inclusive#sum#time.duration' + })) + if __name__ == "__main__": unittest.main()