From 28a6f3b17fef180b2774acc30de08dcb6a4e6463 Mon Sep 17 00:00:00 2001 From: "lubos.ilcik" Date: Tue, 5 Jun 2018 20:29:19 +0200 Subject: [PATCH 01/10] config options split into two files --- programs/witness_node/main.cpp | 43 ++++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/programs/witness_node/main.cpp b/programs/witness_node/main.cpp index 23b571c29d..e703142642 100644 --- a/programs/witness_node/main.cpp +++ b/programs/witness_node/main.cpp @@ -106,8 +106,14 @@ static void load_config_file( const fc::path& config_ini_path, const bpo::option // get the basic options bpo::store(bpo::parse_config_file(config_ini_path.preferred_string().c_str(), - unique_options, true), options); + unique_options, true), options); +} +static void load_logging_config_file +( + const fc::path& config_ini_path +) +{ // try to get logging options from the config file. try { @@ -115,9 +121,9 @@ static void load_config_file( const fc::path& config_ini_path, const bpo::option if (logging_config) fc::configure_logging(*logging_config); } - catch (const fc::exception&) + catch (const fc::exception& ex) { - wlog("Error parsing logging config from config file ${config}, using default config", ("config", config_ini_path.preferred_string())); + wlog("Error parsing logging config from logging config file ${config}, using default config", ("config", config_ini_path.preferred_string())); } } @@ -169,12 +175,25 @@ static void create_new_config_file( const fc::path& config_ini_path, const fc::p } out_cfg << "\n"; } + + out_cfg.close(); +} + +static void create_logging_config_file +( + const fc::path& config_ini_path, + const fc::path& data_dir +) +{ + ilog("Writing new config file at ${path}", ("path", config_ini_path)); + if (!exists(data_dir)) + { + create_directories(data_dir); + } + + std::ofstream out_cfg(config_ini_path.preferred_string()); write_default_logging_config_to_stream(out_cfg); out_cfg.close(); - // read the default logging config we just wrote out to the file and start using it - fc::optional logging_config = load_logging_config_from_ini_file(config_ini_path); - if (logging_config) - fc::configure_logging(*logging_config); } int main(int argc, char** argv) { @@ -239,11 +258,21 @@ int main(int argc, char** argv) { data_dir = fc::current_path() / data_dir; } + // load witness node initial configuration fc::path config_ini_path = data_dir / "config.ini"; if( !fc::exists(config_ini_path) ) create_new_config_file( config_ini_path, data_dir, cfg_options ); load_config_file( config_ini_path, cfg_options, options ); + // load witness node logging configuration + const auto logging_ini_path = data_dir / "logging.ini"; + if (!exists(logging_ini_path)) + { + create_logging_config_file (logging_ini_path, data_dir); + } + + load_logging_config_file( logging_ini_path ); + bpo::notify(options); node->initialize(data_dir, options); node->initialize_plugins( options ); From b8eb753e02989c0e16848fcad73e1eddc9729072 Mon Sep 17 00:00:00 2001 From: "lubos.ilcik" Date: Fri, 8 Jun 2018 05:44:29 +0200 Subject: [PATCH 02/10] fixed styling issues --- programs/witness_node/main.cpp | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/programs/witness_node/main.cpp b/programs/witness_node/main.cpp index e703142642..4cfdd71103 100644 --- a/programs/witness_node/main.cpp +++ b/programs/witness_node/main.cpp @@ -109,10 +109,7 @@ static void load_config_file( const fc::path& config_ini_path, const bpo::option unique_options, true), options); } -static void load_logging_config_file -( - const fc::path& config_ini_path -) +static void load_logging_config_file(const fc::path& config_ini_path) { // try to get logging options from the config file. try @@ -179,11 +176,7 @@ static void create_new_config_file( const fc::path& config_ini_path, const fc::p out_cfg.close(); } -static void create_logging_config_file -( - const fc::path& config_ini_path, - const fc::path& data_dir -) +static void create_logging_config_file(const fc::path& config_ini_path, const fc::path& data_dir) { ilog("Writing new config file at ${path}", ("path", config_ini_path)); if (!exists(data_dir)) From e6896efb1f7d8b3c206c0f1fdb46272d469112fd Mon Sep 17 00:00:00 2001 From: "lubos.ilcik" Date: Fri, 27 Jul 2018 10:01:42 +0200 Subject: [PATCH 03/10] load config options logic extracted in a single function able to call from unit tests --- programs/witness_node/main.cpp | 38 +++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/programs/witness_node/main.cpp b/programs/witness_node/main.cpp index 4cfdd71103..0eda181245 100644 --- a/programs/witness_node/main.cpp +++ b/programs/witness_node/main.cpp @@ -70,7 +70,7 @@ namespace bpo = boost::program_options; void write_default_logging_config_to_stream(std::ostream& out); fc::optional load_logging_config_from_ini_file(const fc::path& config_ini_filename); -class deduplicator +class deduplicator { public: deduplicator() : modifier(nullptr) {} @@ -189,6 +189,24 @@ static void create_logging_config_file(const fc::path& config_ini_path, const fc out_cfg.close(); } +void load_configuration_options(const fc::path& data_dir, const bpo::options_description& cfg_options, bpo::variables_map& options) +{ + // load witness node initial configuration + fc::path config_ini_path = data_dir / "config.ini"; + if( !exists(config_ini_path) ) + create_new_config_file( config_ini_path, data_dir, cfg_options ); + load_config_file( config_ini_path, cfg_options, options ); + + // load witness node logging configuration + const auto logging_ini_path = data_dir / "logging.ini"; + if (!fc::exists(logging_ini_path)) + { + create_logging_config_file (logging_ini_path, data_dir); + } + + load_logging_config_file( logging_ini_path ); +} + int main(int argc, char** argv) { app::application* node = new app::application(); fc::oexception unhandled_exception; @@ -250,21 +268,7 @@ int main(int argc, char** argv) { if( data_dir.is_relative() ) data_dir = fc::current_path() / data_dir; } - - // load witness node initial configuration - fc::path config_ini_path = data_dir / "config.ini"; - if( !fc::exists(config_ini_path) ) - create_new_config_file( config_ini_path, data_dir, cfg_options ); - load_config_file( config_ini_path, cfg_options, options ); - - // load witness node logging configuration - const auto logging_ini_path = data_dir / "logging.ini"; - if (!exists(logging_ini_path)) - { - create_logging_config_file (logging_ini_path, data_dir); - } - - load_logging_config_file( logging_ini_path ); + load_configuration_options(data_dir, cfg_options, options); bpo::notify(options); node->initialize(data_dir, options); @@ -308,7 +312,7 @@ int main(int argc, char** argv) { } } -// logging config is too complicated to be parsed by boost::program_options, +// logging config is too complicated to be parsed by boost::program_options, // so we do it by hand // // Currently, you can only specify the filenames and logging levels, which From 48c61a790b4fd9bc96ccdc9f1cb9c6dc6f4515d3 Mon Sep 17 00:00:00 2001 From: "lubos.ilcik" Date: Sat, 28 Jul 2018 13:30:17 +0200 Subject: [PATCH 04/10] app configuration logic moved to a new file --- libraries/app/CMakeLists.txt | 1 + libraries/app/config_util.cpp | 319 ++++++++++++++++++ .../app/include/graphene/app/config_util.hpp | 34 ++ programs/witness_node/main.cpp | 287 +--------------- 4 files changed, 356 insertions(+), 285 deletions(-) create mode 100644 libraries/app/config_util.cpp create mode 100644 libraries/app/include/graphene/app/config_util.hpp diff --git a/libraries/app/CMakeLists.txt b/libraries/app/CMakeLists.txt index d03f9f111e..19814778a9 100644 --- a/libraries/app/CMakeLists.txt +++ b/libraries/app/CMakeLists.txt @@ -8,6 +8,7 @@ add_library( graphene_app database_api.cpp impacted.cpp plugin.cpp + config_util.cpp ${HEADERS} ${EGENESIS_HEADERS} ) diff --git a/libraries/app/config_util.cpp b/libraries/app/config_util.cpp new file mode 100644 index 0000000000..58b67eda30 --- /dev/null +++ b/libraries/app/config_util.cpp @@ -0,0 +1,319 @@ +/* + * Copyright (c) 2018 Lubos Ilcik, and contributors. + * + * The MIT License + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include + +namespace bpo = boost::program_options; + +class deduplicator +{ +public: + deduplicator() : modifier(nullptr) {} + + deduplicator(const boost::shared_ptr (*mod_fn)(const boost::shared_ptr&)) + : modifier(mod_fn) {} + + const boost::shared_ptr next(const boost::shared_ptr& o) + { + const std::string name = o->long_name(); + if( seen.find( name ) != seen.end() ) + return nullptr; + seen.insert(name); + return modifier ? modifier(o) : o; + } + +private: + boost::container::flat_set seen; + const boost::shared_ptr (*modifier)(const boost::shared_ptr&); +}; + +// logging config is too complicated to be parsed by boost::program_options, +// so we do it by hand +// +// Currently, you can only specify the filenames and logging levels, which +// are all most users would want to change. At a later time, options can +// be added to control rotation intervals, compression, and other seldom- +// used features +static void write_default_logging_config_to_stream(std::ostream& out) +{ + out << "# declare an appender named \"stderr\" that writes messages to the console\n" + "[log.console_appender.stderr]\n" + "stream=std_error\n\n" + "# declare an appender named \"default\" that writes messages to default.log\n" + "[log.file_appender.default]\n" + "# filename can be absolute or relative to this config file\n" + "filename=logs/default/default.log\n" + "# Rotate log every ? minutes, if leave out default to 60\n" + "rotation_interval=60\n" + "# how long will logs be kept (in days), if leave out default to 1\n" + "rotation_limit=7\n\n" + "# declare an appender named \"p2p\" that writes messages to p2p.log\n" + "[log.file_appender.p2p]\n" + "# filename can be absolute or relative to this config file\n" + "filename=logs/p2p/p2p.log\n" + "# Rotate log every ? minutes, if leave out default to 60\n" + "rotation_interval=60\n" + "# how long will logs be kept (in days), if leave out default to 1\n" + "rotation_limit=7\n\n" + "# declare an appender named \"rpc\" that writes messages to rpc.log\n" + "[log.file_appender.rpc]\n" + "# filename can be absolute or relative to this config file\n" + "filename=logs/rpc/rpc.log\n" + "# Rotate log every ? minutes, if leave out default to 60\n" + "rotation_interval=60\n" + "# how long will logs be kept (in days), if leave out default to 1\n" + "rotation_limit=7\n\n" + "# route any messages logged to the default logger to the \"stderr\" appender and\n" + "# \"default\" appender we declared above, if they are info level or higher\n" + "[logger.default]\n" + "level=info\n" + "appenders=stderr,default\n\n" + "# route messages sent to the \"p2p\" logger to the \"p2p\" appender declared above\n" + "[logger.p2p]\n" + "level=warn\n" + "appenders=p2p\n\n" + "# route messages sent to the \"rpc\" logger to the \"rpc\" appender declared above\n" + "[logger.rpc]\n" + "level=error\n" + "appenders=rpc\n\n"; +} + +static fc::optional load_logging_config_from_ini_file(const fc::path& config_ini_filename) +{ + try + { + fc::logging_config logging_config; + bool found_logging_config = false; + + boost::property_tree::ptree config_ini_tree; + boost::property_tree::ini_parser::read_ini(config_ini_filename.preferred_string().c_str(), config_ini_tree); + for (const auto& section : config_ini_tree) + { + const std::string& section_name = section.first; + const boost::property_tree::ptree& section_tree = section.second; + + const std::string console_appender_section_prefix = "log.console_appender."; + const std::string file_appender_section_prefix = "log.file_appender."; + const std::string logger_section_prefix = "logger."; + + if (boost::starts_with(section_name, console_appender_section_prefix)) + { + std::string console_appender_name = section_name.substr(console_appender_section_prefix.length()); + std::string stream_name = section_tree.get("stream"); + + // construct a default console appender config here + // stdout/stderr will be taken from ini file, everything else hard-coded here + fc::console_appender::config console_appender_config; + console_appender_config.level_colors.emplace_back( + fc::console_appender::level_color(fc::log_level::debug, + fc::console_appender::color::green)); + console_appender_config.level_colors.emplace_back( + fc::console_appender::level_color(fc::log_level::warn, + fc::console_appender::color::brown)); + console_appender_config.level_colors.emplace_back( + fc::console_appender::level_color(fc::log_level::error, + fc::console_appender::color::cyan)); + console_appender_config.stream = fc::variant(stream_name).as(GRAPHENE_MAX_NESTED_OBJECTS); + logging_config.appenders.push_back(fc::appender_config(console_appender_name, "console", fc::variant(console_appender_config, GRAPHENE_MAX_NESTED_OBJECTS))); + found_logging_config = true; + } + else if (boost::starts_with(section_name, file_appender_section_prefix)) + { + std::string file_appender_name = section_name.substr(file_appender_section_prefix.length()); + fc::path file_name = section_tree.get("filename"); + if (file_name.is_relative()) + file_name = fc::absolute(config_ini_filename).parent_path() / file_name; + + int interval = section_tree.get_optional("rotation_interval").get_value_or(60); + int limit = section_tree.get_optional("rotation_limit").get_value_or(1); + + // construct a default file appender config here + // filename will be taken from ini file, everything else hard-coded here + fc::file_appender::config file_appender_config; + file_appender_config.filename = file_name; + file_appender_config.flush = true; + file_appender_config.rotate = true; + file_appender_config.rotation_interval = fc::minutes(interval); + file_appender_config.rotation_limit = fc::days(limit); + logging_config.appenders.push_back(fc::appender_config(file_appender_name, "file", fc::variant(file_appender_config, GRAPHENE_MAX_NESTED_OBJECTS))); + found_logging_config = true; + } + else if (boost::starts_with(section_name, logger_section_prefix)) + { + std::string logger_name = section_name.substr(logger_section_prefix.length()); + std::string level_string = section_tree.get("level"); + std::string appenders_string = section_tree.get("appenders"); + fc::logger_config logger_config(logger_name); + logger_config.level = fc::variant(level_string).as(5); + boost::split(logger_config.appenders, appenders_string, + boost::is_any_of(" ,"), + boost::token_compress_on); + logging_config.loggers.push_back(logger_config); + found_logging_config = true; + } + } + if (found_logging_config) + return logging_config; + else + return fc::optional(); + } + FC_RETHROW_EXCEPTIONS(warn, "") +} + +static const boost::shared_ptr new_option_description( const std::string& name, const bpo::value_semantic* value, const std::string& description ) +{ + bpo::options_description helper(""); + helper.add_options()( name.c_str(), value, description.c_str() ); + return helper.options()[0]; +} + + +static void load_config_file(const fc::path& config_ini_path, const bpo::options_description& cfg_options, + bpo::variables_map& options ) +{ + deduplicator dedup; + bpo::options_description unique_options("Graphene Witness Node"); + for( const boost::shared_ptr opt : cfg_options.options() ) + { + const boost::shared_ptr od = dedup.next(opt); + if( !od ) continue; + unique_options.add( od ); + } + + // get the basic options + bpo::store(bpo::parse_config_file(config_ini_path.preferred_string().c_str(), + unique_options, true), options); +} + +static void load_logging_config_file(const fc::path& config_ini_path) +{ + // try to get logging options from the config file. + try + { + fc::optional logging_config = load_logging_config_from_ini_file(config_ini_path); + if (logging_config) + fc::configure_logging(*logging_config); + } + catch (const fc::exception& ex) + { + wlog("Error parsing logging config from logging config file ${config}, using default config", ("config", config_ini_path.preferred_string())); + } +} + +static void create_new_config_file(const fc::path& config_ini_path, const fc::path& data_dir, + const bpo::options_description& cfg_options ) +{ + ilog("Writing new config file at ${path}", ("path", config_ini_path)); + if( !fc::exists(data_dir) ) + fc::create_directories(data_dir); + + auto modify_option_defaults = [](const boost::shared_ptr& o) -> const boost::shared_ptr { + const std::basic_string, std::allocator> & name = o->long_name(); + if( name == "partial-operations" ) + return new_option_description(name, bpo::value()->default_value(true), o->description() ); + if( name == "max-ops-per-account" ) + return new_option_description(name, bpo::value()->default_value(1000), o->description() ); + return o; + }; + deduplicator dedup(modify_option_defaults); + std::ofstream out_cfg(config_ini_path.preferred_string()); + for( const boost::shared_ptr opt : cfg_options.options() ) + { + const boost::shared_ptr od = dedup.next(opt); + if( !od ) continue; + + if( !od->description().empty() ) + out_cfg << "# " << od->description() << "\n"; + boost::any store; + if( !od->semantic()->apply_default(store) ) + out_cfg << "# " << od->long_name() << " = \n"; + else { + auto example = od->format_parameter(); + if( example.empty() ) + // This is a boolean switch + out_cfg << od->long_name() << " = " << "false\n"; + else { + // The string is formatted "arg (=)" + example.erase(0, 6); + example.erase(example.length()-1); + out_cfg << od->long_name() << " = " << example << "\n"; + } + } + out_cfg << "\n"; + } + + out_cfg.close(); +} + +static void create_logging_config_file(const fc::path& config_ini_path, const fc::path& data_dir) +{ + ilog("Writing new config file at ${path}", ("path", config_ini_path)); + if (!exists(data_dir)) + { + create_directories(data_dir); + } + + std::ofstream out_cfg(config_ini_path.preferred_string()); + write_default_logging_config_to_stream(out_cfg); + out_cfg.close(); +} + +namespace graphene { namespace app { + + void load_configuration_options(const fc::path& data_dir, const bpo::options_description& cfg_options, bpo::variables_map& options) + { + // load witness node initial configuration + fc::path config_ini_path = data_dir / "config.ini"; + if( !exists(config_ini_path) ) + create_new_config_file( config_ini_path, data_dir, cfg_options ); + load_config_file( config_ini_path, cfg_options, options ); + + // load witness node logging configuration + const auto logging_ini_path = data_dir / "logging.ini"; + if (!fc::exists(logging_ini_path)) + { + create_logging_config_file (logging_ini_path, data_dir); + } + + load_logging_config_file( logging_ini_path ); + } + +} } // graphene::app \ No newline at end of file diff --git a/libraries/app/include/graphene/app/config_util.hpp b/libraries/app/include/graphene/app/config_util.hpp new file mode 100644 index 0000000000..d7358f228c --- /dev/null +++ b/libraries/app/include/graphene/app/config_util.hpp @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2018 Lubos Ilcik, and contributors. + * + * The MIT License + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#pragma once + +#include +#include + +namespace graphene { namespace app { + + void load_configuration_options(const fc::path &data_dir, const boost::program_options::options_description &cfg_options, + boost::program_options::variables_map &options); + +} } // graphene::app \ No newline at end of file diff --git a/programs/witness_node/main.cpp b/programs/witness_node/main.cpp index 0eda181245..2a04390fd7 100644 --- a/programs/witness_node/main.cpp +++ b/programs/witness_node/main.cpp @@ -22,6 +22,7 @@ * THE SOFTWARE. */ #include +#include #include #include @@ -33,30 +34,18 @@ #include #include -#include #include #include -#include -#include -#include -#include #include - #include -#include -#include -#include -#include #include #include -#include #include #include #include -#include #ifdef WIN32 # include @@ -67,146 +56,6 @@ using namespace graphene; namespace bpo = boost::program_options; -void write_default_logging_config_to_stream(std::ostream& out); -fc::optional load_logging_config_from_ini_file(const fc::path& config_ini_filename); - -class deduplicator -{ - public: - deduplicator() : modifier(nullptr) {} - - deduplicator(const boost::shared_ptr (*mod_fn)(const boost::shared_ptr&)) - : modifier(mod_fn) {} - - const boost::shared_ptr next(const boost::shared_ptr& o) - { - const std::string name = o->long_name(); - if( seen.find( name ) != seen.end() ) - return nullptr; - seen.insert(name); - return modifier ? modifier(o) : o; - } - - private: - boost::container::flat_set seen; - const boost::shared_ptr (*modifier)(const boost::shared_ptr&); -}; - -static void load_config_file( const fc::path& config_ini_path, const bpo::options_description& cfg_options, - bpo::variables_map& options ) -{ - deduplicator dedup; - bpo::options_description unique_options("Graphene Witness Node"); - for( const boost::shared_ptr opt : cfg_options.options() ) - { - const boost::shared_ptr od = dedup.next(opt); - if( !od ) continue; - unique_options.add( od ); - } - - // get the basic options - bpo::store(bpo::parse_config_file(config_ini_path.preferred_string().c_str(), - unique_options, true), options); -} - -static void load_logging_config_file(const fc::path& config_ini_path) -{ - // try to get logging options from the config file. - try - { - fc::optional logging_config = load_logging_config_from_ini_file(config_ini_path); - if (logging_config) - fc::configure_logging(*logging_config); - } - catch (const fc::exception& ex) - { - wlog("Error parsing logging config from logging config file ${config}, using default config", ("config", config_ini_path.preferred_string())); - } -} - -const boost::shared_ptr new_option_description( const std::string& name, const bpo::value_semantic* value, const std::string& description ) -{ - bpo::options_description helper(""); - helper.add_options()( name.c_str(), value, description.c_str() ); - return helper.options()[0]; -} - -static void create_new_config_file( const fc::path& config_ini_path, const fc::path& data_dir, - const bpo::options_description& cfg_options ) -{ - ilog("Writing new config file at ${path}", ("path", config_ini_path)); - if( !fc::exists(data_dir) ) - fc::create_directories(data_dir); - - auto modify_option_defaults = [](const boost::shared_ptr& o) -> const boost::shared_ptr { - const std::string& name = o->long_name(); - if( name == "partial-operations" ) - return new_option_description( name, bpo::value()->default_value(true), o->description() ); - if( name == "max-ops-per-account" ) - return new_option_description( name, bpo::value()->default_value(1000), o->description() ); - return o; - }; - deduplicator dedup(modify_option_defaults); - std::ofstream out_cfg(config_ini_path.preferred_string()); - for( const boost::shared_ptr opt : cfg_options.options() ) - { - const boost::shared_ptr od = dedup.next(opt); - if( !od ) continue; - - if( !od->description().empty() ) - out_cfg << "# " << od->description() << "\n"; - boost::any store; - if( !od->semantic()->apply_default(store) ) - out_cfg << "# " << od->long_name() << " = \n"; - else { - auto example = od->format_parameter(); - if( example.empty() ) - // This is a boolean switch - out_cfg << od->long_name() << " = " << "false\n"; - else { - // The string is formatted "arg (=)" - example.erase(0, 6); - example.erase(example.length()-1); - out_cfg << od->long_name() << " = " << example << "\n"; - } - } - out_cfg << "\n"; - } - - out_cfg.close(); -} - -static void create_logging_config_file(const fc::path& config_ini_path, const fc::path& data_dir) -{ - ilog("Writing new config file at ${path}", ("path", config_ini_path)); - if (!exists(data_dir)) - { - create_directories(data_dir); - } - - std::ofstream out_cfg(config_ini_path.preferred_string()); - write_default_logging_config_to_stream(out_cfg); - out_cfg.close(); -} - -void load_configuration_options(const fc::path& data_dir, const bpo::options_description& cfg_options, bpo::variables_map& options) -{ - // load witness node initial configuration - fc::path config_ini_path = data_dir / "config.ini"; - if( !exists(config_ini_path) ) - create_new_config_file( config_ini_path, data_dir, cfg_options ); - load_config_file( config_ini_path, cfg_options, options ); - - // load witness node logging configuration - const auto logging_ini_path = data_dir / "logging.ini"; - if (!fc::exists(logging_ini_path)) - { - create_logging_config_file (logging_ini_path, data_dir); - } - - load_logging_config_file( logging_ini_path ); -} - int main(int argc, char** argv) { app::application* node = new app::application(); fc::oexception unhandled_exception; @@ -268,7 +117,7 @@ int main(int argc, char** argv) { if( data_dir.is_relative() ) data_dir = fc::current_path() / data_dir; } - load_configuration_options(data_dir, cfg_options, options); + app::load_configuration_options(data_dir, cfg_options, options); bpo::notify(options); node->initialize(data_dir, options); @@ -312,135 +161,3 @@ int main(int argc, char** argv) { } } -// logging config is too complicated to be parsed by boost::program_options, -// so we do it by hand -// -// Currently, you can only specify the filenames and logging levels, which -// are all most users would want to change. At a later time, options can -// be added to control rotation intervals, compression, and other seldom- -// used features -void write_default_logging_config_to_stream(std::ostream& out) -{ - out << "# declare an appender named \"stderr\" that writes messages to the console\n" - "[log.console_appender.stderr]\n" - "stream=std_error\n\n" - "# declare an appender named \"default\" that writes messages to default.log\n" - "[log.file_appender.default]\n" - "# filename can be absolute or relative to this config file\n" - "filename=logs/default/default.log\n" - "# Rotate log every ? minutes, if leave out default to 60\n" - "rotation_interval=60\n" - "# how long will logs be kept (in days), if leave out default to 1\n" - "rotation_limit=7\n\n" - "# declare an appender named \"p2p\" that writes messages to p2p.log\n" - "[log.file_appender.p2p]\n" - "# filename can be absolute or relative to this config file\n" - "filename=logs/p2p/p2p.log\n" - "# Rotate log every ? minutes, if leave out default to 60\n" - "rotation_interval=60\n" - "# how long will logs be kept (in days), if leave out default to 1\n" - "rotation_limit=7\n\n" - "# declare an appender named \"rpc\" that writes messages to rpc.log\n" - "[log.file_appender.rpc]\n" - "# filename can be absolute or relative to this config file\n" - "filename=logs/rpc/rpc.log\n" - "# Rotate log every ? minutes, if leave out default to 60\n" - "rotation_interval=60\n" - "# how long will logs be kept (in days), if leave out default to 1\n" - "rotation_limit=7\n\n" - "# route any messages logged to the default logger to the \"stderr\" appender and\n" - "# \"default\" appender we declared above, if they are info level or higher\n" - "[logger.default]\n" - "level=info\n" - "appenders=stderr,default\n\n" - "# route messages sent to the \"p2p\" logger to the \"p2p\" appender declared above\n" - "[logger.p2p]\n" - "level=warn\n" - "appenders=p2p\n\n" - "# route messages sent to the \"rpc\" logger to the \"rpc\" appender declared above\n" - "[logger.rpc]\n" - "level=error\n" - "appenders=rpc\n\n"; -} - -fc::optional load_logging_config_from_ini_file(const fc::path& config_ini_filename) -{ - try - { - fc::logging_config logging_config; - bool found_logging_config = false; - - boost::property_tree::ptree config_ini_tree; - boost::property_tree::ini_parser::read_ini(config_ini_filename.preferred_string().c_str(), config_ini_tree); - for (const auto& section : config_ini_tree) - { - const std::string& section_name = section.first; - const boost::property_tree::ptree& section_tree = section.second; - - const std::string console_appender_section_prefix = "log.console_appender."; - const std::string file_appender_section_prefix = "log.file_appender."; - const std::string logger_section_prefix = "logger."; - - if (boost::starts_with(section_name, console_appender_section_prefix)) - { - std::string console_appender_name = section_name.substr(console_appender_section_prefix.length()); - std::string stream_name = section_tree.get("stream"); - - // construct a default console appender config here - // stdout/stderr will be taken from ini file, everything else hard-coded here - fc::console_appender::config console_appender_config; - console_appender_config.level_colors.emplace_back( - fc::console_appender::level_color(fc::log_level::debug, - fc::console_appender::color::green)); - console_appender_config.level_colors.emplace_back( - fc::console_appender::level_color(fc::log_level::warn, - fc::console_appender::color::brown)); - console_appender_config.level_colors.emplace_back( - fc::console_appender::level_color(fc::log_level::error, - fc::console_appender::color::cyan)); - console_appender_config.stream = fc::variant(stream_name).as(GRAPHENE_MAX_NESTED_OBJECTS); - logging_config.appenders.push_back(fc::appender_config(console_appender_name, "console", fc::variant(console_appender_config, GRAPHENE_MAX_NESTED_OBJECTS))); - found_logging_config = true; - } - else if (boost::starts_with(section_name, file_appender_section_prefix)) - { - std::string file_appender_name = section_name.substr(file_appender_section_prefix.length()); - fc::path file_name = section_tree.get("filename"); - if (file_name.is_relative()) - file_name = fc::absolute(config_ini_filename).parent_path() / file_name; - - int interval = section_tree.get_optional("rotation_interval").get_value_or(60); - int limit = section_tree.get_optional("rotation_limit").get_value_or(1); - - // construct a default file appender config here - // filename will be taken from ini file, everything else hard-coded here - fc::file_appender::config file_appender_config; - file_appender_config.filename = file_name; - file_appender_config.flush = true; - file_appender_config.rotate = true; - file_appender_config.rotation_interval = fc::minutes(interval); - file_appender_config.rotation_limit = fc::days(limit); - logging_config.appenders.push_back(fc::appender_config(file_appender_name, "file", fc::variant(file_appender_config, GRAPHENE_MAX_NESTED_OBJECTS))); - found_logging_config = true; - } - else if (boost::starts_with(section_name, logger_section_prefix)) - { - std::string logger_name = section_name.substr(logger_section_prefix.length()); - std::string level_string = section_tree.get("level"); - std::string appenders_string = section_tree.get("appenders"); - fc::logger_config logger_config(logger_name); - logger_config.level = fc::variant(level_string).as(5); - boost::split(logger_config.appenders, appenders_string, - boost::is_any_of(" ,"), - boost::token_compress_on); - logging_config.loggers.push_back(logger_config); - found_logging_config = true; - } - } - if (found_logging_config) - return logging_config; - else - return fc::optional(); - } - FC_RETHROW_EXCEPTIONS(warn, "") -} From aabde7f21f69087dcffb8d0466007e04267ea754 Mon Sep 17 00:00:00 2001 From: "lubos.ilcik" Date: Sun, 29 Jul 2018 16:21:35 +0200 Subject: [PATCH 05/10] load_configuration_options basic tests --- tests/app/main.cpp | 104 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/tests/app/main.cpp b/tests/app/main.cpp index c2b0563134..02d212dde4 100644 --- a/tests/app/main.cpp +++ b/tests/app/main.cpp @@ -23,6 +23,7 @@ */ #include #include +#include #include @@ -35,6 +36,8 @@ #include #include +#include +#include #include @@ -44,6 +47,107 @@ #include "../common/genesis_file_util.hpp" using namespace graphene; +namespace bpo = boost::program_options; + +namespace fc { + extern std::unordered_map &get_logger_map(); + extern std::unordered_map &get_appender_map(); +} + +BOOST_AUTO_TEST_CASE(load_configuration_options_test_config_files_created) +{ + fc::temp_directory app_dir(graphene::utilities::temp_directory_path()); + auto dir = app_dir.path(); + auto config_ini_file = dir / "config.ini"; + auto logging_ini_file = dir / "logging.ini"; + + /// create default config options + auto node = new app::application(); + bpo::options_description cli, cfg; + node->set_program_options(cli, cfg); + bpo::options_description cfg_options("Graphene Witness Node"); + cfg_options.add(cfg); + + /// check preconditions + BOOST_CHECK(!fc::exists(config_ini_file)); + BOOST_CHECK(!fc::exists(logging_ini_file)); + + bpo::variables_map options; + app::load_configuration_options(dir, cfg_options, options); + + /// check post-conditions + BOOST_CHECK(fc::exists(config_ini_file)); + BOOST_CHECK(fc::exists(logging_ini_file)); + BOOST_CHECK_GT(fc::file_size(config_ini_file), 0); + BOOST_CHECK_GT(fc::file_size(logging_ini_file), 0); +} + +BOOST_AUTO_TEST_CASE(load_configuration_options_test_config_ini_options) +{ + fc::temp_directory app_dir(graphene::utilities::temp_directory_path()); + auto dir = app_dir.path(); + auto config_ini_file = dir / "config.ini"; + auto logging_ini_file = dir / "logging.ini"; + + /// create config.ini + bpo::options_description cfg_options("config.ini options"); + cfg_options.add_options() + ("option1", bpo::value(), "") + ("option2", bpo::value(), "") + ; + std::ofstream out(config_ini_file.preferred_string()); + out << "option1=is present\n" + "option2=1\n\n"; + out.close(); + + bpo::variables_map options; + app::load_configuration_options(dir, cfg_options, options); + + /// check the options values are parsed into the output map + BOOST_CHECK(!options.empty()); + BOOST_CHECK_EQUAL(options.count("option1"), 1); + BOOST_CHECK_EQUAL(options.count("option2"), 1); + BOOST_CHECK_EQUAL(options["option1"].as(), "is present"); + BOOST_CHECK_EQUAL(options["option2"].as(), 1); +} + +BOOST_AUTO_TEST_CASE(load_configuration_options_test_logging_ini_options) +{ + fc::temp_directory app_dir(graphene::utilities::temp_directory_path()); + auto dir = app_dir.path(); + auto config_ini_file = dir / "config.ini"; + auto logging_ini_file = dir / "logging.ini"; + + /// create logging.ini + /// configure exactly one logger and appender + std::ofstream out(logging_ini_file.preferred_string()); + out << "[log.file_appender.default]\n" + "filename=test.log\n\n" + "[logger.default]\n" + "level=info\n" + "appenders=default\n\n" + ; + out.close(); + + /// clear logger and appender state + fc::get_logger_map().clear(); + fc::get_appender_map().clear(); + BOOST_CHECK(fc::get_logger_map().empty()); + BOOST_CHECK(fc::get_appender_map().empty()); + + bpo::options_description cfg_options("empty"); + bpo::variables_map options; + app::load_configuration_options(dir, cfg_options, options); + + /// check the options values are parsed into the output map + /// this is a little bit tricky since load_configuration_options() doesn't provide output variable for logging_config + auto logger_map = fc::get_logger_map(); + auto appender_map = fc::get_appender_map(); + BOOST_CHECK_EQUAL(logger_map.size(), 1); + BOOST_CHECK(logger_map.count("default")); + BOOST_CHECK_EQUAL(appender_map.size(), 1); + BOOST_CHECK(appender_map.count("default")); +} ///////////// /// @brief create a 2 node network From 0d60379d9f4ebf75596cfcdbdbb5f59cd8d3e005 Mon Sep 17 00:00:00 2001 From: "lubos.ilcik" Date: Mon, 30 Jul 2018 10:43:45 +0200 Subject: [PATCH 06/10] load logging options from legacy config.ini --- libraries/app/config_util.cpp | 45 +++++++++++++++++++------- tests/app/main.cpp | 61 ++++++++++++++++++++++++++++++++++- 2 files changed, 94 insertions(+), 12 deletions(-) diff --git a/libraries/app/config_util.cpp b/libraries/app/config_util.cpp index 58b67eda30..9e84b5ccf6 100644 --- a/libraries/app/config_util.cpp +++ b/libraries/app/config_util.cpp @@ -223,19 +223,23 @@ static void load_config_file(const fc::path& config_ini_path, const bpo::options unique_options, true), options); } -static void load_logging_config_file(const fc::path& config_ini_path) +static bool load_logging_config_file(const fc::path& config_ini_path) { // try to get logging options from the config file. try { fc::optional logging_config = load_logging_config_from_ini_file(config_ini_path); if (logging_config) + { fc::configure_logging(*logging_config); + return true; + } } catch (const fc::exception& ex) { wlog("Error parsing logging config from logging config file ${config}, using default config", ("config", config_ini_path.preferred_string())); } + return false; } static void create_new_config_file(const fc::path& config_ini_path, const fc::path& data_dir, @@ -300,20 +304,39 @@ namespace graphene { namespace app { void load_configuration_options(const fc::path& data_dir, const bpo::options_description& cfg_options, bpo::variables_map& options) { - // load witness node initial configuration - fc::path config_ini_path = data_dir / "config.ini"; - if( !exists(config_ini_path) ) - create_new_config_file( config_ini_path, data_dir, cfg_options ); - load_config_file( config_ini_path, cfg_options, options ); - - // load witness node logging configuration + const auto config_ini_path = data_dir / "config.ini"; const auto logging_ini_path = data_dir / "logging.ini"; - if (!fc::exists(logging_ini_path)) + + if(!exists(config_ini_path) && fc::exists(logging_ini_path)) { - create_logging_config_file (logging_ini_path, data_dir); + // this is an uncommon case + create_new_config_file(config_ini_path, data_dir, cfg_options); } + else if(!exists(config_ini_path)) + { + // create default config.ini and logging.ini + create_new_config_file(config_ini_path, data_dir, cfg_options); + create_logging_config_file(logging_ini_path, data_dir); + } + + // load witness node configuration + load_config_file(config_ini_path, cfg_options, options); - load_logging_config_file( logging_ini_path ); + // load logging configuration + if (fc::exists(logging_ini_path)) + { + load_logging_config_file(logging_ini_path); + } + else + { + // this is the legacy config.ini case + if(!load_logging_config_file(config_ini_path)) + { + // config_ini_path doesn't contain valid logging options - fall back to the defaults + create_logging_config_file(logging_ini_path, data_dir); + load_logging_config_file(logging_ini_path); + } + } } } } // graphene::app \ No newline at end of file diff --git a/tests/app/main.cpp b/tests/app/main.cpp index 02d212dde4..3ac5bc6399 100644 --- a/tests/app/main.cpp +++ b/tests/app/main.cpp @@ -54,7 +54,7 @@ namespace fc { extern std::unordered_map &get_appender_map(); } -BOOST_AUTO_TEST_CASE(load_configuration_options_test_config_files_created) +BOOST_AUTO_TEST_CASE(load_configuration_options_test_config_logging_files_created) { fc::temp_directory app_dir(graphene::utilities::temp_directory_path()); auto dir = app_dir.path(); @@ -100,6 +100,10 @@ BOOST_AUTO_TEST_CASE(load_configuration_options_test_config_ini_options) "option2=1\n\n"; out.close(); + /// check preconditions + BOOST_CHECK(fc::exists(config_ini_file)); + BOOST_CHECK(!fc::exists(logging_ini_file)); + bpo::variables_map options; app::load_configuration_options(dir, cfg_options, options); @@ -109,6 +113,10 @@ BOOST_AUTO_TEST_CASE(load_configuration_options_test_config_ini_options) BOOST_CHECK_EQUAL(options.count("option2"), 1); BOOST_CHECK_EQUAL(options["option1"].as(), "is present"); BOOST_CHECK_EQUAL(options["option2"].as(), 1); + + /// when the config.ini exists and doesn't contain logging configuration while the logging.ini doesn't exist + /// the default logging.ini is created + BOOST_CHECK(fc::exists(logging_ini_file)); } BOOST_AUTO_TEST_CASE(load_configuration_options_test_logging_ini_options) @@ -149,6 +157,57 @@ BOOST_AUTO_TEST_CASE(load_configuration_options_test_logging_ini_options) BOOST_CHECK(appender_map.count("default")); } +BOOST_AUTO_TEST_CASE(load_configuration_options_test_legacy_config_ini_options) +{ + fc::temp_directory app_dir(graphene::utilities::temp_directory_path()); + auto dir = app_dir.path(); + auto config_ini_file = dir / "config.ini"; + auto logging_ini_file = dir / "logging.ini"; + + /// create config.ini + bpo::options_description cfg_options("config.ini options"); + cfg_options.add_options() + ("option1", bpo::value(), "") + ("option2", bpo::value(), "") + ; + std::ofstream out(config_ini_file.preferred_string()); + out << "option1=is present\n" + "option2=1\n\n" + "[log.file_appender.default]\n" + "filename=test.log\n\n" + "[logger.default]\n" + "level=info\n" + "appenders=default\n\n" + ; + out.close(); + + /// clear logger and appender state + fc::get_logger_map().clear(); + fc::get_appender_map().clear(); + BOOST_CHECK(fc::get_logger_map().empty()); + BOOST_CHECK(fc::get_appender_map().empty()); + + bpo::variables_map options; + app::load_configuration_options(dir, cfg_options, options); + + /// check logging.ini not created + BOOST_CHECK(!fc::exists(logging_ini_file)); + + /// check the options values are parsed into the output map + BOOST_CHECK(!options.empty()); + BOOST_CHECK_EQUAL(options.count("option1"), 1); + BOOST_CHECK_EQUAL(options.count("option2"), 1); + BOOST_CHECK_EQUAL(options["option1"].as(), "is present"); + BOOST_CHECK_EQUAL(options["option2"].as(), 1); + + auto logger_map = fc::get_logger_map(); + auto appender_map = fc::get_appender_map(); + BOOST_CHECK_EQUAL(logger_map.size(), 1); + BOOST_CHECK(logger_map.count("default")); + BOOST_CHECK_EQUAL(appender_map.size(), 1); + BOOST_CHECK(appender_map.count("default")); +} + ///////////// /// @brief create a 2 node network ///////////// From 0b5f84d844015f3c7bedbc860c362ede0f5ac9ec Mon Sep 17 00:00:00 2001 From: "lubos.ilcik" Date: Wed, 1 Aug 2018 07:38:13 +0200 Subject: [PATCH 07/10] don't create a logging config file if config_ini_path doesn't contain valid logging options --- libraries/app/config_util.cpp | 7 +------ tests/app/main.cpp | 4 ++-- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/libraries/app/config_util.cpp b/libraries/app/config_util.cpp index 9e84b5ccf6..0b7b867644 100644 --- a/libraries/app/config_util.cpp +++ b/libraries/app/config_util.cpp @@ -330,12 +330,7 @@ namespace graphene { namespace app { else { // this is the legacy config.ini case - if(!load_logging_config_file(config_ini_path)) - { - // config_ini_path doesn't contain valid logging options - fall back to the defaults - create_logging_config_file(logging_ini_path, data_dir); - load_logging_config_file(logging_ini_path); - } + load_logging_config_file(config_ini_path); } } diff --git a/tests/app/main.cpp b/tests/app/main.cpp index 3ac5bc6399..848d60d664 100644 --- a/tests/app/main.cpp +++ b/tests/app/main.cpp @@ -115,8 +115,8 @@ BOOST_AUTO_TEST_CASE(load_configuration_options_test_config_ini_options) BOOST_CHECK_EQUAL(options["option2"].as(), 1); /// when the config.ini exists and doesn't contain logging configuration while the logging.ini doesn't exist - /// the default logging.ini is created - BOOST_CHECK(fc::exists(logging_ini_file)); + /// the logging.ini is not created + BOOST_CHECK(!fc::exists(logging_ini_file)); } BOOST_AUTO_TEST_CASE(load_configuration_options_test_logging_ini_options) From 7e01230e8035ec92039b0e74245961e14d05d1be Mon Sep 17 00:00:00 2001 From: "lubos.ilcik" Date: Wed, 1 Aug 2018 07:43:31 +0200 Subject: [PATCH 08/10] comment moved to right place --- libraries/app/config_util.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/libraries/app/config_util.cpp b/libraries/app/config_util.cpp index 0b7b867644..c626c5fa79 100644 --- a/libraries/app/config_util.cpp +++ b/libraries/app/config_util.cpp @@ -65,9 +65,6 @@ class deduplicator const boost::shared_ptr (*modifier)(const boost::shared_ptr&); }; -// logging config is too complicated to be parsed by boost::program_options, -// so we do it by hand -// // Currently, you can only specify the filenames and logging levels, which // are all most users would want to change. At a later time, options can // be added to control rotation intervals, compression, and other seldom- @@ -116,6 +113,8 @@ static void write_default_logging_config_to_stream(std::ostream& out) "appenders=rpc\n\n"; } +// logging config is too complicated to be parsed by boost::program_options, +// so we do it by hand static fc::optional load_logging_config_from_ini_file(const fc::path& config_ini_filename) { try From a116369044d07ebb09c28dbca982c594e73020e5 Mon Sep 17 00:00:00 2001 From: "lubos.ilcik" Date: Wed, 1 Aug 2018 08:02:34 +0200 Subject: [PATCH 09/10] std::basic_string replaced with std::string --- libraries/app/config_util.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/app/config_util.cpp b/libraries/app/config_util.cpp index c626c5fa79..919f7daf3a 100644 --- a/libraries/app/config_util.cpp +++ b/libraries/app/config_util.cpp @@ -249,7 +249,7 @@ static void create_new_config_file(const fc::path& config_ini_path, const fc::pa fc::create_directories(data_dir); auto modify_option_defaults = [](const boost::shared_ptr& o) -> const boost::shared_ptr { - const std::basic_string, std::allocator> & name = o->long_name(); + const std::string& name = o->long_name(); if( name == "partial-operations" ) return new_option_description(name, bpo::value()->default_value(true), o->description() ); if( name == "max-ops-per-account" ) From 794cb340e5746544f459e157577a7c556dad2d14 Mon Sep 17 00:00:00 2001 From: "lubos.ilcik" Date: Thu, 2 Aug 2018 07:36:25 +0200 Subject: [PATCH 10/10] max-ops-per-account option default value set to 100 --- libraries/app/config_util.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/app/config_util.cpp b/libraries/app/config_util.cpp index 919f7daf3a..341fd00425 100644 --- a/libraries/app/config_util.cpp +++ b/libraries/app/config_util.cpp @@ -253,7 +253,7 @@ static void create_new_config_file(const fc::path& config_ini_path, const fc::pa if( name == "partial-operations" ) return new_option_description(name, bpo::value()->default_value(true), o->description() ); if( name == "max-ops-per-account" ) - return new_option_description(name, bpo::value()->default_value(1000), o->description() ); + return new_option_description(name, bpo::value()->default_value(100), o->description() ); return o; }; deduplicator dedup(modify_option_defaults);