From b65da277bbab8576718ae0eabc2b37db20eba1b9 Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Tue, 15 Sep 2020 17:46:29 +0200 Subject: [PATCH] Parse config files as YAML --- integration/vast.conf | 12 ++- libvast/src/system/configuration.cpp | 107 ++++++++++++++++++++++----- 2 files changed, 94 insertions(+), 25 deletions(-) diff --git a/integration/vast.conf b/integration/vast.conf index 64933cdd025..2a519bf84fb 100644 --- a/integration/vast.conf +++ b/integration/vast.conf @@ -1,8 +1,6 @@ -system = { - disable-metrics = true -} +system: + disable-metrics: true -logger = { - file-verbosity = 'trace' - component-blacklist = [] -} +logger: + file-verbosity: trace + component-blacklist: [] diff --git a/libvast/src/system/configuration.cpp b/libvast/src/system/configuration.cpp index 96592a5eb19..4bc21bd81dc 100644 --- a/libvast/src/system/configuration.cpp +++ b/libvast/src/system/configuration.cpp @@ -13,10 +13,14 @@ #include "vast/system/configuration.hpp" +#include "vast/concept/convertible/to.hpp" #include "vast/config.hpp" +#include "vast/data.hpp" #include "vast/detail/add_message_types.hpp" +#include "vast/detail/append.hpp" #include "vast/detail/assert.hpp" #include "vast/detail/process.hpp" +#include "vast/detail/settings.hpp" #include "vast/detail/string.hpp" #include "vast/detail/system.hpp" #include "vast/path.hpp" @@ -36,7 +40,7 @@ #endif #include -#include +#include namespace vast::system { @@ -79,32 +83,99 @@ caf::error configuration::parse(int argc, char** argv) { command_line.assign(argv + 1, argv + argc); // Move CAF options to the end of the command line, parse them, and then // remove them. - auto is_vast_opt = [](auto& x) { - return !(detail::starts_with(x, "--caf.") - || detail::starts_with(x, "--config=") - || detail::starts_with(x, "--config-file=")); - }; + auto is_vast_opt = [](auto& x) { return !detail::starts_with(x, "--caf."); }; auto caf_opt = std::stable_partition(command_line.begin(), command_line.end(), is_vast_opt); std::vector caf_args; std::move(caf_opt, command_line.end(), std::back_inserter(caf_args)); command_line.erase(caf_opt, command_line.end()); - for (auto& arg : caf_args) { - // Remove caf. prefix for CAF parser. - if (detail::starts_with(arg, "--caf.")) - arg.erase(2, 4); - // Rewrite --config= option to CAF's expexted format. + // If the user provided a config file on the command line, we attempt to + // parse it last. + for (auto& arg : command_line) if (detail::starts_with(arg, "--config=")) - arg.replace(8, 0, "-file"); + config_paths.push_back(arg.substr(9)); + // Parse and merge all configuration files. + caf::settings merged_settings; + for (const auto& config : config_paths) { + if (exists(config)) { + auto contents = load_contents(config); + if (!contents) + return contents.error(); + auto yaml = from_yaml(*contents); + if (!yaml) + return yaml.error(); + auto rec = caf::get_if(&*yaml); + if (!rec) + return caf::make_error(ec::parse_error, "config file not a YAML map"); + auto flat_yaml = flatten(*rec); + // Erase all null values because a caf::config_value has no such notion. + for (auto i = flat_yaml.begin(); i != flat_yaml.end();) { + if (caf::holds_alternative(i->second)) + i = flat_yaml.erase(i); + else + ++i; + } + auto settings = to(flat_yaml); + if (!settings) + return settings.error(); + detail::merge_settings(*settings, merged_settings); + } } - for (const auto& p : config_paths) { - if (auto err = actor_system_config::parse({}, p.str().c_str())) { - err.context() += caf::make_message(p); - return err; + // TODO: Revisit this after we are on CAF 0.18. + // Helper function to parse a config_value with the type information + // contained in an config_option. Because our YAML config only knows about + // strings, but a config_option may require an atom, we have to use a + // heuristic to see whether either type works. + auto parse_config_value + = [](const caf::config_option& opt, + const caf::config_value val) -> caf::expected { + // Hackish way to get a string representation that doesn't add double + // quotes around the value. + auto no_quote_stringify + = detail::overload([](const auto& x) { return caf::deep_to_string(x); }, + [](const std::string& x) { return x; }); + auto str = caf::visit(no_quote_stringify, val); + auto result = opt.parse(str); + if (!result) { + // We now try to parse strings as atom using a regex, since we get + // recursive types like lists for free this way. A string-vs-atom type + // clash is the only instance we currently cannot distinguish. Everything + // else is a true type clash. + // (With CAF 0.18, this heuristic will be obsolete.) + str = std::regex_replace(str, std::regex("\""), "'"); + result = opt.parse(str); + if (!result) + return caf::make_error(ec::type_clash, "failed to parse config option", + caf::deep_to_string(opt.full_name()), str, + "expected", + caf::deep_to_string(opt.type_name())); + } + return result; + }; + for (auto& [key, value] : merged_settings) { + // We have flattened the YAML contents above, so dictionaries cannot occur. + VAST_ASSERT(!caf::holds_alternative(value)); + // Now this is incredibly ugly, but custom_options_ (a config_option_set) + // is the only place that contains the valid type information that our + // config file must abide to. + if (auto option = custom_options_.qualified_name_lookup(key)) { + if (auto x = parse_config_value(*option, value)) + put(content, key, std::move(*x)); + else + return x.error(); + } else { + // If the option is not relevant to CAF's custom options, we just store + // the value directly in the content. + put(content, key, value); } } - // We must clear the config_file_path first so it does not use - // `caf-application.ini` as fallback. + // Try parsing all --caf.* settings. First, strip caf. prefix for the + // CAF parser. + for (auto& arg : caf_args) + if (detail::starts_with(arg, "--caf.")) + arg.erase(2, 4); + // We clear the config_file_path first so it does not use + // caf-application.ini as fallback during actor_system_config::parse(). config_file_path.clear(); return actor_system_config::parse(std::move(caf_args)); }