diff --git a/CHANGELOG.md b/CHANGELOG.md index bf33855b457..f57dfc6abdf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,8 +12,7 @@ Every entry has a category for which we use the following visual abbreviations: ## Unreleased - 🎁 The `import json` command's type restrictions are more relaxed now, and can - additionally convert from JSON strings to VAST internal data types. The new - option `--strict` allows for restoring the old behavior. + additionally convert from JSON strings to VAST internal data types. [#891](https://github.com/tenzir/vast/pull/891) - 🎁 VAST now supports /etc/vast/vast.conf as an additional fallback for the diff --git a/libvast/src/format/json.cpp b/libvast/src/format/json.cpp index d3d46be61fb..3c07ef450d5 100644 --- a/libvast/src/format/json.cpp +++ b/libvast/src/format/json.cpp @@ -37,7 +37,7 @@ namespace vast::format::json { namespace { -struct strict_convert { +struct convert { template using expected = caf::expected; using json = vast::json; @@ -136,18 +136,6 @@ struct strict_convert { return xs; } - template - caf::expected operator()(T, U) const { - VAST_ERROR_ANON("json-reader cannot convert from", - caf::detail::pretty_type_name(typeid(T)), "to", - caf::detail::pretty_type_name(typeid(U))); - return make_error(ec::syntax_error, "invalid json type"); - } -}; - -struct relaxed_convert : strict_convert { - using strict_convert::operator(); - caf::expected operator()(const json::string& str, const bool_type&) const { if (bool x; parsers::json_boolean(str, x)) @@ -192,6 +180,14 @@ struct relaxed_convert : strict_convert { return port{x}; return make_error(ec::convert_error, "cannot convert from", str, "to port"); } + + template + caf::expected operator()(T, U) const { + VAST_ERROR_ANON("json-reader cannot convert from", + caf::detail::pretty_type_name(typeid(T)), "to", + caf::detail::pretty_type_name(typeid(U))); + return make_error(ec::syntax_error, "invalid json type"); + } }; const vast::json* lookup(std::string_view field, const vast::json::object& xs) { @@ -228,7 +224,7 @@ const char* writer::name() const { } caf::error add(table_slice_builder& builder, const vast::json::object& xs, - const record_type& layout, conversion_policy policy) { + const record_type& layout) { for (auto& field : layout.fields) { auto i = lookup(field.name, xs); // Non-existing fields are treated as empty (unset). @@ -238,14 +234,7 @@ caf::error add(table_slice_builder& builder, const vast::json::object& xs, "slice builder"); continue; } - auto x = [&] { - switch (policy) { - case conversion_policy::relaxed: - return caf::visit(relaxed_convert{}, *i, field.type); - case conversion_policy::strict: - return caf::visit(strict_convert{}, *i, field.type); - } - }(); + auto x = caf::visit(convert{}, *i, field.type); if (!x) return make_error(ec::convert_error, x.error().context(), "could not convert", field.name, ":", to_string(*i)); diff --git a/libvast/src/system/application.cpp b/libvast/src/system/application.cpp index 052719c027f..ec531e68a15 100644 --- a/libvast/src/system/application.cpp +++ b/libvast/src/system/application.cpp @@ -179,10 +179,10 @@ auto make_import_command() { source_opts("?import.csv")); import_->add_subcommand("json", "imports JSON with schema", documentation::vast_import_json, - json_source_opts("?import.json")); + source_opts("?import.json")); import_->add_subcommand("suricata", "imports suricata eve json", documentation::vast_import_suricata, - json_source_opts("?import.suricata")); + source_opts("?import.suricata")); import_->add_subcommand("syslog", "imports syslog messages", documentation::vast_import_syslog, source_opts("?import.syslog")); @@ -470,10 +470,6 @@ command::opts_builder source_opts(std::string_view category) { .add("uds,d", "treat -r as listening UNIX domain socket"); } -command::opts_builder json_source_opts(std::string_view category) { - return source_opts(category).add("strict", "strict type matching"); -} - command::opts_builder sink_opts(std::string_view category) { return command::opts(category) .add("write,w", "path to write events to") diff --git a/libvast/test/format/json.cpp b/libvast/test/format/json.cpp index 001c7ae5874..a4c61e4db08 100644 --- a/libvast/test/format/json.cpp +++ b/libvast/test/format/json.cpp @@ -104,7 +104,7 @@ TEST(json to data) { })json"; auto jn = unbox(to(str)); auto xs = caf::get(jn); - format::json::add(builder, xs, flat, format::json::conversion_policy::strict); + format::json::add(builder, xs, flat); auto ptr = builder.finish(); REQUIRE(ptr); CHECK(ptr->at(0, 11) == data{enumeration{2}}); diff --git a/libvast/vast/format/json.hpp b/libvast/vast/format/json.hpp index 34b152080ee..c439e3c04b6 100644 --- a/libvast/vast/format/json.hpp +++ b/libvast/vast/format/json.hpp @@ -31,12 +31,9 @@ #include #include -#include namespace vast::format::json { -enum class conversion_policy { relaxed, strict }; - class writer : public ostream_writer { public: using defaults = vast::defaults::export_::json; @@ -54,10 +51,9 @@ class writer : public ostream_writer { /// @param builder The builder to add the JSON object to. /// @param xs The JSON object to add to *builder. /// @param layout The record type describing *xs*. -/// @param policy Type conversion policy. /// @returns An error iff the operation failed. caf::error add(table_slice_builder& builder, const vast::json::object& xs, - const record_type& layout, conversion_policy policy); + const record_type& layout); /// @relates reader struct default_selector { @@ -123,7 +119,7 @@ class reader final : public multi_layout_reader { /// @param options Additional options. /// @param in The stream of JSON objects. explicit reader(caf::atom_value table_slice_type, - const caf::settings& options, + const caf::settings& /*options*/, std::unique_ptr in = nullptr); void reset(std::unique_ptr in); @@ -151,7 +147,6 @@ class reader final : public multi_layout_reader { mutable size_t num_invalid_lines_ = 0; mutable size_t num_unknown_layouts_ = 0; mutable size_t num_lines_ = 0; - conversion_policy conversion_policy_ = conversion_policy::relaxed; }; // -- implementation ---------------------------------------------------------- @@ -161,12 +156,8 @@ reader::reader(caf::atom_value table_slice_type, const caf::settings& options, std::unique_ptr in) : super(table_slice_type) { - using namespace std::string_literals; if (in != nullptr) reset(std::move(in)); - auto category = vast::defaults::import::json::category; - if (caf::get_or(options, category + ".strict"s, false)) - conversion_policy_ = conversion_policy::strict; } template @@ -265,7 +256,7 @@ caf::error reader::read_impl(size_t max_events, size_t max_slice_size, bptr = builder(*layout); if (bptr == nullptr) return make_error(ec::parse_error, "unable to get a builder"); - if (auto err = add(*bptr, *xs, *layout, conversion_policy_)) { + if (auto err = add(*bptr, *xs, *layout)) { err.context() += caf::make_message("line", lines_->line_number()); return finish(cons, err); } diff --git a/libvast/vast/system/application.hpp b/libvast/vast/system/application.hpp index 86edf141fcc..6b92c5b7b76 100644 --- a/libvast/vast/system/application.hpp +++ b/libvast/vast/system/application.hpp @@ -36,9 +36,6 @@ void render_error(const command& root, const caf::error& err, std::ostream& os); /// @returns default options for source commands. command::opts_builder source_opts(std::string_view category); -/// @returns default options for JSON source commands. -command::opts_builder json_source_opts(std::string_view category); - /// @returns defaults options for sink commands. command::opts_builder sink_opts(std::string_view category); diff --git a/vast.conf b/vast.conf index 09b01b841a9..507d0a11390 100644 --- a/vast.conf +++ b/vast.conf @@ -77,9 +77,6 @@ export { ; The `vast export json` command exports events formatted as JSONL (line- ; delimited JSON). json { - ; Strict type matching. - ;strict = false - ; For additionally available options, see export.ascii. }