From 7aea4fb938b2e74bd415e7361eeb191dbd0df10b Mon Sep 17 00:00:00 2001 From: Alfredo Date: Wed, 3 Oct 2018 08:15:06 -0300 Subject: [PATCH 1/7] add dascoin adaptor --- .../elasticsearch/elasticsearch_plugin.cpp | 17 +++- .../elasticsearch/elasticsearch_plugin.hpp | 88 ++++++++++++++++++- 2 files changed, 103 insertions(+), 2 deletions(-) diff --git a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp index a282b4e40e..d37f002417 100644 --- a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp +++ b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp @@ -58,6 +58,7 @@ class elasticsearch_plugin_impl bool _elasticsearch_visitor = false; std::string _elasticsearch_basic_auth = ""; std::string _elasticsearch_index_prefix = "bitshares-"; + bool _elasticsearch_operation_object = false; CURL *curl; // curl handler vector bulk_lines; // vector of op lines vector prepare; @@ -212,7 +213,17 @@ void elasticsearch_plugin_impl::doOperationHistory(const optional op_in_trx; os.operation_result = fc::json::to_string(oho->result); os.virtual_op = oho->virtual_op; - os.op = fc::json::to_string(oho->op); + + if(_elasticsearch_operation_object) { + oho->op.visit(fc::from_static_variant(os.op_object, FC_PACK_MAX_DEPTH)); + if (os.op_object.is_object()) { + adaptor_struct adaptor; + os.op_object = adaptor.adapt(os.op_object.get_object()); + } + } + else + os.op = fc::json::to_string(oho->op); + } void elasticsearch_plugin_impl::doBlock(uint32_t trx_in_block, const signed_block& b) @@ -418,6 +429,7 @@ void elasticsearch_plugin::plugin_set_program_options( ("elasticsearch-visitor", boost::program_options::value(), "Use visitor to index additional data(slows down the replay(false))") ("elasticsearch-basic-auth", boost::program_options::value(), "Pass basic auth to elasticsearch database('')") ("elasticsearch-index-prefix", boost::program_options::value(), "Add a prefix to the index(bitshares-)") + ("elasticsearch-operation-object", boost::program_options::value(), "Save operation as object(false)") ; cfg.add(cli); } @@ -451,6 +463,9 @@ void elasticsearch_plugin::plugin_initialize(const boost::program_options::varia if (options.count("elasticsearch-index-prefix")) { my->_elasticsearch_index_prefix = options["elasticsearch-index-prefix"].as(); } + if (options.count("elasticsearch-operation-object")) { + my->_elasticsearch_operation_object = options["elasticsearch-operation-object"].as(); + } } void elasticsearch_plugin::plugin_startup() diff --git a/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp b/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp index 088d3c1352..e20079a78e 100644 --- a/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp +++ b/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp @@ -128,6 +128,7 @@ struct operation_history_struct { std::string operation_result; int virtual_op; std::string op; + variant op_object; }; struct block_struct { @@ -183,9 +184,94 @@ struct bulk_struct { optional additional_data; }; +struct adaptor_struct { + variant adapt(const variant_object& op) + { + fc::mutable_variant_object o(op); + vector keys_to_rename; + for (auto i = o.begin(); i != o.end(); ++i) + { + auto& element = (*i).value(); + if (element.is_object()) + { + const string& name = (*i).key(); + auto& vo = element.get_object(); + if (vo.contains(name.c_str())) + keys_to_rename.emplace_back(name); + element = adapt(vo); + } + else if (element.is_array()) + adapt(element.get_array()); + } + for (const auto& i : keys_to_rename) + { + string new_name = i + "_"; + o[new_name] = variant(o[i]); + o.erase(i); + } + + if (o.find("memo") != o.end()) + { + auto& memo = o["memo"]; + if (memo.is_string()) + { + o["memo_"] = variant(o["memo"]); + o.erase("memo"); + } + else if (memo.is_object()) + { + fc::mutable_variant_object tmp(memo.get_object()); + if (tmp.find("nonce") != tmp.end()) + { + tmp["nonce"] = tmp["nonce"].as_string(); + o["memo"] = tmp; + } + } + } + if (o.find("kind") != o.end()) + { + o["kind"] = o["kind"].as_string(); + } + if (o.find("new_parameters") != o.end()) + { + auto& tmp = o["new_parameters"]; + if (tmp.is_object()) + { + fc::mutable_variant_object tmp2(tmp.get_object()); + if (tmp2.find("current_fees") != tmp2.end()) + { + tmp2.erase("current_fees"); + o["new_parameters"] = tmp2; + } + } + } + if (o.find("owner") != o.end() && o["owner"].is_string()) + { + o["owner_"] = o["owner"].as_string(); + o.erase("owner"); + } + variant v; + fc::to_variant(o, v, FC_PACK_MAX_DEPTH); + return v; + } + + void adapt(fc::variants& v) + { + for (auto& array_element : v) + { + if (array_element.is_object()) + array_element = adapt(array_element.get_object()); + else if (array_element.is_array()) + adapt(array_element.get_array()); + else + array_element = array_element.as_string(); + } + } +}; + } } //graphene::elasticsearch -FC_REFLECT( graphene::elasticsearch::operation_history_struct, (trx_in_block)(op_in_trx)(operation_result)(virtual_op)(op) ) +FC_REFLECT( graphene::elasticsearch::operation_history_struct, (trx_in_block)(op_in_trx)(operation_result)(virtual_op)(op)(op_object) ) FC_REFLECT( graphene::elasticsearch::block_struct, (block_num)(block_time)(trx_id) ) FC_REFLECT( graphene::elasticsearch::fee_struct, (asset)(asset_name)(amount)(amount_units) ) FC_REFLECT( graphene::elasticsearch::transfer_struct, (asset)(asset_name)(amount)(amount_units)(from)(to) ) From 51e273e27f230d04684aa306966ac52a01e11244 Mon Sep 17 00:00:00 2001 From: Alfredo Date: Mon, 8 Oct 2018 19:19:28 -0300 Subject: [PATCH 2/7] add legacy_generator, add 2 special cases to string --- libraries/plugins/elasticsearch/elasticsearch_plugin.cpp | 2 +- .../graphene/elasticsearch/elasticsearch_plugin.hpp | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp index d37f002417..573e05b88a 100644 --- a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp +++ b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp @@ -341,7 +341,7 @@ void elasticsearch_plugin_impl::createBulkLine(const account_transaction_history bulk_line_struct.block_data = bs; if(_elasticsearch_visitor) bulk_line_struct.additional_data = vs; - bulk_line = fc::json::to_string(bulk_line_struct); + bulk_line = fc::json::to_string(bulk_line_struct, fc::json::legacy_generator); } void elasticsearch_plugin_impl::prepareBulk(const account_transaction_history_id_type& ath_id) diff --git a/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp b/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp index e20079a78e..29dce5cc53 100644 --- a/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp +++ b/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp @@ -250,6 +250,15 @@ struct adaptor_struct { o["owner_"] = o["owner"].as_string(); o.erase("owner"); } + if (o.find("proposed_ops") != o.end()) + { + o["proposed_ops"] = fc::json::to_string(o["proposed_ops"]); + } + if (o.find("initializer") != o.end()) + { + o["initializer"] = fc::json::to_string(o["initializer"]); + } + variant v; fc::to_variant(o, v, FC_PACK_MAX_DEPTH); return v; From 787f16439befb5e076910910a1b9e0f88f0924ad Mon Sep 17 00:00:00 2001 From: Alfredo Date: Mon, 8 Oct 2018 19:51:47 -0300 Subject: [PATCH 3/7] change identation from 4 to 3 spaces in adaptor --- .../elasticsearch/elasticsearch_plugin.hpp | 156 +++++++++--------- 1 file changed, 78 insertions(+), 78 deletions(-) diff --git a/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp b/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp index 29dce5cc53..3a755356cf 100644 --- a/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp +++ b/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp @@ -187,94 +187,94 @@ struct bulk_struct { struct adaptor_struct { variant adapt(const variant_object& op) { - fc::mutable_variant_object o(op); - vector keys_to_rename; - for (auto i = o.begin(); i != o.end(); ++i) - { - auto& element = (*i).value(); - if (element.is_object()) - { - const string& name = (*i).key(); - auto& vo = element.get_object(); - if (vo.contains(name.c_str())) - keys_to_rename.emplace_back(name); - element = adapt(vo); - } - else if (element.is_array()) - adapt(element.get_array()); - } - for (const auto& i : keys_to_rename) - { - string new_name = i + "_"; - o[new_name] = variant(o[i]); - o.erase(i); - } + fc::mutable_variant_object o(op); + vector keys_to_rename; + for (auto i = o.begin(); i != o.end(); ++i) + { + auto& element = (*i).value(); + if (element.is_object()) + { + const string& name = (*i).key(); + auto& vo = element.get_object(); + if (vo.contains(name.c_str())) + keys_to_rename.emplace_back(name); + element = adapt(vo); + } + else if (element.is_array()) + adapt(element.get_array()); + } + for (const auto& i : keys_to_rename) + { + string new_name = i + "_"; + o[new_name] = variant(o[i]); + o.erase(i); + } - if (o.find("memo") != o.end()) - { - auto& memo = o["memo"]; - if (memo.is_string()) - { - o["memo_"] = variant(o["memo"]); - o.erase("memo"); - } - else if (memo.is_object()) - { - fc::mutable_variant_object tmp(memo.get_object()); - if (tmp.find("nonce") != tmp.end()) - { - tmp["nonce"] = tmp["nonce"].as_string(); - o["memo"] = tmp; - } - } - } - if (o.find("kind") != o.end()) - { - o["kind"] = o["kind"].as_string(); - } - if (o.find("new_parameters") != o.end()) - { - auto& tmp = o["new_parameters"]; - if (tmp.is_object()) - { - fc::mutable_variant_object tmp2(tmp.get_object()); - if (tmp2.find("current_fees") != tmp2.end()) - { - tmp2.erase("current_fees"); - o["new_parameters"] = tmp2; - } - } - } - if (o.find("owner") != o.end() && o["owner"].is_string()) - { - o["owner_"] = o["owner"].as_string(); - o.erase("owner"); - } - if (o.find("proposed_ops") != o.end()) - { - o["proposed_ops"] = fc::json::to_string(o["proposed_ops"]); - } + if (o.find("memo") != o.end()) + { + auto& memo = o["memo"]; + if (memo.is_string()) + { + o["memo_"] = variant(o["memo"]); + o.erase("memo"); + } + else if (memo.is_object()) + { + fc::mutable_variant_object tmp(memo.get_object()); + if (tmp.find("nonce") != tmp.end()) + { + tmp["nonce"] = tmp["nonce"].as_string(); + o["memo"] = tmp; + } + } + } + if (o.find("kind") != o.end()) + { + o["kind"] = o["kind"].as_string(); + } + if (o.find("new_parameters") != o.end()) + { + auto& tmp = o["new_parameters"]; + if (tmp.is_object()) + { + fc::mutable_variant_object tmp2(tmp.get_object()); + if (tmp2.find("current_fees") != tmp2.end()) + { + tmp2.erase("current_fees"); + o["new_parameters"] = tmp2; + } + } + } + if (o.find("owner") != o.end() && o["owner"].is_string()) + { + o["owner_"] = o["owner"].as_string(); + o.erase("owner"); + } + if (o.find("proposed_ops") != o.end()) + { + o["proposed_ops"] = fc::json::to_string(o["proposed_ops"]); + } if (o.find("initializer") != o.end()) { o["initializer"] = fc::json::to_string(o["initializer"]); } - variant v; - fc::to_variant(o, v, FC_PACK_MAX_DEPTH); - return v; + variant v; + fc::to_variant(o, v, FC_PACK_MAX_DEPTH); + return v; } void adapt(fc::variants& v) { - for (auto& array_element : v) - { - if (array_element.is_object()) - array_element = adapt(array_element.get_object()); - else if (array_element.is_array()) - adapt(array_element.get_array()); - else - array_element = array_element.as_string(); - } + for (auto& array_element : v) + { + if (array_element.is_object()) + array_element = adapt(array_element.get_object()); + else if (array_element.is_array()) + adapt(array_element.get_array()); + else + array_element = array_element.as_string(); + } } }; From 110b24ba77bc8c171ef8bb09c53f897b404136eb Mon Sep 17 00:00:00 2001 From: Alfredo Date: Wed, 10 Oct 2018 10:12:09 -0300 Subject: [PATCH 4/7] remove kind check --- .../include/graphene/elasticsearch/elasticsearch_plugin.hpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp b/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp index 3a755356cf..d0e896ce05 100644 --- a/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp +++ b/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp @@ -228,10 +228,6 @@ struct adaptor_struct { } } } - if (o.find("kind") != o.end()) - { - o["kind"] = o["kind"].as_string(); - } if (o.find("new_parameters") != o.end()) { auto& tmp = o["new_parameters"]; From 4f9094ff6697a3fad24c56127aa3d35eb864a490 Mon Sep 17 00:00:00 2001 From: Alfredo Date: Wed, 10 Oct 2018 10:38:59 -0300 Subject: [PATCH 5/7] bump fc --- libraries/fc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/fc b/libraries/fc index 8e1b47a460..62ebc67cb3 160000 --- a/libraries/fc +++ b/libraries/fc @@ -1 +1 @@ -Subproject commit 8e1b47a460d051eb335ee9fa134258af3870cfd8 +Subproject commit 62ebc67cb3bef8742c45c55a8da6eddb7c3bf275 From 53413a121c2f66117619b68c78b598ebe23501dd Mon Sep 17 00:00:00 2001 From: Alfredo Date: Thu, 11 Oct 2018 14:08:43 -0300 Subject: [PATCH 6/7] remove is_object check --- libraries/plugins/elasticsearch/elasticsearch_plugin.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp index 573e05b88a..dcffaea5f9 100644 --- a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp +++ b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp @@ -216,10 +216,8 @@ void elasticsearch_plugin_impl::doOperationHistory(const optional op.visit(fc::from_static_variant(os.op_object, FC_PACK_MAX_DEPTH)); - if (os.op_object.is_object()) { - adaptor_struct adaptor; - os.op_object = adaptor.adapt(os.op_object.get_object()); - } + adaptor_struct adaptor; + os.op_object = adaptor.adapt(os.op_object.get_object()); } else os.op = fc::json::to_string(oho->op); From 2af336308aaa7317c02c7738c4f648372d259f2c Mon Sep 17 00:00:00 2001 From: Alfredo Date: Thu, 11 Oct 2018 15:07:04 -0300 Subject: [PATCH 7/7] remove extra variant conversion --- .../include/graphene/elasticsearch/elasticsearch_plugin.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp b/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp index d0e896ce05..78c1d5cd90 100644 --- a/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp +++ b/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp @@ -215,7 +215,7 @@ struct adaptor_struct { auto& memo = o["memo"]; if (memo.is_string()) { - o["memo_"] = variant(o["memo"]); + o["memo_"] = o["memo"]; o.erase("memo"); } else if (memo.is_object())