From 9d99fce47a93269f9c811bae8a3657567c14d3a6 Mon Sep 17 00:00:00 2001 From: abitmore Date: Fri, 17 Dec 2021 12:10:21 +0000 Subject: [PATCH 01/20] Move some code from hpp file to cpp file --- .../elasticsearch/elasticsearch_plugin.cpp | 155 ++++++++++++++++++ .../elasticsearch/elasticsearch_plugin.hpp | 155 ------------------ 2 files changed, 155 insertions(+), 155 deletions(-) diff --git a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp index 3ae9009ba7..29e56fc6d6 100644 --- a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp +++ b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp @@ -245,6 +245,106 @@ void elasticsearch_plugin_impl::getOperationType(const optional op.which(); } +struct adaptor_struct { + variant adapt(const variant_object& op) + { + fc::mutable_variant_object o(op); + vector keys_to_rename; + for (auto i = o.begin(); i != o.end(); ++i) + { + auto& element = (*i).value(); + if (element.is_object()) + { + const string& name = (*i).key(); + auto& vo = element.get_object(); + if (vo.contains(name.c_str())) + keys_to_rename.emplace_back(name); + element = adapt(vo); + } + else if (element.is_array()) + adapt(element.get_array()); + } + for (const auto& i : keys_to_rename) + { + string new_name = i + "_"; + o[new_name] = variant(o[i]); + o.erase(i); + } + + if (o.find("memo") != o.end()) + { + auto& memo = o["memo"]; + if (memo.is_string()) + { + o["memo_"] = o["memo"]; + o.erase("memo"); + } + else if (memo.is_object()) + { + fc::mutable_variant_object tmp(memo.get_object()); + if (tmp.find("nonce") != tmp.end()) + { + tmp["nonce"] = tmp["nonce"].as_string(); + o["memo"] = tmp; + } + } + } + if (o.find("new_parameters") != o.end()) + { + auto& tmp = o["new_parameters"]; + if (tmp.is_object()) + { + fc::mutable_variant_object tmp2(tmp.get_object()); + if (tmp2.find("current_fees") != tmp2.end()) + { + tmp2.erase("current_fees"); + o["new_parameters"] = tmp2; + } + } + } + if (o.find("owner") != o.end() && o["owner"].is_string()) + { + o["owner_"] = o["owner"].as_string(); + o.erase("owner"); + } + + vector to_string_fields = { + "proposed_ops", + "initializer", + "policy", + "predicates", + "active_special_authority", + "owner_special_authority", + "acceptable_collateral", + "acceptable_borrowers" + }; + for( const auto& name : to_string_fields ) + { + if (o.find(name) != o.end()) + { + o[name] = fc::json::to_string(o[name]); + } + } + + variant v; + fc::to_variant(o, v, FC_PACK_MAX_DEPTH); + return v; + } + + void adapt(fc::variants& v) + { + for (auto& array_element : v) + { + if (array_element.is_object()) + array_element = adapt(array_element.get_object()); + else if (array_element.is_array()) + adapt(array_element.get_array()); + else + array_element = array_element.as_string(); + } + } +}; + void elasticsearch_plugin_impl::doOperationHistory(const optional & oho) { os.trx_in_block = oho->trx_in_block; @@ -271,6 +371,61 @@ void elasticsearch_plugin_impl::doBlock(uint32_t trx_in_block, const signed_bloc bs.trx_id = trx_id; } +struct operation_visitor +{ + typedef void result_type; + + share_type fee_amount; + asset_id_type fee_asset; + + asset_id_type transfer_asset_id; + share_type transfer_amount; + account_id_type transfer_from; + account_id_type transfer_to; + + void operator()( const graphene::chain::transfer_operation& o ) + { + fee_asset = o.fee.asset_id; + fee_amount = o.fee.amount; + + transfer_asset_id = o.amount.asset_id; + transfer_amount = o.amount.amount; + transfer_from = o.from; + transfer_to = o.to; + } + + object_id_type fill_order_id; + account_id_type fill_account_id; + asset_id_type fill_pays_asset_id; + share_type fill_pays_amount; + asset_id_type fill_receives_asset_id; + share_type fill_receives_amount; + double fill_fill_price; + bool fill_is_maker; + + void operator()( const graphene::chain::fill_order_operation& o ) + { + fee_asset = o.fee.asset_id; + fee_amount = o.fee.amount; + + fill_order_id = o.order_id; + fill_account_id = o.account_id; + fill_pays_asset_id = o.pays.asset_id; + fill_pays_amount = o.pays.amount; + fill_receives_asset_id = o.receives.asset_id; + fill_receives_amount = o.receives.amount; + fill_fill_price = o.fill_price.to_real(); + fill_is_maker = o.is_maker; + } + + template + void operator()( const T& o ) + { + fee_asset = o.fee.asset_id; + fee_amount = o.fee.amount; + } +}; + void elasticsearch_plugin_impl::doVisitor(const optional & oho) { graphene::chain::database& db = database(); diff --git a/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp b/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp index 05e24843f0..7aca292b33 100644 --- a/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp +++ b/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp @@ -80,61 +80,6 @@ class elasticsearch_plugin : public graphene::app::plugin }; -struct operation_visitor -{ - typedef void result_type; - - share_type fee_amount; - asset_id_type fee_asset; - - asset_id_type transfer_asset_id; - share_type transfer_amount; - account_id_type transfer_from; - account_id_type transfer_to; - - void operator()( const graphene::chain::transfer_operation& o ) - { - fee_asset = o.fee.asset_id; - fee_amount = o.fee.amount; - - transfer_asset_id = o.amount.asset_id; - transfer_amount = o.amount.amount; - transfer_from = o.from; - transfer_to = o.to; - } - - object_id_type fill_order_id; - account_id_type fill_account_id; - asset_id_type fill_pays_asset_id; - share_type fill_pays_amount; - asset_id_type fill_receives_asset_id; - share_type fill_receives_amount; - double fill_fill_price; - bool fill_is_maker; - - void operator()( const graphene::chain::fill_order_operation& o ) - { - fee_asset = o.fee.asset_id; - fee_amount = o.fee.amount; - - fill_order_id = o.order_id; - fill_account_id = o.account_id; - fill_pays_asset_id = o.pays.asset_id; - fill_pays_amount = o.pays.amount; - fill_receives_asset_id = o.receives.asset_id; - fill_receives_amount = o.receives.amount; - fill_fill_price = o.fill_price.to_real(); - fill_is_maker = o.is_maker; - } - - template - void operator()( const T& o ) - { - fee_asset = o.fee.asset_id; - fee_amount = o.fee.amount; - } -}; - struct operation_history_struct { int trx_in_block; int op_in_trx; @@ -197,106 +142,6 @@ struct bulk_struct { optional additional_data; }; -struct adaptor_struct { - variant adapt(const variant_object& op) - { - fc::mutable_variant_object o(op); - vector keys_to_rename; - for (auto i = o.begin(); i != o.end(); ++i) - { - auto& element = (*i).value(); - if (element.is_object()) - { - const string& name = (*i).key(); - auto& vo = element.get_object(); - if (vo.contains(name.c_str())) - keys_to_rename.emplace_back(name); - element = adapt(vo); - } - else if (element.is_array()) - adapt(element.get_array()); - } - for (const auto& i : keys_to_rename) - { - string new_name = i + "_"; - o[new_name] = variant(o[i]); - o.erase(i); - } - - if (o.find("memo") != o.end()) - { - auto& memo = o["memo"]; - if (memo.is_string()) - { - o["memo_"] = o["memo"]; - o.erase("memo"); - } - else if (memo.is_object()) - { - fc::mutable_variant_object tmp(memo.get_object()); - if (tmp.find("nonce") != tmp.end()) - { - tmp["nonce"] = tmp["nonce"].as_string(); - o["memo"] = tmp; - } - } - } - if (o.find("new_parameters") != o.end()) - { - auto& tmp = o["new_parameters"]; - if (tmp.is_object()) - { - fc::mutable_variant_object tmp2(tmp.get_object()); - if (tmp2.find("current_fees") != tmp2.end()) - { - tmp2.erase("current_fees"); - o["new_parameters"] = tmp2; - } - } - } - if (o.find("owner") != o.end() && o["owner"].is_string()) - { - o["owner_"] = o["owner"].as_string(); - o.erase("owner"); - } - - vector to_string_fields = { - "proposed_ops", - "initializer", - "policy", - "predicates", - "active_special_authority", - "owner_special_authority", - "acceptable_collateral", - "acceptable_borrowers" - }; - for( const auto& name : to_string_fields ) - { - if (o.find(name) != o.end()) - { - o[name] = fc::json::to_string(o[name]); - } - } - - variant v; - fc::to_variant(o, v, FC_PACK_MAX_DEPTH); - return v; - } - - void adapt(fc::variants& v) - { - for (auto& array_element : v) - { - if (array_element.is_object()) - array_element = adapt(array_element.get_object()); - else if (array_element.is_array()) - adapt(array_element.get_array()); - else - array_element = array_element.as_string(); - } - } -}; - } } //graphene::elasticsearch FC_REFLECT_ENUM( graphene::elasticsearch::mode, (only_save)(only_query)(all) ) From 91675d7271726158d421bae21b3454b5df250e2b Mon Sep 17 00:00:00 2001 From: abitmore Date: Fri, 17 Dec 2021 14:49:58 +0000 Subject: [PATCH 02/20] Wrap long lines --- .../elasticsearch/elasticsearch_plugin.hpp | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp b/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp index 7aca292b33..fbc012acba 100644 --- a/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp +++ b/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp @@ -87,6 +87,7 @@ struct operation_history_struct { int virtual_op; std::string op; variant op_object; + variant operation_result_object; }; struct block_struct { @@ -145,12 +146,15 @@ struct bulk_struct { } } //graphene::elasticsearch FC_REFLECT_ENUM( graphene::elasticsearch::mode, (only_save)(only_query)(all) ) -FC_REFLECT( graphene::elasticsearch::operation_history_struct, (trx_in_block)(op_in_trx)(operation_result)(virtual_op)(op)(op_object) ) +FC_REFLECT( graphene::elasticsearch::operation_history_struct, + (trx_in_block)(op_in_trx)(operation_result)(virtual_op)(op)(op_object)(operation_result_object) ) FC_REFLECT( graphene::elasticsearch::block_struct, (block_num)(block_time)(trx_id) ) FC_REFLECT( graphene::elasticsearch::fee_struct, (asset)(asset_name)(amount)(amount_units) ) FC_REFLECT( graphene::elasticsearch::transfer_struct, (asset)(asset_name)(amount)(amount_units)(from)(to) ) -FC_REFLECT( graphene::elasticsearch::fill_struct, (order_id)(account_id)(pays_asset_id)(pays_asset_name)(pays_amount)(pays_amount_units) - (receives_asset_id)(receives_asset_name)(receives_amount)(receives_amount_units)(fill_price) - (fill_price_units)(is_maker)) +FC_REFLECT( graphene::elasticsearch::fill_struct, + (order_id)(account_id)(pays_asset_id)(pays_asset_name)(pays_amount)(pays_amount_units) + (receives_asset_id)(receives_asset_name)(receives_amount)(receives_amount_units)(fill_price) + (fill_price_units)(is_maker) ) FC_REFLECT( graphene::elasticsearch::visitor_struct, (fee_data)(transfer_data)(fill_data) ) -FC_REFLECT( graphene::elasticsearch::bulk_struct, (account_history)(operation_history)(operation_type)(operation_id_num)(block_data)(additional_data) ) +FC_REFLECT( graphene::elasticsearch::bulk_struct, + (account_history)(operation_history)(operation_type)(operation_id_num)(block_data)(additional_data) ) From 03b90574e307fc2343d1a6f212fcc9b500a25399 Mon Sep 17 00:00:00 2001 From: abitmore Date: Fri, 17 Dec 2021 20:28:55 +0000 Subject: [PATCH 03/20] Adapt static_variant and map for ElasticSearch --- .../elasticsearch/elasticsearch_plugin.cpp | 143 +++++++++++++----- 1 file changed, 107 insertions(+), 36 deletions(-) diff --git a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp index 29e56fc6d6..5eb34f2f5d 100644 --- a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp +++ b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp @@ -245,8 +245,13 @@ void elasticsearch_plugin_impl::getOperationType(const optional op.which(); } -struct adaptor_struct { - variant adapt(const variant_object& op) +struct es_data_adaptor { + enum class data_type { + map_type, + static_variant_type, + static_variant_array_type + }; + static variant adapt(const variant_object& op) { fc::mutable_variant_object o(op); vector keys_to_rename; @@ -257,26 +262,37 @@ struct adaptor_struct { { const string& name = (*i).key(); auto& vo = element.get_object(); - if (vo.contains(name.c_str())) + if (vo.contains(name.c_str())) // transfer_operation.amount.amount keys_to_rename.emplace_back(name); element = adapt(vo); } else if (element.is_array()) adapt(element.get_array()); } - for (const auto& i : keys_to_rename) + + for( const auto& i : keys_to_rename ) // transfer_operation.amount { string new_name = i + "_"; o[new_name] = variant(o[i]); o.erase(i); } - if (o.find("memo") != o.end()) + if( o.find("fee") != o.end() ) + { + auto& fee = o["fee"]; + if( fee.is_uint64() ) // fee schedule + { + o["fee_"] = fee; + o.erase("fee"); + } + } + + if( o.find("memo") != o.end() ) { auto& memo = o["memo"]; - if (memo.is_string()) + if (memo.is_string()) // seems unused. TODO remove { - o["memo_"] = o["memo"]; + o["memo_"] = memo; o.erase("memo"); } else if (memo.is_object()) @@ -289,40 +305,32 @@ struct adaptor_struct { } } } - if (o.find("new_parameters") != o.end()) - { - auto& tmp = o["new_parameters"]; - if (tmp.is_object()) - { - fc::mutable_variant_object tmp2(tmp.get_object()); - if (tmp2.find("current_fees") != tmp2.end()) - { - tmp2.erase("current_fees"); - o["new_parameters"] = tmp2; - } - } - } - if (o.find("owner") != o.end() && o["owner"].is_string()) + + if( o.find("owner") != o.end() && o["owner"].is_string() ) // vesting_balance_*_operation.owner { o["owner_"] = o["owner"].as_string(); o.erase("owner"); } - vector to_string_fields = { - "proposed_ops", - "initializer", - "policy", - "predicates", - "active_special_authority", - "owner_special_authority", - "acceptable_collateral", - "acceptable_borrowers" + map to_string_fields = { + { "current_fees", data_type::static_variant_array_type }, + { "proposed_ops", data_type::static_variant_array_type }, + { "initializer", data_type::static_variant_type }, + { "policy", data_type::static_variant_type }, + { "predicates", data_type::static_variant_array_type }, + { "active_special_authority", data_type::static_variant_type }, + { "owner_special_authority", data_type::static_variant_type }, + { "acceptable_collateral", data_type::map_type }, + { "acceptable_borrowers", data_type::map_type } }; - for( const auto& name : to_string_fields ) + for( const auto& pair : to_string_fields ) { - if (o.find(name) != o.end()) + const auto& name = pair.first; + if( o.find(name) != o.end() ) { - o[name] = fc::json::to_string(o[name]); + const auto& value = o[name]; + o[name + "_object"] = adapt( value.get_array(), pair.second ); + o[name] = fc::json::to_string(value); } } @@ -331,7 +339,66 @@ struct adaptor_struct { return v; } - void adapt(fc::variants& v) + static variant adapt( const fc::variants& v, data_type type ) + { + if( data_type::map_type == type ) + return adapt_map(v); + if( data_type::static_variant_type == type ) + return adapt_static_variant(v); + + // static_variant array + fc::variants vs; + vs.reserve( v.size() ); + for( const auto& item : v ) + { + vs.push_back( adapt_static_variant( item.get_array() ) ); + } + + variant nv; + fc::to_variant(vs, nv, FC_PACK_MAX_DEPTH); + return nv; + } + + static void extract_data_from_variant( const variant& v, fc::mutable_variant_object& mv, const string& prefix ) + { + if( v.is_object() ) + mv[prefix + "_object"] = adapt( v.get_object() ); + else if( v.is_int64() || v.is_uint64() ) + mv[prefix + "_int"] = v; + else if( v.is_bool() ) + mv[prefix + "_bool"] = v; + else + mv[prefix + "_string"] = fc::json::to_string( v ); + // Note: we don't use double or array here, and we convert null and blob to string + } + + static variant adapt_map( const fc::variants& v ) + { + FC_ASSERT( v.size() == 2, "Internal error" ); + fc::mutable_variant_object mv; + + extract_data_from_variant( v[0], mv, "key" ); + extract_data_from_variant( v[1], mv, "data" ); + + variant nv; + fc::to_variant( mv, nv, FC_PACK_MAX_DEPTH ); + return nv; + } + + static variant adapt_static_variant( const fc::variants& v ) + { + FC_ASSERT( v.size() == 2, "Internal error" ); + fc::mutable_variant_object mv; + + mv["which"] = v[0]; + extract_data_from_variant( v[1], mv, "data" ); + + variant nv; + fc::to_variant( mv, nv, FC_PACK_MAX_DEPTH ); + return nv; + } + + static void adapt(fc::variants& v) { for (auto& array_element : v) { @@ -353,9 +420,13 @@ void elasticsearch_plugin_impl::doOperationHistory(const optional virtual_op; if(_elasticsearch_operation_object) { + // op oho->op.visit(fc::from_static_variant(os.op_object, FC_PACK_MAX_DEPTH)); - adaptor_struct adaptor; - os.op_object = adaptor.adapt(os.op_object.get_object()); + os.op_object = es_data_adaptor::adapt(os.op_object.get_object()); + // operation_result + variant v; + fc::to_variant( oho->result, v, FC_PACK_MAX_DEPTH ); + os.operation_result_object = es_data_adaptor::adapt_static_variant( v.get_array() ); } if(_elasticsearch_operation_string) os.op = fc::json::to_string(oho->op); From 0947fed2e907da29121f2049542d4da78d4f5c83 Mon Sep 17 00:00:00 2001 From: abitmore Date: Fri, 17 Dec 2021 21:47:59 +0000 Subject: [PATCH 04/20] Add logging --- libraries/plugins/elasticsearch/elasticsearch_plugin.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp index 5eb34f2f5d..d4e57b02b9 100644 --- a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp +++ b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp @@ -413,7 +413,7 @@ struct es_data_adaptor { }; void elasticsearch_plugin_impl::doOperationHistory(const optional & oho) -{ +{ try { os.trx_in_block = oho->trx_in_block; os.op_in_trx = oho->op_in_trx; os.operation_result = fc::json::to_string(oho->result); @@ -430,7 +430,7 @@ void elasticsearch_plugin_impl::doOperationHistory(const optional op); -} +} FC_CAPTURE_LOG_AND_RETHROW( (oho) ) } void elasticsearch_plugin_impl::doBlock(uint32_t trx_in_block, const signed_block& b) { From d03ef2def2543d96f05cdab0b30c9c45e87afb6c Mon Sep 17 00:00:00 2001 From: abitmore Date: Fri, 17 Dec 2021 22:24:20 +0000 Subject: [PATCH 05/20] Fix adaption of proposed_ops for ElasticSearch --- .../elasticsearch/elasticsearch_plugin.cpp | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp index d4e57b02b9..e70f8ee83b 100644 --- a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp +++ b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp @@ -249,7 +249,7 @@ struct es_data_adaptor { enum class data_type { map_type, static_variant_type, - static_variant_array_type + array_type }; static variant adapt(const variant_object& op) { @@ -313,11 +313,12 @@ struct es_data_adaptor { } map to_string_fields = { - { "current_fees", data_type::static_variant_array_type }, - { "proposed_ops", data_type::static_variant_array_type }, + { "current_fees", data_type::array_type }, + { "op", data_type::static_variant_type }, // proposal_create_op.proposed_ops[*].op + { "proposed_ops", data_type::array_type }, { "initializer", data_type::static_variant_type }, { "policy", data_type::static_variant_type }, - { "predicates", data_type::static_variant_array_type }, + { "predicates", data_type::array_type }, { "active_special_authority", data_type::static_variant_type }, { "owner_special_authority", data_type::static_variant_type }, { "acceptable_collateral", data_type::map_type }, @@ -346,12 +347,17 @@ struct es_data_adaptor { if( data_type::static_variant_type == type ) return adapt_static_variant(v); - // static_variant array + // array_type fc::variants vs; vs.reserve( v.size() ); for( const auto& item : v ) { - vs.push_back( adapt_static_variant( item.get_array() ) ); + if( item.is_array() ) // static_variant array + vs.push_back( adapt_static_variant( item.get_array() ) ); + else if( item.is_object() ) // object array + vs.push_back( adapt( item.get_object() ) ); + else + wlog( "Type of item is unexpected: ${item}", ("item", item) ); } variant nv; From 7152dbb948e890c3692c7511210ad265f2b9d7fa Mon Sep 17 00:00:00 2001 From: abitmore Date: Fri, 17 Dec 2021 22:37:14 +0000 Subject: [PATCH 06/20] Fix adaption of fee schedule update op for ES --- libraries/plugins/elasticsearch/elasticsearch_plugin.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp index e70f8ee83b..5d22beac3a 100644 --- a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp +++ b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp @@ -313,7 +313,7 @@ struct es_data_adaptor { } map to_string_fields = { - { "current_fees", data_type::array_type }, + { "parameters", data_type::array_type }, // in committee proposals, current_fees.parameters { "op", data_type::static_variant_type }, // proposal_create_op.proposed_ops[*].op { "proposed_ops", data_type::array_type }, { "initializer", data_type::static_variant_type }, From aff8e72a26d688c2de3e80cfe99b92d0c90d5e22 Mon Sep 17 00:00:00 2001 From: abitmore Date: Fri, 17 Dec 2021 23:06:47 +0000 Subject: [PATCH 07/20] Fix adaption of arrays for ES --- .../elasticsearch/elasticsearch_plugin.cpp | 51 +++++++++++-------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp index 5d22beac3a..c437b55405 100644 --- a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp +++ b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp @@ -254,20 +254,44 @@ struct es_data_adaptor { static variant adapt(const variant_object& op) { fc::mutable_variant_object o(op); + + map to_string_fields = { + { "parameters", data_type::array_type }, // in committee proposals, current_fees.parameters + { "op", data_type::static_variant_type }, // proposal_create_op.proposed_ops[*].op + { "proposed_ops", data_type::array_type }, + { "initializer", data_type::static_variant_type }, + { "policy", data_type::static_variant_type }, + { "predicates", data_type::array_type }, + { "active_special_authority", data_type::static_variant_type }, + { "owner_special_authority", data_type::static_variant_type }, + { "acceptable_collateral", data_type::map_type }, + { "acceptable_borrowers", data_type::map_type } + }; + map original_arrays; vector keys_to_rename; for (auto i = o.begin(); i != o.end(); ++i) { + const string& name = (*i).key(); auto& element = (*i).value(); if (element.is_object()) { - const string& name = (*i).key(); auto& vo = element.get_object(); if (vo.contains(name.c_str())) // transfer_operation.amount.amount keys_to_rename.emplace_back(name); element = adapt(vo); } else if (element.is_array()) - adapt(element.get_array()); + { + auto& array = element.get_array(); + if( to_string_fields.find(name) != to_string_fields.end() ) + { + // make a backup and convert to string + original_arrays[name] = array; + element = fc::json::to_string(element); + } + else + adapt(array); + } } for( const auto& i : keys_to_rename ) // transfer_operation.amount @@ -312,27 +336,12 @@ struct es_data_adaptor { o.erase("owner"); } - map to_string_fields = { - { "parameters", data_type::array_type }, // in committee proposals, current_fees.parameters - { "op", data_type::static_variant_type }, // proposal_create_op.proposed_ops[*].op - { "proposed_ops", data_type::array_type }, - { "initializer", data_type::static_variant_type }, - { "policy", data_type::static_variant_type }, - { "predicates", data_type::array_type }, - { "active_special_authority", data_type::static_variant_type }, - { "owner_special_authority", data_type::static_variant_type }, - { "acceptable_collateral", data_type::map_type }, - { "acceptable_borrowers", data_type::map_type } - }; - for( const auto& pair : to_string_fields ) + for( const auto& pair : original_arrays ) { const auto& name = pair.first; - if( o.find(name) != o.end() ) - { - const auto& value = o[name]; - o[name + "_object"] = adapt( value.get_array(), pair.second ); - o[name] = fc::json::to_string(value); - } + auto& value = pair.second; + auto type = to_string_fields[name]; + o[name + "_object"] = adapt( value, type ); } variant v; From 345942b0732cc26dbe75addcfbf1e8955210ca90 Mon Sep 17 00:00:00 2001 From: abitmore Date: Sat, 18 Dec 2021 00:28:22 +0000 Subject: [PATCH 08/20] Fix adaption of maps for ES --- .../elasticsearch/elasticsearch_plugin.cpp | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp index c437b55405..b9dba95ee5 100644 --- a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp +++ b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp @@ -247,9 +247,9 @@ void elasticsearch_plugin_impl::getOperationType(const optional Date: Sat, 18 Dec 2021 16:17:14 +0000 Subject: [PATCH 09/20] Remove unnecessary code --- .../elasticsearch/elasticsearch_plugin.cpp | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp index b9dba95ee5..2e60966eed 100644 --- a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp +++ b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp @@ -301,25 +301,10 @@ struct es_data_adaptor { o.erase(i); } - if( o.find("fee") != o.end() ) - { - auto& fee = o["fee"]; - if( fee.is_uint64() ) // fee schedule - { - o["fee_"] = fee; - o.erase("fee"); - } - } - if( o.find("memo") != o.end() ) { auto& memo = o["memo"]; - if (memo.is_string()) // seems unused. TODO remove - { - o["memo_"] = memo; - o.erase("memo"); - } - else if (memo.is_object()) + if (memo.is_object()) { fc::mutable_variant_object tmp(memo.get_object()); if (tmp.find("nonce") != tmp.end()) From ab64829fc6985ca5efc4b134be9c3420b49bb8d7 Mon Sep 17 00:00:00 2001 From: abitmore Date: Wed, 22 Dec 2021 22:06:32 +0000 Subject: [PATCH 10/20] Fix code smells --- .../elasticsearch/elasticsearch_plugin.cpp | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp index ec0bc9bd42..44c4ee3c77 100644 --- a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp +++ b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp @@ -273,7 +273,7 @@ struct es_data_adaptor { { fc::mutable_variant_object o(op); - map to_string_fields = { + static const map> to_string_fields = { { "parameters", data_type::array_type }, // in committee proposals, current_fees.parameters { "op", data_type::static_variant_type }, // proposal_create_op.proposed_ops[*].op { "proposed_ops", data_type::array_type }, @@ -285,20 +285,20 @@ struct es_data_adaptor { { "acceptable_collateral", data_type::map_type }, { "acceptable_borrowers", data_type::map_type } }; - map original_arrays; + map> original_arrays; vector keys_to_rename; - for (auto i = o.begin(); i != o.end(); ++i) + for( auto& i : o ) { - const string& name = (*i).key(); - auto& element = (*i).value(); - if (element.is_object()) + const string& name = i.key(); + auto& element = i.value(); + if( element.is_object() ) { - auto& vo = element.get_object(); - if (vo.contains(name.c_str())) // transfer_operation.amount.amount + const auto& vo = element.get_object(); + if( vo.contains(name.c_str()) ) // transfer_operation.amount.amount keys_to_rename.emplace_back(name); element = adapt(vo); } - else if (element.is_array()) + else if( element.is_array() ) { auto& array = element.get_array(); if( to_string_fields.find(name) != to_string_fields.end() ) @@ -343,7 +343,7 @@ struct es_data_adaptor { { const auto& name = pair.first; auto& value = pair.second; - auto type = to_string_fields[name]; + auto type = to_string_fields.at(name); o[name + "_object"] = adapt( value, type ); } @@ -465,7 +465,7 @@ void elasticsearch_plugin_impl::doBlock(uint32_t trx_in_block, const signed_bloc struct operation_visitor { - typedef void result_type; + using result_type = void; share_type fee_amount; asset_id_type fee_asset; From 38d5b285f11e5502531396cb0c192dd1b7d4f2e3 Mon Sep 17 00:00:00 2001 From: abitmore Date: Wed, 22 Dec 2021 22:12:43 +0000 Subject: [PATCH 11/20] Simplify code about adaption of nonce for ES --- .../plugins/elasticsearch/elasticsearch_plugin.cpp | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp index 44c4ee3c77..5bca4cf0c9 100644 --- a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp +++ b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp @@ -319,18 +319,9 @@ struct es_data_adaptor { o.erase(i); } - if( o.find("memo") != o.end() ) + if( o.find("nonce") != o.end() ) { - auto& memo = o["memo"]; - if (memo.is_object()) - { - fc::mutable_variant_object tmp(memo.get_object()); - if (tmp.find("nonce") != tmp.end()) - { - tmp["nonce"] = tmp["nonce"].as_string(); - o["memo"] = tmp; - } - } + o["nonce"] = o["nonce"].as_string(); } if( o.find("owner") != o.end() && o["owner"].is_string() ) // vesting_balance_*_operation.owner From ba394221688fe9802e0271afc45f0ab37fa7b42f Mon Sep 17 00:00:00 2001 From: abitmore Date: Fri, 24 Dec 2021 16:08:09 +0000 Subject: [PATCH 12/20] Adapt more maps as objects for ES --- .../elasticsearch/elasticsearch_plugin.cpp | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp index 5bca4cf0c9..e54f64c054 100644 --- a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp +++ b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp @@ -273,6 +273,16 @@ struct es_data_adaptor { { fc::mutable_variant_object o(op); + // Note: these fields are maps, but were stored in ES as flattened arrays + static const map> flattened_fields = { + { "account_auths", data_type::map_type }, + { "address_auths", data_type::map_type }, + { "key_auths", data_type::map_type } + }; + // Note: + // object arrays listed in this map are stored redundantly in ES, with one instance as a nested object and + // the other as a string for backward compatibility, + // object arrays not listed in this map are stored as nested objects only. static const map> to_string_fields = { { "parameters", data_type::array_type }, // in committee proposals, current_fees.parameters { "op", data_type::static_variant_type }, // proposal_create_op.proposed_ops[*].op @@ -307,6 +317,13 @@ struct es_data_adaptor { original_arrays[name] = array; element = fc::json::to_string(element); } + else if( flattened_fields.find(name) != flattened_fields.end() ) + { + // make a backup and adapt the original + auto backup = array; + original_arrays[name] = backup; + adapt(array); + } else adapt(array); } @@ -334,7 +351,9 @@ struct es_data_adaptor { { const auto& name = pair.first; auto& value = pair.second; - auto type = to_string_fields.at(name); + auto type = data_type::map_type; + if( to_string_fields.find(name) != to_string_fields.end() ) + type = to_string_fields.at(name); o[name + "_object"] = adapt( value, type ); } From e28cb307ef252554f16f92eac0615c8c8b5c7f37 Mon Sep 17 00:00:00 2001 From: abitmore Date: Fri, 24 Dec 2021 18:51:12 +0000 Subject: [PATCH 13/20] Fix adaption of strings in maps or static variants --- libraries/plugins/elasticsearch/elasticsearch_plugin.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp index e54f64c054..9521d185c3 100644 --- a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp +++ b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp @@ -398,6 +398,8 @@ struct es_data_adaptor { mv[prefix + "_int"] = v; else if( v.is_bool() ) mv[prefix + "_bool"] = v; + else if( v.is_string() ) + mv[prefix + "_string"] = v.get_string(); else mv[prefix + "_string"] = fc::json::to_string( v ); // Note: we don't use double or array here, and we convert null and blob to string From 79993e2e4035d9077ce50ada19d0c2d96fe60229 Mon Sep 17 00:00:00 2001 From: abitmore Date: Fri, 24 Dec 2021 20:13:26 +0000 Subject: [PATCH 14/20] Move es_data_adaptor into utilities namespace --- .../elasticsearch/elasticsearch_plugin.cpp | 186 +----------------- libraries/utilities/elasticsearch.cpp | 177 +++++++++++++++++ .../graphene/utilities/elasticsearch.hpp | 27 +++ 3 files changed, 206 insertions(+), 184 deletions(-) diff --git a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp index 9521d185c3..ba02c4d074 100644 --- a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp +++ b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp @@ -263,188 +263,6 @@ void elasticsearch_plugin_impl::getOperationType(const optional op.which(); } -struct es_data_adaptor { - enum class data_type { - static_variant_type, - map_type, - array_type // can be simple arrays, object arrays, static_variant arrays, or even nested arrays - }; - static variant adapt(const variant_object& op) - { - fc::mutable_variant_object o(op); - - // Note: these fields are maps, but were stored in ES as flattened arrays - static const map> flattened_fields = { - { "account_auths", data_type::map_type }, - { "address_auths", data_type::map_type }, - { "key_auths", data_type::map_type } - }; - // Note: - // object arrays listed in this map are stored redundantly in ES, with one instance as a nested object and - // the other as a string for backward compatibility, - // object arrays not listed in this map are stored as nested objects only. - static const map> to_string_fields = { - { "parameters", data_type::array_type }, // in committee proposals, current_fees.parameters - { "op", data_type::static_variant_type }, // proposal_create_op.proposed_ops[*].op - { "proposed_ops", data_type::array_type }, - { "initializer", data_type::static_variant_type }, - { "policy", data_type::static_variant_type }, - { "predicates", data_type::array_type }, - { "active_special_authority", data_type::static_variant_type }, - { "owner_special_authority", data_type::static_variant_type }, - { "acceptable_collateral", data_type::map_type }, - { "acceptable_borrowers", data_type::map_type } - }; - map> original_arrays; - vector keys_to_rename; - for( auto& i : o ) - { - const string& name = i.key(); - auto& element = i.value(); - if( element.is_object() ) - { - const auto& vo = element.get_object(); - if( vo.contains(name.c_str()) ) // transfer_operation.amount.amount - keys_to_rename.emplace_back(name); - element = adapt(vo); - } - else if( element.is_array() ) - { - auto& array = element.get_array(); - if( to_string_fields.find(name) != to_string_fields.end() ) - { - // make a backup and convert to string - original_arrays[name] = array; - element = fc::json::to_string(element); - } - else if( flattened_fields.find(name) != flattened_fields.end() ) - { - // make a backup and adapt the original - auto backup = array; - original_arrays[name] = backup; - adapt(array); - } - else - adapt(array); - } - } - - for( const auto& i : keys_to_rename ) // transfer_operation.amount - { - string new_name = i + "_"; - o[new_name] = variant(o[i]); - o.erase(i); - } - - if( o.find("nonce") != o.end() ) - { - o["nonce"] = o["nonce"].as_string(); - } - - if( o.find("owner") != o.end() && o["owner"].is_string() ) // vesting_balance_*_operation.owner - { - o["owner_"] = o["owner"].as_string(); - o.erase("owner"); - } - - for( const auto& pair : original_arrays ) - { - const auto& name = pair.first; - auto& value = pair.second; - auto type = data_type::map_type; - if( to_string_fields.find(name) != to_string_fields.end() ) - type = to_string_fields.at(name); - o[name + "_object"] = adapt( value, type ); - } - - variant v; - fc::to_variant(o, v, FC_PACK_MAX_DEPTH); - return v; - } - - static variant adapt( const fc::variants& v, data_type type ) - { - if( data_type::static_variant_type == type ) - return adapt_static_variant(v); - - // map_type or array_type - fc::variants vs; - vs.reserve( v.size() ); - for( const auto& item : v ) - { - if( item.is_array() ) - { - if( data_type::map_type == type ) - vs.push_back( adapt_map_item( item.get_array() ) ); - else // assume it is a static_variant array - vs.push_back( adapt_static_variant( item.get_array() ) ); - } - else if( item.is_object() ) // object array - vs.push_back( adapt( item.get_object() ) ); - else - wlog( "Type of item is unexpected: ${item}", ("item", item) ); - } - - variant nv; - fc::to_variant(vs, nv, FC_PACK_MAX_DEPTH); - return nv; - } - - static void extract_data_from_variant( const variant& v, fc::mutable_variant_object& mv, const string& prefix ) - { - if( v.is_object() ) - mv[prefix + "_object"] = adapt( v.get_object() ); - else if( v.is_int64() || v.is_uint64() ) - mv[prefix + "_int"] = v; - else if( v.is_bool() ) - mv[prefix + "_bool"] = v; - else if( v.is_string() ) - mv[prefix + "_string"] = v.get_string(); - else - mv[prefix + "_string"] = fc::json::to_string( v ); - // Note: we don't use double or array here, and we convert null and blob to string - } - - static variant adapt_map_item( const fc::variants& v ) - { - FC_ASSERT( v.size() == 2, "Internal error" ); - fc::mutable_variant_object mv; - - extract_data_from_variant( v[0], mv, "key" ); - extract_data_from_variant( v[1], mv, "data" ); - - variant nv; - fc::to_variant( mv, nv, FC_PACK_MAX_DEPTH ); - return nv; - } - - static variant adapt_static_variant( const fc::variants& v ) - { - FC_ASSERT( v.size() == 2, "Internal error" ); - fc::mutable_variant_object mv; - - mv["which"] = v[0]; - extract_data_from_variant( v[1], mv, "data" ); - - variant nv; - fc::to_variant( mv, nv, FC_PACK_MAX_DEPTH ); - return nv; - } - - static void adapt(fc::variants& v) - { - for (auto& array_element : v) - { - if (array_element.is_object()) - array_element = adapt(array_element.get_object()); - else if (array_element.is_array()) - adapt(array_element.get_array()); - else - array_element = array_element.as_string(); - } - } -}; - void elasticsearch_plugin_impl::doOperationHistory(const optional & oho) { try { os.trx_in_block = oho->trx_in_block; @@ -455,11 +273,11 @@ void elasticsearch_plugin_impl::doOperationHistory(const optional op.visit(fc::from_static_variant(os.op_object, FC_PACK_MAX_DEPTH)); - os.op_object = es_data_adaptor::adapt(os.op_object.get_object()); + os.op_object = graphene::utilities::es_data_adaptor::adapt( os.op_object.get_object() ); // operation_result variant v; fc::to_variant( oho->result, v, FC_PACK_MAX_DEPTH ); - os.operation_result_object = es_data_adaptor::adapt_static_variant( v.get_array() ); + os.operation_result_object = graphene::utilities::es_data_adaptor::adapt_static_variant( v.get_array() ); } if(_elasticsearch_operation_string) os.op = fc::json::to_string(oho->op); diff --git a/libraries/utilities/elasticsearch.cpp b/libraries/utilities/elasticsearch.cpp index 9a268b766e..b6b6182d37 100644 --- a/libraries/utilities/elasticsearch.cpp +++ b/libraries/utilities/elasticsearch.cpp @@ -26,6 +26,7 @@ #include #include #include +#include size_t WriteCallback(void *contents, size_t size, size_t nmemb, void *userp) { @@ -216,4 +217,180 @@ std::string doCurl(CurlRequest& curl) return CurlReadBuffer; } +fc::variant es_data_adaptor::adapt(const fc::variant_object& op) +{ + fc::mutable_variant_object o(op); + + // Note: these fields are maps, but were stored in ES as flattened arrays + static const std::map> flattened_fields = { + { "account_auths", data_type::map_type }, + { "address_auths", data_type::map_type }, + { "key_auths", data_type::map_type } + }; + // Note: + // object arrays listed in this map are stored redundantly in ES, with one instance as a nested object and + // the other as a string for backward compatibility, + // object arrays not listed in this map are stored as nested objects only. + static const std::map> to_string_fields = { + { "parameters", data_type::array_type }, // in committee proposals, current_fees.parameters + { "op", data_type::static_variant_type }, // proposal_create_op.proposed_ops[*].op + { "proposed_ops", data_type::array_type }, + { "initializer", data_type::static_variant_type }, + { "policy", data_type::static_variant_type }, + { "predicates", data_type::array_type }, + { "active_special_authority", data_type::static_variant_type }, + { "owner_special_authority", data_type::static_variant_type }, + { "acceptable_collateral", data_type::map_type }, + { "acceptable_borrowers", data_type::map_type } + }; + std::map> original_arrays; + std::vector keys_to_rename; + for( auto& i : o ) + { + const std::string& name = i.key(); + auto& element = i.value(); + if( element.is_object() ) + { + const auto& vo = element.get_object(); + if( vo.contains(name.c_str()) ) // transfer_operation.amount.amount + keys_to_rename.emplace_back(name); + element = adapt(vo); + } + else if( element.is_array() ) + { + auto& array = element.get_array(); + if( to_string_fields.find(name) != to_string_fields.end() ) + { + // make a backup and convert to string + original_arrays[name] = array; + element = fc::json::to_string(element); + } + else if( flattened_fields.find(name) != flattened_fields.end() ) + { + // make a backup and adapt the original + auto backup = array; + original_arrays[name] = backup; + adapt(array); + } + else + adapt(array); + } + } + + for( const auto& i : keys_to_rename ) // transfer_operation.amount + { + std::string new_name = i + "_"; + o[new_name] = fc::variant(o[i]); + o.erase(i); + } + + if( o.find("nonce") != o.end() ) + { + o["nonce"] = o["nonce"].as_string(); + } + + if( o.find("owner") != o.end() && o["owner"].is_string() ) // vesting_balance_*_operation.owner + { + o["owner_"] = o["owner"].as_string(); + o.erase("owner"); + } + + for( const auto& pair : original_arrays ) + { + const auto& name = pair.first; + auto& value = pair.second; + auto type = data_type::map_type; + if( to_string_fields.find(name) != to_string_fields.end() ) + type = to_string_fields.at(name); + o[name + "_object"] = adapt( value, type ); + } + + fc::variant v; + fc::to_variant(o, v, FC_PACK_MAX_DEPTH); + return v; +} + +fc::variant es_data_adaptor::adapt( const fc::variants& v, data_type type ) +{ + if( data_type::static_variant_type == type ) + return adapt_static_variant(v); + + // map_type or array_type + fc::variants vs; + vs.reserve( v.size() ); + for( const auto& item : v ) + { + if( item.is_array() ) + { + if( data_type::map_type == type ) + vs.push_back( adapt_map_item( item.get_array() ) ); + else // assume it is a static_variant array + vs.push_back( adapt_static_variant( item.get_array() ) ); + } + else if( item.is_object() ) // object array + vs.push_back( adapt( item.get_object() ) ); + else + wlog( "Type of item is unexpected: ${item}", ("item", item) ); + } + + fc::variant nv; + fc::to_variant(vs, nv, FC_PACK_MAX_DEPTH); + return nv; +} + +void es_data_adaptor::extract_data_from_variant( + const fc::variant& v, fc::mutable_variant_object& mv, const std::string& prefix ) +{ + if( v.is_object() ) + mv[prefix + "_object"] = adapt( v.get_object() ); + else if( v.is_int64() || v.is_uint64() ) + mv[prefix + "_int"] = v; + else if( v.is_bool() ) + mv[prefix + "_bool"] = v; + else if( v.is_string() ) + mv[prefix + "_string"] = v.get_string(); + else + mv[prefix + "_string"] = fc::json::to_string( v ); + // Note: we don't use double or array here, and we convert null and blob to string +} + +fc::variant es_data_adaptor::adapt_map_item( const fc::variants& v ) +{ + FC_ASSERT( v.size() == 2, "Internal error" ); + fc::mutable_variant_object mv; + + extract_data_from_variant( v[0], mv, "key" ); + extract_data_from_variant( v[1], mv, "data" ); + + fc::variant nv; + fc::to_variant( mv, nv, FC_PACK_MAX_DEPTH ); + return nv; +} + +fc::variant es_data_adaptor::adapt_static_variant( const fc::variants& v ) +{ + FC_ASSERT( v.size() == 2, "Internal error" ); + fc::mutable_variant_object mv; + + mv["which"] = v[0]; + extract_data_from_variant( v[1], mv, "data" ); + + fc::variant nv; + fc::to_variant( mv, nv, FC_PACK_MAX_DEPTH ); + return nv; +} + +void es_data_adaptor::adapt(fc::variants& v) +{ + for (auto& array_element : v) + { + if (array_element.is_object()) + array_element = adapt(array_element.get_object()); + else if (array_element.is_array()) + adapt(array_element.get_array()); + else + array_element = array_element.as_string(); + } +} + } } // end namespace graphene::utilities diff --git a/libraries/utilities/include/graphene/utilities/elasticsearch.hpp b/libraries/utilities/include/graphene/utilities/elasticsearch.hpp index 2fb29a0967..e64f59719d 100644 --- a/libraries/utilities/include/graphene/utilities/elasticsearch.hpp +++ b/libraries/utilities/include/graphene/utilities/elasticsearch.hpp @@ -66,4 +66,31 @@ namespace graphene { namespace utilities { std::string joinBulkLines(const std::vector& bulk); long getResponseCode(CURL *handler); +struct es_data_adaptor +{ + enum class data_type + { + static_variant_type, + map_type, + array_type // can be simple arrays, object arrays, static_variant arrays, or even nested arrays + }; + + static fc::variant adapt( const fc::variant_object& op ); + + static fc::variant adapt( const fc::variants& v, data_type type ); + + static fc::variant adapt_map_item( const fc::variants& v ); + + static fc::variant adapt_static_variant( const fc::variants& v ); + + /// In-place update + static void adapt( fc::variants& v ); + + /// Extract data from @p v into @p mv + static void extract_data_from_variant( const fc::variant& v, + fc::mutable_variant_object& mv, + const std::string& prefix ); + +}; + } } // end namespace graphene::utilities From 0eed9237c3a2f0d3ae351ea7c682792ef6656734 Mon Sep 17 00:00:00 2001 From: abitmore Date: Fri, 24 Dec 2021 22:39:54 +0000 Subject: [PATCH 15/20] Merge adaptor_struct in es_objects into utilities --- libraries/plugins/es_objects/es_objects.cpp | 3 +- .../graphene/es_objects/es_objects.hpp | 57 ------------------- libraries/utilities/elasticsearch.cpp | 2 + 3 files changed, 3 insertions(+), 59 deletions(-) diff --git a/libraries/plugins/es_objects/es_objects.cpp b/libraries/plugins/es_objects/es_objects.cpp index 93a4687f36..791df773df 100644 --- a/libraries/plugins/es_objects/es_objects.cpp +++ b/libraries/plugins/es_objects/es_objects.cpp @@ -254,10 +254,9 @@ void es_objects_plugin_impl::prepareTemplate(T blockchain_object, string index_n bulk_header["_id"] = string(blockchain_object.id); } - adaptor_struct adaptor; fc::variant blockchain_object_variant; fc::to_variant( blockchain_object, blockchain_object_variant, GRAPHENE_NET_MAX_NESTED_OBJECTS ); - fc::mutable_variant_object o = adaptor.adapt(blockchain_object_variant.get_object()); + fc::mutable_variant_object o( utilities::es_data_adaptor::adapt( blockchain_object_variant.get_object() ) ); o["object_id"] = string(blockchain_object.id); o["block_time"] = block_time; diff --git a/libraries/plugins/es_objects/include/graphene/es_objects/es_objects.hpp b/libraries/plugins/es_objects/include/graphene/es_objects/es_objects.hpp index dff4812498..cfae63d77b 100644 --- a/libraries/plugins/es_objects/include/graphene/es_objects/es_objects.hpp +++ b/libraries/plugins/es_objects/include/graphene/es_objects/es_objects.hpp @@ -53,61 +53,4 @@ class es_objects_plugin : public graphene::app::plugin std::unique_ptr my; }; -struct adaptor_struct { - fc::mutable_variant_object adapt(const variant_object &obj) { - fc::mutable_variant_object o(obj); - vector keys_to_rename; - for (auto i = o.begin(); i != o.end(); ++i) { - auto &element = (*i).value(); - if (element.is_object()) { - const string &name = (*i).key(); - auto &vo = element.get_object(); - if (vo.contains(name.c_str())) - keys_to_rename.emplace_back(name); - element = adapt(vo); - } else if (element.is_array()) - adapt(element.get_array()); - } - for (const auto &i : keys_to_rename) { - string new_name = i + "_"; - o[new_name] = variant(o[i]); - o.erase(i); - } - if (o.find("owner") != o.end() && o["owner"].is_string()) - { - o["owner_"] = o["owner"].as_string(); - o.erase("owner"); - } - if (o.find("active_special_authority") != o.end()) - { - o["active_special_authority"] = fc::json::to_string(o["active_special_authority"]); - } - if (o.find("owner_special_authority") != o.end()) - { - o["owner_special_authority"] = fc::json::to_string(o["owner_special_authority"]); - } - if (o.find("feeds") != o.end()) - { - o["feeds"] = fc::json::to_string(o["feeds"]); - } - if (o.find("operations") != o.end()) - { - o["operations"] = fc::json::to_string(o["operations"]); - } - - return o; - } - - void adapt(fc::variants &v) { - for (auto &array_element : v) { - if (array_element.is_object()) - array_element = adapt(array_element.get_object()); - else if (array_element.is_array()) - adapt(array_element.get_array()); - else - array_element = array_element.as_string(); - } - } -}; - } } //graphene::es_objects diff --git a/libraries/utilities/elasticsearch.cpp b/libraries/utilities/elasticsearch.cpp index b6b6182d37..756d7c0352 100644 --- a/libraries/utilities/elasticsearch.cpp +++ b/libraries/utilities/elasticsearch.cpp @@ -235,11 +235,13 @@ fc::variant es_data_adaptor::adapt(const fc::variant_object& op) { "parameters", data_type::array_type }, // in committee proposals, current_fees.parameters { "op", data_type::static_variant_type }, // proposal_create_op.proposed_ops[*].op { "proposed_ops", data_type::array_type }, + { "operations", data_type::array_type }, // proposal_object.operations { "initializer", data_type::static_variant_type }, { "policy", data_type::static_variant_type }, { "predicates", data_type::array_type }, { "active_special_authority", data_type::static_variant_type }, { "owner_special_authority", data_type::static_variant_type }, + { "feeds", data_type::map_type }, // asset_bitasset_data_object.feeds { "acceptable_collateral", data_type::map_type }, { "acceptable_borrowers", data_type::map_type } }; From 67e6733569b0c571fff441718fb4f9da193ae0e1 Mon Sep 17 00:00:00 2001 From: abitmore Date: Fri, 24 Dec 2021 23:59:35 +0000 Subject: [PATCH 16/20] Try to fix genesis data process in es_objects --- libraries/plugins/es_objects/es_objects.cpp | 63 ++++++++++----------- 1 file changed, 29 insertions(+), 34 deletions(-) diff --git a/libraries/plugins/es_objects/es_objects.cpp b/libraries/plugins/es_objects/es_objects.cpp index 791df773df..f7d4263b62 100644 --- a/libraries/plugins/es_objects/es_objects.cpp +++ b/libraries/plugins/es_objects/es_objects.cpp @@ -54,6 +54,7 @@ class es_objects_plugin_impl void remove_from_database(object_id_type id, std::string index); friend class graphene::es_objects::es_objects_plugin; + friend struct genesis_inserter; private: es_objects_plugin& _self; @@ -85,6 +86,31 @@ class es_objects_plugin_impl void init_program_options(const boost::program_options::variables_map& options); }; +struct genesis_inserter +{ + template + static void insert_genesis_objects( bool b, const string& prefix, es_objects_plugin_impl* my, DB& db ) + { + if( !b ) + return; + + db.get_index( SpaceID, TypeID ).inspect_all_objects( [my, &db, &prefix](const graphene::db::object &o) { + auto a = static_cast(&o); + my->prepareTemplate(*a, prefix); + }); + + graphene::utilities::ES es; + es.curl = my->curl; + es.bulk_lines = my->bulk; + es.elasticsearch_url = my->_es_objects_elasticsearch_url; + es.auth = my->_es_objects_auth; + if (!graphene::utilities::SendBulk(std::move(es))) + FC_THROW_EXCEPTION(graphene::chain::plugin_exception, "Error inserting genesis data."); + else + my->bulk.clear(); + } +}; + bool es_objects_plugin_impl::genesis() { ilog("elasticsearch OBJECTS: inserting data from genesis"); @@ -94,40 +120,9 @@ bool es_objects_plugin_impl::genesis() block_number = db.head_block_num(); block_time = db.head_block_time(); - if (_es_objects_accounts) { - auto &index_accounts = db.get_index(1, 2); - index_accounts.inspect_all_objects([this, &db](const graphene::db::object &o) { - auto obj = db.find_object(o.id); - auto a = static_cast(obj); - prepareTemplate(*a, "account"); - }); - } - if (_es_objects_assets) { - auto &index_assets = db.get_index(1, 3); - index_assets.inspect_all_objects([this, &db](const graphene::db::object &o) { - auto obj = db.find_object(o.id); - auto a = static_cast(obj); - prepareTemplate(*a, "asset"); - }); - } - if (_es_objects_balances) { - auto &index_balances = db.get_index(2, 5); - index_balances.inspect_all_objects([this, &db](const graphene::db::object &o) { - auto obj = db.find_object(o.id); - auto b = static_cast(obj); - prepareTemplate(*b, "balance"); - }); - } - - graphene::utilities::ES es; - es.curl = curl; - es.bulk_lines = bulk; - es.elasticsearch_url = _es_objects_elasticsearch_url; - es.auth = _es_objects_auth; - if (!graphene::utilities::SendBulk(std::move(es))) - FC_THROW_EXCEPTION(graphene::chain::plugin_exception, "Error inserting genesis data."); - else - bulk.clear(); + genesis_inserter::insert_genesis_objects<1,2,account_object >( _es_objects_accounts, "account", this, db ); + genesis_inserter::insert_genesis_objects<1,3,asset_object >( _es_objects_assets, "asset", this, db ); + genesis_inserter::insert_genesis_objects<2,5,account_balance_object>( _es_objects_balances, "balance", this, db ); return true; } From 952a326895dff096e2ad8e91ab75264e275314b5 Mon Sep 17 00:00:00 2001 From: abitmore Date: Sat, 25 Dec 2021 00:31:52 +0000 Subject: [PATCH 17/20] Insert genesis objects into ES in smaller bulks --- libraries/plugins/es_objects/es_objects.cpp | 25 ++++++++++++--------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/libraries/plugins/es_objects/es_objects.cpp b/libraries/plugins/es_objects/es_objects.cpp index f7d4263b62..833fb25761 100644 --- a/libraries/plugins/es_objects/es_objects.cpp +++ b/libraries/plugins/es_objects/es_objects.cpp @@ -62,6 +62,7 @@ class es_objects_plugin_impl std::string _es_objects_auth = ""; uint32_t _es_objects_bulk_replay = 10000; uint32_t _es_objects_bulk_sync = 100; + uint32_t limit_documents = _es_objects_bulk_replay; bool _es_objects_proposals = true; bool _es_objects_accounts = true; bool _es_objects_assets = true; @@ -98,16 +99,6 @@ struct genesis_inserter auto a = static_cast(&o); my->prepareTemplate(*a, prefix); }); - - graphene::utilities::ES es; - es.curl = my->curl; - es.bulk_lines = my->bulk; - es.elasticsearch_url = my->_es_objects_elasticsearch_url; - es.auth = my->_es_objects_auth; - if (!graphene::utilities::SendBulk(std::move(es))) - FC_THROW_EXCEPTION(graphene::chain::plugin_exception, "Error inserting genesis data."); - else - my->bulk.clear(); } }; @@ -137,7 +128,6 @@ bool es_objects_plugin_impl::index_database(const vector& ids, s if(block_number > _es_objects_start_es_after_block) { // check if we are in replay or in sync and change number of bulk documents accordingly - uint32_t limit_documents = 0; if ((fc::time_point::now() - block_time) < fc::seconds(30)) limit_documents = _es_objects_bulk_sync; else @@ -262,6 +252,19 @@ void es_objects_plugin_impl::prepareTemplate(T blockchain_object, string index_n prepare = graphene::utilities::createBulk(bulk_header, std::move(data)); std::move(prepare.begin(), prepare.end(), std::back_inserter(bulk)); prepare.clear(); + + if( curl && bulk.size() >= limit_documents ) // send data to elasticsearch when bulk is too large + { + graphene::utilities::ES es; + es.curl = curl; + es.bulk_lines = bulk; + es.elasticsearch_url = _es_objects_elasticsearch_url; + es.auth = _es_objects_auth; + if (!graphene::utilities::SendBulk(std::move(es))) + FC_THROW_EXCEPTION(graphene::chain::plugin_exception, "Error sending bulk data."); + else + bulk.clear(); + } } es_objects_plugin_impl::~es_objects_plugin_impl() From 4f4f8a41ada4e3de702ccf2fb2de63084cb3adb6 Mon Sep 17 00:00:00 2001 From: abitmore Date: Sat, 25 Dec 2021 00:45:21 +0000 Subject: [PATCH 18/20] Insert bitasset data in genesis into ES --- libraries/plugins/es_objects/es_objects.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/libraries/plugins/es_objects/es_objects.cpp b/libraries/plugins/es_objects/es_objects.cpp index 833fb25761..339bef138c 100644 --- a/libraries/plugins/es_objects/es_objects.cpp +++ b/libraries/plugins/es_objects/es_objects.cpp @@ -90,7 +90,7 @@ class es_objects_plugin_impl struct genesis_inserter { template - static void insert_genesis_objects( bool b, const string& prefix, es_objects_plugin_impl* my, DB& db ) + static void insert( bool b, const string& prefix, es_objects_plugin_impl* my, DB& db ) { if( !b ) return; @@ -111,9 +111,10 @@ bool es_objects_plugin_impl::genesis() block_number = db.head_block_num(); block_time = db.head_block_time(); - genesis_inserter::insert_genesis_objects<1,2,account_object >( _es_objects_accounts, "account", this, db ); - genesis_inserter::insert_genesis_objects<1,3,asset_object >( _es_objects_assets, "asset", this, db ); - genesis_inserter::insert_genesis_objects<2,5,account_balance_object>( _es_objects_balances, "balance", this, db ); + genesis_inserter::insert<1,2,account_object >( _es_objects_accounts, "account", this, db ); + genesis_inserter::insert<1,3,asset_object >( _es_objects_assets, "asset", this, db ); + genesis_inserter::insert<2,4,asset_bitasset_data_object >( _es_objects_asset_bitasset, "bitasset", this, db ); + genesis_inserter::insert<2,5,account_balance_object >( _es_objects_balances, "balance", this, db ); return true; } From 695187c031f8afbb8045c60a3c9baa360421641a Mon Sep 17 00:00:00 2001 From: abitmore Date: Sat, 25 Dec 2021 13:42:04 +0000 Subject: [PATCH 19/20] Move program options of es_objects into a struct --- libraries/plugins/es_objects/es_objects.cpp | 113 ++++++++++++-------- 1 file changed, 69 insertions(+), 44 deletions(-) diff --git a/libraries/plugins/es_objects/es_objects.cpp b/libraries/plugins/es_objects/es_objects.cpp index 339bef138c..3e70e28688 100644 --- a/libraries/plugins/es_objects/es_objects.cpp +++ b/libraries/plugins/es_objects/es_objects.cpp @@ -57,25 +57,34 @@ class es_objects_plugin_impl friend struct genesis_inserter; private: + struct plugin_options + { + std::string _es_objects_elasticsearch_url = "http://localhost:9200/"; + std::string _es_objects_auth = ""; + uint32_t _es_objects_bulk_replay = 10000; + uint32_t _es_objects_bulk_sync = 100; + bool _es_objects_proposals = true; + bool _es_objects_accounts = true; + bool _es_objects_assets = true; + bool _es_objects_balances = true; + bool _es_objects_limit_orders = false; + bool _es_objects_asset_bitasset = true; + std::string _es_objects_index_prefix = "objects-"; + uint32_t _es_objects_start_es_after_block = 0; + bool _es_objects_keep_only_current = true; + + void init(const boost::program_options::variables_map& options); + }; + es_objects_plugin& _self; - std::string _es_objects_elasticsearch_url = "http://localhost:9200/"; - std::string _es_objects_auth = ""; - uint32_t _es_objects_bulk_replay = 10000; - uint32_t _es_objects_bulk_sync = 100; - uint32_t limit_documents = _es_objects_bulk_replay; - bool _es_objects_proposals = true; - bool _es_objects_accounts = true; - bool _es_objects_assets = true; - bool _es_objects_balances = true; - bool _es_objects_limit_orders = false; - bool _es_objects_asset_bitasset = true; - std::string _es_objects_index_prefix = "objects-"; - uint32_t _es_objects_start_es_after_block = 0; + plugin_options _options; + + uint32_t limit_documents = _options._es_objects_bulk_replay; + CURL *curl; // curl handler - vector bulk; + vector bulk; vector prepare; - bool _es_objects_keep_only_current = true; uint32_t block_number; fc::time_point_sec block_time; @@ -89,13 +98,22 @@ class es_objects_plugin_impl struct genesis_inserter { - template - static void insert( bool b, const string& prefix, es_objects_plugin_impl* my, DB& db ) + es_objects_plugin_impl* my; + graphene::chain::database &db; + + explicit genesis_inserter( es_objects_plugin_impl* _my ) + : my(_my), db( my->_self.database() ) + { // Nothing to do + } + + template + void insert( bool b, const string& prefix ) { if( !b ) return; - db.get_index( SpaceID, TypeID ).inspect_all_objects( [my, &db, &prefix](const graphene::db::object &o) { + db.get_index( ObjType::space_id, ObjType::type_id ).inspect_all_objects( + [this, &prefix](const graphene::db::object &o) { auto a = static_cast(&o); my->prepareTemplate(*a, prefix); }); @@ -111,10 +129,12 @@ bool es_objects_plugin_impl::genesis() block_number = db.head_block_num(); block_time = db.head_block_time(); - genesis_inserter::insert<1,2,account_object >( _es_objects_accounts, "account", this, db ); - genesis_inserter::insert<1,3,asset_object >( _es_objects_assets, "asset", this, db ); - genesis_inserter::insert<2,4,asset_bitasset_data_object >( _es_objects_asset_bitasset, "bitasset", this, db ); - genesis_inserter::insert<2,5,account_balance_object >( _es_objects_balances, "balance", this, db ); + genesis_inserter inserter( this ); + + inserter.insert( _options._es_objects_accounts, "account" ); + inserter.insert( _options._es_objects_assets, "asset" ); + inserter.insert( _options._es_objects_asset_bitasset, "bitasset" ); + inserter.insert( _options._es_objects_balances, "balance" ); return true; } @@ -126,17 +146,17 @@ bool es_objects_plugin_impl::index_database(const vector& ids, s block_time = db.head_block_time(); block_number = db.head_block_num(); - if(block_number > _es_objects_start_es_after_block) { + if(block_number > _options._es_objects_start_es_after_block) { // check if we are in replay or in sync and change number of bulk documents accordingly if ((fc::time_point::now() - block_time) < fc::seconds(30)) - limit_documents = _es_objects_bulk_sync; + limit_documents = _options._es_objects_bulk_sync; else - limit_documents = _es_objects_bulk_replay; + limit_documents = _options._es_objects_bulk_replay; for (auto const &value: ids) { - if (value.is() && _es_objects_proposals) { + if (value.is() && _options._es_objects_proposals) { auto obj = db.find_object(value); auto p = static_cast(obj); if (p != nullptr) { @@ -145,7 +165,7 @@ bool es_objects_plugin_impl::index_database(const vector& ids, s else prepareTemplate(*p, "proposal"); } - } else if (value.is() && _es_objects_accounts) { + } else if (value.is() && _options._es_objects_accounts) { auto obj = db.find_object(value); auto a = static_cast(obj); if (a != nullptr) { @@ -154,7 +174,7 @@ bool es_objects_plugin_impl::index_database(const vector& ids, s else prepareTemplate(*a, "account"); } - } else if (value.is() && _es_objects_assets) { + } else if (value.is() && _options._es_objects_assets) { auto obj = db.find_object(value); auto a = static_cast(obj); if (a != nullptr) { @@ -163,7 +183,7 @@ bool es_objects_plugin_impl::index_database(const vector& ids, s else prepareTemplate(*a, "asset"); } - } else if (value.is() && _es_objects_balances) { + } else if (value.is() && _options._es_objects_balances) { auto obj = db.find_object(value); auto b = static_cast(obj); if (b != nullptr) { @@ -172,7 +192,7 @@ bool es_objects_plugin_impl::index_database(const vector& ids, s else prepareTemplate(*b, "balance"); } - } else if (value.is() && _es_objects_limit_orders) { + } else if (value.is() && _options._es_objects_limit_orders) { auto obj = db.find_object(value); auto l = static_cast(obj); if (l != nullptr) { @@ -181,7 +201,7 @@ bool es_objects_plugin_impl::index_database(const vector& ids, s else prepareTemplate(*l, "limitorder"); } - } else if (value.is() && _es_objects_asset_bitasset) { + } else if (value.is() && _options._es_objects_asset_bitasset) { auto obj = db.find_object(value); auto ba = static_cast(obj); if (ba != nullptr) { @@ -198,8 +218,8 @@ bool es_objects_plugin_impl::index_database(const vector& ids, s graphene::utilities::ES es; es.curl = curl; es.bulk_lines = bulk; - es.elasticsearch_url = _es_objects_elasticsearch_url; - es.auth = _es_objects_auth; + es.elasticsearch_url = _options._es_objects_elasticsearch_url; + es.auth = _options._es_objects_auth; if (!graphene::utilities::SendBulk(std::move(es))) return false; @@ -213,11 +233,11 @@ bool es_objects_plugin_impl::index_database(const vector& ids, s void es_objects_plugin_impl::remove_from_database( object_id_type id, std::string index) { - if(_es_objects_keep_only_current) + if(_options._es_objects_keep_only_current) { fc::mutable_variant_object delete_line; delete_line["_id"] = string(id); - delete_line["_index"] = _es_objects_index_prefix + index; + delete_line["_index"] = _options._es_objects_index_prefix + index; if(!is_es_version_7_or_above) delete_line["_type"] = "_doc"; fc::mutable_variant_object final_delete_line; @@ -232,10 +252,10 @@ template void es_objects_plugin_impl::prepareTemplate(T blockchain_object, string index_name) { fc::mutable_variant_object bulk_header; - bulk_header["_index"] = _es_objects_index_prefix + index_name; + bulk_header["_index"] = _options._es_objects_index_prefix + index_name; if(!is_es_version_7_or_above) bulk_header["_type"] = "_doc"; - if(_es_objects_keep_only_current) + if(_options._es_objects_keep_only_current) { bulk_header["_id"] = string(blockchain_object.id); } @@ -259,8 +279,8 @@ void es_objects_plugin_impl::prepareTemplate(T blockchain_object, string index_n graphene::utilities::ES es; es.curl = curl; es.bulk_lines = bulk; - es.elasticsearch_url = _es_objects_elasticsearch_url; - es.auth = _es_objects_auth; + es.elasticsearch_url = _options._es_objects_elasticsearch_url; + es.auth = _options._es_objects_auth; if (!graphene::utilities::SendBulk(std::move(es))) FC_THROW_EXCEPTION(graphene::chain::plugin_exception, "Error sending bulk data."); else @@ -326,6 +346,11 @@ void es_objects_plugin::plugin_set_program_options( } void detail::es_objects_plugin_impl::init_program_options(const boost::program_options::variables_map& options) +{ + _options.init( options ); +} + +void detail::es_objects_plugin_impl::plugin_options::init(const boost::program_options::variables_map& options) { if (options.count("es-objects-elasticsearch-url") > 0) { _es_objects_elasticsearch_url = options["es-objects-elasticsearch-url"].as(); @@ -373,7 +398,7 @@ void es_objects_plugin::plugin_initialize(const boost::program_options::variable my->init_program_options( options ); database().applied_block.connect([this](const signed_block &b) { - if(b.block_num() == 1 && my->_es_objects_start_es_after_block == 0) { + if( 1U == b.block_num() && 0 == my->_options._es_objects_start_es_after_block ) { if (!my->genesis()) FC_THROW_EXCEPTION(graphene::chain::plugin_exception, "Error populating genesis data."); } @@ -405,12 +430,12 @@ void es_objects_plugin::plugin_initialize(const boost::program_options::variable graphene::utilities::ES es; es.curl = my->curl; - es.elasticsearch_url = my->_es_objects_elasticsearch_url; - es.auth = my->_es_objects_auth; - es.auth = my->_es_objects_index_prefix; + es.elasticsearch_url = my->_options._es_objects_elasticsearch_url; + es.auth = my->_options._es_objects_auth; + es.auth = my->_options._es_objects_index_prefix; if(!graphene::utilities::checkES(es)) - FC_THROW( "ES database is not up in url ${url}", ("url", my->_es_objects_elasticsearch_url) ); + FC_THROW( "ES database is not up in url ${url}", ("url", my->_options._es_objects_elasticsearch_url) ); graphene::utilities::checkESVersion7OrAbove(es, my->is_es_version_7_or_above); } From b02337e2b0dd3b5c2af13268bd0bd3bd3ea6e4dc Mon Sep 17 00:00:00 2001 From: abitmore Date: Sat, 25 Dec 2021 22:15:31 +0000 Subject: [PATCH 20/20] Adapt htlc_preimage_hash as an object for ES --- libraries/utilities/elasticsearch.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libraries/utilities/elasticsearch.cpp b/libraries/utilities/elasticsearch.cpp index 756d7c0352..75c4abe5cf 100644 --- a/libraries/utilities/elasticsearch.cpp +++ b/libraries/utilities/elasticsearch.cpp @@ -241,6 +241,7 @@ fc::variant es_data_adaptor::adapt(const fc::variant_object& op) { "predicates", data_type::array_type }, { "active_special_authority", data_type::static_variant_type }, { "owner_special_authority", data_type::static_variant_type }, + { "htlc_preimage_hash", data_type::static_variant_type }, { "feeds", data_type::map_type }, // asset_bitasset_data_object.feeds { "acceptable_collateral", data_type::map_type }, { "acceptable_borrowers", data_type::map_type } @@ -353,7 +354,8 @@ void es_data_adaptor::extract_data_from_variant( mv[prefix + "_string"] = v.get_string(); else mv[prefix + "_string"] = fc::json::to_string( v ); - // Note: we don't use double or array here, and we convert null and blob to string + // Note: we don't use double or array here, and we convert null and blob to string, + // and static_variants (i.e. in custom authorities) and maps (if any) are converted to strings too. } fc::variant es_data_adaptor::adapt_map_item( const fc::variants& v )