From a1851cd21c9a8ce11bb8a7cf2f46e4856ed08eba Mon Sep 17 00:00:00 2001 From: Andrew Falaleev Date: Wed, 8 May 2019 15:58:51 +0700 Subject: [PATCH 1/2] Pass ram usage to transaction context. #616 --- contracts/eosio.bios/eosio.bios.abi | 1 + contracts/eosio.system/eosio.system.abi | 1 + contracts/eosiolib/privileged.hpp | 4 + libraries/chain/chaindb/cache_map.cpp | 82 +++++++++++++++---- libraries/chain/chaindb/controller.cpp | 4 + libraries/chain/controller.cpp | 77 +++++++++-------- libraries/chain/eosio_contract_abi.cpp | 1 + .../include/cyberway/chaindb/cache_map.hpp | 1 + .../chain/include/cyberway/chaindb/common.hpp | 2 + libraries/chain/include/eosio/chain/block.hpp | 3 +- .../include/eosio/chain/chain_config.hpp | 12 ++- .../chain/include/eosio/chain/config.hpp | 2 + .../chain/include/eosio/chain/controller.hpp | 30 ++++++- .../include/eosio/chain/genesis_state.hpp | 2 + .../include/eosio/chain/resource_limits.hpp | 2 +- .../eosio/chain/transaction_context.hpp | 5 ++ libraries/chain/resource_limits.cpp | 4 +- libraries/chain/transaction_context.cpp | 22 ++++- .../eosio/event_engine_plugin/messages.hpp | 2 + .../state_history_serialization.hpp | 1 + .../state_history_plugin_abi.cpp | 1 + 21 files changed, 199 insertions(+), 60 deletions(-) diff --git a/contracts/eosio.bios/eosio.bios.abi b/contracts/eosio.bios/eosio.bios.abi index 90eba90f9f1..83cf6d13e41 100644 --- a/contracts/eosio.bios/eosio.bios.abi +++ b/contracts/eosio.bios/eosio.bios.abi @@ -68,6 +68,7 @@ {"name":"target_block_cpu_usage_pct", "type":"uint32"}, {"name":"max_transaction_cpu_usage", "type":"uint32"}, {"name":"min_transaction_cpu_usage", "type":"uint32"}, + {"name":"min_transaction_ram_usage", "type":"uint64"}, {"name":"max_transaction_lifetime", "type":"uint32"}, {"name":"deferred_trx_expiration_window", "type":"uint32"}, {"name":"max_transaction_delay", "type":"uint32"}, diff --git a/contracts/eosio.system/eosio.system.abi b/contracts/eosio.system/eosio.system.abi index 3fb29567e54..fff95f51ec0 100644 --- a/contracts/eosio.system/eosio.system.abi +++ b/contracts/eosio.system/eosio.system.abi @@ -234,6 +234,7 @@ {"name":"target_block_cpu_usage_pct", "type":"uint32"}, {"name":"max_transaction_cpu_usage", "type":"uint32"}, {"name":"min_transaction_cpu_usage", "type":"uint32"}, + {"name":"min_transaction_ram_usage", "type":"uint64"}, {"name":"max_transaction_lifetime", "type":"uint32"}, {"name":"deferred_trx_expiration_window", "type":"uint32"}, {"name":"max_transaction_delay", "type":"uint32"}, diff --git a/contracts/eosiolib/privileged.hpp b/contracts/eosiolib/privileged.hpp index 3091acf8b3b..dba89144292 100644 --- a/contracts/eosiolib/privileged.hpp +++ b/contracts/eosiolib/privileged.hpp @@ -46,6 +46,8 @@ namespace eosio { uint32_t min_transaction_cpu_usage; + uint64_t min_transaction_ram_usage; + /** * The numerator for the discount on cpu usage for CFA's @@ -103,6 +105,8 @@ namespace eosio { (max_block_cpu_usage)(target_block_cpu_usage_pct) (max_transaction_cpu_usage)(min_transaction_cpu_usage) + (min_transaction_ram_usage) + (max_transaction_lifetime)(deferred_trx_expiration_window)(max_transaction_delay) (max_inline_action_size)(max_inline_action_depth)(max_authority_depth) ) diff --git a/libraries/chain/chaindb/cache_map.cpp b/libraries/chain/chaindb/cache_map.cpp index c45ae6c1c9e..18c7c9f9f89 100644 --- a/libraries/chain/chaindb/cache_map.cpp +++ b/libraries/chain/chaindb/cache_map.cpp @@ -225,13 +225,55 @@ namespace cyberway { namespace chaindb { }; // struct pending_cache_object_state struct pending_cache_cell final: public cache_cell { - pending_cache_cell(cache_map_impl& m, const revision_t r) + pending_cache_cell(cache_map_impl& m, const revision_t r, const uint64_t d) : cache_cell(m, cache_cell::Pending), - revision(r) { + revision_(r), + max_distance_(d) { + assert(revision_ >= start_revision); + assert(max_distance_ > 0); } void emplace(cache_object_ptr obj_ptr, const bool is_deleted) { - size += obj_ptr->service().size; + assert(obj_ptr); + assert(!obj_ptr->has_cell() || obj_ptr->cell().kind == cache_cell::LRU); + + auto& state = emplace_impl(std::move(obj_ptr), is_deleted); + add_ram_usage(state); + } + + void emplace(cache_object_ptr obj_ptr, const bool is_deleted, cache_object_state* prev_state) { + assert(obj_ptr); + assert(obj_ptr->cell().kind == cache_cell::Pending); + + // squash case - don't add ram usage + auto& state = emplace_impl(std::move(obj_ptr), is_deleted); + state.prev_state = prev_state; + } + + void squash_revision() { + revision_--; + assert(revision_ >= start_revision); + } + + revision_t revision() { + return revision_; + } + + std::deque state_list; // Expansion of a std::deque is cheaper than the expansion of a std::vector + + private: + void add_ram_usage(const pending_cache_object_state& state) { + auto delta = max_distance_; + if (state.prev_state) { + delta = pos - state.prev_state->cell->pos; + } + + CYBERWAY_CACHE_ASSERT(UINT64_MAX - size >= delta, "Pending delta would overflow UINT64_MAX"); + size += delta; + } + + pending_cache_object_state& emplace_impl(cache_object_ptr obj_ptr, const bool is_deleted) { + assert(obj_ptr); state_list.emplace_back(*this, std::move(obj_ptr)); @@ -240,15 +282,12 @@ namespace cyberway { namespace chaindb { state.prev_state = prev_state; state.is_deleted = is_deleted; - } - void emplace(cache_object_ptr obj_ptr, const bool is_deleted, cache_object_state* prev_state) { - emplace(std::move(obj_ptr), is_deleted); - state_list.back().prev_state = prev_state; + return state; } - revision_t revision = impossible_revision; - std::deque state_list; // Expansion of a std::deque is cheaper than the expansion of a std::vector + revision_t revision_ = impossible_revision; + const uint64_t max_distance_ = 0; }; // struct pending_cache_cell //--------------------------------------- @@ -507,8 +546,12 @@ namespace cyberway { namespace chaindb { } } + uint64_t calc_ram_bytes(const revision_t revision) { + return get_pending_cell(revision).size; + } + void start_session(const revision_t revision) { - pending_cell_list_.emplace_back(*this, revision); + pending_cell_list_.emplace_back(*this, revision, ram_limit_); auto& pending = pending_cell_list_.back(); if (!lru_cell_list_.empty()) { auto& lru = lru_cell_list_.back(); @@ -546,13 +589,12 @@ namespace cyberway { namespace chaindb { auto itr = pending_cell_list_.end(); --itr; auto& src_cell =*itr; - CYBERWAY_CACHE_ASSERT(revision == src_cell.revision, "Wrong revision of top pending caches"); + CYBERWAY_CACHE_ASSERT(revision == src_cell.revision(), "Wrong revision of top pending caches"); --itr; auto& dst_cell =*itr; - CYBERWAY_CACHE_ASSERT(revision - 1 == dst_cell.revision, "Wrong revision of pending caches"); + CYBERWAY_CACHE_ASSERT(revision - 1 == dst_cell.revision(), "Wrong revision of pending caches"); - if (pending_cell_list_.size() == 2 && dst_cell.state_list.empty()) { - assert(pending_cell_list_.front().revision == revision -1); - src_cell.revision = dst_cell.revision; + if (pending_cell_list_.begin() == itr && dst_cell.state_list.empty()) { + src_cell.squash_revision(); pending_cell_list_.pop_front(); return; } @@ -570,7 +612,7 @@ namespace cyberway { namespace chaindb { void undo_session(const revision_t revision) { if (!has_pending_cell()) { - // pop_block() + // case of pop_block return; } @@ -746,9 +788,9 @@ namespace cyberway { namespace chaindb { pending_cache_cell& get_pending_cell(const revision_t revision) { auto pending_ptr = find_pending_cell(); - CYBERWAY_CACHE_ASSERT(pending_ptr && pending_ptr->revision == revision, + CYBERWAY_CACHE_ASSERT(pending_ptr && pending_ptr->revision() == revision, "Wrong pending revision ${pending_revision} != ${revision}", - ("pending_revision", pending_ptr ? pending_ptr->revision : impossible_revision)("revision", revision)); + ("pending_revision", pending_ptr ? pending_ptr->revision() : impossible_revision)("revision", revision)); return *pending_ptr; } @@ -1066,6 +1108,10 @@ namespace cyberway { namespace chaindb { impl_->set_revision(obj.service, rev); } + uint64_t cache_map::calc_ram_bytes(const revision_t revision) const { + return impl_->calc_ram_bytes(revision); + } + void cache_map::start_session(const revision_t revision) const { impl_->start_session(revision); } diff --git a/libraries/chain/chaindb/controller.cpp b/libraries/chain/chaindb/controller.cpp index 052bd57c722..e82240e154f 100644 --- a/libraries/chain/chaindb/controller.cpp +++ b/libraries/chain/chaindb/controller.cpp @@ -1012,4 +1012,8 @@ namespace cyberway { namespace chaindb { apply_ = false; } + uint64_t chaindb_session::calc_ram_bytes() const { + return controller_.impl_->cache_.calc_ram_bytes(revision_); + } + } } // namespace cyberway::chaindb diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 228b0c60b92..c2ee74d9ec5 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -735,8 +735,8 @@ struct controller_impl { fc::time_point deadline, fc::time_point start, uint32_t& cpu_time_to_bill_us, // only set on failure - uint32_t billed_cpu_time_us, - bool explicit_billed_cpu_time = false + uint64_t& ram_to_bill_bytes, // only set on failure + const billed_bw_usage& billed ) { signed_transaction etrx; @@ -748,8 +748,10 @@ struct controller_impl { transaction_context trx_context( self, etrx, etrx.id(), start ); trx_context.deadline = deadline; - trx_context.explicit_billed_cpu_time = explicit_billed_cpu_time; - trx_context.billed_cpu_time_us = billed_cpu_time_us; + trx_context.explicit_billed_cpu_time = billed.explicit_usage; + trx_context.billed_cpu_time_us = billed.cpu_time_us; + trx_context.explicit_billed_ram_bytes = billed.explicit_usage; + trx_context.billed_ram_bytes = billed.ram_bytes; transaction_trace_ptr trace = trx_context.trace; try { trx_context.init_for_implicit_trx(); @@ -760,7 +762,7 @@ struct controller_impl { auto restore = make_block_restore_point(); trace->receipt = push_receipt( gtrx.trx_id, transaction_receipt::soft_fail, - trx_context.billed_cpu_time_us, trace->net_usage ); + trx_context.billed_cpu_time_us, trace->net_usage, trx_context.billed_ram_bytes ); fc::move_append( pending->_actions, move(trx_context.executed) ); trx_context.squash(); @@ -768,6 +770,7 @@ struct controller_impl { return trace; } catch( const fc::exception& e ) { cpu_time_to_bill_us = trx_context.update_billed_cpu_time( fc::time_point::now() ); + ram_to_bill_bytes = trx_context.update_billed_ram_bytes(); trace->except = e; trace->except_ptr = std::current_exception(); } @@ -800,11 +803,11 @@ struct controller_impl { || failure_is_subjective(e); } - transaction_trace_ptr push_scheduled_transaction( const transaction_id_type& trxid, fc::time_point deadline, uint32_t billed_cpu_time_us, bool explicit_billed_cpu_time = false ) { + transaction_trace_ptr push_scheduled_transaction( const transaction_id_type& trxid, fc::time_point deadline, const billed_bw_usage& billed ) { auto idx = chaindb.get_index(); auto itr = idx.find( trxid ); EOS_ASSERT( itr != idx.end(), unknown_transaction_exception, "unknown transaction" ); - return push_scheduled_transaction( *itr, deadline, billed_cpu_time_us, explicit_billed_cpu_time ); + return push_scheduled_transaction( *itr, deadline, billed ); } // TODO: request bw, why provided? @@ -844,7 +847,7 @@ struct controller_impl { // return {trx_context.get_provided_bandwith(), trx_context.get_net_usage(), trx_context.get_cpu_usage()}; // } - transaction_trace_ptr push_scheduled_transaction( const generated_transaction_object& gto, fc::time_point deadline, uint32_t billed_cpu_time_us, bool explicit_billed_cpu_time = false ) + transaction_trace_ptr push_scheduled_transaction( const generated_transaction_object& gto, fc::time_point deadline, const billed_bw_usage& billed ) try { maybe_session undo_session; if ( !self.skip_db_sessions() ) @@ -879,7 +882,7 @@ struct controller_impl { trace->block_time = self.pending_block_time(); trace->producer_block_id = self.pending_producer_block_id(); trace->scheduled = true; - trace->receipt = push_receipt( gtrx.trx_id, transaction_receipt::expired, billed_cpu_time_us, 0 ); // expire the transaction + trace->receipt = push_receipt( gtrx.trx_id, transaction_receipt::expired, billed.cpu_time_us, 0, billed.ram_bytes ); // expire the transaction emit( self.accepted_transaction, trx ); emit( self.applied_transaction, trace ); undo_session.squash(); @@ -891,13 +894,16 @@ struct controller_impl { }); in_trx_requiring_checks = true; - uint32_t cpu_time_to_bill_us = billed_cpu_time_us; + uint32_t cpu_time_to_bill_us = billed.cpu_time_us; + uint64_t ram_to_bill_bytes = billed.ram_bytes; transaction_context trx_context( self, dtrx, gtrx.trx_id ); trx_context.leeway = fc::microseconds(0); // avoid stealing cpu resource trx_context.deadline = deadline; - trx_context.explicit_billed_cpu_time = explicit_billed_cpu_time; - trx_context.billed_cpu_time_us = billed_cpu_time_us; + trx_context.explicit_billed_cpu_time = billed.explicit_usage; + trx_context.billed_cpu_time_us = billed.cpu_time_us; + trx_context.explicit_billed_ram_bytes = billed.explicit_usage; + trx_context.billed_ram_bytes = billed.ram_bytes; trace = trx_context.trace; try { //TODO: request bw, why provided? @@ -915,7 +921,8 @@ struct controller_impl { trace->receipt = push_receipt( gtrx.trx_id, transaction_receipt::executed, trx_context.billed_cpu_time_us, - trace->net_usage ); + trace->net_usage, + trx_context.billed_ram_bytes); fc::move_append( pending->_actions, move(trx_context.executed) ); @@ -930,6 +937,7 @@ struct controller_impl { return trace; } catch( const fc::exception& e ) { cpu_time_to_bill_us = trx_context.update_billed_cpu_time( fc::time_point::now() ); + ram_to_bill_bytes = trx_context.update_billed_ram_bytes(); trace->except = e; trace->except_ptr = std::current_exception(); trace->elapsed = fc::time_point::now() - trx_context.start; @@ -942,7 +950,7 @@ struct controller_impl { // Attempt error handling for the generated transaction. auto error_trace = apply_onerror( gtrx, deadline, trx_context.pseudo_start, - cpu_time_to_bill_us, billed_cpu_time_us, explicit_billed_cpu_time); + cpu_time_to_bill_us, ram_to_bill_bytes, billed); error_trace->failed_dtrx_trace = trace; trace = error_trace; if( !trace->except_ptr ) { @@ -958,7 +966,7 @@ struct controller_impl { // subjectivity changes based on producing vs validating bool subjective = false; - if (explicit_billed_cpu_time) { + if (billed.explicit_usage) { subjective = failure_is_subjective(*trace->except); } else { subjective = scheduled_failure_is_subjective(*trace->except); @@ -967,7 +975,7 @@ struct controller_impl { if ( !subjective ) { // hard failure logic - if( !explicit_billed_cpu_time ) { + if( !billed.explicit_usage ) { auto& rl = self.get_mutable_resource_limits_manager(); rl.update_account_usage( trx_context.bill_to_accounts, block_timestamp_type(self.pending_block_time()).slot ); int64_t account_cpu_limit = trx_context.get_min_cpu_limit(); @@ -975,11 +983,13 @@ struct controller_impl { cpu_time_to_bill_us = static_cast( std::min( std::min( static_cast(cpu_time_to_bill_us), account_cpu_limit ), trx_context.initial_objective_duration_limit.count() ) ); + + // TODO: CyberWay #616 should be here correction of ram_to_bill_bytes? } - resource_limits.add_transaction_usage( trx_context.bill_to_accounts, cpu_time_to_bill_us, 0, self.pending_block_time() ); // Should never fail + resource_limits.add_transaction_usage( trx_context.bill_to_accounts, cpu_time_to_bill_us, 0, ram_to_bill_bytes, self.pending_block_time() ); // Should never fail - trace->receipt = push_receipt(gtrx.trx_id, transaction_receipt::hard_fail, cpu_time_to_bill_us, 0); + trace->receipt = push_receipt(gtrx.trx_id, transaction_receipt::hard_fail, cpu_time_to_bill_us, 0, ram_to_bill_bytes); emit( self.accepted_transaction, trx ); emit( self.applied_transaction, trace ); @@ -999,13 +1009,14 @@ struct controller_impl { */ template const transaction_receipt& push_receipt( const T& trx, transaction_receipt_header::status_enum status, - uint64_t cpu_usage_us, uint64_t net_usage ) { + uint64_t cpu_usage_us, uint64_t net_usage, uint64_t ram_bytes ) { uint64_t net_usage_words = net_usage / 8; EOS_ASSERT( net_usage_words*8 == net_usage, transaction_exception, "net_usage is not divisible by 8" ); pending->_pending_block_state->block->transactions.emplace_back( trx ); transaction_receipt& r = pending->_pending_block_state->block->transactions.back(); r.cpu_usage_us = cpu_usage_us; r.net_usage_words = net_usage_words; + r.ram_kbytes = ram_bytes >> 10; r.status = status; return r; } @@ -1017,15 +1028,14 @@ struct controller_impl { */ transaction_trace_ptr push_transaction( const transaction_metadata_ptr& trx, fc::time_point deadline, - uint32_t billed_cpu_time_us, - bool explicit_billed_cpu_time = false ) + const billed_bw_usage& billed ) { EOS_ASSERT(deadline != fc::time_point(), transaction_exception, "deadline cannot be uninitialized"); transaction_trace_ptr trace; try { auto start = fc::time_point::now(); - if( !explicit_billed_cpu_time ) { + if( !billed.explicit_usage ) { fc::microseconds already_consumed_time( EOS_PERCENT(trx->sig_cpu_usage.count(), conf.sig_cpu_bill_pct) ); if( start.time_since_epoch() < already_consumed_time ) { @@ -1041,8 +1051,10 @@ struct controller_impl { trx_context.leeway = *subjective_cpu_leeway; } trx_context.deadline = deadline; - trx_context.explicit_billed_cpu_time = explicit_billed_cpu_time; - trx_context.billed_cpu_time_us = billed_cpu_time_us; + trx_context.explicit_billed_cpu_time = billed.explicit_usage; + trx_context.billed_cpu_time_us = billed.cpu_time_us; + trx_context.explicit_billed_ram_bytes = billed.explicit_usage; + trx_context.billed_ram_bytes = billed.ram_bytes; trace = trx_context.trace; try { //TODO: request bw, why provided? @@ -1085,13 +1097,14 @@ struct controller_impl { transaction_receipt::status_enum s = (trx_context.delay == fc::seconds(0)) ? transaction_receipt::executed : transaction_receipt::delayed; - trace->receipt = push_receipt(*trx->packed_trx, s, trx_context.billed_cpu_time_us, trace->net_usage); + trace->receipt = push_receipt(*trx->packed_trx, s, trx_context.billed_cpu_time_us, trace->net_usage, trx_context.billed_ram_bytes); pending->_pending_block_state->trxs.emplace_back(trx); } else { transaction_receipt_header r; r.status = transaction_receipt::executed; r.cpu_usage_us = trx_context.billed_cpu_time_us; r.net_usage_words = trace->net_usage / 8; + r.ram_kbytes = trx_context.billed_ram_bytes >> 10; trace->receipt = r; } @@ -1237,9 +1250,9 @@ struct controller_impl { for( const auto& receipt : b->transactions ) { auto num_pending_receipts = pending->_pending_block_state->block->transactions.size(); if( receipt.trx.contains() ) { - trace = push_transaction( packed_transactions.at(packed_idx++), fc::time_point::maximum(), receipt.cpu_usage_us, true ); + trace = push_transaction( packed_transactions.at(packed_idx++), fc::time_point::maximum(), {receipt} ); } else if( receipt.trx.contains() ) { - trace = push_scheduled_transaction( receipt.trx.get(), fc::time_point::maximum(), receipt.cpu_usage_us, true ); + trace = push_scheduled_transaction( receipt.trx.get(), fc::time_point::maximum(), {receipt} ); } else { EOS_ASSERT( false, block_validate_exception, "encountered unexpected receipt type" ); } @@ -1616,7 +1629,7 @@ struct controller_impl { in_trx_requiring_checks = old_value; }); in_trx_requiring_checks = true; - auto trace = push_transaction( onbtrx, fc::time_point::maximum(), self.get_global_properties().configuration.min_transaction_cpu_usage, true ); + auto trace = push_transaction( onbtrx, fc::time_point::maximum(), { self.get_global_properties().configuration } ); if(trace && trace->except) { edump((*trace)); @@ -1721,17 +1734,17 @@ void controller::push_block( std::future& block_state_future ) my->push_block( block_state_future ); } -transaction_trace_ptr controller::push_transaction(const transaction_metadata_ptr& trx, fc::time_point deadline, uint32_t billed_cpu_time_us ) { +transaction_trace_ptr controller::push_transaction(const transaction_metadata_ptr& trx, fc::time_point deadline, const billed_bw_usage& billed ) { validate_db_available_size(); EOS_ASSERT( get_read_mode() != chain::db_read_mode::READ_ONLY, transaction_type_exception, "push transaction not allowed in read-only mode" ); EOS_ASSERT( trx && !trx->implicit && !trx->scheduled, transaction_type_exception, "Implicit/Scheduled transaction not allowed" ); - return my->push_transaction(trx, deadline, billed_cpu_time_us, billed_cpu_time_us > 0 ); + return my->push_transaction(trx, deadline, billed ); } -transaction_trace_ptr controller::push_scheduled_transaction( const transaction_id_type& trxid, fc::time_point deadline, uint32_t billed_cpu_time_us ) +transaction_trace_ptr controller::push_scheduled_transaction( const transaction_id_type& trxid, fc::time_point deadline, const billed_bw_usage& billed ) { validate_db_available_size(); - return my->push_scheduled_transaction( trxid, deadline, billed_cpu_time_us, billed_cpu_time_us > 0 ); + return my->push_scheduled_transaction( trxid, deadline, billed ); } diff --git a/libraries/chain/eosio_contract_abi.cpp b/libraries/chain/eosio_contract_abi.cpp index 97780f2b3ef..bce69229ca4 100644 --- a/libraries/chain/eosio_contract_abi.cpp +++ b/libraries/chain/eosio_contract_abi.cpp @@ -207,6 +207,7 @@ abi_def eosio_contract_abi(abi_def eos_abi) {"target_block_cpu_usage_pct", "uint32"}, {"max_transaction_cpu_usage", "uint32"}, {"min_transaction_cpu_usage", "uint32"}, + {"min_transaction_ram_usage", "uint64"}, {"max_transaction_lifetime", "uint32"}, {"deferred_trx_expiration_window", "uint32"}, {"max_transaction_delay", "uint32"}, diff --git a/libraries/chain/include/cyberway/chaindb/cache_map.hpp b/libraries/chain/include/cyberway/chaindb/cache_map.hpp index 4a864ed36ca..fc358696e00 100644 --- a/libraries/chain/include/cyberway/chaindb/cache_map.hpp +++ b/libraries/chain/include/cyberway/chaindb/cache_map.hpp @@ -22,6 +22,7 @@ namespace cyberway { namespace chaindb { void remove(const table_info&, primary_key_t) const; void set_revision(const object_value&, revision_t) const; + uint64_t calc_ram_bytes(revision_t) const; void start_session(revision_t) const; void push_session(revision_t) const; void squash_session(revision_t) const; diff --git a/libraries/chain/include/cyberway/chaindb/common.hpp b/libraries/chain/include/cyberway/chaindb/common.hpp index 4d44df9ad58..c2eeee8c83e 100644 --- a/libraries/chain/include/cyberway/chaindb/common.hpp +++ b/libraries/chain/include/cyberway/chaindb/common.hpp @@ -110,6 +110,8 @@ namespace cyberway { namespace chaindb { /** Undo changes made in this session */ void undo(); + uint64_t calc_ram_bytes() const; + revision_t revision() const { return revision_; } diff --git a/libraries/chain/include/eosio/chain/block.hpp b/libraries/chain/include/eosio/chain/block.hpp index 807ab80ce22..9638f9bf8f9 100644 --- a/libraries/chain/include/eosio/chain/block.hpp +++ b/libraries/chain/include/eosio/chain/block.hpp @@ -29,6 +29,7 @@ namespace eosio { namespace chain { fc::enum_type status; uint32_t cpu_usage_us = 0; ///< total billed CPU usage (microseconds) fc::unsigned_int net_usage_words; ///< total billed NET usage, so we can reconstruct resource state when skipping context free data... hard failures... + uint64_t ram_kbytes = 0; ///< total billed RAM usage (kbytes) }; struct transaction_receipt : public transaction_receipt_header { @@ -83,6 +84,6 @@ namespace eosio { namespace chain { FC_REFLECT_ENUM( eosio::chain::transaction_receipt::status_enum, (executed)(soft_fail)(hard_fail)(delayed)(expired) ) -FC_REFLECT(eosio::chain::transaction_receipt_header, (status)(cpu_usage_us)(net_usage_words) ) +FC_REFLECT(eosio::chain::transaction_receipt_header, (status)(cpu_usage_us)(net_usage_words)(ram_kbytes) ) FC_REFLECT_DERIVED(eosio::chain::transaction_receipt, (eosio::chain::transaction_receipt_header), (trx) ) FC_REFLECT_DERIVED(eosio::chain::signed_block, (eosio::chain::signed_block_header), (transactions)(archive_records)(block_extensions) ) diff --git a/libraries/chain/include/eosio/chain/chain_config.hpp b/libraries/chain/include/eosio/chain/chain_config.hpp index 37b73cb51fe..9cfe30adc30 100644 --- a/libraries/chain/include/eosio/chain/chain_config.hpp +++ b/libraries/chain/include/eosio/chain/chain_config.hpp @@ -30,6 +30,8 @@ struct chain_config { uint32_t max_transaction_cpu_usage; ///< the maximum billable cpu usage (in microseconds) that the chain will allow regardless of account limits uint32_t min_transaction_cpu_usage; ///< the minimum billable cpu usage (in microseconds) that the chain requires + uint64_t min_transaction_ram_usage; ///< the minimum billable ram usage (in bytes) that the chain requires + uint32_t max_transaction_lifetime; ///< the maximum number of seconds that an input transaction's expiration can be ahead of the time of the block in which it is first included uint32_t deferred_trx_expiration_window; ///< the number of seconds after the time a deferred transaction can first execute until it expires uint32_t max_transaction_delay; ///< the maximum number of seconds that can be imposed as a delay requirement by authorization checks @@ -53,6 +55,8 @@ struct chain_config { << "Max Transaction CPU Usage: " << c.max_transaction_cpu_usage << ", " << "Min Transaction CPU Usage: " << c.min_transaction_cpu_usage << ", " + << "Min Transaction RAM Usage: " << c.min_transaction_ram_usage << ", " + << "Max Transaction Lifetime: " << c.max_transaction_lifetime << ", " << "Deferred Transaction Expiration Window: " << c.deferred_trx_expiration_window << ", " << "Max Transaction Delay: " << c.max_transaction_delay << ", " @@ -72,7 +76,8 @@ struct chain_config { lhs.max_block_cpu_usage, lhs.target_block_cpu_usage_pct, lhs.max_transaction_cpu_usage, - lhs.max_transaction_cpu_usage, + lhs.min_transaction_cpu_usage, + lhs.min_transaction_ram_usage, lhs.max_transaction_lifetime, lhs.deferred_trx_expiration_window, lhs.max_transaction_delay, @@ -91,7 +96,8 @@ struct chain_config { rhs.max_block_cpu_usage, rhs.target_block_cpu_usage_pct, rhs.max_transaction_cpu_usage, - rhs.max_transaction_cpu_usage, + rhs.min_transaction_cpu_usage, + rhs.min_transaction_ram_usage, rhs.max_transaction_lifetime, rhs.deferred_trx_expiration_window, rhs.max_transaction_delay, @@ -115,6 +121,8 @@ FC_REFLECT(eosio::chain::chain_config, (max_block_cpu_usage)(target_block_cpu_usage_pct) (max_transaction_cpu_usage)(min_transaction_cpu_usage) + (min_transaction_ram_usage) + (max_transaction_lifetime)(deferred_trx_expiration_window)(max_transaction_delay) (max_inline_action_size)(max_inline_action_depth)(max_authority_depth) diff --git a/libraries/chain/include/eosio/chain/config.hpp b/libraries/chain/include/eosio/chain/config.hpp index 377d167e70b..232275c8514 100644 --- a/libraries/chain/include/eosio/chain/config.hpp +++ b/libraries/chain/include/eosio/chain/config.hpp @@ -80,6 +80,8 @@ const static uint32_t default_target_block_cpu_usage_pct = 10 * perce const static uint32_t default_max_transaction_cpu_usage = 3*default_max_block_cpu_usage/4; /// max trx cpu usage in microseconds const static uint32_t default_min_transaction_cpu_usage = 100; /// min trx cpu usage in microseconds (10000 TPS equiv) +const static uint64_t default_min_transaction_ram_usage = 1024; /// 1 kb + const static uint32_t default_max_trx_lifetime = 60*60; // 1 hour const static uint32_t default_deferred_trx_expiration_window = 10*60; // 10 minutes const static uint32_t default_max_trx_delay = 45*24*3600; // 45 days diff --git a/libraries/chain/include/eosio/chain/controller.hpp b/libraries/chain/include/eosio/chain/controller.hpp index 7567394463c..11ba004137d 100644 --- a/libraries/chain/include/eosio/chain/controller.hpp +++ b/libraries/chain/include/eosio/chain/controller.hpp @@ -61,6 +61,32 @@ namespace eosio { namespace chain { LIGHT }; + struct billed_bw_usage { + const uint32_t cpu_time_us = 0; + const uint64_t ram_bytes = 0; + const bool explicit_usage = false; + + billed_bw_usage() = default; + + billed_bw_usage(const uint32_t us, const uint64_t kbytes) + : cpu_time_us(us), + ram_bytes(kbytes << 10), + explicit_usage(us > 0 || kbytes > 0) { + } + + billed_bw_usage(const transaction_receipt& r) + : cpu_time_us(r.cpu_usage_us), + ram_bytes(r.ram_kbytes << 10), + explicit_usage(true) { + } + + billed_bw_usage(const chain_config& c) + : cpu_time_us(c.min_transaction_cpu_usage), + ram_bytes(c.min_transaction_ram_usage), + explicit_usage(true) { + } + }; + class controller { public: @@ -145,13 +171,13 @@ namespace eosio { namespace chain { /** * */ - transaction_trace_ptr push_transaction( const transaction_metadata_ptr& trx, fc::time_point deadline, uint32_t billed_cpu_time_us = 0 ); + transaction_trace_ptr push_transaction( const transaction_metadata_ptr& trx, fc::time_point deadline, const billed_bw_usage& = {} ); /** * Attempt to execute a specific transaction in our deferred trx database * */ - transaction_trace_ptr push_scheduled_transaction( const transaction_id_type& scheduled, fc::time_point deadline, uint32_t billed_cpu_time_us = 0 ); + transaction_trace_ptr push_scheduled_transaction( const transaction_id_type& scheduled, fc::time_point deadline, const billed_bw_usage& = {} ); void finalize_block(); void sign_block( const std::function& signer_callback ); diff --git a/libraries/chain/include/eosio/chain/genesis_state.hpp b/libraries/chain/include/eosio/chain/genesis_state.hpp index 0c68dfe9ccc..25dc79a4126 100644 --- a/libraries/chain/include/eosio/chain/genesis_state.hpp +++ b/libraries/chain/include/eosio/chain/genesis_state.hpp @@ -34,6 +34,8 @@ struct genesis_state { .max_transaction_cpu_usage = config::default_max_transaction_cpu_usage, .min_transaction_cpu_usage = config::default_min_transaction_cpu_usage, + .min_transaction_ram_usage = config::default_min_transaction_ram_usage, + .max_transaction_lifetime = config::default_max_trx_lifetime, .deferred_trx_expiration_window = config::default_deferred_trx_expiration_window, .max_transaction_delay = config::default_max_trx_delay, diff --git a/libraries/chain/include/eosio/chain/resource_limits.hpp b/libraries/chain/include/eosio/chain/resource_limits.hpp index 115389a2ae9..e1889ad7eca 100644 --- a/libraries/chain/include/eosio/chain/resource_limits.hpp +++ b/libraries/chain/include/eosio/chain/resource_limits.hpp @@ -80,7 +80,7 @@ namespace resource_limits { void set_block_parameters( const elastic_limit_parameters& cpu_limit_parameters, const elastic_limit_parameters& net_limit_parameters ); void update_account_usage( const flat_set& accounts, uint32_t ordinal ); - void add_transaction_usage( const flat_set& accounts, uint64_t cpu_usage, uint64_t net_usage, fc::time_point now ); + void add_transaction_usage( const flat_set& accounts, uint64_t cpu_usage, uint64_t net_usage, uint64_t ram_usage, fc::time_point now ); void add_storage_usage( const storage_payer_info& ); diff --git a/libraries/chain/include/eosio/chain/transaction_context.hpp b/libraries/chain/include/eosio/chain/transaction_context.hpp index 0b3cc5be831..8d7c8544ea2 100644 --- a/libraries/chain/include/eosio/chain/transaction_context.hpp +++ b/libraries/chain/include/eosio/chain/transaction_context.hpp @@ -106,6 +106,8 @@ namespace eosio { namespace chain { int64_t get_billed_cpu_time( fc::time_point now )const; uint32_t update_billed_cpu_time( fc::time_point now ); + uint64_t update_billed_ram_bytes(); + int64_t get_min_cpu_limit()const; void add_storage_usage( const storage_payer_info& ); @@ -176,6 +178,9 @@ namespace eosio { namespace chain { int64_t billed_cpu_time_us = 0; bool explicit_billed_cpu_time = false; + uint64_t billed_ram_bytes = 0; + bool explicit_billed_ram_bytes = false; + private: bool is_initialized = false; diff --git a/libraries/chain/resource_limits.cpp b/libraries/chain/resource_limits.cpp index 21db4fdfe7c..572adc08afd 100644 --- a/libraries/chain/resource_limits.cpp +++ b/libraries/chain/resource_limits.cpp @@ -111,7 +111,7 @@ void resource_limits_manager::update_account_usage(const flat_set& } -void resource_limits_manager::add_transaction_usage(const flat_set& accounts, uint64_t cpu_usage, uint64_t net_usage, fc::time_point now) { +void resource_limits_manager::add_transaction_usage(const flat_set& accounts, uint64_t cpu_usage, uint64_t net_usage, uint64_t ram_usage, fc::time_point now) { auto state_table = _chaindb.get_table(); const auto& state = state_table.get(); const auto& config = _chaindb.get(); @@ -127,6 +127,7 @@ void resource_limits_manager::add_transaction_usage(const flat_set usage_table.modify( usage, [&]( auto& bu ) { bu.net_usage.add( net_usage, time_slot, config.account_net_usage_average_window ); bu.cpu_usage.add( cpu_usage, time_slot, config.account_cpu_usage_average_window ); + // TODO: Cyberway #616 }); EOS_ASSERT(get_account_balance(now.sec_since_epoch(), a, prices).stake >= 0, resource_exhausted_exception, "authorizing account '${n}' has insufficient resources for this transaction", ("n", name(a))); @@ -136,6 +137,7 @@ void resource_limits_manager::add_transaction_usage(const flat_set state_table.modify(state, [&](resource_limits_state_object& rls){ rls.pending_cpu_usage += cpu_usage; rls.pending_net_usage += net_usage; + // TODO: CyberWay #616 }); EOS_ASSERT( state.pending_cpu_usage <= config.cpu_limit_parameters.max, block_resource_exhausted, "Block has insufficient cpu resources" ); diff --git a/libraries/chain/transaction_context.cpp b/libraries/chain/transaction_context.cpp index be41f1690e3..b7033643ff8 100644 --- a/libraries/chain/transaction_context.cpp +++ b/libraries/chain/transaction_context.cpp @@ -399,10 +399,13 @@ namespace bacc = boost::accumulators; } void transaction_context::finalize() { - validate_bw_usage(); + validate_bw_usage(); - control.get_mutable_resource_limits_manager().add_transaction_usage( bill_to_accounts, static_cast(billed_cpu_time_us), net_usage, - control.pending_block_time()); // Should never fail + control.get_mutable_resource_limits_manager().add_transaction_usage( bill_to_accounts, + static_cast(billed_cpu_time_us), + net_usage, + billed_ram_bytes, + control.pending_block_time()); // Should never fail } void transaction_context::validate_bw_usage() { @@ -419,6 +422,8 @@ namespace bacc = boost::accumulators; int64_t block_time = control.pending_block_time().sec_since_epoch(); auto& rl = control.get_mutable_resource_limits_manager(); + update_billed_ram_bytes(); + net_usage = ((net_usage + 7)/8)*8; // Round up to nearest multiple of word size (8 bytes) eager_net_limit = net_limit; @@ -563,6 +568,17 @@ namespace bacc = boost::accumulators; return static_cast(billed_cpu_time_us); } + uint64_t transaction_context::update_billed_ram_bytes() { + if( !explicit_billed_ram_bytes ) { + const auto& cfg = control.get_global_properties().configuration; + billed_ram_bytes = chaindb_undo_session->calc_ram_bytes(); + billed_ram_bytes = ((billed_ram_bytes + 1023) >> 10) << 10; // Round up to nearest kbytes + billed_ram_bytes = std::max(billed_ram_bytes, cfg.min_transaction_ram_usage); + explicit_billed_ram_bytes = true; + } + return billed_ram_bytes; + } + // TODO: requested bw, why provided ? // uint64_t transaction_context::get_provided_net_limit(account_name account) const { // const auto provided_bw_it = provided_bandwith_.find(account); diff --git a/plugins/event_engine_plugin/include/eosio/event_engine_plugin/messages.hpp b/plugins/event_engine_plugin/include/eosio/event_engine_plugin/messages.hpp index 49c79911ac8..cc12b54f11f 100644 --- a/plugins/event_engine_plugin/include/eosio/event_engine_plugin/messages.hpp +++ b/plugins/event_engine_plugin/include/eosio/event_engine_plugin/messages.hpp @@ -44,12 +44,14 @@ namespace eosio { fc::enum_type status; uint32_t cpu_usage_us; fc::unsigned_int net_usage_words; + uint64_t ram_kbytes; TrxReceipt(const chain::transaction_id_type &id, const chain::transaction_receipt &receipt) : id(id) , status(receipt.status) , cpu_usage_us(receipt.cpu_usage_us) , net_usage_words(receipt.net_usage_words) + , ram_kbytes(receipt.ram_kbytes) { } }; diff --git a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp index 04662b8615d..1c9c21a3a03 100644 --- a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp +++ b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp @@ -249,6 +249,7 @@ datastream& operator<<(datastream& ds, const history_serial_wrapper(obj.obj.target_block_cpu_usage_pct)); fc::raw::pack(ds, as_type(obj.obj.max_transaction_cpu_usage)); fc::raw::pack(ds, as_type(obj.obj.min_transaction_cpu_usage)); + fc::raw::pack(ds, as_type(obj.obj.min_transaction_ram_usage)); fc::raw::pack(ds, as_type(obj.obj.max_transaction_lifetime)); fc::raw::pack(ds, as_type(obj.obj.deferred_trx_expiration_window)); fc::raw::pack(ds, as_type(obj.obj.max_transaction_delay)); diff --git a/plugins/state_history_plugin/state_history_plugin_abi.cpp b/plugins/state_history_plugin/state_history_plugin_abi.cpp index bdedcc81cd9..04e58d89aea 100644 --- a/plugins/state_history_plugin/state_history_plugin_abi.cpp +++ b/plugins/state_history_plugin/state_history_plugin_abi.cpp @@ -288,6 +288,7 @@ extern const char* const state_history_plugin_abi = R"({ { "type": "uint32", "name": "target_block_cpu_usage_pct" }, { "type": "uint32", "name": "max_transaction_cpu_usage" }, { "type": "uint32", "name": "min_transaction_cpu_usage" }, + { "type": "uint64", "name": "min_transaction_ram_usage" }, { "type": "uint32", "name": "max_transaction_lifetime" }, { "type": "uint32", "name": "deferred_trx_expiration_window" }, { "type": "uint32", "name": "max_transaction_delay" }, From 2844572a6cb101bd4132d3cf9f27725a45808f53 Mon Sep 17 00:00:00 2001 From: Andrew Falaleev Date: Wed, 8 May 2019 16:32:53 +0700 Subject: [PATCH 2/2] Add support of subjective RAM to tests. #616 --- libraries/testing/include/eosio/testing/tester.hpp | 7 +++++-- libraries/testing/tester.cpp | 12 ++++++++---- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/libraries/testing/include/eosio/testing/tester.hpp b/libraries/testing/include/eosio/testing/tester.hpp index 9320d2c481a..98e7c9cd28c 100644 --- a/libraries/testing/include/eosio/testing/tester.hpp +++ b/libraries/testing/include/eosio/testing/tester.hpp @@ -93,6 +93,7 @@ namespace eosio { namespace testing { static const uint32_t DEFAULT_EXPIRATION_DELTA = 6; static const uint32_t DEFAULT_BILLED_CPU_TIME_US = 2000; + static const uint64_t DEFAULT_BILLED_RAM_BYTES = 1024; static const fc::microseconds abi_serializer_max_time; virtual ~base_tester() {}; @@ -117,8 +118,10 @@ namespace eosio { namespace testing { void produce_min_num_of_blocks_to_spend_time_wo_inactive_prod(const fc::microseconds target_elapsed_time = fc::microseconds()); signed_block_ptr push_block(signed_block_ptr b); - transaction_trace_ptr push_transaction( packed_transaction& trx, fc::time_point deadline = fc::time_point::maximum(), uint32_t billed_cpu_time_us = DEFAULT_BILLED_CPU_TIME_US ); - transaction_trace_ptr push_transaction( signed_transaction& trx, fc::time_point deadline = fc::time_point::maximum(), uint32_t billed_cpu_time_us = DEFAULT_BILLED_CPU_TIME_US ); + transaction_trace_ptr push_transaction( packed_transaction& trx, fc::time_point deadline = fc::time_point::maximum(), + uint32_t billed_cpu_time_us = DEFAULT_BILLED_CPU_TIME_US, uint64_t billed_ram_bytes = DEFAULT_BILLED_RAM_BYTES ); + transaction_trace_ptr push_transaction( signed_transaction& trx, fc::time_point deadline = fc::time_point::maximum(), + uint32_t billed_cpu_time_us = DEFAULT_BILLED_CPU_TIME_US, uint64_t billed_ram_bytes = DEFAULT_BILLED_RAM_BYTES ); action_result push_action(action&& cert_act, uint64_t authorizer); // TODO/QUESTION: Is this needed? transaction_trace_ptr push_action( const account_name& code, diff --git a/libraries/testing/tester.cpp b/libraries/testing/tester.cpp index 48430d55042..e9073a22a0e 100644 --- a/libraries/testing/tester.cpp +++ b/libraries/testing/tester.cpp @@ -350,12 +350,14 @@ namespace eosio { namespace testing { transaction_trace_ptr base_tester::push_transaction( packed_transaction& trx, fc::time_point deadline, - uint32_t billed_cpu_time_us + uint32_t billed_cpu_time_us, + uint64_t billed_ram_bytes ) { try { if( !control->pending_block_state() ) _start_block(control->head_block_time() + fc::microseconds(config::block_interval_us)); - auto r = control->push_transaction( std::make_shared(std::make_shared(trx)), deadline, billed_cpu_time_us ); + auto r = control->push_transaction( std::make_shared(std::make_shared(trx)), deadline, + { billed_cpu_time_us, billed_ram_bytes } ); if( r->except_ptr ) std::rethrow_exception( r->except_ptr ); if( r->except ) throw *r->except; return r; @@ -363,7 +365,8 @@ namespace eosio { namespace testing { transaction_trace_ptr base_tester::push_transaction( signed_transaction& trx, fc::time_point deadline, - uint32_t billed_cpu_time_us + uint32_t billed_cpu_time_us, + uint64_t billed_ram_bytes ) { try { if( !control->pending_block_state() ) @@ -374,7 +377,8 @@ namespace eosio { namespace testing { c = packed_transaction::zlib; } - auto r = control->push_transaction( std::make_shared(trx,c), deadline, billed_cpu_time_us ); + auto r = control->push_transaction( std::make_shared(trx,c), deadline, + { billed_cpu_time_us, billed_ram_bytes } ); if( r->except_ptr ) std::rethrow_exception( r->except_ptr ); if( r->except) throw *r->except; return r;