Merge branch 'master' into gh-4133-contract_code_endpoint

apps/aecore/include/aec_block_insertion.hrl

@@ -10,6 +10,7 @@
 -record(node, { header :: aec_headers:header()
               , hash   :: binary()
               , type   :: key | micro
+              , txs    :: undefined | list(aetx_sign:signed_tx())
               }).
 
 %% Metadata about the given fork

apps/aecore/src/aec_chain_state.erl

@@ -78,13 +78,15 @@
 -export([ calculate_state_for_new_keyblock/4
         , find_common_ancestor/2
         , get_key_block_hash_at_height/1
+        , key_block_hashes_at_height/1
         , get_n_key_headers_backward_from/2
         , hash_is_connected_to_genesis/1
         , hash_is_in_main_chain/1
         , hash_is_in_main_chain/2
         , insert_block/1
         , insert_block/2
         , insert_block_conductor/2
+        , repair_block_state/1
         , gossip_allowed_height_from_top/0
         , proof_of_fraud_report_delay/0
         , get_fork_result/2
@@ -142,6 +144,15 @@
 get_key_block_hash_at_height(Height) when is_integer(Height), Height >= 0 ->
     get_key_block_hash_at_height(Height, new_state_from_persistence()).
 
+-spec key_block_hashes_at_height(aec_blocks:height()) -> [binary()].
+key_block_hashes_at_height(Height) when is_integer(Height), Height >= 0 ->
+    aec_db:ensure_transaction(
+        fun() ->
+                [Hash ||
+                 {Hash, _} <-
+                 aec_db:find_key_headers_and_hash_at_height(Height)]
+        end).
+
 -spec get_n_key_headers_backward_from(aec_headers:header(), non_neg_integer()) ->
                                              {ok, [aec_headers:header()]} | 'error'.
 get_n_key_headers_backward_from(Header, N) ->
@@ -178,6 +189,11 @@ insert_block_strip_res({ok, _, _, Events}) -> {ok, Events};
 insert_block_strip_res({pof, _, _, PoF, Events}) -> {pof, PoF, Events};
 insert_block_strip_res(Other) -> Other.
 
+repair_block_state(Block) ->
+    {Time, Res} = timer:tc(fun() -> internal_repair_block_state(Block) end),
+    lager:info("Repair block state (~p us): ~p", [Time, Res]),
+    Res.
+
 do_insert_block(Block, Origin) ->
     aec_blocks:assert_block(Block),
     Node = wrap_block(Block),
@@ -434,7 +450,18 @@ node_is_genesis(Node) ->
 
 wrap_block(Block) ->
     Header = aec_blocks:to_header(Block),
-    wrap_header(Header).
+    {ok, Hash} = aec_headers:hash_header(Header),
+    Type = aec_headers:type(Header),
+    Txs = block_txs(Type, Block),
+    #node{ header = Header
+         , hash = Hash
+         , type = Type
+         , txs = Txs }.
+
+block_txs(micro, Block) ->
+    aec_blocks:txs(Block);
+block_txs(key, _) ->
+    [].
 
 fake_key_node(PrevNode, Height, Miner, Beneficiary, Protocol) ->
     PrevKeyHash = case node_type(PrevNode) of
@@ -534,10 +561,13 @@ internal_insert(Node, Block, Origin) ->
                 false ->
                     internal_insert_normal(Node, Block, Origin)
             end;
-        {ok, Node} ->
-            {error, already_in_db};
-        {ok, Old} ->
-            {error, {same_key_different_content, Node, Old}}
+        {ok, Found} ->
+            case {node_header(Node), node_header(Found)} of
+                {Same, Same} ->
+                    {error, already_in_db};
+                {New, Old} ->
+                    {error, {same_key_different_content, New, Old}}
+            end
     end.
 
 internal_insert_genesis(Node, Block) ->
@@ -590,6 +620,32 @@ internal_insert_normal(Node, Block, Origin) ->
             end
     end.
 
+internal_repair_block_state(Block) ->
+    Fun = fun() ->
+                  internal_repair_block_state_(Block)
+          end,
+    aec_block_insertion:start_state_transition(Fun).
+
+internal_repair_block_state_(Block) ->
+    Node = wrap_block(Block),
+    BlockHash = node_hash(Node),
+    case db_get_top_block_node() of
+        undefined ->
+            {error, no_top_block};
+        TopNode ->
+            case hash_is_in_main_chain(BlockHash, node_hash(TopNode)) of
+                true ->
+                    State = build_repair_state(Node, TopNode),
+                    repair_state_tree(Node, State);
+                false ->
+                    {error, not_in_main_chain}
+            end
+    end.
+
+build_repair_state(_Node, TopNode) ->
+    #{ top_block_node => TopNode
+     }.
+
 maybe_cache_new_top({ok, true, _PrevKeyHdr, _Events}, Node) ->
     cache_new_top(Node);
 maybe_cache_new_top({pof, true, _PrevKeyHdr, _PoF, _Events}, Node) ->
@@ -674,6 +730,10 @@ update_state_tree(Node, State, Ctx) ->
     OldTopNode = get_top_block_node(State),
     handle_top_block_change(OldTopNode, NewTopDifficulty, Node, Events, State3).
 
+repair_state_tree(Node, State) ->
+    {ok, TreesIn, ForkInfoIn} = get_state_trees_in(Node, true),
+    apply_and_repair_trees(Node, TreesIn, ForkInfoIn, State).
+
 update_state_tree(Node, TreesIn, ForkInfo, State) ->
     case db_find_state(node_hash(Node), true) of
         {ok, FoundTrees, FoundForkInfo} ->
@@ -736,6 +796,16 @@ get_state_trees_in(Node, PrevNode, DirtyBackend) ->
         error -> error
     end.
 
+%% This is modeled after apply_and_store_state_trees/4, but we don't want
+%% to update pof, not risk invalidating the cache, and only touch up the trees
+%% value of the block state.
+%%
+apply_and_repair_trees(Node, TreesIn, ForkInfoIn, State) ->
+    {Trees, _Fees, _Events} = apply_node_transactions(Node, TreesIn, ForkInfoIn, State),
+    assert_state_hash_valid(Trees, Node),
+    aec_db:update_trees_and_block_state(node_hash(Node), Trees),
+    ok.
+
 apply_and_store_state_trees(Node, TreesIn, ForkInfoIn, State) ->
     {Trees, Fees, Events} = apply_node_transactions(Node, TreesIn, ForkInfoIn, State),
     assert_state_hash_valid(Trees, Node),
@@ -964,7 +1034,10 @@ calculate_gas_fee(Calls) ->
 
 
 apply_micro_block_transactions(Node, FeesIn, Trees) ->
-    Txs = db_get_txs(node_hash(Node)),
+    Txs = case Node#node.txs of
+              undefined -> db_get_txs(node_hash(Node));
+              Txs1      -> Txs1
+          end,
     KeyHeader = db_get_header(node_prev_key_hash(Node)),
     Env = aetx_env:tx_env_from_key_header(KeyHeader, node_prev_key_hash(Node),
                                           node_time(Node), node_prev_hash(Node)),

apps/aecore/src/aec_conductor.erl

@@ -1390,20 +1390,22 @@ handle_successfully_added_block(Block, Hash, false, _, Events, State, Origin) ->
     maybe_publish_tx_events(Events, Hash, Origin),
     maybe_publish_block(Origin, Block),
     State1 = maybe_consensus_change(State, Block),
+    [ maybe_garbage_collect(Block, Hash, false) || aec_blocks:type(Block) == key ],
     {ok, State1};
 handle_successfully_added_block(Block, Hash, true, PrevKeyHeader, Events, State, Origin) ->
     maybe_publish_tx_events(Events, Hash, Origin),
     maybe_publish_block(Origin, Block),
     State1 = maybe_consensus_change(State, Block),
-    ConsensusModule = consensus_module(State),
+    ConsensusModule = consensus_module(State1),
     case preempt_on_new_top(State1, Block, Hash, Origin) of
         {micro_changed, State2 = #state{ consensus = Cons }} ->
             {ok, setup_loop(State2, false, Cons#consensus.leader, Origin)};
         {changed, BlockType, NewTopBlock, State2} ->
             (BlockType == key) andalso
                 aec_metrics:try_update(
                   [ae,epoch,aecore,blocks,key,info], info_value(NewTopBlock)),
-            [ maybe_garbage_collect(NewTopBlock) || BlockType == key ],
+            [ maybe_garbage_collect(NewTopBlock, Hash, true)
+              || BlockType == key ],
             IsLeader = is_leader(NewTopBlock, PrevKeyHeader, ConsensusModule),
             case IsLeader of
                 true ->
@@ -1506,15 +1508,15 @@ get_pending_key_block(TopHash, State) ->
 %%
 %% To avoid starting of the GC process just for EUNIT
 -ifdef(EUNIT).
-maybe_garbage_collect(_) -> nop.
+maybe_garbage_collect(_, _, _) -> nop.
 -else.
 
 %% This should be called when there are no processes modifying the block state
 %% (e.g. aec_conductor on specific places)
-maybe_garbage_collect(Block) ->
+maybe_garbage_collect(Block, Hash, TopChange) ->
     T0 = erlang:system_time(microsecond),
     Header = aec_blocks:to_header(Block),
-    Res = aec_db_gc:maybe_garbage_collect(Header),
+    Res = aec_db_gc:maybe_garbage_collect(Header, Hash, TopChange),
     T1 = erlang:system_time(microsecond),
     lager:debug("Result -> ~p (time: ~p us)", [Res, T1-T0]),
     Res.

apps/aecore/src/aec_db.erl

@@ -28,6 +28,7 @@
          write_block/1,
          write_block/2,
          write_block_state/6,
+         update_trees_and_block_state/2,
          write_discovered_pof/2,
          write_genesis_hash/1,
          write_top_block_node/2,
@@ -92,6 +93,7 @@
 -export([ tree_table_name/1
         , new_tree_context/2
         , lookup_tree_node/2
+        , lookup_tree_node/3
         , enter_tree_node/3
         , node_is_in_primary/2
         ]).
@@ -922,6 +924,17 @@ write_block_state(Hash, Trees, AccDifficulty, ForkId, Fees, Fraud) ->
            write(BlockState)
        end).
 
+update_trees_and_block_state(Hash, Trees) ->
+    ensure_transaction(
+      fun() ->
+              [#aec_block_state{} = St] =
+                  read(aec_block_state, Hash),
+              Trees1 = aec_trees:serialize_for_db(
+                         aec_trees:commit_to_db(Trees)),
+              BlockState = St#aec_block_state{ value = Trees1 },
+              write(BlockState)
+      end).
+
 -spec secondary_state_tab(tree_name()) -> table_name().
 secondary_state_tab(Tree) ->
     Tab = tree_table_name(Tree),
@@ -969,16 +982,16 @@ read_last_gc_switch(Default) ->
     lager:debug("<-- last GC Height: ~p", [R]),
     R.
 
-write_last_gc_scan(Height) ->
-    lager:debug("Last complete GC scan: ~p", [Height]),
-    ?t(write(#aec_chain_state{key = last_gc_scan, value = Height})).
+write_last_gc_scan(Value) ->
+    lager:debug("Last complete GC scan: ~p", [Value]),
+    ?t(write(#aec_chain_state{key = last_gc_scan, value = Value})).
 
 read_last_gc_scan() ->
     R = ?t(case read(aec_chain_state, last_gc_scan) of
                [] ->
-                   0;
-               [#aec_chain_state{value = Height}] ->
-                   Height
+                   error;
+               [#aec_chain_state{value = Value}] ->
+                   {ok, Value}
            end),
     lager:debug("<-- last height of complete GC scan: ~p", [R]),
     R.
@@ -1222,15 +1235,18 @@ new_tree_context(Mode, Tree) when Mode == dirty;
 
 %% Behaves like gb_trees:lookup(Key, Tree).
 -spec lookup_tree_node(hash(), tree_context()) -> none | {value, value()}.
-lookup_tree_node(Hash, #tree{table = T, mode = ActivityType}) ->
+lookup_tree_node(Hash, Ctxt) ->
+    lookup_tree_node_legacy(Hash, Ctxt).
+
+lookup_tree_node_legacy(Hash, #tree{table = T, mode = ActivityType}) ->
     ensure_activity(ActivityType,
                     fun() ->
                             lookup_tree_node_(Hash, T, T)
                     end);
-lookup_tree_node(Hash, #tree_gc{ primary   = Prim
-                               , secondary = Sec
-                               , record    = Rec
-                               , mode      = ActivityType }) ->
+lookup_tree_node_legacy(Hash, #tree_gc{ primary   = Prim
+                                      , secondary = Sec
+                                      , record    = Rec
+                                      , mode      = ActivityType }) ->
     ensure_activity(ActivityType,
                     fun() ->
                             case lookup_tree_node_(Hash, Prim, Rec) of
@@ -1247,6 +1263,38 @@ lookup_tree_node(Hash, #tree_gc{ primary   = Prim
                             end
                     end).
 
+%% This version takes a map (context) as 3rd argument and returns the result as part
+%% of that map, together with information on whether the node was promoted.
+lookup_tree_node(Hash, #tree{table = T, mode = ActivityType}, Map) when is_map(Map) ->
+    ensure_activity(ActivityType,
+                    fun() ->
+                            Map#{result => lookup_tree_node_(Hash, T, T),
+                                 source => T}
+                    end);
+lookup_tree_node(Hash, #tree_gc{ primary   = Prim
+                               , secondary = Sec
+                               , record    = Rec
+                               , mode      = ActivityType }, Map) when is_map(Map) ->
+    ensure_activity(ActivityType,
+                    fun() ->
+                            case lookup_tree_node_(Hash, Prim, Rec) of
+                                {value, _} = Res ->
+                                    Map#{result => Res, source => Prim, promoted => false};
+                                none ->
+                                    case lookup_tree_node_(Hash, Sec, Rec) of
+                                        none ->
+                                            Map#{result => none,
+                                                 source => Sec,
+                                                 promoted => false};
+                                        {value, V} = Res ->
+                                            promote_tree_node(Hash, V, Prim, Rec),
+                                            Map#{result => Res,
+                                                 source => Prim,
+                                                 promoted => true}
+                                    end
+                            end
+                    end).
+
 lookup_tree_node_(Hash, T, Rec) ->
     case read(T, Hash) of
         [Obj] -> {value, get_tree_value(Rec, Obj)};

apps/aecore/src/aec_db_backends.erl

@@ -30,6 +30,7 @@
 %% Callbacks for aeu_mp_trees_db
 -export([ mpt_db_drop_cache/1
         , mpt_db_get/2
+        , mpt_db_get/3
         , mpt_db_put/3
         ]).
 
@@ -125,6 +126,11 @@ mpt_db_get(Key, {gb_trees, Tree}) ->
 mpt_db_get(Key, Handle) ->
     aec_db:lookup_tree_node(Key, Handle).
 
+mpt_db_get(Key, {gb_trees, Tree}, Map) when is_map(Map) ->
+    Map#{result => gb_trees:lookup(Key, Tree)};
+mpt_db_get(Key, Handle, Map) when is_map(Map) ->
+    aec_db:lookup_tree_node(Key, Handle, Map).
+
 mpt_db_put(Key, Val, {gb_trees, Tree}) ->
     {gb_trees, gb_trees:enter(Key, Val, Tree)};
 mpt_db_put(Key, Val, Handle) ->