HC: Lazy leader (#4183)

HC Lazy leader implementation

apps/aecontract/test/aecontract_staking_contract_SUITE.erl

@@ -559,8 +559,7 @@ single_validator_gets_elected_every_time(_Config) ->
         fun(Height, TreesAccum) ->
             %% Alice is expected to be next
             TxEnvPrev = aetx_env:set_height(TxEnv, Height - 1),
-            {ok, _ , {tuple, {{address, Alice}, _}}} = elect_next_(Alice, TxEnvPrev,
-                                                                   TreesAccum),
+            {ok, _ , {tuple, {{address, Alice}, _}}} = elect_next_(Alice, TxEnvPrev, TreesAccum),
             TxEnv1 = aetx_env:set_height(TxEnv, Height),
             {ok, TreesAccum1, {tuple, {}}} = elect_(?OWNER_PUBKEY, TxEnv1, TreesAccum),
             {ok, _, {address, Alice}} = leader_(Alice, TxEnv1, TreesAccum1),

apps/aecore/src/aec_block_insertion.erl

@@ -80,7 +80,8 @@ node_prev_key_hash(#node{header = H}) -> aec_headers:prev_key_hash(H).
 node_height(#node{header = H}) -> aec_headers:height(H).
 
 node_difficulty(#node{type = micro}) -> 0;
-node_difficulty(#node{header = H}) -> aec_headers:difficulty(H).
+node_difficulty(#node{header = H}) ->
+    aec_headers:difficulty(H).
 
 node_target(#node{header = H}) -> aec_headers:target(H).
 

apps/aecore/src/aec_chain_state.erl

@@ -286,7 +286,8 @@ calculate_state_for_new_keyblock(PrevHash, Miner, Beneficiary, Protocol) ->
                 case get_state_trees_in(Node, true) of
                     error -> error;
                     {ok, TreesIn, ForkInfoIn} ->
-                        {Trees,_Fees,_Events} = apply_node_transactions(Node, TreesIn,
+                        {Trees,_Fees,_Events} = apply_node_transactions(Node,
+                                                                        PrevNode, TreesIn,
                                                                         ForkInfoIn, State),
                         {ok, Trees}
                 end
@@ -635,8 +636,9 @@ internal_repair_block_state_(Block) ->
         TopNode ->
             case hash_is_in_main_chain(BlockHash, node_hash(TopNode)) of
                 true ->
+                    PrevNode = db_get_node(node_prev_hash(Node)),
                     State = build_repair_state(Node, TopNode),
-                    repair_state_tree(Node, State);
+                    repair_state_tree(Node, PrevNode, State);
                 false ->
                     {error, not_in_main_chain}
             end
@@ -730,14 +732,15 @@ update_state_tree(Node, State, Ctx) ->
     OldTopNode = get_top_block_node(State),
     handle_top_block_change(OldTopNode, NewTopDifficulty, Node, Events, State3).
 
-repair_state_tree(Node, State) ->
+repair_state_tree(Node, PrevNode, State) ->
     {ok, TreesIn, ForkInfoIn} = get_state_trees_in(Node, true),
-    apply_and_repair_trees(Node, TreesIn, ForkInfoIn, State).
+    apply_and_repair_trees(Node, PrevNode, TreesIn, ForkInfoIn, State).
 
 update_state_tree(Node, TreesIn, ForkInfo, State) ->
+    PrevNode = db_get_node(node_prev_hash(Node)),
     case db_find_state(node_hash(Node), true) of
         {ok, FoundTrees, FoundForkInfo} ->
-            {Trees, _Fees, Events} = apply_node_transactions(Node, TreesIn,
+            {Trees, _Fees, Events} = apply_node_transactions(Node, PrevNode, TreesIn,
                                                              ForkInfo, State),
             case aec_trees:hash(Trees) =:= aec_trees:hash(FoundTrees) of
                 true -> %% race condition, we've already inserted this block
@@ -752,7 +755,8 @@ update_state_tree(Node, TreesIn, ForkInfo, State) ->
                     error({found_already_calculated_state, node_hash(Node)})
             end;
         error ->
-            {DifficultyOut, Events} = apply_and_store_state_trees(Node, TreesIn,
+            {DifficultyOut, Events} = apply_and_store_state_trees(Node,
+                                                                  PrevNode, TreesIn,
                                                                   ForkInfo, State),
             State1 = set_top_block_node(Node, State),
             {State1, DifficultyOut, Events}
@@ -800,14 +804,14 @@ get_state_trees_in(Node, PrevNode, DirtyBackend) ->
 %% to update pof, not risk invalidating the cache, and only touch up the trees
 %% value of the block state.
 %%
-apply_and_repair_trees(Node, TreesIn, ForkInfoIn, State) ->
-    {Trees, _Fees, _Events} = apply_node_transactions(Node, TreesIn, ForkInfoIn, State),
+apply_and_repair_trees(Node, PrevNode, TreesIn, ForkInfoIn, State) ->
+    {Trees, _Fees, _Events} = apply_node_transactions(Node, PrevNode, TreesIn, ForkInfoIn, State),
     assert_state_hash_valid(Trees, Node),
     aec_db:update_trees_and_block_state(node_hash(Node), Trees),
     ok.
 
-apply_and_store_state_trees(Node, TreesIn, ForkInfoIn, State) ->
-    {Trees, Fees, Events} = apply_node_transactions(Node, TreesIn, ForkInfoIn, State),
+apply_and_store_state_trees(Node, PrevNode, TreesIn, ForkInfoIn, State) ->
+    {Trees, Fees, Events} = apply_node_transactions(Node, PrevNode, TreesIn, ForkInfoIn, State),
     assert_state_hash_valid(Trees, Node),
     DifficultyOut = ForkInfoIn#fork_info.difficulty + node_difficulty(Node),
     Fraud = update_fraud_info(ForkInfoIn, Node, State),
@@ -902,15 +906,16 @@ validate_generation_leader(Node, Trees, Env) ->
     case node_is_key_block(Node) of
         true  ->
             ConsensusModule = aec_block_insertion:node_consensus(Node),
-            case ConsensusModule:is_leader_valid(Node, Trees, Env) of
+            PrevKeyNode = db_get_node(node_prev_key_hash(Node)),
+            case ConsensusModule:is_leader_valid(Node, Trees, Env, PrevKeyNode) of
                 true -> ok;
                 false -> {error, invalid_leader}
             end;
         false ->
             ok
     end.
 
-apply_node_transactions(Node, Trees, ForkInfo, State) ->
+apply_node_transactions(Node, PrevNode, Trees, ForkInfo, State) ->
     Consensus = aec_block_insertion:node_consensus(Node),
     case node_is_micro_block(Node) of
         true ->
@@ -932,7 +937,7 @@ apply_node_transactions(Node, Trees, ForkInfo, State) ->
             Trees2 = if Consensus =:= PrevConsensus -> Trees1;
                         true -> Consensus:state_pre_transform_key_node_consensus_switch(Node, Trees1)
                      end,
-            Trees3 = Consensus:state_pre_transform_key_node(Node, Trees2),
+            Trees3 = Consensus:state_pre_transform_key_node(Node, PrevNode, Trees2),
             %% leader generation happens after pre_transformations
             case validate_generation_leader(Node, Trees3, Env) of
                 ok ->

apps/aecore/src/aec_conductor.erl

@@ -534,7 +534,33 @@ get_beneficiary(Consensus) ->
     Consensus:beneficiary().
 
 get_next_beneficiary(Consensus) ->
-    Consensus:next_beneficiary().
+    TopHeader = aec_chain:top_header(),
+    get_next_beneficiary(Consensus, TopHeader).
+
+get_next_beneficiary(Consensus, TopHeader) ->
+    case Consensus:next_beneficiary() of
+        {ok, _L} = OK -> OK;
+        {error, not_in_cache} = Err ->
+            %%timer:sleep(1000), %% TODO: make this configurable
+            Err;
+        {error, not_leader} = NotLeader ->
+            case Consensus:allow_lazy_leader() of
+                {true, LazyLeaderTimeDelta} ->
+                    LastBlockTime = aec_headers:time_in_msecs(TopHeader),
+                    Now = aeu_time:now_in_msecs(),
+                    TimeDelta = Now - LastBlockTime,
+                    case TimeDelta > LazyLeaderTimeDelta of
+                        true ->
+                            case Consensus:pick_lazy_leader() of
+                                error -> NotLeader;
+                                {ok, _L} = OK -> OK
+                            end;
+                        false ->
+                            NotLeader
+                    end;
+                false -> NotLeader
+            end
+    end.
 
 get_beneficiary() ->
     TopHeader = aec_chain:top_header(),
@@ -544,7 +570,7 @@ get_beneficiary() ->
 get_next_beneficiary() ->
     TopHeader = aec_chain:top_header(),
     Consensus = aec_consensus:get_consensus_module_at_height(aec_headers:height(TopHeader) + 1),
-    get_next_beneficiary(Consensus).
+    get_next_beneficiary(Consensus, TopHeader).
 
 note_rollback(Info) ->
     gen_server:call(?SERVER, {note_rollback, Info}).

apps/aecore/src/aec_consensus.erl

@@ -167,7 +167,7 @@
 %% Performs initial state transformation when the previous block used a different consensus algorithm
 -callback state_pre_transform_key_node_consensus_switch(#node{}, aec_trees:trees()) -> aec_trees:trees() | no_return().
 %% State pre transformations on every keyblock
--callback state_pre_transform_key_node(#node{}, aec_trees:trees()) -> aec_trees:trees() | no_return().
+-callback state_pre_transform_key_node(#node{}, #node{}, aec_trees:trees()) -> aec_trees:trees() | no_return().
 %% State pre transformations on every microblock
 -callback state_pre_transform_micro_node(#node{}, aec_trees:trees()) -> aec_trees:trees() | no_return().
 
@@ -186,10 +186,12 @@
 %% rewards and signing
 -callback beneficiary() -> {ok, binary() | fun(() -> binary())} | {error, atom()}.
 -callback next_beneficiary() -> {ok, binary() | fun(() -> binary())} | {error, atom()}.
+-callback allow_lazy_leader() -> false | {true, integer()}.
+-callback pick_lazy_leader() -> error | {ok, aec_keys:pubkey()}.
 -callback get_sign_module() -> sign_module().
 -callback get_type() -> pow | pos.
 -callback get_block_producer_configs() -> list().
--callback is_leader_valid(#node{}, aec_trees:trees(), aetx_env:env()) -> boolean().
+-callback is_leader_valid(#node{}, aec_trees:trees(), aetx_env:env(), #node{}) -> boolean().
 
 %% -------------------------------------------------------------------
 %% Block sealing

apps/aecore/src/aec_consensus_bitcoin_ng.erl

@@ -31,7 +31,7 @@
         , dirty_validate_micro_node_with_ctx/3
         %% State transition
         , state_pre_transform_key_node_consensus_switch/2
-        , state_pre_transform_key_node/2
+        , state_pre_transform_key_node/3
         , state_pre_transform_micro_node/2
         %% Block rewards
         , state_grant_reward/4
@@ -56,10 +56,12 @@
         %% rewards and signing
         , beneficiary/0
         , next_beneficiary/0
+        , allow_lazy_leader/0
+        , pick_lazy_leader/0
         , get_sign_module/0
         , get_type/0
         , get_block_producer_configs/0
-        , is_leader_valid/3
+        , is_leader_valid/4
         ]).
 
 -export([ get_whitelist/0
@@ -417,7 +419,7 @@ time_diff_greater_than_minimal(Node, PrevNode) ->
 %% -------------------------------------------------------------------
 %% Custom state transitions
 state_pre_transform_key_node_consensus_switch(_Node, Trees) -> Trees.
-state_pre_transform_key_node(_Node, Trees) -> Trees.
+state_pre_transform_key_node(_Node, _PrevNode, Trees) -> Trees.
 state_pre_transform_micro_node(_Node, Trees) -> Trees.
 
 %% -------------------------------------------------------------------
@@ -523,13 +525,16 @@ beneficiary() ->
 
 next_beneficiary() -> beneficiary().
 
+allow_lazy_leader() -> false.
+pick_lazy_leader() -> error.
+
 get_sign_module() -> aec_keys.
 
 get_type() -> pow.
 
 get_block_producer_configs() -> aec_mining:get_miner_configs().
 
-is_leader_valid(_Node, _Trees, _TxEnv) ->
+is_leader_valid(_Node, _Trees, _TxEnv, _PrevNode) ->
     true.
 
 load_whitelist() ->

apps/aecore/src/aec_consensus_common_tests.erl

@@ -37,7 +37,7 @@
         , dirty_validate_micro_node_with_ctx/3
         %% State transition
         , state_pre_transform_key_node_consensus_switch/2
-        , state_pre_transform_key_node/2
+        , state_pre_transform_key_node/3
         , state_pre_transform_micro_node/2
         %% Block rewards
         , state_grant_reward/4
@@ -62,10 +62,12 @@
         %% rewards and signing
         , beneficiary/0
         , next_beneficiary/0
+        , allow_lazy_leader/0
+        , pick_lazy_leader/0
         , get_sign_module/0
         , get_type/0
         , get_block_producer_configs/0
-        , is_leader_valid/3
+        , is_leader_valid/4
         ]).
 
 -include_lib("aecontract/include/hard_forks.hrl").
@@ -112,11 +114,11 @@ client_request({mine_blocks, NumBlocksToMine, Type}) ->
             {ok, [client_request(emit_mb) || _ <- lists:seq(1, NumBlocksToMine)]}
     end;
 client_request(mine_micro_block_emptying_mempool_or_fail) ->
-    KB = client_request(emit_kb),
+    MaybeKB = ensure_leader(),
     MB = client_request(emit_mb),
     %% If instant mining is enabled then we can't have microforks :)
     {ok, []} = aec_tx_pool:peek(infinity),
-    {ok, [KB, MB]};
+    {ok, MaybeKB ++ [MB]};
 client_request({mine_blocks_until_txs_on_chain, TxHashes, Max}) ->
     mine_blocks_until_txs_on_chain(TxHashes, Max, []).
 
@@ -164,7 +166,7 @@ dirty_validate_micro_node_with_ctx(_Node, _Block, _Ctx) -> ok.
 %% -------------------------------------------------------------------
 %% Custom state transitions
 state_pre_transform_key_node_consensus_switch(_Node, Trees) -> Trees.
-state_pre_transform_key_node(_Node, Trees) -> Trees.
+state_pre_transform_key_node(_Node, _PrevNode, Trees) -> Trees.
 state_pre_transform_micro_node(_Node, Trees) -> Trees.
 
 %% -------------------------------------------------------------------
@@ -229,13 +231,27 @@ beneficiary() -> aec_consensus_bitcoin_ng:beneficiary().
 
 next_beneficiary() -> aec_consensus_bitcoin_ng:next_beneficiary().
 
+allow_lazy_leader() -> false.
+
+pick_lazy_leader() -> error.
+
 get_sign_module() -> aec_consensus_bitcoin_ng:get_sign_module().
 
 get_type() -> aec_consensus_bitcoin_ng:get_type().
 
 get_block_producer_configs() -> aec_consensus_bitcoin_ng:get_block_producer_configs().
 
-is_leader_valid(_Node, _Trees, _TxEnv) ->
+is_leader_valid(_Node, _Trees, _TxEnv, _PrevNode) ->
     true.
 
+ensure_leader() ->
+    case aec_conductor:is_leader() of
+        false ->
+            KB = client_request(emit_kb),
+            [KB];
+        true ->
+            []
+    end.
+
+
 -endif.