Merge 11e7051 into c137028

helium · Sep 4, 2018 · 9626932 · 9626932
2 parents c137028 + 11e7051
commit 9626932
Show file tree

Hide file tree

Showing 3 changed files with 89 additions and 33 deletions.
diff --git a/src/hbbft_acs.erl b/src/hbbft_acs.erl
@@ -114,7 +114,7 @@ handle_msg(Data = #acs_data{n=N, f=F}, J, {{bba, I}, BBAMsg}) ->
     case hbbft_bba:handle_msg(BBA#bba_state.bba_data, J, BBAMsg) of
         {NewBBA, {send, ToSend}} ->
             {store_bba_state(Data, I, NewBBA), {send, hbbft_utils:wrap({bba, I}, ToSend)}};
-        {NewBBA, {result, B}} ->
+        {NewBBA, {result_and_send, B, {send, ToSend0}}} ->
             NewData = store_bba_state(store_bba_result(Data, I, B), I, NewBBA),
             %% Figure4, Bullet3
             %% upon delivery of value 1 from at least N − f instances of BA , provide input 0 to each instance of BA that has not yet been provided input.
@@ -135,7 +135,7 @@ handle_msg(Data = #acs_data{n=N, f=F}, J, {{bba, I}, BBAMsg}) ->
                                                                   true ->
                                                                       Acc
                                                               end
-                                                      end, {NewData, []}, lists:seq(0, N - 1)),
+                                                      end, {NewData, hbbft_utils:wrap({bba, I}, ToSend0)}, lists:seq(0, N - 1)),
                     %% each BBA is independant, so the total ordering here is unimportant
                     {NextData#acs_data{done=true}, {send, lists:flatten(Replies)}};
                 false ->

diff --git a/src/hbbft_bba.erl b/src/hbbft_bba.erl
@@ -11,9 +11,11 @@
           output :: undefined | 0 | 1,
           f :: non_neg_integer(),
           n :: pos_integer(),
-          witness = maps:new() :: #{non_neg_integer() => 0 | 1},
-          aux_witness = maps:new() :: #{non_neg_integer() => 0 | 1},
+          %% XXX I think some of these witnesses can hold more values...
+          bval_witness = maps:new() :: #{non_neg_integer() => 0 | 1 | 2 | 3},
+          aux_witness = maps:new() :: #{non_neg_integer() => 0 | 1 | 2 | 3},
           conf_witness = maps:new() :: #{non_neg_integer() => 0 | 1},
+          terminate_witness = maps:new() :: #{non_neg_integer() => 0 | 1},
           aux_sent = false :: boolean(),
           conf_sent = false :: boolean(),
           coin_sent = false :: boolean(),
@@ -29,9 +31,10 @@
           output :: undefined | 0 | 1,
           f :: non_neg_integer(),
           n :: pos_integer(),
-          witness = maps:new() :: #{non_neg_integer() => 0 | 1},
+          bval_witness = maps:new() :: #{non_neg_integer() => 0 | 1},
           aux_witness = maps:new() :: #{non_neg_integer() => 0 | 1},
           conf_witness = maps:new() :: #{non_neg_integer() => 0 | 1},
+          terminate_witness = maps:new() :: #{non_neg_integer() => 0 | 1},
           aux_sent = false :: boolean(),
           conf_sent = false :: boolean(),
           broadcasted = 2#0 :: 0 | 1 | 2 | 3,
@@ -44,10 +47,11 @@
 -type bval_msg() :: {bval, non_neg_integer(), 0 | 1}.
 -type aux_msg() :: {aux, non_neg_integer(), 0 | 1}.
 -type conf_msg() :: {conf, non_neg_integer(), 0 | 1}.
+-type term_msg() :: {term, 0 | 1}.
 -type coin_msg() :: {{coin, non_neg_integer()}, hbbft_cc:share_msg()}.
 -type msgs() :: bval_msg() | aux_msg() | conf_msg() | coin_msg().
 
--export_type([bba_data/0, bba_serialized_data/0, bval_msg/0, aux_msg/0, coin_msg/0, msgs/0, conf_msg/0]).
+-export_type([bba_data/0, bba_serialized_data/0, bval_msg/0, aux_msg/0, coin_msg/0, term_msg/0, msgs/0, conf_msg/0]).
 
 -spec status(bba_data()) -> map().
 status(BBAData) ->
@@ -60,7 +64,7 @@ status(BBAData) ->
       output => BBAData#bba_data.output,
       conf_witness => BBAData#bba_data.conf_witness,
       aux_witness => BBAData#bba_data.aux_witness,
-      witness => BBAData#bba_data.witness,
+      bval_witness => BBAData#bba_data.bval_witness,
       bin_values => BBAData#bba_data.bin_values,
       broadcasted => BBAData#bba_data.broadcasted
      }.
@@ -86,7 +90,7 @@ input(Data = #bba_data{state=done}, _BInput) ->
                  conf_msg()) -> {bba_data(), ok} |
                                 {bba_data(), defer} |
                                 {bba_data(), {send, [hbbft_utils:multicast(bval_msg() | aux_msg() | conf_msg() | coin_msg())]}} |
-                                {bba_data(), {result, 0 | 1}}.
+                                {bba_data(), {result_and_send, 0 | 1, {send, [hbbft_utils:multicast(term_msg())]}}}.
 handle_msg(Data = #bba_data{state=done}, _J, _BInput) ->
     {Data, ok};
 handle_msg(Data = #bba_data{round=R}, J, {bval, R, V}) ->
@@ -120,6 +124,18 @@ handle_msg(Data = #bba_data{round=R, coin=Coin}, J, {{coin, R}, CMsg}) when Coin
 handle_msg(Data = #bba_data{round=R, coin=Coin}, J, Msg = {{coin, R}, _CMsg}) when Coin == undefined ->
     %% we have not called input() yet this round, so we need to manually init the coin
     handle_msg(Data#bba_data{coin=maybe_init_coin(Data)}, J, Msg);
+handle_msg(Data, J, {term, B}) when B == 0; B == 1 ->
+    %% Another peer has output a value, we can't expect them to send us any more values so we
+    %% retain the value they sent us as the virtual value for any subsequent BVAL/AUX/CONF messages
+    NewData = add_terminate(J, B, Data),
+    %% if we have F+1 Terminate witnesses for a value, we can also terminate with that value
+    case maps:get({val, B}, NewData#bba_data.terminate_witness, 0) of
+        Count when Count >= Data#bba_data.f + 1 ->
+            {NewData#bba_data{state=done}, {result_and_send, B, {send, [{multicast, {term, B}}]}}};
+        _ ->
+            %% TODO we need to check bval/aux/conf message thresholds in light of the new TERM message
+            {NewData, ok}
+    end;
 handle_msg(Data, _J, _Msg) ->
     {Data, ok}.
 
@@ -128,25 +144,25 @@ handle_msg(Data, _J, _Msg) ->
 -spec bval(bba_data(), non_neg_integer(), 0 | 1) -> {bba_data(), {send, [hbbft_utils:multicast(aux_msg() | coin_msg())]}}.
 bval(Data=#bba_data{n=N, f=F}, Id, V) ->
     %% add to witnesses
-    Witness = add_witness(Id, V, Data#bba_data.witness),
+    Witness = add_witness(Id, V, Data#bba_data.bval_witness, true),
     WitnessCount = maps:get({val, V}, Witness, 0),
 
     {NewData, ToSend} = case WitnessCount >= F+1 andalso not has(V, Data#bba_data.broadcasted) of
                             true ->
                                 %% add to broadcasted
-                                NewData0 = Data#bba_data{witness=Witness,
+                                NewData0 = Data#bba_data{bval_witness=Witness,
                                                          broadcasted=add(V, Data#bba_data.broadcasted)},
                                 {NewData0, [{multicast, {bval, Data#bba_data.round, V}}]};
                             false ->
-                                {Data#bba_data{witness=Witness}, []}
+                                {Data#bba_data{bval_witness=Witness}, []}
                         end,
 
     %% - upon receiving BVALr (b) messages from 2 f + 1 nodes,
     %% bin_values_r := bin_valuesr ∪ {b}
     case WitnessCount >= 2*F+1 of
         true ->
             %% add to binvalues
-            NewData2 = Data#bba_data{witness=Witness,
+            NewData2 = Data#bba_data{bval_witness=Witness,
                                      bin_values=add(V, NewData#bba_data.bin_values)},
             {NewData3, ToSend2} = case NewData2#bba_data.aux_sent == false of
                                       true ->
@@ -185,7 +201,7 @@ bval(Data=#bba_data{n=N, f=F}, Id, V) ->
 
 -spec aux(bba_data(), non_neg_integer(), 0 | 1) -> {bba_data(), ok | {send, [hbbft_utils:multicast(conf_msg())]}}.
 aux(Data = #bba_data{n=N, f=F}, Id, V) ->
-    Witness = add_witness(Id, V, Data#bba_data.aux_witness),
+    Witness = add_witness(Id, V, Data#bba_data.aux_witness, true),
     NewData = Data#bba_data{aux_witness = Witness},
     case threshold(N, F, NewData, aux) of
         true->
@@ -234,7 +250,7 @@ serialize(#bba_data{state=State,
                     output=Output,
                     f=F,
                     n=N,
-                    witness=Witness,
+                    bval_witness=Witness,
                     aux_witness=AuxWitness,
                     conf_witness=ConfWitness,
                     aux_sent=AuxSent,
@@ -252,7 +268,7 @@ serialize(#bba_data{state=State,
                          output=Output,
                          f=F,
                          n=N,
-                         witness=Witness,
+                         bval_witness=Witness,
                          aux_witness=AuxWitness,
                          conf_witness=ConfWitness,
                          aux_sent=AuxSent,
@@ -268,7 +284,7 @@ deserialize(#bba_serialized_data{state=State,
                                  output=Output,
                                  f=F,
                                  n=N,
-                                 witness=Witness,
+                                 bval_witness=Witness,
                                  aux_witness=AuxWitness,
                                  conf_witness=ConfWitness,
                                  aux_sent=AuxSent,
@@ -287,7 +303,7 @@ deserialize(#bba_serialized_data{state=State,
               output=Output,
               f=F,
               n=N,
-              witness=Witness,
+              bval_witness=Witness,
               aux_witness=AuxWitness,
               conf_witness=ConfWitness,
               aux_sent=AuxSent,
@@ -301,36 +317,39 @@ deserialize(#bba_serialized_data{state=State,
 -spec threshold(pos_integer(), non_neg_integer(), bba_data(), aux | conf) -> boolean().
 threshold(N, F, Data, Msg) ->
     case Msg of
-        aux -> check(N, F, Data#bba_data.bin_values, Data#bba_data.aux_witness);
-        conf -> check(N, F, Data#bba_data.bin_values, Data#bba_data.conf_witness, fun subset/2)
+        aux -> check(N, F, Data#bba_data.bin_values, Data#bba_data.aux_witness, Data#bba_data.terminate_witness);
+        conf -> check(N, F, Data#bba_data.bin_values, Data#bba_data.conf_witness, Data#bba_data.terminate_witness, fun subset/2)
     end.
 
--spec check(pos_integer(), non_neg_integer(), 0 | 1 | 2 | 3, #{non_neg_integer() => 0 | 1}) -> boolean().
-check(N, F, ToCheck, Witness) ->
+-spec check(pos_integer(), non_neg_integer(), 0 | 1 | 2 | 3, #{non_neg_integer() => 0 | 1}, #{}) -> boolean().
+check(N, F, ToCheck, Witness, Terms) ->
     case ToCheck of
         0 ->
             false;
         1 ->
-            maps:get({val, 0}, Witness, 0) >= N - F;
+            maps:get({val, 0}, Witness, 0) + maps:get({val, 0}, Terms, 0) >= N - F;
         2 ->
-            maps:get({val, 1}, Witness, 0) >= N - F;
+            maps:get({val, 1}, Witness, 0) + maps:get({val, 1}, Terms, 0) >= N - F;
         3 ->
             %% both
             Both = maps:get({val, both}, Witness, 0),
-            Zeros = maps:get({val, 0}, Witness, 0),
-            Ones = maps:get({val, 1}, Witness, 0),
+            Zeros = maps:get({val, 0}, Witness, 0) + maps:get({val, 0}, Terms, 0),
+            Ones = maps:get({val, 1}, Witness, 0) + maps:get({val, 1}, Terms, 0),
             Both + (Zeros - Both) + (Ones - Both) >= N - F
     end.
 
--spec check(pos_integer(), non_neg_integer(), 0 | 1 | 2 | 3, #{non_neg_integer() => 0 | 1}, fun((0|1|2|3, 0|1|2|3) -> boolean())) -> boolean().
-check(N, F, ToCheck, Map, Fun) ->
-    maps:fold(fun(_, V, Acc) ->
+-spec check(pos_integer(), non_neg_integer(), 0 | 1 | 2 | 3, #{non_neg_integer() => 0 | 1}, #{}, fun((0|1|2|3, 0|1|2|3) -> boolean())) -> boolean().
+check(N, F, ToCheck, Map, Terms, Fun) ->
+    maps:fold(fun({val, _}, _, Acc) ->
+                      %% don't count memoized fields
+                      Acc;
+                 (_, V, Acc) ->
                       case Fun(V, ToCheck) of
                           true ->
                               Acc + 1;
                           false -> Acc
                       end
-              end, 0, Map) >= N - F.
+              end, 0, maps:merge(Map, Terms)) >= N - F.
 
 maybe_init_coin(Data) ->
     case Data#bba_data.coin of
@@ -366,10 +385,16 @@ rand_val(2#11) -> hd(hbbft_utils:random_n(1, [0, 1])).
 val(2#1) -> 0;
 val(2#10) -> 1.
 
-add_witness(Id, Value, Witness) ->
+add_witness(Id, Value, Witness, AllowUnion) ->
     Old = maps:get(Id, Witness, 0),
     New = add(Value, Old),
     case Old /= New of
+        true when New == 3, AllowUnion == false ->
+            %% not allowed to update or merge the value
+            Witness;
+        true when Old /= 0, AllowUnion == false ->
+            %% now allowed to update a value
+            Witness;
         true when New == 3 ->
             OldCount = maps:get({val, Value}, Witness, 0),
             OldBothCount = maps:get({val, both}, Witness, 0),
@@ -381,6 +406,35 @@ add_witness(Id, Value, Witness) ->
             Witness
     end.
 
+remove_witness(ID, Witness) ->
+    Old = maps:get(ID, Witness, 0),
+    case Old of
+        0 ->
+            Witness;
+        3 ->
+            OldCount0 = maps:get({val, 0}, Witness, 1),
+            OldCount1 = maps:get({val, 1}, Witness, 1),
+            OldBothCount = maps:get({val, both}, Witness, 1),
+            maps:merge(maps:remove(ID, Witness), #{{val, 0} => OldCount0 - 1, {val, 1} => OldCount1 - 1, {val, both} => OldBothCount - 1});
+        Val ->
+            OldCount = maps:get({val, val(Val)}, Witness, 1),
+            maps:merge(maps:remove(ID, Witness), #{{val, val(Val)} => OldCount - 1})
+    end.
+
+%count_witness(Value, Witness, Terminate) ->
+    %% add_terminate deletes other witness values, so we don't have to
+    %% worry about double counting here
+    %maps:get({val, Value}, Witness) + maps:get({val, Value}, Terminate).
+
+add_terminate(Id, Value, Data) ->
+    TerminateWitness = add_witness(Id, Value, Data#bba_data.terminate_witness, false),
+    %% XXX can we naively delete bval/aux/conf witness values here? what if the terminate witness
+    %% value isn't the same as the aux witness value?
+    BValWitness = remove_witness(Id, Data#bba_data.bval_witness),
+    AuxWitness = remove_witness(Id, Data#bba_data.aux_witness),
+    ConfWitness = remove_witness(Id, Data#bba_data.conf_witness),
+    Data#bba_data{bval_witness=BValWitness, aux_witness=AuxWitness, conf_witness=ConfWitness, terminate_witness=TerminateWitness}.
+
 schedule(0) -> 1;
 schedule(1) -> 1;
 schedule(2) -> 0;
@@ -409,15 +463,15 @@ check_coin_flip(Data, Flip) ->
                 true ->
                     %% we are done
                     NewData = Data#bba_data{state=done},
-                    {NewData, {result, B}};
+                    {NewData, {result_and_send, B, {send, [{multicast, {term, B}}]}} };
                 false ->
                     %% increment round and continue
                     NewData = init(Data#bba_data.secret_key, Data#bba_data.n, Data#bba_data.f),
-                    input(NewData#bba_data{round=Data#bba_data.round + 1, output=Output}, B)
+                    input(NewData#bba_data{round=Data#bba_data.round + 1, output=Output, terminate_witness=Data#bba_data.terminate_witness}, B)
             end;
         false ->
             %% else estr+1 := s%2
             B = Flip,
             NewData = init(Data#bba_data.secret_key, Data#bba_data.n, Data#bba_data.f),
-            input(NewData#bba_data{round=Data#bba_data.round + 1}, B)
+            input(NewData#bba_data{round=Data#bba_data.round + 1, terminate_witness=Data#bba_data.terminate_witness}, B)
     end.
diff --git a/test/hbbft_test_utils.erl b/test/hbbft_test_utils.erl
@@ -10,6 +10,8 @@ do_send_outer(Mod, [H|T], States, Acc) ->
     {R, NewStates} = do_send(Mod, H, [], States),
     do_send_outer(Mod, T++R, NewStates, Acc).
 
+do_send(Mod, {Id, {result_and_send, Result, ToSend}}, Acc, States) ->
+    do_send(Mod, {Id, ToSend}, [{result, {Id, Result}} | Acc], States);
 do_send(_Mod, {Id, {result, Result}}, Acc, States) ->
     {[{result, {Id, Result}} | Acc], States};
 do_send(_Mod, {_, ok}, Acc, States) ->