Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Revise replicator and repairer#1

  • Loading branch information...
commit f76197920cbda4ea1f0f15f19fce1470f51804af 1 parent 6fc1ce3
@yosukehara yosukehara authored
View
13 include/leo_storage.hrl
@@ -81,6 +81,19 @@
-define(ERROR_COULD_MATCH, "Could not match").
+%% @doc request parameter for READ
+%%
+-record(read_parameter, {
+ ref :: reference(),
+ addr_id :: integer(),
+ key :: string(),
+ etag = [] :: string(),
+ start_pos = 0 :: integer(),
+ end_pos = 0 :: integer(),
+ quorum :: integer(),
+ req_id :: integer()
+ }).
+
%% @doc Queue's Message.
%%
-record(inconsistent_data_message, {
View
242 src/leo_storage_handler_object.erl
@@ -36,7 +36,7 @@
-include_lib("leo_statistics/include/leo_statistics.hrl").
-include_lib("eunit/include/eunit.hrl").
--export([get/1, get/3, get/4, get/5,
+-export([get/1, get/2, get/3, get/4, get/5,
put/1, put/2, put/3,
delete/1, delete/2, head/2,
copy/3,
@@ -50,19 +50,10 @@
-define(DEF_DELIMITER, <<"/">>).
--record(read_parameter, {
- addr_id :: integer(),
- key :: string(),
- start_pos = 0 :: integer(),
- end_pos = 0 :: integer(),
- quorum :: integer(),
- req_id :: integer()
- }).
-
%%--------------------------------------------------------------------
%% API - GET
%%--------------------------------------------------------------------
-%% @doc get object (from storage-node).
+%% @doc get object (from storage-node#1).
%%
-spec(get({reference(), string()}) ->
{ok, reference(), binary(), binary(), binary()} |
@@ -82,13 +73,34 @@ get({Ref, Key}) ->
{error, Ref, ?ERROR_COULD_NOT_GET_REDUNDANCY}
end.
+%% @doc get object (from storage-node#2).
+%%
+-spec(get(#read_parameter{}, list(#redundant_node{})) ->
+ {ok, reference(), binary(), binary(), binary()} |
+ {error, reference(), any()}).
+get(#read_parameter{addr_id = AddrId} = ReadParameter, []) ->
+ case leo_redundant_manager_api:get_redundancies_by_addr_id(get, AddrId) of
+ {ok, #redundancies{nodes = Redundancies,
+ r = ReadQuorum}} ->
+ get(ReadParameter#read_parameter{quorum = ReadQuorum},
+ Redundancies);
+ _Error ->
+ {error, ?ERROR_COULD_NOT_GET_REDUNDANCY}
+ end;
+get(ReadParameter, Redundancies) ->
+ _ = leo_statistics_req_counter:increment(?STAT_REQ_GET),
+ read_and_repair(ReadParameter, Redundancies).
+
%% @doc Retrieve an object which is requested from gateway.
%%
-spec(get(integer(), string(), integer()) ->
{ok, #metadata{}, binary()} |
{error, any()}).
get(AddrId, Key, ReqId) ->
- get(AddrId, Key, 0, 0, ReqId).
+ get(#read_parameter{ref = make_ref(),
+ addr_id = AddrId,
+ key = Key,
+ req_id = ReqId}, []).
%% @doc Retrieve an object which is requested from gateway w/etag.
%%
@@ -97,20 +109,11 @@ get(AddrId, Key, ReqId) ->
{ok, match} |
{error, any()}).
get(AddrId, Key, ETag, ReqId) ->
- case leo_object_storage_api:head({AddrId, Key}) of
- {ok, MetaBin} ->
- Metadata = binary_to_term(MetaBin),
- case (Metadata#metadata.checksum == ETag) of
- true ->
- {ok, match};
- false ->
- get(AddrId, Key, ReqId)
- end;
- not_found = Cause ->
- {error, Cause};
- Error ->
- Error
- end.
+ get(#read_parameter{ref = make_ref(),
+ addr_id = AddrId,
+ key = Key,
+ etag = ETag,
+ req_id = ReqId}, []).
%% @doc Retrieve a part of an object.
%%
@@ -118,28 +121,12 @@ get(AddrId, Key, ETag, ReqId) ->
{ok, #metadata{}, binary()} |
{error, any()}).
get(AddrId, Key, StartPos, EndPos, ReqId) ->
- _ = leo_statistics_req_counter:increment(?STAT_REQ_GET),
-
- Ret = case leo_redundant_manager_api:get_redundancies_by_addr_id(get, AddrId) of
- {ok, #redundancies{nodes = Redundancies,
- r = ReadQuorum}} ->
- ReadParameter = #read_parameter{addr_id = AddrId,
- key = Key,
- start_pos = StartPos,
- end_pos = EndPos,
- quorum = ReadQuorum,
- req_id = ReqId},
- read_and_repair(ReadParameter, Redundancies);
- _Error ->
- {error, ?ERROR_COULD_NOT_GET_REDUNDANCY}
- end,
-
- case Ret of
- {ok, NewMeta, #object{data = Bin}} ->
- {ok, NewMeta, Bin};
- Error ->
- Error
- end.
+ get(#read_parameter{ref = make_ref(),
+ addr_id = AddrId,
+ key = Key,
+ start_pos = StartPos,
+ end_pos = EndPos,
+ req_id = ReqId}, []).
%% @doc read data (common).
@@ -513,67 +500,140 @@ find_uploaded_objects_by_key(OriginalKey) ->
read_and_repair(_, []) ->
{error, ?ERROR_COULD_NOT_GET_DATA};
-read_and_repair(#read_parameter{addr_id = AddrId,
+read_and_repair(#read_parameter{ref = Ref,
+ addr_id = AddrId,
key = Key,
+ etag = [],
start_pos = StartPos,
- end_pos = EndPos,
- quorum = ReadQuorum,
- req_id = ReqId} = ReadParameter, [_|T] = Redundancies) ->
- Ref = make_ref(),
-
- case get_fun(Ref, AddrId, Key, StartPos, EndPos) of
- {ok, Ref, Metadata, Object} when T =:= [] ->
- {ok, Metadata, Object};
- {ok, Ref, Metadata, Object} when T =/= [] ->
- F = fun(ok) ->
- {ok, Metadata, Object};
- ({error,_Cause}) ->
- {error, ?ERROR_RECOVER_FAILURE}
- end,
- leo_storage_read_repairer:repair(
- ReadQuorum -1, Redundancies, Metadata, ReqId, F);
-
- {error, Ref, not_found = Cause} ->
- {error, Cause};
- {error, Ref, _Cause} ->
- case (erlang:length(T) >= ReadQuorum) of
- true ->
- read_and_repair(ReadParameter, T);
- false ->
- {error, ?ERROR_COULD_NOT_GET_DATA}
- end
- end.
+ end_pos = EndPos} = ReadParameter,
+ [#redundant_node{node = Node,
+ available = true}|T]
+ ) when Node == erlang:node() ->
+ read_and_repair_1(
+ get_fun(Ref, AddrId, Key, StartPos, EndPos), ReadParameter, T);
+
+read_and_repair(#read_parameter{ref = Ref,
+ addr_id = AddrId,
+ key = Key,
+ etag = ETag,
+ start_pos = StartPos,
+ end_pos = EndPos} = ReadParameter,
+ [#redundant_node{node = Node,
+ available = true}|T]
+ ) when Node == erlang:node(),
+ ETag /= [] ->
+ %% Retrieve an head of object,
+ %% then compare it with requested 'Etag'
+ Ret = case leo_object_storage_api:head({AddrId, Key}) of
+ {ok, MetaBin} ->
+ Metadata = binary_to_term(MetaBin),
+ case Metadata#metadata.checksum of
+ ETag ->
+ {ok, match};
+ _ ->
+ []
+ end;
+ _ ->
+ []
+ end,
+
+ %% If the result is 'match', then response it,
+ %% not the case, retrieve an object by key
+ case Ret of
+ {ok, match} = Reply ->
+ Reply;
+ _ ->
+ read_and_repair_1(
+ get_fun(Ref, AddrId, Key, StartPos, EndPos), ReadParameter, T)
+ end;
+
+read_and_repair(#read_parameter{ref = Ref} = ReadParameter,
+ [#redundant_node{node = Node,
+ available = true}|_] = Redundancies) ->
+ RPCKey = rpc:async_call(Node, ?MODULE, get, [ReadParameter, Redundancies]),
+ Reply = case rpc:nb_yield(RPCKey, ?DEF_REQ_TIMEOUT) of
+ {value, {ok, Ref,_,_} = Ret} ->
+ Ret;
+ {value, {error, Cause}} ->
+ {error, Ref, Cause};
+ {value, {badrpc, Cause}} ->
+ {error, Ref, Cause};
+ timeout = Cause ->
+ {error, Ref, Cause};
+ {badrpc, Cause} ->
+ {error, Ref, Cause}
+ end,
+ read_and_repair_1(Reply, ReadParameter, []).
+
+
+%% @private
+read_and_repair_1({ok, Ref, Metadata, Object},
+ #read_parameter{ref = Ref}, []) ->
+ {ok, Metadata, Object};
+
+read_and_repair_1({ok, Ref, Metadata, Object},
+ #read_parameter{ref = Ref,
+ quorum = Quorum} = ReadParameter, Redundancies) ->
+ Fun = fun(ok) ->
+ {ok, Metadata, Object};
+ ({error,_Cause}) ->
+ {error, ?ERROR_RECOVER_FAILURE}
+ end,
+ ReadParameter_1 = ReadParameter#read_parameter{quorum = Quorum - 1},
+ leo_storage_read_repairer:repair(ReadParameter_1, Redundancies, Metadata, Fun);
+
+read_and_repair_1({error, Ref, timeout = Cause},
+ #read_parameter{ref = Ref}, _Redundancies) ->
+ {error, Cause};
+read_and_repair_1({error, Ref, Cause},
+ #read_parameter{ref = Ref}, []) ->
+ {error, Cause};
+
+read_and_repair_1({error, Ref,_Cause},
+ #read_parameter{ref = Ref,
+ quorum = ReadQuorum} = ReadParameter, Redundancies) ->
+ NumOfNodes = erlang:length([N || #redundant_node{node = N,
+ can_read_repair = true}
+ <- Redundancies]),
+ case (NumOfNodes >= ReadQuorum) of
+ true ->
+ read_and_repair(ReadParameter, Redundancies);
+ false ->
+ {error, ?ERROR_COULD_NOT_GET_DATA}
+ end;
+read_and_repair_1(_,_,_) ->
+ {error, invalid_request}.
%% @doc Replicate an object from local-node to remote node
%% @private
-spec(replicate(replication(), put | delete, integer(), #object{}) ->
ok | {error, any()}).
-replicate(?REP_LOCAL, Method, AddrId, Object0) ->
+replicate(?REP_LOCAL, Method, AddrId, Object_1) ->
case leo_redundant_manager_api:get_redundancies_by_addr_id(put, AddrId) of
{ok, #redundancies{nodes = Redundancies,
w = WriteQuorum,
d = DeleteQuorum,
ring_hash = RingHash}} ->
- Object1 = Object0#object{ring_hash = RingHash},
+ Object_2 = Object_1#object{ring_hash = RingHash},
Quorum = case Method of
?CMD_PUT -> WriteQuorum;
?CMD_DELETE -> DeleteQuorum
end,
- F = fun({ok, ETag}) when Method == ?CMD_PUT ->
- {ok, ETag};
- ({ok,_ETag}) when Method == ?CMD_DELETE ->
- ok;
- ({error, Cause}) ->
- case lists:keyfind(not_found, 2, Cause) of
- false ->
- {error, ?ERROR_REPLICATE_FAILURE};
- _ ->
- {error, not_found}
- end
- end,
- leo_storage_replicator:replicate(Quorum, Redundancies, Object1, F);
+ Fun = fun({ok, ETag}) when Method == ?CMD_PUT ->
+ {ok, ETag};
+ ({ok,_ETag}) when Method == ?CMD_DELETE ->
+ ok;
+ ({error, Cause}) ->
+ case lists:keyfind(not_found, 2, Cause) of
+ false ->
+ {error, ?ERROR_REPLICATE_FAILURE};
+ _ ->
+ {error, not_found}
+ end
+ end,
+ leo_storage_replicator:replicate(Quorum, Redundancies, Object_2, Fun);
_Error ->
{error, ?ERROR_META_NOT_FOUND}
end;
View
47 src/leo_storage_read_repairer.erl
@@ -34,32 +34,37 @@
-include_lib("leo_redundant_manager/include/leo_redundant_manager.hrl").
%% API
--export([repair/5]).
+-export([repair/4]).
--record(req_params, {addr_id = 0 :: integer(),
- key :: string(),
- read_quorum = 0 :: integer(),
- redundancies = [] :: list(),
- metadata :: #metadata{},
- rpc_key :: rpc:key(),
- req_id = 0 :: integer()
- }).
+-record(state, {addr_id = 0 :: integer(),
+ key :: string(),
+ read_quorum = 0 :: integer(),
+ redundancies = [] :: list(),
+ metadata :: #metadata{},
+ rpc_key :: rpc:key(),
+ req_id = 0 :: integer()
+ }).
%%--------------------------------------------------------------------
%% API
%%--------------------------------------------------------------------
%% @doc Repair an object.
%% @end
--spec(repair(pos_integer(), list(), #metadata{}, integer(), function()) ->
+-spec(repair(#read_parameter{}, #redundant_node{}, #metadata{}, function()) ->
{ok, reference()} | {error, reference(), any()}).
-repair(ReadQuorum, Nodes, Metadata, ReqId, Callback) ->
+repair(#read_parameter{quorum = ReadQuorum,
+ req_id = ReqId}, Redundancies, Metadata, Callback) ->
From = self(),
AddrId = Metadata#metadata.addr_id,
Key = Metadata#metadata.key,
- Params = #req_params{read_quorum = ReadQuorum,
- redundancies = Nodes,
- metadata = Metadata,
- req_id = ReqId},
+ Params = #state{read_quorum = ReadQuorum,
+ redundancies = Redundancies,
+ metadata = Metadata,
+ req_id = ReqId},
+ NumOfNodes = erlang:length(
+ [N || #redundant_node{node = N,
+ can_read_repair = true}
+ <- Redundancies]),
lists:foreach(
fun(#redundant_node{node = Node,
available = true,
@@ -71,8 +76,8 @@ repair(ReadQuorum, Nodes, Metadata, ReqId, Callback) ->
end);
(#redundant_node{}) ->
void
- end, Nodes),
- loop(ReadQuorum, From, erlang:length(Nodes), {ReqId, Key, []}, Callback).
+ end, Redundancies),
+ loop(ReadQuorum, From, NumOfNodes, {ReqId, Key, []}, Callback).
%% @doc Waiting for messages (compare a metadata)
@@ -106,11 +111,11 @@ loop(R, From, NumOfNodes, {ReqId, Key, Errors} = Args, Callback) ->
%%--------------------------------------------------------------------
%% @doc Compare local-metadata with remote-metadata
%% @private
--spec(compare(pid(), pid(), atom(), #req_params{}) ->
+-spec(compare(pid(), pid(), atom(), #state{}) ->
ok).
-compare(Pid, RPCKey, Node, #req_params{metadata = #metadata{addr_id = AddrId,
- key = Key,
- clock = Clock}}) ->
+compare(Pid, RPCKey, Node, #state{metadata = #metadata{addr_id = AddrId,
+ key = Key,
+ clock = Clock}}) ->
Ret = case rpc:nb_yield(RPCKey, ?DEF_REQ_TIMEOUT) of
{value, {ok, #metadata{clock = RemoteClock}}} when Clock == RemoteClock ->
ok;
View
190 test/leo_storage_handler_object_tests.erl
@@ -109,6 +109,7 @@ teardown({_, Node1}) ->
%% @doc get/1
%% @private
get_a0_({Node0, Node1}) ->
+ %% leo_redundant_manager_api
meck:new(leo_redundant_manager_api),
meck:expect(leo_redundant_manager_api, get_redundancies_by_key,
fun(get, _Key) ->
@@ -117,6 +118,7 @@ get_a0_({Node0, Node1}) ->
n = 2, r = 1, w = 1, d = 1}}
end),
+ %% leo_object_storage_api
meck:new(leo_object_storage_api),
meck:expect(leo_object_storage_api, get,
fun(_Key, _StartPos, _EndPos) ->
@@ -135,6 +137,7 @@ get_a0_({Node0, Node1}) ->
%% @doc get/1
%% @private
get_a1_({Node0, Node1}) ->
+ %% leo_redundant_manager_api
meck:new(leo_redundant_manager_api),
meck:expect(leo_redundant_manager_api, get_redundancies_by_key,
fun(get, _AddrId) ->
@@ -143,6 +146,7 @@ get_a1_({Node0, Node1}) ->
n = 2, r = 1, w = 1, d = 1}}
end),
+ %% leo_object_storage_api
meck:new(leo_object_storage_api),
meck:expect(leo_object_storage_api, get,
fun(_Key, _StartPos, _EndPos) ->
@@ -158,19 +162,32 @@ get_a1_({Node0, Node1}) ->
%% @doc get/3
%% @private
get_b0_({Node0, Node1}) ->
+ %% leo_redundant_manager_api
+ Fun1 = fun(get, _AddrId) ->
+ {ok, #redundancies{id = 0,
+ nodes = [#redundant_node{node = Node0,
+ available = true},
+ #redundant_node{node = Node1,
+ available = true}],
+ n = 2, r = 1, w = 1, d = 1}}
+ end,
meck:new(leo_redundant_manager_api),
- meck:expect(leo_redundant_manager_api, get_redundancies_by_addr_id,
- fun(get, _AddrId) ->
- {ok, #redundancies{id = 0,
- nodes = [{Node0,true}, {Node1,true}],
- n = 2, r = 1, w = 1, d = 1}}
- end),
+ meck:expect(leo_redundant_manager_api, get_redundancies_by_addr_id, Fun1),
+ ok = rpc:call(Node1, meck, new, [leo_redundant_manager_api, [no_link]]),
+ ok = rpc:call(Node1, meck, expect, [leo_redundant_manager_api, get_redundancies_by_addr_id, Fun1]),
+
+ %% leo_object_storage_api
+ Fun2 = fun(_Key, _StartPos, _EndPos) ->
+ not_found
+ end,
meck:new(leo_object_storage_api),
- meck:expect(leo_object_storage_api, get,
- fun(_Key, _StartPos, _EndPos) ->
- not_found
- end),
+ meck:expect(leo_object_storage_api, get, Fun2),
+
+ ok = rpc:call(Node1, meck, new, [leo_object_storage_api, [no_link]]),
+ ok = rpc:call(Node1, meck, expect, [leo_object_storage_api, get, Fun2]),
+ ok = rpc:call(Node1, meck, new, [leo_statistics_req_counter, [no_link]]),
+ ok = rpc:call(Node1, meck, expect, [leo_statistics_req_counter, increment, fun(_) -> ok end]),
Res = leo_storage_handler_object:get(0, ?TEST_KEY_0, 0),
?assertEqual({error,not_found}, Res),
@@ -180,26 +197,44 @@ get_b0_({Node0, Node1}) ->
%% @doc get/3
%% @private
get_b1_({Node0, Node1}) ->
+ %% leo_redundant_manager_api
+ Fun1 = fun(get, _AddrId) ->
+ {ok, #redundancies{id = 0,
+ nodes = [#redundant_node{node = Node0,
+ available = true},
+ #redundant_node{node = Node1,
+ available = true}],
+ n = 2, r = 1, w = 1, d = 1}}
+ end,
meck:new(leo_redundant_manager_api),
- meck:expect(leo_redundant_manager_api, get_redundancies_by_addr_id,
- fun(get, _AddrId) ->
- {ok, #redundancies{id = 0,
- nodes = [{Node0,true}, {Node1,true}], %% two
- n = 2, r = 1, w = 1, d = 1}}
- end),
+ meck:expect(leo_redundant_manager_api, get_redundancies_by_addr_id, Fun1),
+ ok = rpc:call(Node1, meck, new, [leo_redundant_manager_api, [no_link]]),
+ ok = rpc:call(Node1, meck, expect, [leo_redundant_manager_api, get_redundancies_by_addr_id, Fun1]),
+
+ %% leo_object_storage_api
+ Fun2 = fun(_Key, _StartPos, _EndPos) ->
+ {ok, ?TEST_META_0, []}
+ end,
meck:new(leo_object_storage_api),
- meck:expect(leo_object_storage_api, get,
- fun(_Key, _StartPos, _EndPos) ->
- {ok, ?TEST_META_0, []}
- end),
+ meck:expect(leo_object_storage_api, get, Fun2),
+
+ ok = rpc:call(Node1, meck, new, [leo_object_storage_api, [no_link]]),
+ ok = rpc:call(Node1, meck, expect, [leo_object_storage_api, get, Fun2]),
+ ok = rpc:call(Node1, meck, new, [leo_statistics_req_counter, [no_link]]),
+ ok = rpc:call(Node1, meck, expect, [leo_statistics_req_counter, increment, fun(_) -> ok end]),
+
+ Fun3 = fun(_R,_Nodes,_Metadata,_Callback) ->
+ {error,not_found}
+ end,
meck:new(leo_storage_read_repairer),
- meck:expect(leo_storage_read_repairer, repair,
- fun(_R, _Node,_Metadata,_ReqId,_Callback) ->
- {error,not_found}
- end),
+ meck:expect(leo_storage_read_repairer, repair, Fun3),
+ ok = rpc:call(Node1, meck, new, [leo_storage_read_repairer, [no_link]]),
+ ok = rpc:call(Node1, meck, expect, [leo_storage_read_repairer, repair, Fun3]),
+
+ %% Execute
Res = leo_storage_handler_object:get(0, ?TEST_KEY_0, 0),
?assertEqual({error,not_found}, Res),
@@ -210,20 +245,24 @@ get_b1_({Node0, Node1}) ->
%% @doc get/3
%% @private
get_b2_({Node0, _Node1}) ->
+ %% leo_redundant_manager_api
meck:new(leo_redundant_manager_api),
meck:expect(leo_redundant_manager_api, get_redundancies_by_addr_id,
fun(get, _AddrId) ->
{ok, #redundancies{id = 0,
- nodes = [{Node0,true}], %% one
+ nodes = [#redundant_node{node = Node0,
+ available = true}],
n = 2, r = 1, w = 1, d = 1}}
end),
+ %% leo_object_storage_api
meck:new(leo_object_storage_api),
meck:expect(leo_object_storage_api, get,
fun(_Key, _StartPos, _EndPos) ->
{ok, ?TEST_META_0, []}
end),
+ %% leo_storage_read_repairer
meck:new(leo_storage_read_repairer),
meck:expect(leo_storage_read_repairer, repair,
fun(_R, _Node,_Metadata,_ReqId,_Callback) ->
@@ -240,27 +279,45 @@ get_b2_({Node0, _Node1}) ->
%% @doc get/3
%% @private
get_b3_({Node0, Node1}) ->
+ %% leo_redundant_manager_api
+ Fun1 = fun(get, _AddrId) ->
+ {ok, #redundancies{id = 0,
+ nodes = [#redundant_node{node = Node0,
+ available = true},
+ #redundant_node{node = Node1,
+ available = true}],
+ n = 2, r = 1, w = 1, d = 1}}
+ end,
meck:new(leo_redundant_manager_api),
- meck:expect(leo_redundant_manager_api, get_redundancies_by_addr_id,
- fun(get, _AddrId) ->
- {ok, #redundancies{id = 0,
- nodes = [{Node0,true}, {Node1,true}], %% two
- n = 2, r = 1, w = 1, d = 1}}
- end),
+ meck:expect(leo_redundant_manager_api, get_redundancies_by_addr_id, Fun1),
+
+ ok = rpc:call(Node1, meck, new, [leo_redundant_manager_api, [no_link]]),
+ ok = rpc:call(Node1, meck, expect, [leo_redundant_manager_api, get_redundancies_by_addr_id, Fun1]),
+ %% leo_object_storage_api
+ Fun2 = fun(_Key, _StartPos, _EndPos) ->
+ {ok, ?TEST_META_0, []}
+ end,
meck:new(leo_object_storage_api),
- meck:expect(leo_object_storage_api, get,
- fun(_Key, _StartPos, _EndPos) ->
- {ok, ?TEST_META_0, []}
- end),
+ meck:expect(leo_object_storage_api, get, Fun2),
+ ok = rpc:call(Node1, meck, new, [leo_object_storage_api, [no_link]]),
+ ok = rpc:call(Node1, meck, expect, [leo_object_storage_api, get, Fun2]),
+ ok = rpc:call(Node1, meck, new, [leo_statistics_req_counter, [no_link]]),
+ ok = rpc:call(Node1, meck, expect, [leo_statistics_req_counter, increment, fun(_) -> ok end]),
+
+ %% leo_storage_read_repairer
+ Fun3 = fun(_R, _Nodes, _Metadata,_Callback) ->
+ {ok, ?TEST_META_0, ?TEST_BIN}
+ end,
meck:new(leo_storage_read_repairer),
- meck:expect(leo_storage_read_repairer, repair,
- fun(_R, _Node, _Metadata, _ReqId,_Callback) ->
- {ok, ?TEST_META_0, ?TEST_BIN}
- end),
+ meck:expect(leo_storage_read_repairer, repair, Fun3),
+
+ ok = rpc:call(Node1, meck, new, [leo_storage_read_repairer, [no_link]]),
+ ok = rpc:call(Node1, meck, expect, [leo_storage_read_repairer, repair, Fun3]),
+ %% Execute
Res = leo_storage_handler_object:get(0, ?TEST_KEY_0, 0),
?assertEqual({ok, ?TEST_META_0, ?TEST_BIN}, Res),
@@ -275,24 +332,39 @@ get_c0_({Node0, Node1}) ->
Checksum0 = 12345,
Checksum1 = 12340,
+ %% leo_redundant_manager_api
+ Fun1 = fun(get, _AddrId) ->
+ {ok, #redundancies{id = 0,
+ nodes = [#redundant_node{node = Node0,
+ available = true},
+ #redundant_node{node = Node1,
+ available = true}],
+ n = 2, r = 1, w = 1, d = 1}}
+ end,
meck:new(leo_redundant_manager_api),
- meck:expect(leo_redundant_manager_api, get_redundancies_by_addr_id,
- fun(get, _AddrId) ->
- {ok, #redundancies{id = 0,
- nodes = [{Node0,true}, {Node1,true}],
- n = 2, r = 1, w = 1, d = 1}}
- end),
-
+ meck:expect(leo_redundant_manager_api, get_redundancies_by_addr_id, Fun1),
+
+ ok = rpc:call(Node1, meck, new, [leo_redundant_manager_api, [no_link]]),
+ ok = rpc:call(Node1, meck, expect, [leo_redundant_manager_api, get_redundancies_by_addr_id, Fun1]),
+
+ %% leo_object_storage_api
+ Fun2 = fun(_Key, _StartPos, _EndPos) ->
+ not_found
+ end,
+ Fun3 = fun(_Key) ->
+ {ok, term_to_binary(#metadata{checksum = Checksum0})}
+ end,
meck:new(leo_object_storage_api),
- meck:expect(leo_object_storage_api, get,
- fun(_Key, _StartPos, _EndPos) ->
- not_found
- end),
- meck:expect(leo_object_storage_api, head,
- fun(_Key) ->
- {ok, term_to_binary(#metadata{checksum = Checksum0})}
- end),
+ meck:expect(leo_object_storage_api, get, Fun2),
+ meck:expect(leo_object_storage_api, head, Fun3),
+ ok = rpc:call(Node1, meck, new, [leo_object_storage_api, [no_link]]),
+ ok = rpc:call(Node1, meck, expect, [leo_object_storage_api, get, Fun2]),
+ ok = rpc:call(Node1, meck, expect, [leo_object_storage_api, head, Fun3]),
+ ok = rpc:call(Node1, meck, new, [leo_statistics_req_counter, [no_link]]),
+ ok = rpc:call(Node1, meck, expect, [leo_statistics_req_counter, increment, fun(_) -> ok end]),
+
+ %% Execute
Res0 = leo_storage_handler_object:get(0, ?TEST_KEY_0, Checksum0, 0),
?assertEqual({ok,match}, Res0),
@@ -317,7 +389,10 @@ put_0_({Node0, Node1}) ->
meck:expect(leo_redundant_manager_api, get_redundancies_by_addr_id,
fun(put, _AddrId) ->
{ok, #redundancies{id = 0,
- nodes = [{Node0,true}, {Node1,true}],
+ nodes = [#redundant_node{node = Node0,
+ available = true},
+ #redundant_node{node = Node1,
+ available = true}],
n = 2, r = 1, w = 1, d = 1}}
end),
@@ -382,7 +457,10 @@ delete_0_({Node0, Node1}) ->
meck:expect(leo_redundant_manager_api, get_redundancies_by_addr_id,
fun(put, _AddrId) ->
{ok, #redundancies{id = 0,
- nodes = [{Node0,true}, {Node1,true}],
+ nodes = [#redundant_node{node = Node0,
+ available = true},
+ #redundant_node{node = Node1,
+ available = true}],
n = 2, r = 1, w = 1, d = 1}}
end),
View
15 test/leo_storage_read_repairer_tests.erl
@@ -24,8 +24,9 @@
%% @end
%%====================================================================
-module(leo_storage_read_repairer_tests).
--author('yosuke hara').
+-author('Yosuke Hara').
+-include("leo_storage.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("leo_object_storage/include/leo_object_storage.hrl").
-include_lib("leo_redundant_manager/include/leo_redundant_manager.hrl").
@@ -99,7 +100,9 @@ regular_({Test0Node, Test1Node}) ->
Nodes = [#redundant_node{node = Test0Node, available = true},
#redundant_node{node = Test1Node, available = true}],
F = fun(_) -> ok end,
- ok = leo_storage_read_repairer:repair(1, Nodes, ?TEST_META_1, 0, F),
+ ok = leo_storage_read_repairer:repair(
+ #read_parameter{quorum = 1,
+ req_id = 0}, Nodes, ?TEST_META_1, F),
ok.
fail_1_({Test0Node, Test1Node}) ->
@@ -120,7 +123,9 @@ fail_1_({Test0Node, Test1Node}) ->
Nodes = [#redundant_node{node = Test0Node, available = true},
#redundant_node{node = Test1Node, available = true}],
F = fun(_) -> ok end,
- ok = leo_storage_read_repairer:repair(1, Nodes, ?TEST_META_1, 0, F),
+ ok = leo_storage_read_repairer:repair(
+ #read_parameter{quorum = 1,
+ req_id = 0}, Nodes, ?TEST_META_1, F),
ok.
fail_2_({Test0Node, Test1Node}) ->
@@ -141,7 +146,9 @@ fail_2_({Test0Node, Test1Node}) ->
Nodes = [#redundant_node{node = Test0Node, available = true},
#redundant_node{node = Test1Node, available = true}],
F = fun(_) -> ok end,
- ok = leo_storage_read_repairer:repair(1, Nodes, ?TEST_META_2, 0, F),
+ ok = leo_storage_read_repairer:repair(
+ #read_parameter{quorum = 1,
+ req_id = 0}, Nodes, ?TEST_META_2, F),
ok.
-endif.
Please sign in to comment.
Something went wrong with that request. Please try again.