Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP

Loading…

Qc sync range #8

Merged
merged 4 commits into from

2 participants

@jlouis

This patch enables yet another call from LSM Btree to go through the PropEr/Triq tests, namely the sync_range call.

While here, fix a race condition found when trying to run 12000 tests.

jlouis added some commits
@jlouis jlouis Provide a target for an eunit_console. 6293ada
@jlouis jlouis Build a record for the trees.
This is a needed step to support range-queries since we may collect
range queries at a later time, interleaved with the normal operations.
e003786
@jlouis jlouis Make the sync_range call test work.
Currently, we are also synchronizing all the key collection in the
call. But in the future we might want to interleave that with other
operations.
bb17564
@jlouis jlouis Fix a bug in lsm_tree:close/1.
There is a race condition based on the monitor set in a call. We might
get a normal exit from the monitor message deep inside gen_server.
This has to be handled. I've seen this race in my QC tests.
baa779d
@krestenkrab

Seems like there must be a standard pattern for "cleanup and stop gen_server" than what I do here...

@krestenkrab krestenkrab merged commit 90ba15f into krestenkrab:master
@krestenkrab
Owner

Awesome work. Thanks!

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Commits on Jan 21, 2012
  1. @jlouis
  2. @jlouis

    Build a record for the trees.

    jlouis authored
    This is a needed step to support range-queries since we may collect
    range queries at a later time, interleaved with the normal operations.
  3. @jlouis

    Make the sync_range call test work.

    jlouis authored
    Currently, we are also synchronizing all the key collection in the
    call. But in the future we might want to interleave that with other
    operations.
  4. @jlouis

    Fix a bug in lsm_tree:close/1.

    jlouis authored
    There is a race condition based on the monitor set in a call. We might
    get a normal exit from the monitor message deep inside gen_server.
    This has to be handled. I've seen this race in my QC tests.
This page is out of date. Refresh to see the latest.
View
3  Makefile
@@ -17,6 +17,9 @@ clean:
eunit: compile clean-test-btrees
@$(REBAR) eunit skip_deps=true
+eunit_console:
+ erl -pa .eunit deps/*/ebin
+
clean-test-btrees:
rm -fr .eunit/Btree_* .eunit/simple
View
4 src/lsm_btree.erl
@@ -23,7 +23,9 @@ close(Ref) ->
gen_server:call(Ref, close)
catch
exit:{noproc,_} -> ok;
- exit:noproc -> ok
+ exit:noproc -> ok;
+ %% Handle the case where the monitor triggers
+ exit:{normal, _} -> ok
end.
View
22 test/lsm_btree_drv.erl
@@ -12,6 +12,7 @@
lookup_fail/2,
open/1, close/1,
put/3,
+ sync_range/3,
stop/0]).
%% gen_server callbacks
@@ -49,6 +50,9 @@ close(N) ->
put(N, K, V) ->
call({put, N, K, V}).
+sync_range(T, LK, HK) ->
+ call({sync_range, T, LK, HK}).
+
stop() ->
call(stop).
@@ -72,6 +76,12 @@ handle_call({close, N}, _, #state { btrees = D} = State) ->
Otherwise ->
{reply, {error, Otherwise}, State}
end;
+handle_call({sync_range, Name, LoKey, HiKey}, _From,
+ #state { btrees = D} = State) ->
+ Tree = dict:fetch(Name, D),
+ {ok, Ref} = lsm_btree:sync_range(Tree, LoKey, HiKey),
+ Result = sync_range_gather(Ref),
+ {reply, Result, State};
handle_call({put, N, K, V}, _, #state { btrees = D} = State) ->
Tree = dict:fetch(N, D),
case lsm_btree:put(Tree, K, V) of
@@ -119,3 +129,15 @@ cleanup_trees(#state { btrees = BTs }) ->
BTs).
+sync_range_gather(Ref) ->
+ sync_range_gather(Ref, []).
+
+sync_range_gather(Ref, Acc) ->
+ receive
+ {fold_result, Ref, K, V} ->
+ sync_range_gather(Ref, [{K, V} | Acc]);
+ {fold_done, Ref} ->
+ {ok, Acc}
+ after 3000 ->
+ {error, timeout}
+ end.
View
154 test/lsm_btree_tests.erl
@@ -20,6 +20,7 @@
next_state/3, postcondition/3,
precondition/2]).
+-record(tree, { elements = dict:new() }).
-record(state, { open = dict:new(),
closed = dict:new() }).
-define(SERVER, lsm_btree_drv).
@@ -32,6 +33,8 @@ full_test_() ->
[
?_test(test_tree_simple_1()),
?_test(test_tree_simple_2()),
+ ?_test(test_tree_simple_3()),
+ ?_test(test_tree_simple_4()),
?_test(test_tree()),
{timeout, 120, ?_test(test_qc())}
]}.
@@ -72,16 +75,20 @@ g_open_tree(Open) ->
%% Pick a name of a non-empty Btree
g_non_empty_btree(Open) ->
- ?LET(TreesWithKeys, dict:filter(fun(_K, D) -> dict:size(D) > 0 end, Open),
+ ?LET(TreesWithKeys, dict:filter(fun(_K, #tree { elements = D}) ->
+ dict:size(D) > 0
+ end,
+ Open),
oneof(dict:fetch_keys(TreesWithKeys))).
g_existing_key(Name, Open) ->
- oneof(dict:fetch_keys(dict:fetch(Name, Open))).
+ #tree { elements = Elems } = dict:fetch(Name, Open),
+ oneof(dict:fetch_keys(Elems)).
g_non_existing_key(Name, Open) ->
?SUCHTHAT(Key, g_fail_key(),
begin
- D = dict:fetch(Name, Open),
+ #tree { elements = D } = dict:fetch(Name, Open),
not dict:is_key(Key, D)
end).
@@ -93,7 +100,7 @@ btree_name(I) ->
initial_state() ->
ClosedBTrees = lists:foldl(fun(N, Closed) ->
dict:store(btree_name(N),
- dict:new(),
+ #tree { },
Closed)
end,
dict:new(),
@@ -103,41 +110,43 @@ initial_state() ->
command(#state { open = Open, closed = Closed } = S) ->
frequency(
- [ {20, {call, ?SERVER, open, [oneof(dict:fetch_keys(Closed))]}}
- || closed_dicts(S)] ++
- [ {20, {call, ?SERVER, close, [oneof(dict:fetch_keys(Open))]}}
- || open_dicts(S)] ++
- [ {2000, {call, ?SERVER, put, cmd_put_args(S)}}
- || open_dicts(S)] ++
- [ {1500, {call, ?SERVER, lookup_fail, cmd_lookup_fail_args(S)}}
- || open_dicts(S)] ++
- [ {1500, {call, ?SERVER, lookup_exist, cmd_lookup_args(S)}}
- || open_dicts(S), open_dicts_with_keys(S)] ++
- [ {500, {call, ?SERVER, delete_exist, cmd_delete_args(S)}}
- || open_dicts(S), open_dicts_with_keys(S)]).
+ [ {20, {call, ?SERVER, open, [oneof(dict:fetch_keys(Closed))]}}
+ || closed_dicts(S)]
+ ++ [ {20, {call, ?SERVER, close, [oneof(dict:fetch_keys(Open))]}}
+ || open_dicts(S)]
+ ++ [ {2000, {call, ?SERVER, put, cmd_put_args(S)}}
+ || open_dicts(S)]
+ ++ [ {1500, {call, ?SERVER, lookup_fail, cmd_lookup_fail_args(S)}}
+ || open_dicts(S)]
+ ++ [ {1500, {call, ?SERVER, lookup_exist, cmd_lookup_args(S)}}
+ || open_dicts(S), open_dicts_with_keys(S)]
+ ++ [ {500, {call, ?SERVER, delete_exist, cmd_delete_args(S)}}
+ || open_dicts(S), open_dicts_with_keys(S)]
+ ++ [ {250, {call, ?SERVER, sync_range, cmd_sync_range_args(S)}}
+ || open_dicts(S), open_dicts_with_keys(S)]
+ ).
%% Precondition (abstract)
-precondition(#state { open = _Open}, {call, ?SERVER, delete_exist,
- [_Name, _K]}) ->
- %% No need to check since we limit this in the command/1 generator
- true;
-precondition(#state { open = _Open }, {call, ?SERVER, lookup_fail,
- [_Name, _K]}) ->
- %% We can always try to look up some non-existing key
- true;
-precondition(#state { open = _Open }, {call, ?SERVER, lookup_exist,
- [_Name, _K]}) ->
- %% No need to check since we limit this in the command/1 generator
- true;
+precondition(S, {call, ?SERVER, sync_range, [_Tree, _K1, _K2]}) ->
+ open_dicts(S) andalso open_dicts_with_keys(S);
+precondition(S, {call, ?SERVER, delete_exist, [_Name, _K]}) ->
+ open_dicts(S) andalso open_dicts_with_keys(S);
+precondition(S, {call, ?SERVER, lookup_fail, [_Name, _K]}) ->
+ open_dicts(S);
+precondition(S, {call, ?SERVER, lookup_exist, [_Name, _K]}) ->
+ open_dicts(S) andalso open_dicts_with_keys(S);
precondition(#state { open = Open }, {call, ?SERVER, put, [Name, _K, _V]}) ->
dict:is_key(Name, Open);
-precondition(#state { open = Open, closed = Closed }, {call, ?SERVER, open, [Name]}) ->
+precondition(#state { open = Open, closed = Closed },
+ {call, ?SERVER, open, [Name]}) ->
(not (dict:is_key(Name, Open))) and (dict:is_key(Name, Closed));
-precondition(#state { open = Open, closed = Closed }, {call, ?SERVER, close, [Name]}) ->
+precondition(#state { open = Open, closed = Closed },
+ {call, ?SERVER, close, [Name]}) ->
(dict:is_key(Name, Open)) and (not dict:is_key(Name, Closed)).
-
%% Next state manipulation (abstract / concrete)
+next_state(S, _Res, {call, ?SERVER, sync_range, [_Tree, _K1, _K2]}) ->
+ S;
next_state(S, _Res, {call, ?SERVER, lookup_fail, [_Name, _Key]}) ->
S;
next_state(S, _Res, {call, ?SERVER, lookup_exist, [_Name, _Key]}) ->
@@ -145,31 +154,42 @@ next_state(S, _Res, {call, ?SERVER, lookup_exist, [_Name, _Key]}) ->
next_state(#state { open = Open} = S, _Res,
{call, ?SERVER, delete_exist, [Name, Key]}) ->
S#state { open = dict:update(Name,
- fun(Dict) ->
- dict:erase(Key, Dict)
+ fun(#tree { elements = Dict}) ->
+ #tree { elements =
+ dict:erase(Key, Dict)}
end,
Open)};
next_state(#state { open = Open} = S, _Res,
{call, ?SERVER, put, [Name, Key, Value]}) ->
- S#state { open = dict:update(Name,
- fun(Dict) ->
- dict:store(Key, Value, Dict)
- end,
- Open)};
-next_state(#state { open = Open, closed=Closed} = S, _Res, {call, ?SERVER, open, [Name]}) ->
+ S#state { open = dict:update(
+ Name,
+ fun(#tree { elements = Dict}) ->
+ #tree { elements =
+ dict:store(Key, Value, Dict) }
+ end,
+ Open)};
+next_state(#state { open = Open, closed=Closed} = S,
+ _Res, {call, ?SERVER, open, [Name]}) ->
S#state { open = dict:store(Name, dict:fetch(Name, Closed) , Open),
closed = dict:erase(Name, Closed) };
-next_state(#state { open = Open, closed=Closed} = S, _Res, {call, ?SERVER, close, [Name]}) ->
+next_state(#state { open = Open, closed=Closed} = S, _Res,
+ {call, ?SERVER, close, [Name]}) ->
S#state { closed = dict:store(Name, dict:fetch(Name, Open) , Closed),
open = dict:erase(Name, Open) }.
%% Postcondition check (concrete)
+postcondition(#state { open = Open},
+ {call, ?SERVER, sync_range, [Tree, K1, K2]}, {ok, Result}) ->
+ #tree { elements = TDict } = dict:fetch(Tree, Open),
+ lists:sort(dict_range_query(TDict, K1, K2))
+ == lists:sort(Result);
postcondition(_S,
{call, ?SERVER, lookup_fail, [_Name, _Key]}, notfound) ->
true;
postcondition(#state { open = Open },
{call, ?SERVER, lookup_exist, [Name, Key]}, {ok, Value}) ->
- dict:fetch(Key, dict:fetch(Name, Open)) == Value;
+ #tree { elements = Elems } = dict:fetch(Name, Open),
+ dict:fetch(Key, Elems) == Value;
postcondition(_S, {call, ?SERVER, delete_exist, [_Name, _Key]}, ok) ->
true;
postcondition(_S, {call, ?SERVER, put, [_Name, _Key, _Value]}, ok) ->
@@ -211,8 +231,28 @@ test_tree_simple_2() ->
ok = lsm_btree:delete(Tree, <<"ã">>),
ok = lsm_btree:close(Tree).
-test_tree() ->
+test_tree_simple_3() ->
+ {ok, Tree} = lsm_btree:open("simple"),
+ ok = lsm_btree:put(Tree, <<"X">>, <<"Y">>),
+ {ok, Ref} = lsm_btree:sync_range(Tree, <<"X">>, <<"X">>),
+ ?assertEqual(ok,
+ receive
+ {fold_done, Ref} -> ok
+ after 1000 -> {error, timeout}
+ end),
+ ok = lsm_btree:close(Tree).
+
+test_tree_simple_4() ->
+ Key = <<56,11,62,42,35,163,16,100,9,224,8,228,130,94,198,2,126,117,243,
+ 1,122,175,79,159,212,177,30,153,71,91,85,233,41,199,190,58,3,
+ 173,220,9>>,
+ Value = <<212,167,12,6,105,152,17,80,243>>,
+ {ok, Tree} = lsm_btree:open("simple"),
+ ok = lsm_btree:put(Tree, Key, Value),
+ ?assertEqual({ok, Value}, lsm_btree:lookup(Tree, Key)),
+ ok = lsm_btree:close(Tree).
+test_tree() ->
application:start(sasl),
{ok, Tree} = lsm_btree:open("simple2"),
@@ -289,22 +329,35 @@ cmd_delete_args(#state { open = Open}) ->
?LET(Key, g_existing_key(Name, Open),
[Name, Key])).
+cmd_sync_range_args(#state { open = Open }) ->
+ ?LET(Tree, g_non_empty_btree(Open),
+ ?LET({K1, K2}, {g_existing_key(Tree, Open),
+ g_existing_key(Tree, Open)},
+ [Tree, K1, K2])).
%% Context management
%% ----------------------------------------------------------------------
-cleanup_test_trees(#state { open = Open}) ->
- [cleanup_tree(N) || N <- dict:fetch_keys(Open)].
+cleanup_test_trees(#state { open = Open, closed = Closed }) ->
+ [cleanup_tree(N) || N <- dict:fetch_keys(Open)],
+ [cleanup_tree(N) || N <- dict:fetch_keys(Closed)].
cleanup_tree(Tree) ->
- {ok, FileNames} = file:list_dir(Tree),
- [ok = file:delete(filename:join([Tree, Fname])) || Fname <- FileNames],
- file:del_dir(Tree).
+ case file:list_dir(Tree) of
+ {error, enoent} ->
+ ok;
+ {ok, FileNames} ->
+ [ok = file:delete(filename:join([Tree, Fname]))
+ || Fname <- FileNames],
+ file:del_dir(Tree)
+ end.
%% Various Helper routines
%% ----------------------------------------------------------------------
open_dicts_with_keys(#state { open = Open}) ->
- lists:any(fun({_, D}) -> dict:size(D) > 0 end,
+ lists:any(fun({_, #tree { elements = D}}) ->
+ dict:size(D) > 0
+ end,
dict:to_list(Open)).
open_dicts(#state { open = Open}) ->
@@ -313,4 +366,9 @@ open_dicts(#state { open = Open}) ->
closed_dicts(#state { closed = Closed}) ->
dict:size(Closed) > 0.
+dict_range_query(Dict, LowKey, HighKey) ->
+ [{K, V} || {K, V} <- dict:to_list(Dict),
+ K >= LowKey,
+ K < HighKey].
+
Something went wrong with that request. Please try again.