Skip to content
Browse files

More changes

  • Loading branch information...
1 parent 51d1cab commit 8765905c614a2d14c77bea6ceb32e791190e368e @bsparrow435 bsparrow435 committed Nov 19, 2012
Showing with 19 additions and 25 deletions.
  1. +19 −22 src/bitcask.erl
  2. +0 −3 src/bitcask_merge_worker.erl
View
41 src/bitcask.erl
@@ -32,7 +32,7 @@
fold_keys/3, fold_keys/5,
fold/3, fold/5,
iterator/3, iterator_next/1, iterator_release/1,
- merge/1, merge/2, merge/3, merge/4,
+ merge/1, merge/2, merge/3,
needs_merge/1,
is_empty_estimate/1,
status/1]).
@@ -448,45 +448,41 @@ iterator_release(Ref) ->
%% into a more compact form.
-spec merge(Dirname::string()) -> ok.
merge(Dirname) ->
- merge(Dirname, [], readable_files(Dirname)).
+ merge(Dirname, [], {readable_files(Dirname), []}).
%% @doc Merge several data files within a bitcask datastore
%% into a more compact form.
-spec merge(Dirname::string(), Opts::[_]) -> ok.
merge(Dirname, Opts) ->
- merge(Dirname, Opts, readable_files(Dirname)).
+ merge(Dirname, Opts, {readable_files(Dirname), []}).
%% @doc Merge several data files within a bitcask datastore
%% into a more compact form.
-spec merge(Dirname::string(), Opts::[_], FilesToMerge::[string()]) -> ok.
-merge(_Dirname, _Opts, []) ->
+merge(_Dirname, _Opts, {[],_}) ->
ok;
-merge(Dirname, Opts, FilesToMerge0) ->
+merge(Dirname, Opts, {FilesToMerge0, ExpiredFiles0}) ->
%% Make sure bitcask app is started so we can pull defaults from env
ok = start_app(),
-
%% Filter the files to merge and ensure that they all exist. It's
%% possible in some circumstances that we'll get an out-of-date
%% list of files.
FilesToMerge = [F || F <- FilesToMerge0,
filelib:is_file(F)],
- merge1(Dirname, Opts, FilesToMerge, []).
-
-%% @doc
--spec merge(Dirname::string(), Opts::[_], FilesToMerge::[string()], ExpiredFiles::[string()]) -> ok.
-merge(Dirname, Opts, FilesToMerge0, []) ->
- merge(Dirname, Opts, FilesToMerge0);
-merge(Dirname, Opts, FilesToMerge0, ExpiredFiles0) ->
+ ExpiredFiles = [F || F <- ExpiredFiles0,
+ filelib:is_file(F)],
+ merge1(Dirname, Opts, FilesToMerge, ExpiredFiles);
+merge(_Dirname, _Opts, []) ->
+ ok;
+merge(Dirname, Opts, FilesToMerge0) ->
%% Make sure bitcask app is started so we can pull defaults from env
ok = start_app(),
%% Filter the files to merge and ensure that they all exist. It's
%% possible in some circumstances that we'll get an out-of-date
%% list of files.
FilesToMerge = [F || F <- FilesToMerge0,
filelib:is_file(F)],
- ExpiredFiles = [F || F <- ExpiredFiles0,
- filelib:is_file(F)],
- merge1(Dirname, Opts, FilesToMerge, ExpiredFiles).
+ merge1(Dirname, Opts, FilesToMerge, []).
%% Inner merge function, assumes that bitcask is running and all files exist.
@@ -582,7 +578,6 @@ merge1(Dirname, Opts, FilesToMerge, ExpiredFiles) ->
end, [], InFiles1)),
InExpiredFiles = [F#file_status.filename || F <- Summary,
F#file_status.newest_tstamp < expiry_time(Opts)],
- expiry_merge(InExpiredFiles, LiveKeyDir),
%% Setup our first output merge file and update the merge lock accordingly
{ok, Outfile} = bitcask_fileops:create_file(Dirname, Opts),
@@ -607,6 +602,7 @@ merge1(Dirname, Opts, FilesToMerge, ExpiredFiles) ->
opts = Opts },
%% Finally, start the merge process
+ expiry_merge(InExpiredFiles, LiveKeyDir),
State1 = merge_files(State),
%% Make sure to close the final output file
@@ -708,7 +704,7 @@ needs_merge(Ref) ->
end,
FileNames = [Filename || {Filename, _Reasons} <- MergableFiles],
ExpiredFiles = [Filename || {Filename, [{oldest_tstamp,_,_}]} <- MergableFiles],
- {true, FileNames, ExpiredFiles};
+ {true, {FileNames, ExpiredFiles}};
false ->
false
end.
@@ -1020,6 +1016,7 @@ merge_files(#mstate { dirname = Dirname,
[File#filestate.filename, Dirname, Error]),
State
after
+ error_logger:info_msg("BRIAN SPARROW! Just folded over ~p\n", [File]),
catch bitcask_fileops:close(File)
end,
merge_files(State2#mstate { input_files = Rest }).
@@ -1315,10 +1312,10 @@ expiry_merge([File | Files], LiveKeyDir) ->
bitcask_nifs:keydir_remove(LiveKeyDir, K, Tstamp, FileId, Offset),
Acc
end,
- case bitcask_fileops:fold_keys(FileDesc, Fun, accumulator, hintfile) of
+ case bitcask_fileops:fold_keys(FileDesc, Fun, ok, hintfile) of
{error, Stuff} ->
error_logger:error_msg("Ohhh nooo! something went wrong: ~p\n", Stuff);
- Result ->
+ ok ->
error_logger:info_msg("YAARRRRR all keys expired in: ~p\n", [File]),
bitcask_fileops:delete(FileDesc),
bitcask_fileops:close(FileDesc)
@@ -1725,9 +1722,9 @@ delete_partial_merge_test() ->
%% selective merge, hit all of the files with deletes but not
%% all of the ones with deleted data
timer:sleep(1100),
- ok = merge("/tmp/bc.test.pardel",[],lists:reverse(lists:nthtail(2,
+ ok = merge("/tmp/bc.test.pardel",[],{lists:reverse(lists:nthtail(2,
lists:reverse(readable_files(
- "/tmp/bc.test.pardel"))))),
+ "/tmp/bc.test.pardel")))),[]}),
%% Verify we've now only got one item left
B2 = bitcask:open("/tmp/bc.test.pardel"),
View
3 src/bitcask_merge_worker.erl
@@ -61,9 +61,6 @@ merge(Dir, Opts) ->
merge(Dir, Opts, Files) ->
gen_server:call(?MODULE, {merge, [Dir, Opts, Files]}, infinity).
-merge(Dir, Opts, Files, ExpiredFiles) ->
- gen_server:call(?MODULE, {merge, [Dir, Opts, Files, ExpiredFiles]}, infinity).
-
%% ====================================================================
%% gen_server callbacks
%% ====================================================================

0 comments on commit 8765905

Please sign in to comment.
Something went wrong with that request. Please try again.