Dss timeshift crc

src/bitcask.erl

@@ -1052,7 +1052,7 @@ do_put(Key, Value, #bc_state{write_file = WriteFile} = State, Retries, _LastErr)
             %% wrap to a new file with a greater file_id and rewrite
             %% the key there.  Limit the number of recursions in case
             %% there is a different issue with the keydir.
-            State3 = wrap_write_file(State2),
+            State3 = wrap_write_file(State2#bc_state { write_file = WriteFile2 }),
             do_put(Key, Value, State3, Retries - 1, already_exist)
     end.
 

src/bitcask_fileops.erl

@@ -53,8 +53,8 @@
 -ifdef(EQC).
 -include_lib("eqc/include/eqc.hrl").
 -include_lib("eqc/include/eqc_fsm.hrl").
--compile(export_all).
 -endif.
+-compile(export_all).
 -include_lib("eunit/include/eunit.hrl").
 -endif.
 
@@ -81,34 +81,33 @@ open_file(Filename) ->
 -spec close(#filestate{} | fresh | undefined) -> ok.
 close(fresh) -> ok;
 close(undefined) -> ok;
-close(#filestate{ fd = FD, hintfd = HintFd }) ->
+close(State = #filestate{ fd = FD }) ->
+    S2 = close_hintfile(State),
     bitcask_nifs:file_close(FD),
-    case HintFd of
-        undefined ->
-            ok;
-        _ ->
-            bitcask_nifs:file_close(HintFd)
-    end,
     ok.
 
 %% @doc Close a file for writing, but leave it open for reads.
 -spec close_for_writing(#filestate{} | fresh | undefined) -> #filestate{} | ok.
 close_for_writing(fresh) -> ok;
 close_for_writing(undefined) -> ok;
-close_for_writing(State = 
-                  #filestate{ mode = read_write, fd = Fd, 
-                              hintfd = HintFd, hintcrc = HintCRC }) ->
-    %% Write out CRC check at end of hint file.  Write with an empty key,
-    %% zero timestamp and offset as large as the file format supports so
-    %% opening with an older version of bitcask will just reject the 
-    %% record at the end of the hintfile and otherwise work normally.
+close_for_writing(State = #filestate{ mode = read_write, fd = Fd }) ->
+    S2 = close_hintfile(State),
+    bitcask_nifs:file_sync(Fd),
+    S2#filestate { mode = read_only }.
+
+close_hintfile(State = #filestate { hintfd = undefined }) ->
+    State;
+close_hintfile(State = #filestate { hintfd = HintFd, hintcrc = HintCRC }) ->
+    %% Write out CRC check at end of hint file.  Write with an empty key, zero
+    %% timestamp and offset as large as the file format supports so opening with
+    %% an older version of bitcask will just reject the record at the end of the
+    %% hintfile and otherwise work normally.
     Iolist = hintfile_entry(<<>>, 0, {?MAXOFFSET, HintCRC}),
     ok = bitcask_nifs:file_write(HintFd, Iolist),
-
-    bitcask_nifs:file_sync(Fd),
     bitcask_nifs:file_sync(HintFd),
     bitcask_nifs:file_close(HintFd),
-    State#filestate { mode = read_only, hintfd = undefined }.
+    State#filestate { hintfd = undefined, hintcrc = 0 }.
+
 
 %% Build a list of {tstamp, filename} for all files in the directory that
 %% match our regex. 

test/bitcask_qc.erl

@@ -196,7 +196,20 @@ prop_merge() ->
                          true
                      after
                          bitcask:close(Ref)
-                     end
+                     end,
+
+                     %% For each of the data files, validate that it has a valid hint file
+                     Validate = fun(Fname) ->
+                                        {ok, S} = bitcask_fileops:open_file(Fname),
+                                        try
+                                            ?assertEqual(true, bitcask_fileops:has_valid_hintfile(S))
+                                        after
+                                            bitcask_fileops:close(S)
+                                        end
+                                end,
+                     [Validate(Fname) || {_Ts, Fname} <-
+                                             bitcask_fileops:data_file_tstamps("/tmp/bc.prop.merge")],
+                     true
                  end)).
 
 

test/bitcask_timeshift.erl

@@ -0,0 +1,80 @@
+%% -------------------------------------------------------------------
+%%
+%% bitcask: Eric Brewer-inspired key/value store
+%%
+%% Copyright (c) 2012 Basho Technologies, Inc. All Rights Reserved.
+%%
+%% This file is provided to you under the Apache License,
+%% Version 2.0 (the "License"); you may not use this file
+%% except in compliance with the License.  You may obtain
+%% a copy of the License at
+%%
+%%   http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing,
+%% software distributed under the License is distributed on an
+%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+%% KIND, either express or implied.  See the License for the
+%% specific language governing permissions and limitations
+%% under the License.
+%%
+%% -------------------------------------------------------------------
+-module(bitcask_timeshift).
+
+-include_lib("eunit/include/eunit.hrl").
+-include("bitcask.hrl").
+
+-compile(export_all).
+
+current_tstamp() ->
+    case erlang:get(meck_tstamp) of
+        undefined ->
+            erlang:error(uninitialized_meck_tstamp);
+        Value ->
+            Value
+    end.
+
+next_tstamp() ->
+    Ts = case erlang:get(meck_tstamp) of
+             undefined ->
+                 1;
+             Tstamp ->
+                 Tstamp + erlang:get(meck_tstamp_step)
+         end,
+    erlang:put(meck_tstamp, Ts),
+    Ts.
+
+set_tstamp(Tstamp) ->
+    erlang:put(meck_tstamp, Tstamp).
+
+set_tstamp_step(Step) ->
+    erlang:put(meck_tstamp_step, Step).
+
+timeshift_test() ->
+    try
+        meck:new(bitcask_time, [passthrough]),
+        meck:expect(bitcask_time, tstamp, fun next_tstamp/0),
+        set_tstamp(100),
+        set_tstamp_step(-1),
+
+        ?cmd("rm -rf /tmp/bc.timeshift"),
+        Bref = bitcask:open("/tmp/bc.timeshift", [read_write]),
+        ok = bitcask:put(Bref, <<"k1">>, <<"v1">>),
+        {error, _} = bitcask:put(Bref, <<"k1">>, <<"v2">>),
+        bitcask:close(Bref),
+
+        %% For each of the data files, validate that it has a valid hint file
+        Validate = fun(Fname) ->
+                           {ok, S} = bitcask_fileops:open_file(Fname),
+                           try
+                               ?assert(bitcask_fileops:has_valid_hintfile(S))
+                           after
+                               bitcask_fileops:close(S)
+                           end
+                   end,
+        [Validate(Fname) || {_Ts, Fname} <-
+                                bitcask_fileops:data_file_tstamps("/tmp/bc.timeshift")]
+
+    after
+        meck:unload()
+    end.