Permalink
Browse files

Merge pull request #66 from basho/slf-crc-error-spam

Fix log spam introduced by branch 'gh62-badrecord-mstate'
  • Loading branch information...
2 parents f927f1e + 3c16f75 commit 859013d1c80228f23c2470ad7c6c4348235f7486 @slfritchie slfritchie committed Nov 14, 2012
Showing with 41 additions and 6 deletions.
  1. +27 −0 src/bitcask.erl
  2. +14 −6 src/bitcask_fileops.erl
View
@@ -1671,6 +1671,33 @@ truncated_datafile_test() ->
{1, [{_, _, _, 513}]} = bitcask:status(B2),
ok.
+trailing_junk_big_datafile_test() ->
+ Dir = "/tmp/bc.test.trailingdata",
+ NumKeys = 400,
+ os:cmd("rm -rf " ++ Dir),
+ os:cmd("mkdir " ++ Dir),
+ B1 = bitcask:open(Dir, [read_write, {max_file_size, 1024*1024*1024}]),
+ [ok = bitcask:put(B1, <<"k", X:32>>, <<X:1024>>) || X <- lists:seq(1, NumKeys)],
+ ok = bitcask:close(B1),
+
+ [DataFile|_] = filelib:wildcard(Dir ++ "/*.data"),
+ {ok, FH} = file:open(DataFile, [read, write]),
+ {ok, _} = file:position(FH, 40*1024),
+ ok = file:write(FH, <<0:(40*1024*8)>>),
+ ok = file:close(FH),
+
+ %% Merge everything
+ ok = merge(Dir),
+
+ B2 = bitcask:open(Dir, [read_write]),
+ KeyList = bitcask:fold(B2, fun(K, _V, Acc0) -> [K|Acc0] end, []),
+ true = length(KeyList) < NumKeys,
+ ArbKey = 5, % get arbitrary key near start
+ {ok, <<ArbKey:1024>>} = bitcask:get(B2, <<"k", ArbKey:32>>),
+ ok = bitcask:close(B2),
+
+ ok.
+
truncated_merge_test() ->
Dir = "/tmp/bc.test.truncmerge",
os:cmd("rm -rf " ++ Dir),
View
@@ -210,7 +210,7 @@ fold(#filestate { fd=Fd, filename=Filename, tstamp=FTStamp }, Fun, Acc) ->
case bitcask_nifs:file_read(Fd, ?HEADER_SIZE) of
{ok, <<_Crc:?CRCSIZEFIELD, _Tstamp:?TSTAMPFIELD, _KeySz:?KEYSIZEFIELD,
_ValueSz:?VALSIZEFIELD>> = H} ->
- fold_loop(Fd, Filename, FTStamp, H, 0, Fun, Acc);
+ fold_loop(Fd, Filename, FTStamp, H, 0, Fun, Acc, 0);
{ok, OtherBytes} ->
error_logger:error_msg("~s:fold: ~s: expected ~p bytes but got "
"only ~p bytes, skipping\n",
@@ -322,27 +322,35 @@ has_valid_hintfile(State) ->
%% Internal functions
%% ===================================================================
-fold_loop(Fd, Filename, FTStamp, Header, Offset, Fun, Acc0) ->
+fold_loop(_Fd, Filename, _FTStamp, _Header, Offset, _Fun, Acc, 20) ->
+ error_logger:error_msg("fold_loop: CRC error limit at file ~p offset ~p\n",
+ [Filename, Offset]),
+ Acc;
+fold_loop(Fd, Filename, FTStamp, Header, Offset, Fun, Acc0, CrcSkipCount) ->
<<Crc32:?CRCSIZEFIELD, Tstamp:?TSTAMPFIELD, KeySz:?KEYSIZEFIELD,
ValueSz:?VALSIZEFIELD>> = Header,
<<_:4/binary, HeaderMinusCRC/binary>> = Header,
TotalSz = KeySz + ValueSz + ?HEADER_SIZE,
case bitcask_nifs:file_read(Fd, TotalSz) of
{ok, <<Key:KeySz/bytes, Value:ValueSz/bytes, Rest/binary>>} ->
- Acc = case erlang:crc32([HeaderMinusCRC, Key, Value]) of
- Crc32 ->
+ CrcMatch = erlang:crc32([HeaderMinusCRC, Key, Value]) =:= Crc32,
+ Acc = case CrcMatch of
+ true ->
PosInfo = {Filename, FTStamp, Offset, TotalSz},
Fun(Key, Value, Tstamp, PosInfo, Acc0);
- _ ->
+ false ->
error_logger:error_msg(
"fold_loop: CRC error at file ~s offset ~p, "
"skipping ~p bytes\n", [Filename, Offset, TotalSz]),
Acc0
end,
case Rest of
<<NextHeader:?HEADER_SIZE/bytes>> ->
+ NewCrcSkipCount = if (not CrcMatch) -> CrcSkipCount + 1;
+ true -> CrcSkipCount
+ end,
fold_loop(Fd, Filename, FTStamp, NextHeader,
- Offset + TotalSz, Fun, Acc);
+ Offset + TotalSz, Fun, Acc, NewCrcSkipCount);
<<>> ->
Acc;
Tail ->

0 comments on commit 859013d

Please sign in to comment.