Skip to content
Browse files

JIRA: ARAKOON-229 backport

  • Loading branch information...
1 parent 3cc2115 commit ef5c0c2c78b849755b3f05e46e91d8d2445a0672 Romain Slootmaekers committed Aug 2, 2011
Showing with 40 additions and 6 deletions.
  1. +6 −6 src/tlog/compression.ml
  2. +34 −0 src/tlog/tlc2_test.ml
View
12 src/tlog/compression.ml
@@ -31,7 +31,8 @@ let tlog_name archive_name =
let compress_tlog tlog_name archive_name =
- let limit = 1024 * 1024 in
+ let limit = 896 * 1024 in
+ let buffer_size = limit + (64 * 1024) in
Lwt_io.with_file ~mode:Lwt_io.input tlog_name
(fun ic ->
Lwt_io.with_file ~mode:Lwt_io.output archive_name
@@ -43,10 +44,9 @@ let compress_tlog tlog_name archive_name =
Tlogcommon.entry_to buffer i u;
let (last_i':Sn.t) = if i > last_i then i else last_i
in
- if Buffer.length buffer < limit then
- fill_buffer (buffer:Buffer.t) last_i' (counter+1)
- else
- Lwt.return (last_i',counter)
+ if Buffer.length buffer < limit || counter = 0
+ then fill_buffer (buffer:Buffer.t) last_i' (counter+1)
+ else Lwt.return (last_i',counter)
)
(function
| End_of_file -> Lwt.return (last_i,counter)
@@ -59,7 +59,7 @@ let compress_tlog tlog_name archive_name =
Llio.output_int64 oc last_i >>= fun () ->
Llio.output_string oc output
in
- let buffer = Buffer.create limit in
+ let buffer = Buffer.create buffer_size in
let rec loop () =
fill_buffer buffer (-1L) 0 >>= fun (last_i,counter) ->
if counter = 0
View
34 src/tlog/tlc2_test.ml
@@ -24,6 +24,7 @@ open OUnit
open Lwt
open Update
open Extra
+open Unix
let wrap_tlc = Tlogcollection_test.wrap Tlc2.make_tlc2
@@ -168,6 +169,38 @@ let test_iterate6 (dn,factory) =
OUnit.assert_equal ~printer:string_of_int 38 !sum;
Lwt.return ()
+let test_compression_bug (dn, factory) =
+ let () = Tlogcommon.tlogEntriesPerFile := 10 in
+ let v = String.create (1024 * 1024) in
+ factory dn >>= fun tlc ->
+ let n = 12 in
+ let rec loop i =
+ if i = n then Lwt.return ()
+ else
+ let key = Printf.sprintf "test_compression_bug_%i" i in
+ let update = Update.Set(key, v) in
+ let sni = Sn.of_int i in
+ tlc # log_update sni update >>= fun _ ->
+ loop (i+1)
+ in
+ tlc # log_update 0L (Update.Set("xxx","XXX")) >>= fun _ ->
+ loop 1 >>= fun () ->
+ tlc # close () >>= fun () ->
+ Lwt_unix.sleep 20.0 >>= fun () ->
+ Lwt_preemptive.detach Unix.stat (dn ^ "/000.tlf") >>= fun stat ->
+ OUnit.assert_bool "file should have size >0" (stat.st_size > 0);
+ let entries = ref [] in
+ factory dn >>= fun tlc2 ->
+ tlc2 # iterate 0L (Sn.of_int n)
+ (fun (i,u) -> entries := i :: !entries;
+ Lwt_log.debug_f "ENTRY: i=%Li" i)
+ >>= fun () ->
+ OUnit.assert_equal
+ ~printer:string_of_int
+ ~msg:"tlc has a hole" n (List.length !entries);
+ Lwt.return ()
+
+
let suite = "tlc2" >:::[
"regexp" >:: wrap_tlc Tlogcollection_test.test_regexp;
"empty_collection" >:: wrap_tlc Tlogcollection_test.test_empty_collection;
@@ -185,4 +218,5 @@ let suite = "tlc2" >:::[
"test_rollover_1002" >:: wrap_tlc Tlogcollection_test.test_rollover_1002;
"test_rollover_boundary" >:: wrap_tlc test_validate_at_rollover_boundary;
"test_interrupted_rollover" >:: wrap_tlc test_interrupted_rollover;
+ "test_compression_bug" >:: wrap_tlc test_compression_bug;
]

0 comments on commit ef5c0c2

Please sign in to comment.
Something went wrong with that request. Please try again.