Skip to content

Commit

Permalink
irmin-pack: eio backend
Browse files Browse the repository at this point in the history
  • Loading branch information
clecat authored and art-w committed May 15, 2024
1 parent 9d606d7 commit 9046cf5
Show file tree
Hide file tree
Showing 104 changed files with 2,831 additions and 1,946 deletions.
38 changes: 14 additions & 24 deletions bench/irmin-pack/bench_common.ml
Original file line number Diff line number Diff line change
Expand Up @@ -60,20 +60,11 @@ let random_string n = String.init n (fun _i -> random_char ())
let random_blob () = random_string 10 |> Bytes.of_string
let random_key () = random_string 5

let default_artefacts_dir =
let ( / ) = Filename.concat in
Unix.getcwd () / "_artefacts" / Uuidm.to_string (Uuidm.v `V4)
let default_artefacts_dir cwd =
Eio.Path.(cwd / "_artefacts" / Uuidm.to_string (Uuidm.v `V4))

let prepare_artefacts_dir path =
let rec mkdir_p path =
if Sys.file_exists path then ()
else
let path' = Filename.dirname path in
if path' = path then failwith "Failed to prepare result dir";
mkdir_p path';
Unix.mkdir path 0o755
in
mkdir_p path
Eio.Path.mkdirs ~exists_ok:true ~perm:0o755 path

let with_timer f =
let t0 = Sys.time () in
Expand Down Expand Up @@ -121,27 +112,26 @@ end

module FSHelper = struct
let file f =
try (Unix.stat f).st_size with Unix.Unix_error (Unix.ENOENT, _, _) -> 0
(* in MiB *)
try
Eio.Switch.run @@ fun sw ->
let f = Eio.Path.open_in ~sw f in
Optint.Int63.to_int (Eio.File.size f)
with Eio.Exn.Io (Eio.Fs.E (Not_found _), _) -> 0

let dict root = file (Irmin_pack.Layout.V1_and_v2.dict ~root) / 1024 / 1024
let pack root = file (Irmin_pack.Layout.V1_and_v2.pack ~root) / 1024 / 1024

let index root =
let index_dir = Filename.concat root "index" in
let a = file (Filename.concat index_dir "data") in
let b = file (Filename.concat index_dir "log") in
let c = file (Filename.concat index_dir "log_async") in
let index_dir = Eio.Path.(root / "index") in
let a = file Eio.Path.(index_dir / "data") in
let b = file Eio.Path.(index_dir / "log") in
let c = file Eio.Path.(index_dir / "log_async") in
(a + b + c) / 1024 / 1024

let size root = dict root + pack root + index root
let get_size root = size root

let rm_dir root =
if Sys.file_exists root then (
let cmd = Printf.sprintf "rm -rf %s" root in
[%logs.info "exec: %s" cmd];
let _ = Sys.command cmd in
())
let rm_dir root = Eio.Path.rmtree ~missing_ok:true root
end

module Generate_trees
Expand Down
8 changes: 4 additions & 4 deletions bench/irmin-pack/bench_common.mli
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@

module Mtime : module type of Import.Mtime

val default_artefacts_dir : string
val prepare_artefacts_dir : string -> unit
val default_artefacts_dir : Eio.Fs.dir_ty Eio.Path.t -> Eio.Fs.dir_ty Eio.Path.t
val prepare_artefacts_dir : Eio.Fs.dir_ty Eio.Path.t -> unit
val reporter : ?prefix:string -> unit -> Logs.reporter
val setup_log : Fmt.style_renderer option -> Logs.level option -> unit
val reset_stats : unit -> unit
Expand All @@ -36,8 +36,8 @@ module Conf : Irmin_pack.Conf.S
module Schema : Irmin.Schema.S

module FSHelper : sig
val rm_dir : string -> unit
val get_size : string -> int
val rm_dir : Eio.Fs.dir_ty Eio.Path.t -> unit
val get_size : Eio.Fs.dir_ty Eio.Path.t -> int
end

module Generate_trees
Expand Down
2 changes: 1 addition & 1 deletion bench/irmin-pack/dune
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@
(executable
(name trace_stats)
(modules trace_stats)
(libraries cmdliner irmin_traces))
(libraries cmdliner irmin_traces eio_main))

;; Require the executables to compile during tests

Expand Down
22 changes: 16 additions & 6 deletions bench/irmin-pack/main.ml
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,6 @@
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*)

let config ~root = Irmin_pack.config ~fresh:false root

module Config = struct
let entries = 2
let stable_hash = 3
Expand All @@ -33,15 +31,19 @@ module Bench = Irmin_bench.Make (KV)

let file f =
(* in MiB *)
try (Unix.stat f).st_size / 1024 / 1024
with Unix.Unix_error (Unix.ENOENT, _, _) -> 0
try
Eio.Switch.run @@ fun sw ->
let open Optint.Int63 in
let f = Eio.Path.open_in ~sw f in
Infix.(to_int @@ (Eio.File.size f / of_int 1024 / of_int 1024))
with Eio.Exn.Io (Eio.Fs.E (Not_found _), _) -> 0

let index root =
let rec aux acc i =
if i = 256 then acc
else
let filename = Format.sprintf "store.index.%d" i in
let s = file (Filename.concat root filename) in
let s = file Eio.Path.(root / filename) in
aux (acc + s) (i + 1)
in
aux 0 0
Expand All @@ -52,4 +54,12 @@ let size ~root =
|> List.map file
|> List.fold_left ( + ) index_size

let () = Bench.run ~config ~size
let () =
Eio_main.run @@ fun env ->
Eio.Switch.run @@ fun sw ->
let fs = Eio.Stdenv.cwd env in
let config ~root =
Irmin_pack.config ~sw ~fs ~fresh:false Eio.Path.(fs / root)
in
let size ~root = size ~root:Eio.Path.(fs / root) in
Bench.run ~config ~size
2 changes: 1 addition & 1 deletion bench/irmin-pack/trace_collection.ml
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@ module Make_stat (Store : Irmin.Generic_key.KV) = struct
}
end

let create_file : string -> Def.config -> string -> t =
let create_file : Eio.Fs.dir_ty Eio.Path.t -> Def.config -> string -> t =
fun path config store_path ->
let header =
Def.
Expand Down
5 changes: 4 additions & 1 deletion bench/irmin-pack/trace_common.ml
Original file line number Diff line number Diff line change
Expand Up @@ -228,8 +228,10 @@ module Io (Ff : File_format) = struct
in
Seq.unfold produce_row ()

let open_reader : string -> Ff.Latest.header * Ff.Latest.row Seq.t =
let open_reader :
Eio.Fs.dir_ty Eio.Path.t -> Ff.Latest.header * Ff.Latest.row Seq.t =
fun path ->
let path = Eio.Path.native_exn path in
let chan = open_in_bin path in
let len = LargeFile.in_channel_length chan in
if len < 12L then
Expand Down Expand Up @@ -260,6 +262,7 @@ module Io (Ff : File_format) = struct
type writer = { path : string; channel : out_channel; buffer : Buffer.t }

let create_file path header =
let path = Eio.Path.native_exn path in
let channel = open_out path in
let buffer = Buffer.create 0 in
output_string channel (Magic.to_string Ff.magic);
Expand Down
37 changes: 23 additions & 14 deletions bench/irmin-pack/trace_replay.ml
Original file line number Diff line number Diff line change
Expand Up @@ -370,8 +370,8 @@ module Make (Store : Store) = struct
let really_add_volume = time_to_add_volume in
(really_wait_gc, really_start_gc, really_split, really_add_volume)

let add_commits config repo commit_seq on_commit on_end stats check_hash
empty_blobs =
let add_commits ~fs ~domain_mgr config repo commit_seq on_commit on_end stats
check_hash empty_blobs =
let max_ncommits = config.number_of_commits_to_replay in
with_progress_bar ~message:"Replaying trace" ~n:max_ncommits ~unit:"commit"
@@ fun prog ->
Expand Down Expand Up @@ -444,7 +444,7 @@ module Make (Store : Store) = struct
commit_duration duration finalise_duration]
| Error s -> failwith s
in
Store.gc_run ~finished repo gc_commit_key)
Store.gc_run ~fs ~domain_mgr ~finished repo gc_commit_key)
in
let () = add_operations t repo ops i stats check_hash empty_blobs in
t.latest_commit_idx <- i;
Expand All @@ -465,8 +465,14 @@ module Make (Store : Store) = struct
in
aux commit_seq 0

let run : type a. _ -> a config -> a =
fun ext_config config ->
let run :
type a.
fs:Eio.Fs.dir_ty Eio.Path.t ->
domain_mgr:_ Eio.Domain_manager.t ->
_ ->
a config ->
a =
fun ~fs ~domain_mgr ext_config config ->
let check_hash =
config.path_conversion = `None
&& config.inode_config = (32, 256)
Expand All @@ -475,14 +481,15 @@ module Make (Store : Store) = struct
[%logs.app
"Will %scheck commit hashes against reference."
(if check_hash then "" else "NOT ")];
Eio.Switch.run @@ fun sw ->
let commit_seq =
open_commit_sequence config.number_of_commits_to_replay
config.path_conversion config.replay_trace_path
in
let root = Filename.concat config.artefacts_path "root" in
let repo, on_commit, on_end = Store.create_repo ~root ext_config in
let root = Eio.Path.(config.artefacts_path / "root") in
let repo, on_commit, on_end = Store.create_repo ~sw ~fs ~root ext_config in
prepare_artefacts_dir config.artefacts_path;
let stat_path = Filename.concat config.artefacts_path "stat_trace.repr" in
let stat_path = Eio.Path.(config.artefacts_path / "stat_trace.repr") in
let c =
let entries, stable_hash = config.inode_config in
Trace_definitions.Stat_trace.
Expand All @@ -491,19 +498,21 @@ module Make (Store : Store) = struct
`Replay
{
path_conversion = config.path_conversion;
artefacts_dir = config.artefacts_path;
artefacts_dir = Eio.Path.native_exn config.artefacts_path;
};
inode_config = (entries, entries, stable_hash);
store_type = config.store_type;
}
in
let stats = Stat_collector.create_file stat_path c root in
let stats =
Stat_collector.create_file stat_path c (Eio.Path.native_exn root)
in
Irmin_pack.Stats.reset_stats ();
Fun.protect
(fun () ->
let block_count =
add_commits config repo commit_seq on_commit on_end stats check_hash
config.empty_blobs
add_commits ~fs ~domain_mgr config repo commit_seq on_commit on_end
stats check_hash config.empty_blobs
in
[%logs.app "Closing repo..."];
let () = Store.Repo.close repo in
Expand All @@ -515,7 +524,7 @@ module Make (Store : Store) = struct
Trace_stat_summary.summarise ~block_count stat_path)
~finally:(fun () ->
if config.keep_stat_trace then (
[%logs.app "Stat trace kept at %s" stat_path];
Unix.chmod stat_path 0o444)
[%logs.app "Stat trace kept at %s" (Eio.Path.native_exn stat_path)];
Unix.chmod (Eio.Path.native_exn stat_path) 0o444)
else Stat_collector.remove stats)
end
26 changes: 21 additions & 5 deletions bench/irmin-pack/trace_replay_intf.ml
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@ module Config = struct
path_conversion : [ `None | `V1 | `V0_and_v1 | `V0 ];
inode_config : int * int;
store_type : [ `Pack | `Pack_layered | `Pack_mem ];
replay_trace_path : string;
artefacts_path : string;
replay_trace_path : Eio.Fs.dir_ty Eio.Path.t;
artefacts_path : Eio.Fs.dir_ty Eio.Path.t;
keep_store : bool;
keep_stat_trace : bool;
empty_blobs : bool;
Expand Down Expand Up @@ -99,15 +99,26 @@ module type Store = sig
type on_commit := int -> Hash.t -> unit
type on_end := unit -> unit

val create_repo : root:string -> store_config -> Repo.t * on_commit * on_end
val create_repo :
sw:Eio.Switch.t ->
fs:Eio.Fs.dir_ty Eio.Path.t ->
root:Eio.Fs.dir_ty Eio.Path.t ->
store_config ->
Repo.t * on_commit * on_end

val split : repo -> unit
val add_volume : repo -> unit
val gc_wait : repo -> unit

type stats := Irmin_pack_unix.Stats.Latest_gc.stats

val gc_run :
?finished:((stats, string) result -> unit) -> repo -> commit_key -> unit
fs:Eio.Fs.dir_ty Eio.Path.t ->
domain_mgr:_ Eio.Domain_manager.t ->
?finished:((stats, string) result -> unit) ->
repo ->
commit_key ->
unit
end

module type Sigs = sig
Expand All @@ -124,6 +135,11 @@ module type Sigs = sig
with type 'a return_type = 'a return_type
and type 'a config = 'a config

val run : Store.store_config -> 'a config -> 'a
val run :
fs:Eio.Fs.dir_ty Eio.Path.t ->
domain_mgr:_ Eio.Domain_manager.t ->
Store.store_config ->
'a config ->
'a
end
end
1 change: 1 addition & 0 deletions bench/irmin-pack/trace_stat_summary.ml
Original file line number Diff line number Diff line change
Expand Up @@ -1098,6 +1098,7 @@ let summarise ?block_count trace_stat_path =
(* Section 4/4 - Conversion from summary to json file *)

let save_to_json v path =
let path = Eio.Path.native_exn path in
let j = Fmt.str "%a\n" (Irmin.Type.pp_json t) v in
let chan = open_out path in
output_string chan j;
Expand Down
Loading

0 comments on commit 9046cf5

Please sign in to comment.