Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

A new benchsuite/ repertory for performance measurements.

The idea is to support objective discussion about different
implementation choices for a given module.

In the present case, I wish to experiment with the Map/PMap code, and
must consider performance implications of the potential changes.
  • Loading branch information...
commit 5b196799ee93f8e08c30645c04f9ad9938eeb847 1 parent dc778a5
@gasche gasche authored
View
5 Makefile
@@ -36,6 +36,7 @@ NATIVE_INSTALL_FILES = _build/src/*.cmx _build/src/*.a _build/src/*.cmxa
# What to build
TARGETS = syntax.otarget byte.otarget src/batteries_help.cmo META
TEST_TARGETS = testsuite/main.byte qtest/test_runner.byte
+BENCH_TARGETS = benchsuite/test_maps.native
ifeq ($(BATTERIES_NATIVE_SHLIB), yes)
EXT = native
@@ -108,6 +109,10 @@ test: src/batCamomile.ml $(patsubst src/%.ml,qtest/%_t.ml, $(TESTABLE)) qtest/te
$(OCAMLBUILD) $(TARGETS) $(TEST_TARGETS)
$(foreach TEST, $(TEST_TARGETS), echo "Running $(TEST)"; _build/$(TEST); echo; )
+bench:
+ $(OCAMLBUILD) $(TARGETS) $(BENCH_TARGETS)
+ $(foreach BENCH, $(BENCH_TARGETS), _build/$(BENCH); )
+
release: test
git archive --format=tar --prefix=batteries-$(VERSION)/ HEAD \
| gzip > batteries-$(VERSION).tar.gz
View
2  README
@@ -1 +1 @@
-See README.md
+See README.md
View
2  README.md
@@ -26,11 +26,13 @@ You will need the following libraries:
* [Camomile][] >= 0.7
* GNU make
* [OUnit][] to build and run the tests
+* [ocaml-benchmark][] to build and run the performance tests (optional)
[Findlib]: http://projects.camlcity.org/projects/findlib.html/
[OCaml]: http://caml.inria.fr/ocaml/release.en.html
[Camomile]: http://camomile.sourceforge.net/
[OUnit]: http://ounit.forge.ocamlcore.org/
+[ocaml-benchmark]: http://ocaml-benchmark.forge.ocamlcore.org/
### Configuration and Installation
View
3  _tags
@@ -3,4 +3,5 @@
"src": include
"libs" or "libs/estring": include
"testsuite": include
-"qtest": include
+"qtest": include
+"benchsuite": include
View
7 benchsuite/README
@@ -0,0 +1,7 @@
+The purpose of this directory is to provide performance evaluations of
+Batteries functions implementations. This is specially useful when
+testing changes against an upstream library such as INRIA's stdlib or
+Extlib.
+The benchmarks rely on the Ocaml [benchmark] library.
+
+ [benchmark] http://ocaml-benchmark.sourceforge.net/
View
1  benchsuite/_tags
@@ -0,0 +1 @@
+<*>: pkg_benchmark
View
164 benchsuite/test_maps.ml
@@ -0,0 +1,164 @@
+(* The purpose of this test is to compare different implementation of
+ the Map associative data structure. *)
+
+let total_length = 500_000
+
+let (-|) = BatStd.(-|)
+let (<|) = BatStd.(<|)
+
+module MapBench (M : sig val input_length : int end) = struct
+ let input_length = M.input_length
+
+ let repeat = 3
+
+ let nb_iter =
+ max 10 (total_length / input_length)
+
+ let () = Printf.printf "%d iterations\n" nb_iter
+
+ let random_input () =
+ BatList.init input_length (fun _ -> Random.int input_length)
+
+ module StdMap = BatMap.Make(BatInt)
+
+ module PMap = BatPMap
+
+ let same_elts stdmap pmap =
+ BatList.of_enum (StdMap.enum stdmap)
+ = BatList.of_enum (PMap.enum pmap)
+
+ (* A benchmark for key insertion *)
+ let create_input_keys =
+ random_input ()
+ let create_input_values =
+ random_input ()
+
+ let create_std_map () =
+ BatList.fold_left2
+ (fun t k v -> StdMap.add k v t)
+ StdMap.empty
+ create_input_keys
+ create_input_values
+
+ let create_poly_map () =
+ BatList.fold_left2
+ (fun t k v -> PMap.add k v t)
+ PMap.empty
+ create_input_keys
+ create_input_values
+
+ let std_created_map = create_std_map ()
+ let poly_created_map = create_poly_map ()
+
+ let () =
+ assert (same_elts std_created_map poly_created_map)
+
+ let samples_create =
+ Benchmark.latencyN ~repeat (Int64.of_int nb_iter) [
+ "stdmap create", ignore -| create_std_map, ();
+ "pmap create", ignore -| create_poly_map, ();
+ ]
+
+ (* A benchmark for fast import *)
+ let key_val_list =
+ BatList.combine
+ create_input_keys
+ create_input_values
+
+ let import_std_map () =
+ StdMap.of_enum (BatList.enum key_val_list)
+
+ let import_poly_map () =
+ PMap.of_enum (BatList.enum key_val_list)
+
+ let () =
+ let std_imported_map = import_std_map () in
+ assert (same_elts std_imported_map poly_created_map);
+ let poly_imported_map = import_poly_map () in
+ assert (same_elts std_created_map poly_imported_map);
+ ()
+
+ let samples_import =
+ Benchmark.latencyN ~repeat (Int64.of_int nb_iter) [
+ "stdmap import", ignore -| import_std_map, ();
+ "pmap import", ignore -| import_poly_map, ();
+ ]
+
+ (* A benchmark for key lookup *)
+ let lookup_keys =
+ random_input ()
+
+ let lookup_std_map () =
+ List.iter
+ (fun k -> ignore (StdMap.mem k std_created_map))
+ lookup_keys
+
+ let lookup_poly_map () =
+ List.iter
+ (fun k -> ignore (StdMap.mem k std_created_map))
+ lookup_keys
+
+ let samples_lookup =
+ Benchmark.latencyN ~repeat (Int64.of_int nb_iter) [
+ "stdmap lookup", lookup_std_map, ();
+ "pmap lookup", lookup_poly_map, ();
+ ]
+
+ (* A benchmark for key removal *)
+ let remove_keys =
+ random_input ()
+
+ let remove_std_map () =
+ List.fold_left
+ (fun t k -> StdMap.remove k t)
+ std_created_map
+ remove_keys
+
+ let remove_poly_map () =
+ List.fold_left
+ (fun t k -> PMap.remove k t)
+ poly_created_map
+ remove_keys
+
+ let () =
+ assert (same_elts (remove_std_map ()) (remove_poly_map ()))
+
+ let samples_remove =
+ Benchmark.latencyN ~repeat (Int64.of_int nb_iter) [
+ "stdmap remove", ignore -| remove_std_map, ();
+ "pmap remove", ignore -| remove_poly_map, ();
+ ]
+
+
+ let () =
+ List.iter
+ (print_newline -| Benchmark.tabulate)
+ [
+ samples_create;
+ samples_import;
+ samples_lookup;
+ samples_remove;
+ ]
+end
+
+let big_length = 100_000
+let small_length = 500
+
+let () =
+ Printf.printf "Test with small maps (length = %d)\n" small_length;
+ let () =
+ let module M = MapBench(struct let input_length = small_length end) in
+ () in
+
+ print_newline ();
+ print_newline ();
+
+ Printf.printf "Test with big maps (length = %d)\n" big_length;
+ let () =
+ let module M = MapBench(struct let input_length = big_length end) in
+ () in
+
+
+ ()
+
+
View
12 myocamlbuild.ml
@@ -63,10 +63,14 @@ let _ = dispatch begin function
flag ["ocaml"; "link"] & S[A"-package"; A packs];
flag ["ocaml"; "infer_interface"] & S[A"-package"; A packs];
- flag ["ocaml"; "infer_interface"; "pkg_oUnit"] & S[A"-package"; A"oUnit"];
- flag ["ocaml"; "ocamldep"; "pkg_oUnit"] & S[A"-package"; A"oUnit"];
- flag ["ocaml"; "compile"; "pkg_oUnit"] & S[A"-package"; A"oUnit"];
- flag ["ocaml"; "link"; "pkg_oUnit"] & S[A"-package"; A"oUnit"];
+ List.iter
+ (fun pkg ->
+ flag ["ocaml"; "infer_interface"; "pkg_"^pkg] & S[A"-package"; A pkg];
+ flag ["ocaml"; "ocamldep"; "pkg_"^pkg] & S[A"-package"; A pkg];
+ flag ["ocaml"; "compile"; "pkg_"^pkg] & S[A"-package"; A pkg];
+ flag ["ocaml"; "link"; "pkg_"^pkg] & S[A"-package"; A pkg];
+ ())
+ ["oUnit"; "benchmark"];
(* DON'T USE TAG 'thread', USE 'threads'
for compatibility with ocamlbuild *)
Please sign in to comment.
Something went wrong with that request. Please try again.