diff --git a/lib/benchee.ex b/lib/benchee.ex index b73e4467..d576d702 100644 --- a/lib/benchee.ex +++ b/lib/benchee.ex @@ -20,35 +20,27 @@ for {module, moduledoc} <- [{Benchee, elixir_doc}, {:benchee, erlang_doc}] do alias Benchee.Formatter @doc """ - Run benchmark jobs defined by a map and optionally provide configuration - options. + Runs the given benchmarks, calculates statistics based on the results and + outputs results with the configured formatters. - Runs the given benchmarks and prints the results on the console. - - * jobs - a map from descriptive benchmark job name to a function to be - executed and benchmarked - * configuration - configuration options to alter what Benchee does, see - `Benchee.Configuration.init/1` for documentation of the available options. + Benchmarks are defined as a map where the keys are a name for the given + function and the values are the functions to benchmark. Users can configure + the run by passing a keyword list as the second argument. For more + information on configuration see `Benchee.Configuration.init/1`. ## Examples - Benchee.run(%{"My Benchmark" => fn -> 1 + 1 end, - "My other benchmrk" => fn -> "1" ++ "1" end}, time: 3) - # Prints a summary of the benchmark to the console - + Benchee.run( + %{ + "My Benchmark" => fn -> 1 + 1 end, + "My other benchmrk" => fn -> [1] ++ [1] end + }, + warmup: 2, + time: 3 + ) """ - def run(jobs, config \\ []) - - def run(jobs, config) when is_list(config) do - do_run(jobs, config) - end - - def run(config, jobs) when is_map(jobs) do - # pre 0.6.0 way of passing in the config first and as a map - do_run(jobs, config) - end - - defp do_run(jobs, config) do + @spec run(map, keyword) :: any + def run(jobs, config \\ []) when is_list(config) do config |> Benchee.init() |> Benchee.system() diff --git a/lib/benchee/benchmark.ex b/lib/benchee/benchmark.ex index 1871fb4b..7ed18410 100644 --- a/lib/benchee/benchmark.ex +++ b/lib/benchee/benchmark.ex @@ -6,8 +6,7 @@ defmodule Benchee.Benchmark do alias Benchee.Benchmark.{Runner, Scenario, ScenarioContext} alias Benchee.Output.BenchmarkPrinter, as: Printer - alias Benchee.Suite - alias Benchee.Utility.DeepConvert + alias Benchee.{Suite, Utility.DeepConvert} @type job_name :: String.t() | atom @no_input :__no_input @@ -48,19 +47,6 @@ defmodule Benchee.Benchmark do %Suite{suite | scenarios: List.flatten([scenarios | new_scenarios])} end - defp build_scenarios_for_job(job_name, function, config) - - defp build_scenarios_for_job(job_name, function, nil) do - [ - build_scenario(%{ - job_name: job_name, - function: function, - input: @no_input, - input_name: @no_input - }) - ] - end - defp build_scenarios_for_job(job_name, function, %{inputs: nil}) do [ build_scenario(%{ diff --git a/lib/benchee/configuration.ex b/lib/benchee/configuration.ex index 3681b842..1694dcba 100644 --- a/lib/benchee/configuration.ex +++ b/lib/benchee/configuration.ex @@ -3,8 +3,6 @@ defmodule Benchee.Configuration do Functions to handle the configuration of Benchee, exposes `init/1` function. """ - alias Benchee.Formatters.{Console, CSV, HTML, JSON} - alias Benchee.{ Configuration, Conversion.Duration, @@ -19,7 +17,7 @@ defmodule Benchee.Configuration do warmup: 2, memory_time: 0.0, pre_check: false, - formatters: [Console], + formatters: [{Console, %{comparison: true, extended_statistics: false}}], percentiles: [50, 99], print: %{ benchmarking: true, @@ -29,16 +27,7 @@ defmodule Benchee.Configuration do inputs: nil, save: false, load: false, - # formatters should end up here but known once are still picked up at - # the top level for now - formatter_options: %{ - console: %{ - comparison: true, - extended_statistics: false - } - }, unit_scaling: :best, - # If you/your plugin/whatever needs it your data can go here assigns: %{}, before_each: nil, after_each: nil, @@ -53,12 +42,11 @@ defmodule Benchee.Configuration do warmup: number, memory_time: number, pre_check: boolean, - formatters: [(Suite.t() -> Suite.t())], + formatters: [(Suite.t() -> Suite.t()) | {atom, map}], print: map, inputs: %{Suite.key() => any} | [{String.t(), any}] | nil, save: map | false, load: String.t() | [String.t()] | false, - formatter_options: map, unit_scaling: Scale.scaling_strategy(), assigns: map, before_each: fun | nil, @@ -265,9 +253,7 @@ defmodule Benchee.Configuration do ...> warmup: 0.2, ...> formatters: [&IO.puts/1], ...> print: [fast_warning: false], - ...> console: [comparison: false], ...> inputs: %{"Small" => 5, "Big" => 9999}, - ...> formatter_options: [some: "option"], ...> unit_scaling: :smallest) %Benchee.Suite{ configuration: @@ -284,13 +270,6 @@ defmodule Benchee.Configuration do fast_warning: false, configuration: true }, - formatter_options: %{ - console: %{ - comparison: false, - extended_statistics: false - }, - some: "option" - }, percentiles: [50, 99], unit_scaling: :smallest, assigns: %{}, @@ -311,7 +290,6 @@ defmodule Benchee.Configuration do config |> standardized_user_configuration |> merge_with_defaults - |> formatter_options_to_tuples |> convert_time_to_nano_s |> update_measure_memory |> save_option_conversion @@ -322,37 +300,9 @@ defmodule Benchee.Configuration do defp standardized_user_configuration(config) do config |> DeepConvert.to_map([:formatters, :inputs]) - |> translate_formatter_keys |> force_string_input_keys end - # backwards compatible translation of formatter keys to go into - # formatter_options now - @formatter_keys [:console, :csv, :json, :html] - defp translate_formatter_keys(config) do - {formatter_options, config} = Map.split(config, @formatter_keys) - DeepMerge.deep_merge(%{formatter_options: formatter_options}, config) - end - - # backwards compatible formatter definition without leaving the burden on every formatter - defp formatter_options_to_tuples(config) do - update_in(config, [Access.key(:formatters), Access.all()], fn - Console -> formatter_configuration_from_options(config, Console, :console) - CSV -> formatter_configuration_from_options(config, CSV, :csv) - JSON -> formatter_configuration_from_options(config, JSON, :json) - HTML -> formatter_configuration_from_options(config, HTML, :html) - formatter -> formatter - end) - end - - defp formatter_configuration_from_options(config, module, legacy_option_key) do - if Map.has_key?(config.formatter_options, legacy_option_key) do - {module, config.formatter_options[legacy_option_key]} - else - module - end - end - defp force_string_input_keys(config = %{inputs: inputs}) do standardized_inputs = inputs @@ -408,22 +358,13 @@ defmodule Benchee.Configuration do """) end - defp save_option_conversion(config = %{save: false}) do - config - end + defp save_option_conversion(config = %{save: false}), do: config defp save_option_conversion(config = %{save: save_values}) do save_options = Map.merge(save_defaults(), save_values) - - tagged_save_options = %{ - tag: save_options.tag, - path: save_options.path - } - - %__MODULE__{ - config - | formatters: config.formatters ++ [{Benchee.Formatters.TaggedSave, tagged_save_options}] - } + tagged_save_options = %{tag: save_options.tag, path: save_options.path} + formatters = config.formatters ++ [{Benchee.Formatters.TaggedSave, tagged_save_options}] + %__MODULE__{config | formatters: formatters} end defp save_defaults do diff --git a/lib/benchee/formatter.ex b/lib/benchee/formatter.ex index 8ff483fe..6e3e756c 100644 --- a/lib/benchee/formatter.ex +++ b/lib/benchee/formatter.ex @@ -32,8 +32,6 @@ defmodule Benchee.Formatter do """ @callback write(any, options) :: :ok | {:error, String.t()} - @typep module_configuration :: module | {module, options} - @doc """ Format and output all configured formatters and formatting functions. @@ -54,7 +52,7 @@ defmodule Benchee.Formatter do {parallelizable, serial} = formatters |> Enum.map(&normalize_module_configuration/1) - |> Enum.split_with(&is_formatter_module?/1) + |> Enum.split_with(&is_tuple/1) # why do we ignore this suite? It shouldn't be changed anyway. # We assign it because dialyzer would complain otherwise :D @@ -65,25 +63,37 @@ defmodule Benchee.Formatter do suite end - @default_opts %{} - defp normalize_module_configuration(module_configuration) - defp normalize_module_configuration({module, opts}), do: {module, DeepConvert.to_map(opts)} + defp normalize_module_configuration(formatter) when is_function(formatter, 1), do: formatter - defp normalize_module_configuration(formatter) when is_atom(formatter) do - {formatter, @default_opts} + defp normalize_module_configuration({module, opts}) do + normalize_module_configuration(module, DeepConvert.to_map(opts)) end - defp normalize_module_configuration(formatter), do: formatter + defp normalize_module_configuration(module) when is_atom(module) do + normalize_module_configuration(module, %{}) + end - defp is_formatter_module?({formatter, _options}) when is_atom(formatter) do - module_attributes = formatter.module_info(:attributes) + defp normalize_module_configuration(module, opts) do + if formatter_module?(module) do + {module, opts} + else + raise_behaviour_not_implemented(module) + end + end - module_attributes + defp formatter_module?(module) do + :attributes + |> module.module_info() |> Keyword.get(:behaviour, []) |> Enum.member?(Benchee.Formatter) end - defp is_formatter_module?(_), do: false + defp raise_behaviour_not_implemented(module) do + raise """ + The module you're attempting to use as a formatter - #{module} - does + not implement the `Benchee.Formatter` behaviour. + """ + end @doc """ Output a suite with a given formatter and options. @@ -105,7 +115,6 @@ defmodule Benchee.Formatter do # Invokes `format/2` and `write/2` as defined by the `Benchee.Formatter` # behaviour. The output for all formatters is generated in parallel, and then # the results of that formatting are written in sequence. - @spec parallel_output(Suite.t(), [module_configuration]) :: Suite.t() defp parallel_output(suite, module_configurations) do module_configurations |> Parallel.map(fn {module, options} -> {module, options, module.format(suite, options)} end) diff --git a/test/benchee/configuration_test.exs b/test/benchee/configuration_test.exs index 89c35c3a..40cf9b08 100644 --- a/test/benchee/configuration_test.exs +++ b/test/benchee/configuration_test.exs @@ -11,104 +11,58 @@ defmodule Benchee.ConfigurationTest do describe "init/1" do test "crashes for values that are going to be ignored" do - assert_raise KeyError, fn -> - init(runntime: 2) - end + assert_raise KeyError, fn -> init(runntime: 2) end end test "converts maps to lists and input keys to strings" do - suite = init(inputs: %{"map" => %{}, list: []}) - - assert %Suite{configuration: %{inputs: [{"list", []}, {"map", %{}}]}} = suite + assert %Suite{configuration: %{inputs: [{"list", []}, {"map", %{}}]}} = + init(inputs: %{"map" => %{}, list: []}) end - test "doesn't convert input lists to maps" do - suite = init(inputs: [{"map", %{}}, {:list, []}]) - assert %Suite{configuration: %{inputs: [{"map", %{}}, {"list", []}]}} = suite + test "doesn't convert input lists to maps and retains the order of input lists" do + assert %Suite{configuration: %{inputs: [{"map", %{}}, {"list", []}]}} = + init(inputs: [{"map", %{}}, {:list, []}]) end test "loses duplicated inputs keys after normalization" do - suite = init(inputs: %{"map" => %{}, map: %{}}) - - assert %Suite{configuration: %{inputs: inputs}} = suite - assert [{"map", %{}}] == inputs + assert %Suite{configuration: %{inputs: [{"map", %{}}]}} = + init(inputs: %{"map" => %{}, map: %{}}) end test "uses information from :save to setup the external term formattter" do - suite = init(save: [path: "save_one.benchee", tag: "master"]) - - assert suite.configuration.formatters == [ - {Benchee.Formatters.Console, %{comparison: true, extended_statistics: false}}, - {Benchee.Formatters.TaggedSave, %{path: "save_one.benchee", tag: "master"}} - ] + assert %Suite{ + configuration: %{ + formatters: [ + {Benchee.Formatters.Console, %{comparison: true, extended_statistics: false}}, + {Benchee.Formatters.TaggedSave, %{path: "save_one.benchee", tag: "master"}} + ] + } + } = init(save: [path: "save_one.benchee", tag: "master"]) end test ":save tag defaults to date" do - suite = init(save: [path: "save_one.benchee"]) - - [_, {_, etf_options}] = suite.configuration.formatters - - assert etf_options.tag =~ ~r/\d\d\d\d-\d\d?-\d\d?--\d\d?-\d\d?-\d\d?/ - assert etf_options.path == "save_one.benchee" - end - - test "takes formatter_options to build tuple list" do - suite = - init( - formatter_options: %{console: %{foo: :bar}}, - formatters: [Benchee.Formatters.Console] - ) - - assert [{Benchee.Formatters.Console, %{foo: :bar}}] = suite.configuration.formatters - end - - test "formatters already specified as a tuple are left alone" do - suite = - init( - formatter_options: %{console: %{foo: :bar}}, - formatters: [{Benchee.Formatters.Console, %{a: :b}}] - ) - - assert [{Benchee.Formatters.Console, %{a: :b}}] == suite.configuration.formatters - end - - test "legacy formatter options default to just the module if no options are given" do - suite = init(formatters: [Benchee.Formatter.CSV]) + assert %Suite{configuration: %{formatters: [_, {_, %{tag: tag, path: "save_one.benchee"}}]}} = + init(save: [path: "save_one.benchee"]) - assert [Benchee.Formatter.CSV] == suite.configuration.formatters + assert tag =~ ~r/\d\d\d\d-\d\d?-\d\d?--\d\d?-\d\d?-\d\d?/ end end describe ".deep_merge behaviour" do test "it can be adjusted with a map" do - user_options = %{ - time: 10, - formatter_options: %{ - custom: %{option: true}, - console: %{extended_statistics: true} - } - } + user_options = %{time: 10} result = deep_merge(@default_config, user_options) - expected = %Configuration{ - time: 10, - formatter_options: %{ - custom: %{option: true}, - console: %{ - comparison: true, - extended_statistics: true - } - } - } + expected = %Configuration{time: 10} assert expected == result end test "it just replaces when given another configuration" do - other_config = %Configuration{formatter_options: %{some: %{value: true}}} + other_config = %Configuration{} result = deep_merge(@default_config, other_config) - expected = %Configuration{formatter_options: %{some: %{value: true}}} + expected = %Configuration{} assert ^expected = result end diff --git a/test/benchee_test.exs b/test/benchee_test.exs index c34e1e92..99593b98 100644 --- a/test/benchee_test.exs +++ b/test/benchee_test.exs @@ -79,23 +79,6 @@ defmodule BencheeTest do assert Regex.match?(body_regex("Magic"), output) end - test "integration high level README example old school map config" do - output = - capture_io(fn -> - list = Enum.to_list(1..10_000) - map_fun = fn i -> [i, i * i] end - - map_config = Enum.into(@test_configuration, %{}) - - Benchee.run(map_config, %{ - "flat_map" => fn -> Enum.flat_map(list, map_fun) end, - "map.flatten" => fn -> list |> Enum.map(map_fun) |> List.flatten() end - }) - end) - - readme_sample_asserts(output) - end - test "integration high level README example but with formatter options" do output = capture_io(fn -> @@ -233,7 +216,7 @@ defmodule BencheeTest do @test_configuration, time: 0.01, warmup: 0, - console: [comparison: false] + formatters: [{Console, %{comparison: false}}] ) ) end) @@ -281,7 +264,7 @@ defmodule BencheeTest do readme_sample_asserts(output) end - test "formatters can be supplied with the Formatter.output/3 function" do + test "formatters can be supplied as a function with arity 1" do output = capture_io(fn -> list = Enum.to_list(1..10_000)