Merge 1545357 into a65bb81

bencheeorg · Dec 18, 2018 · 5649242 · 5649242
2 parents a65bb81 + 1545357
commit 5649242
Show file tree

Hide file tree

Showing 7 changed files with 73 additions and 207 deletions.
diff --git a/.tool-versions b/.tool-versions
@@ -1,2 +1,2 @@
-elixir 1.7.1
+elixir 1.7.3-otp-21
 erlang 21.0
diff --git a/lib/benchee.ex b/lib/benchee.ex
@@ -20,35 +20,27 @@ for {module, moduledoc} <- [{Benchee, elixir_doc}, {:benchee, erlang_doc}] do
     alias Benchee.Formatter
 
     @doc """
-    Run benchmark jobs defined by a map and optionally provide configuration
-    options.
+    Runs the given benchmarks, calculates statistics based on the results and
+    outputs results with the configured formatters.
 
-    Runs the given benchmarks and prints the results on the console.
-
-    * jobs - a map from descriptive benchmark job name to a function to be
-    executed and benchmarked
-    * configuration - configuration options to alter what Benchee does, see
-    `Benchee.Configuration.init/1` for documentation of the available options.
+    Benchmarks are defined as a map where the keys are a name for the given
+    function and the values are the functions to benchmark. Users can configure
+    the run by passing a keyword list as the second argument. For more
+    information on configuration see `Benchee.Configuration.init/1`.
 
     ## Examples
 
-        Benchee.run(%{"My Benchmark" => fn -> 1 + 1 end,
-                      "My other benchmrk" => fn -> "1" ++ "1" end}, time: 3)
-        # Prints a summary of the benchmark to the console
-
+        Benchee.run(
+          %{
+            "My Benchmark" => fn -> 1 + 1 end,
+            "My other benchmrk" => fn -> [1] ++ [1] end
+          },
+          warmup: 2,
+          time: 3
+        )
     """
-    def run(jobs, config \\ [])
-
-    def run(jobs, config) when is_list(config) do
-      do_run(jobs, config)
-    end
-
-    def run(config, jobs) when is_map(jobs) do
-      # pre 0.6.0 way of passing in the config first and as a map
-      do_run(jobs, config)
-    end
-
-    defp do_run(jobs, config) do
+    @spec run(map, keyword) :: any
+    def run(jobs, config \\ []) when is_list(config) do
       config
       |> Benchee.init()
       |> Benchee.system()

diff --git a/lib/benchee/benchmark.ex b/lib/benchee/benchmark.ex
@@ -6,8 +6,7 @@ defmodule Benchee.Benchmark do
 
   alias Benchee.Benchmark.{Runner, Scenario, ScenarioContext}
   alias Benchee.Output.BenchmarkPrinter, as: Printer
-  alias Benchee.Suite
-  alias Benchee.Utility.DeepConvert
+  alias Benchee.{Suite, Utility.DeepConvert}
 
   @type job_name :: String.t() | atom
   @no_input :__no_input
@@ -48,19 +47,6 @@ defmodule Benchee.Benchmark do
     %Suite{suite | scenarios: List.flatten([scenarios | new_scenarios])}
   end
 
-  defp build_scenarios_for_job(job_name, function, config)
-
-  defp build_scenarios_for_job(job_name, function, nil) do
-    [
-      build_scenario(%{
-        job_name: job_name,
-        function: function,
-        input: @no_input,
-        input_name: @no_input
-      })
-    ]
-  end
-
   defp build_scenarios_for_job(job_name, function, %{inputs: nil}) do
     [
       build_scenario(%{

diff --git a/lib/benchee/configuration.ex b/lib/benchee/configuration.ex
@@ -3,8 +3,6 @@ defmodule Benchee.Configuration do
   Functions to handle the configuration of Benchee, exposes `init/1` function.
   """
 
-  alias Benchee.Formatters.{Console, CSV, HTML, JSON}
-
   alias Benchee.{
     Configuration,
     Conversion.Duration,
@@ -19,7 +17,7 @@ defmodule Benchee.Configuration do
             warmup: 2,
             memory_time: 0.0,
             pre_check: false,
-            formatters: [Console],
+            formatters: [{Console, %{comparison: true, extended_statistics: false}}],
             percentiles: [50, 99],
             print: %{
               benchmarking: true,
@@ -29,16 +27,7 @@ defmodule Benchee.Configuration do
             inputs: nil,
             save: false,
             load: false,
-            # formatters should end up here but known once are still picked up at
-            # the top level for now
-            formatter_options: %{
-              console: %{
-                comparison: true,
-                extended_statistics: false
-              }
-            },
             unit_scaling: :best,
-            # If you/your plugin/whatever needs it your data can go here
             assigns: %{},
             before_each: nil,
             after_each: nil,
@@ -53,12 +42,11 @@ defmodule Benchee.Configuration do
           warmup: number,
           memory_time: number,
           pre_check: boolean,
-          formatters: [(Suite.t() -> Suite.t())],
+          formatters: [(Suite.t() -> Suite.t()) | {atom, map}],
           print: map,
           inputs: %{Suite.key() => any} | [{Suite.key(), any}] | nil,
           save: map | false,
           load: String.t() | [String.t()] | false,
-          formatter_options: map,
           unit_scaling: Scale.scaling_strategy(),
           assigns: map,
           before_each: fun | nil,
@@ -265,9 +253,7 @@ defmodule Benchee.Configuration do
       ...>   warmup: 0.2,
       ...>   formatters: [&IO.puts/1],
       ...>   print: [fast_warning: false],
-      ...>   console: [comparison: false],
       ...>   inputs: %{"Small" => 5, "Big" => 9999},
-      ...>   formatter_options: [some: "option"],
       ...>   unit_scaling: :smallest)
       %Benchee.Suite{
         configuration:
@@ -284,13 +270,6 @@ defmodule Benchee.Configuration do
               fast_warning: false,
               configuration: true
             },
-            formatter_options: %{
-              console: %{
-                comparison: false,
-                extended_statistics: false
-              },
-              some: "option"
-            },
             percentiles: [50, 99],
             unit_scaling: :smallest,
             assigns: %{},
@@ -311,7 +290,6 @@ defmodule Benchee.Configuration do
       config
       |> standardized_user_configuration
       |> merge_with_defaults
-      |> formatter_options_to_tuples
       |> convert_time_to_nano_s
       |> update_measure_memory
       |> save_option_conversion
@@ -322,37 +300,9 @@ defmodule Benchee.Configuration do
   defp standardized_user_configuration(config) do
     config
     |> DeepConvert.to_map([:formatters, :inputs])
-    |> translate_formatter_keys()
     |> standardize_inputs()
   end
 
-  # backwards compatible translation of formatter keys to go into
-  # formatter_options now
-  @formatter_keys [:console, :csv, :json, :html]
-  defp translate_formatter_keys(config) do
-    {formatter_options, config} = Map.split(config, @formatter_keys)
-    DeepMerge.deep_merge(%{formatter_options: formatter_options}, config)
-  end
-
-  # backwards compatible formatter definition without leaving the burden on every formatter
-  defp formatter_options_to_tuples(config) do
-    update_in(config, [Access.key(:formatters), Access.all()], fn
-      Console -> formatter_configuration_from_options(config, Console, :console)
-      CSV -> formatter_configuration_from_options(config, CSV, :csv)
-      JSON -> formatter_configuration_from_options(config, JSON, :json)
-      HTML -> formatter_configuration_from_options(config, HTML, :html)
-      formatter -> formatter
-    end)
-  end
-
-  defp formatter_configuration_from_options(config, module, legacy_option_key) do
-    if Map.has_key?(config.formatter_options, legacy_option_key) do
-      {module, config.formatter_options[legacy_option_key]}
-    else
-      module
-    end
-  end
-
   defp standardize_inputs(config = %{inputs: inputs}) do
     standardized_inputs =
       inputs
@@ -410,22 +360,13 @@ defmodule Benchee.Configuration do
     """)
   end
 
-  defp save_option_conversion(config = %{save: false}) do
-    config
-  end
+  defp save_option_conversion(config = %{save: false}), do: config
 
   defp save_option_conversion(config = %{save: save_values}) do
     save_options = Map.merge(save_defaults(), save_values)
-
-    tagged_save_options = %{
-      tag: save_options.tag,
-      path: save_options.path
-    }
-
-    %__MODULE__{
-      config
-      | formatters: config.formatters ++ [{Benchee.Formatters.TaggedSave, tagged_save_options}]
-    }
+    tagged_save_options = %{tag: save_options.tag, path: save_options.path}
+    formatters = config.formatters ++ [{Benchee.Formatters.TaggedSave, tagged_save_options}]
+    %__MODULE__{config | formatters: formatters}
   end
 
   defp save_defaults do

diff --git a/lib/benchee/formatter.ex b/lib/benchee/formatter.ex
@@ -32,8 +32,6 @@ defmodule Benchee.Formatter do
   """
   @callback write(any, options) :: :ok | {:error, String.t()}
 
-  @typep module_configuration :: module | {module, options}
-
   @doc """
   Format and output all configured formatters and formatting functions.
 
@@ -54,7 +52,7 @@ defmodule Benchee.Formatter do
     {parallelizable, serial} =
       formatters
       |> Enum.map(&normalize_module_configuration/1)
-      |> Enum.split_with(&is_formatter_module?/1)
+      |> Enum.split_with(&is_tuple/1)
 
     # why do we ignore this suite? It shouldn't be changed anyway.
     # We assign it because dialyzer would complain otherwise :D
@@ -65,25 +63,38 @@ defmodule Benchee.Formatter do
     suite
   end
 
-  @default_opts %{}
-  defp normalize_module_configuration(module_configuration)
-  defp normalize_module_configuration({module, opts}), do: {module, DeepConvert.to_map(opts)}
+  defp normalize_module_configuration(formatter) when is_function(formatter, 1), do: formatter
 
-  defp normalize_module_configuration(formatter) when is_atom(formatter) do
-    {formatter, @default_opts}
+  defp normalize_module_configuration({module, opts}) do
+    normalize_module_configuration(module, DeepConvert.to_map(opts))
   end
 
-  defp normalize_module_configuration(formatter), do: formatter
+  defp normalize_module_configuration(module) when is_atom(module) do
+    normalize_module_configuration(module, %{})
+  end
 
-  defp is_formatter_module?({formatter, _options}) when is_atom(formatter) do
-    module_attributes = formatter.module_info(:attributes)
+  defp normalize_module_configuration(module, opts) do
+    if formatter_module?(module) do
+      {module, opts}
+    else
+      raise_behaviour_not_implemented(module)
+    end
+  end
 
-    module_attributes
+  defp formatter_module?(module) do
+    :attributes
+    |> module.module_info()
     |> Keyword.get(:behaviour, [])
     |> Enum.member?(Benchee.Formatter)
   end
 
-  defp is_formatter_module?(_), do: false
+  @spec raise_behaviour_not_implemented(atom) :: no_return()
+  defp raise_behaviour_not_implemented(module) do
+    raise """
+    The module you're attempting to use as a formatter - #{module} - does
+    not implement the `Benchee.Formatter` behaviour.
+    """
+  end
 
   @doc """
   Output a suite with a given formatter and options.
@@ -105,7 +116,6 @@ defmodule Benchee.Formatter do
   # Invokes `format/2` and `write/2` as defined by the `Benchee.Formatter`
   # behaviour. The output for all formatters is generated in parallel, and then
   # the results of that formatting are written in sequence.
-  @spec parallel_output(Suite.t(), [module_configuration]) :: Suite.t()
   defp parallel_output(suite, module_configurations) do
     module_configurations
     |> Parallel.map(fn {module, options} -> {module, options, module.format(suite, options)} end)