Merge 43d8b7d into de9e994

.exguard.exs

@@ -3,6 +3,12 @@ use ExGuard.Config
 project_files = ~r{\.(erl|ex|exs|eex|xrl|yrl)\z}i
 deps = ~r{deps}
 
+guard("compile and warn", run_on_start: true)
+|> command("MIX_ENV=test mix compile --warnings-as-errors")
+|> watch(project_files)
+|> ignore(deps)
+|> notification(:auto)
+
 guard("credo", run_on_start: true)
 |> command("mix credo --strict")
 |> watch(project_files)

lib/benchee/benchmark/repeated_measurement.ex

@@ -39,7 +39,7 @@ defmodule Benchee.Benchmark.RepeatedMeasurement do
     run_time = measure_iteration(scenario, scenario_context, collector)
 
     if run_time >= @minimum_execution_time do
-      {num_iterations, adjust_for_iterations(run_time, num_iterations)}
+      {num_iterations, report_time(run_time, num_iterations)}
     else
       if fast_warning, do: printer.fast_warning()
 
@@ -52,6 +52,14 @@ defmodule Benchee.Benchmark.RepeatedMeasurement do
     end
   end
 
+  # we need to convert the time here since we measure native time to see when we have enough
+  # repetitions but the first time is used in the actual samples
+  defp report_time(measurement, num_iterations) do
+    measurement
+    |> :erlang.convert_time_unit(:native, :nanosecond)
+    |> adjust_for_iterations(num_iterations)
+  end
+
   defp adjust_for_iterations(measurement, 1), do: measurement
   defp adjust_for_iterations(measurement, num_iterations), do: measurement / num_iterations
 

lib/benchee/formatters/console/helpers.ex

@@ -49,4 +49,49 @@ defmodule Benchee.Formatters.Console.Helpers do
 
   @spec descriptor(String.t()) :: String.t()
   def descriptor(header_str), do: "\n#{header_str}: \n"
+
+  def format_comparison(
+        name,
+        statistics,
+        display_value,
+        comparison_name,
+        display_unit,
+        label_width,
+        column_width
+      ) do
+    "~*s~*s ~ts"
+    |> :io_lib.format([
+      -label_width,
+      name,
+      column_width,
+      display_value,
+      comparison_display(statistics, comparison_name, display_unit)
+    ])
+    |> to_string
+  end
+
+  defp comparison_display(%Statistics{relative_more: nil, absolute_difference: nil}, _, _), do: ""
+
+  defp comparison_display(statistics, comparison_name, unit) do
+    "- #{comparison_text(statistics, comparison_name)} - #{
+      absolute_difference_text(statistics, unit)
+    }\n"
+  end
+
+  defp comparison_text(%Statistics{relative_more: :infinity}, name), do: " 	∞ x #{name}"
+  defp comparison_text(%Statistics{relative_more: nil}, _), do: "N/A"
+
+  defp comparison_text(statistics, comparison_name) do
+    "~.2fx ~s"
+    |> :io_lib.format([statistics.relative_more, comparison_name])
+    |> to_string
+  end
+
+  defp absolute_difference_text(statistics, unit) do
+    formatted_value = Format.format({Scale.scale(statistics.absolute_difference, unit), unit})
+
+    # currently the fastest/least consuming is always first so everything else eats more
+    # resources hence this is always +
+    "+#{formatted_value}"
+  end
 end

lib/benchee/formatters/console/memory.ex

@@ -106,7 +106,7 @@ defmodule Benchee.Formatters.Console.Memory do
   defp scenario_reports([scenario | other_scenarios], units, label_width, true) do
     [
       reference_report(scenario, units, label_width),
-      comparisons(scenario, units, label_width, other_scenarios),
+      comparisons(other_scenarios, units, label_width),
       "\n**All measurements for memory usage were the same**\n"
     ]
   end
@@ -209,7 +209,7 @@ defmodule Benchee.Formatters.Console.Memory do
     [
       Helpers.descriptor("Comparison"),
       reference_report(scenario, units, label_width)
-      | comparisons(scenario, units, label_width, other_scenarios)
+      | comparisons(other_scenarios, units, label_width)
     ]
   end
 
@@ -227,52 +227,27 @@ defmodule Benchee.Formatters.Console.Memory do
     |> to_string
   end
 
-  @spec comparisons(Scenario.t(), unit_per_statistic, integer, [Scenario.t()]) :: [String.t()]
-  defp comparisons(scenario, units, label_width, scenarios_to_compare) do
-    %Scenario{memory_usage_data: %{statistics: reference_stats}} = scenario
-
+  @spec comparisons([Scenario.t()], unit_per_statistic, integer) :: [String.t()]
+  defp comparisons(scenarios_to_compare, units, label_width) do
     Enum.map(
       scenarios_to_compare,
-      fn scenario = %Scenario{memory_usage_data: %{statistics: job_stats}} ->
-        slower = calculate_slower_value(job_stats.median, reference_stats.median)
-
-        format_comparison(scenario, units, label_width, slower)
+      fn scenario ->
+        statistics = scenario.memory_usage_data.statistics
+        memory_format = memory_output(statistics.average, units.memory)
+
+        Helpers.format_comparison(
+          scenario.name,
+          statistics,
+          memory_format,
+          "memory usage",
+          units.memory,
+          label_width,
+          @median_width
+        )
       end
     )
   end
 
-  defp calculate_slower_value(job_median, reference_median)
-       when job_median == 0 or is_nil(job_median) or reference_median == 0 or
-              is_nil(reference_median) do
-    @na
-  end
-
-  defp calculate_slower_value(job_median, reference_median) do
-    job_median / reference_median
-  end
-
-  defp format_comparison(scenario, %{memory: memory_unit}, label_width, @na) do
-    %Scenario{name: name, memory_usage_data: %{statistics: %Statistics{median: median}}} =
-      scenario
-
-    median_format = memory_output(median, memory_unit)
-
-    "~*s~*s\n"
-    |> :io_lib.format([-label_width, name, @median_width, median_format])
-    |> to_string
-  end
-
-  defp format_comparison(scenario, %{memory: memory_unit}, label_width, slower) do
-    %Scenario{name: name, memory_usage_data: %{statistics: %Statistics{median: median}}} =
-      scenario
-
-    median_format = memory_output(median, memory_unit)
-
-    "~*s~*s - ~.2fx memory usage\n"
-    |> :io_lib.format([-label_width, name, @median_width, median_format, slower])
-    |> to_string
-  end
-
   defp memory_output(nil, _unit), do: "N/A"
 
   defp memory_output(memory, unit) do

lib/benchee/formatters/console/run_time.ex

@@ -239,7 +239,7 @@ defmodule Benchee.Formatters.Console.RunTime do
     [
       Helpers.descriptor("Comparison"),
       reference_report(scenario, units, label_width)
-      | comparisons(scenario, units, label_width, other_scenarios)
+      | comparisons(other_scenarios, units, label_width)
     ]
   end
 
@@ -251,28 +251,27 @@ defmodule Benchee.Formatters.Console.RunTime do
     |> to_string
   end
 
-  @spec comparisons(Scenario.t(), unit_per_statistic, integer, [Scenario.t()]) :: [String.t()]
-  defp comparisons(scenario, units, label_width, scenarios_to_compare) do
-    %Scenario{run_time_data: %{statistics: reference_stats}} = scenario
-
+  @spec comparisons([Scenario.t()], unit_per_statistic, integer) :: [String.t()]
+  defp comparisons(scenarios_to_compare, units, label_width) do
     Enum.map(
       scenarios_to_compare,
-      fn scenario = %Scenario{run_time_data: %{statistics: job_stats}} ->
-        slower = reference_stats.ips / job_stats.ips
-        format_comparison(scenario, units, label_width, slower)
+      fn scenario ->
+        statistics = scenario.run_time_data.statistics
+        ips_format = Helpers.count_output(statistics.ips, units.ips)
+
+        Helpers.format_comparison(
+          scenario.name,
+          statistics,
+          ips_format,
+          "slower",
+          units.run_time,
+          label_width,
+          @ips_width
+        )
       end
     )
   end
 
-  defp format_comparison(scenario, %{ips: ips_unit}, label_width, slower) do
-    %Scenario{name: name, run_time_data: %{statistics: %Statistics{ips: ips}}} = scenario
-    ips_format = Helpers.count_output(ips, ips_unit)
-
-    "~*s~*s - ~.2fx slower\n"
-    |> :io_lib.format([-label_width, name, @ips_width, ips_format, slower])
-    |> to_string
-  end
-
   defp duration_output(duration, unit) do
     Duration.format({Duration.scale(duration, unit), unit})
   end

lib/benchee/statistics.ex

@@ -21,6 +21,9 @@ defmodule Benchee.Statistics do
     :mode,
     :minimum,
     :maximum,
+    :relative_more,
+    :relative_less,
+    :absolute_difference,
     sample_size: 0
   ]
 
@@ -37,6 +40,9 @@ defmodule Benchee.Statistics do
           mode: mode,
           minimum: number,
           maximum: number,
+          relative_more: float | nil | :infinity,
+          relative_less: float | nil | :infinity,
+          absolute_difference: float | nil,
           sample_size: integer
         }
 
@@ -140,21 +146,28 @@ defmodule Benchee.Statistics do
   def statistics(suite = %Suite{scenarios: scenarios}) do
     config = suite.configuration
 
-    percentiles = Enum.uniq([50 | config.percentiles])
-
     scenarios_with_statistics =
-      Parallel.map(scenarios, fn scenario ->
-        run_time_stats = scenario.run_time_data.samples |> job_statistics(percentiles) |> add_ips
-        memory_stats = job_statistics(scenario.memory_usage_data.samples, percentiles)
+      scenarios
+      |> calculate_per_scenario_statistics(config)
+      |> sort()
+      |> calculate_relative_statistics(config.inputs)
 
-        %{
-          scenario
-          | run_time_data: %{scenario.run_time_data | statistics: run_time_stats},
-            memory_usage_data: %{scenario.memory_usage_data | statistics: memory_stats}
-        }
-      end)
+    %Suite{suite | scenarios: scenarios_with_statistics}
+  end
+
+  defp calculate_per_scenario_statistics(scenarios, config) do
+    percentiles = Enum.uniq([50 | config.percentiles])
+
+    Parallel.map(scenarios, fn scenario ->
+      run_time_stats = scenario.run_time_data.samples |> job_statistics(percentiles) |> add_ips
+      memory_stats = job_statistics(scenario.memory_usage_data.samples, percentiles)
 
-    %Suite{suite | scenarios: sort(scenarios_with_statistics)}
+      %{
+        scenario
+        | run_time_data: %{scenario.run_time_data | statistics: run_time_stats},
+          memory_usage_data: %{scenario.memory_usage_data | statistics: memory_stats}
+      }
+    end)
   end
 
   @doc """
@@ -215,17 +228,17 @@ defmodule Benchee.Statistics do
     %__MODULE__{sample_size: 0}
   end
 
-  def job_statistics(measurements, percentiles) do
-    total = Enum.sum(measurements)
-    num_iterations = length(measurements)
+  def job_statistics(samples, percentiles) do
+    total = Enum.sum(samples)
+    num_iterations = length(samples)
     average = total / num_iterations
-    deviation = standard_deviation(measurements, average, num_iterations)
+    deviation = standard_deviation(samples, average, num_iterations)
     standard_dev_ratio = if average == 0, do: 0, else: deviation / average
-    percentiles = Percentile.percentiles(measurements, percentiles)
+    percentiles = Percentile.percentiles(samples, percentiles)
     median = Map.fetch!(percentiles, 50)
-    mode = Mode.mode(measurements)
-    minimum = Enum.min(measurements)
-    maximum = Enum.max(measurements)
+    mode = Mode.mode(samples)
+    minimum = Enum.min(samples)
+    maximum = Enum.max(samples)
 
     %__MODULE__{
       average: average,
@@ -266,6 +279,65 @@ defmodule Benchee.Statistics do
     }
   end
 
+  defp calculate_relative_statistics([], _inputs), do: []
+
+  defp calculate_relative_statistics(scenarios, inputs) do
+    scenarios
+    |> scenarios_by_input(inputs)
+    |> Enum.flat_map(fn scenarios_with_same_input ->
+      {reference, others} = split_reference_scenario(scenarios_with_same_input)
+      others_with_relative = statistics_relative_to(others, reference)
+      [reference | others_with_relative]
+    end)
+  end
+
+  defp scenarios_by_input(scenarios, nil), do: [scenarios]
+
+  # we can't just group_by `input_name` because that'd lose the order of inputs which might
+  # be important
+  defp scenarios_by_input(scenarios, inputs) do
+    Enum.map(inputs, fn {input_name, _} ->
+      Enum.filter(scenarios, fn scenario -> scenario.input_name == input_name end)
+    end)
+  end
+
+  # right now we take the first scenario as we sorted them and it is the fastest,
+  # whenever we implement #179 though this becomesd more involved
+  defp split_reference_scenario(scenarios) do
+    [reference | others] = scenarios
+    {reference, others}
+  end
+
+  defp statistics_relative_to(scenarios, reference) do
+    Enum.map(scenarios, fn scenario ->
+      scenario
+      |> update_in([Access.key!(:run_time_data), Access.key!(:statistics)], fn statistics ->
+        add_relative_statistics(statistics, reference.run_time_data.statistics)
+      end)
+      |> update_in([Access.key!(:memory_usage_data), Access.key!(:statistics)], fn statistics ->
+        add_relative_statistics(statistics, reference.memory_usage_data.statistics)
+      end)
+    end)
+  end
+
+  # we might not run time/memory --> we shouldn't crash then ;)
+  defp add_relative_statistics(statistics = %{average: nil}, _reference), do: statistics
+  defp add_relative_statistics(statistics, %{average: nil}), do: statistics
+
+  defp add_relative_statistics(statistics, reference_statistics) do
+    %__MODULE__{
+      statistics
+      | relative_more: zero_safe_division(statistics.average, reference_statistics.average),
+        relative_less: zero_safe_division(reference_statistics.average, statistics.average),
+        absolute_difference: statistics.average - reference_statistics.average
+    }
+  end
+
+  defp zero_safe_division(0.0, 0.0), do: 1.0
+  defp zero_safe_division(_, 0), do: :infinity
+  defp zero_safe_division(_, 0.0), do: :infinity
+  defp zero_safe_division(a, b), do: a / b
+
   @doc """
   Calculate additional percentiles and add them to the
   `run_time_data.statistics`. Should only be used after `statistics/1`, to

samples/fast.exs

@@ -0,0 +1,12 @@
+list = Enum.to_list(1..10_000)
+map_fun = fn i -> [i, i * i] end
+
+Benchee.run(
+  %{
+    "flat_map" => fn -> Enum.flat_map(list, map_fun) end,
+    "map.flatten" => fn -> list |> Enum.map(map_fun) |> List.flatten() end
+  },
+  warmup: 0.1,
+  time: 0.3,
+  memory_time: 0.3
+)