Merge 7fbd009 into aaf7b8b

lib/benchee/configuration.ex

@@ -18,6 +18,7 @@ defmodule Benchee.Configuration do
             memory_time: 0.0,
             pre_check: false,
             formatters: [Console],
+            percentiles: [50, 99],
             print: %{
               benchmarking: true,
               configuration: true,
@@ -126,6 +127,9 @@ defmodule Benchee.Configuration do
       (x times slower than) is shown (true/false). Enabled by default.
       * `extended_statistics` - display more statistics, aka `minimum`,
       `maximum`, `sample_size` and `mode`. Disabled by default.
+    * `percentiles` - if you are using extended statistics and want to see the
+    results for certain percentiles of results beyond just the median.
+    Defaults to [50, 99] to calculate the 50th and 99th percentiles.
     * `:unit_scaling` - the strategy for choosing a unit for durations and
     counts. May or may not be implemented by a given formatter (The console
     formatter implements it). When scaling a value, Benchee finds the "best fit"
@@ -165,8 +169,12 @@ defmodule Benchee.Configuration do
               configuration: true
             },
             formatter_options: %{
-              console: %{comparison: true, extended_statistics: false}
+              console: %{
+                comparison: true,
+                extended_statistics: false
+              }
             },
+            percentiles: [50, 99],
             unit_scaling: :best,
             assigns: %{},
             before_each: nil,
@@ -195,8 +203,12 @@ defmodule Benchee.Configuration do
               configuration: true
             },
             formatter_options: %{
-              console: %{comparison: true, extended_statistics: false}
+              console: %{
+                comparison: true,
+                extended_statistics: false
+              }
             },
+            percentiles: [50, 99],
             unit_scaling: :best,
             assigns: %{},
             before_each: nil,
@@ -225,8 +237,12 @@ defmodule Benchee.Configuration do
               configuration: true
             },
             formatter_options: %{
-              console: %{comparison: true, extended_statistics: false}
+              console: %{
+                comparison: true,
+                extended_statistics: false
+              }
             },
+            percentiles: [50, 99],
             unit_scaling: :best,
             assigns: %{},
             before_each: nil,
@@ -264,9 +280,13 @@ defmodule Benchee.Configuration do
               configuration: true
             },
             formatter_options: %{
-              console: %{comparison: false, extended_statistics: false},
+              console: %{
+                comparison: false,
+                extended_statistics: false
+              },
               some: "option"
             },
+            percentiles: [50, 99],
             unit_scaling: :smallest,
             assigns: %{},
             before_each: nil,

lib/benchee/statistics.ex

@@ -151,10 +151,14 @@ defmodule Benchee.Statistics do
   """
   @spec statistics(Suite.t()) :: Suite.t()
   def statistics(suite = %Suite{scenarios: scenarios}) do
+    config = suite.configuration
+
+    percentiles = Enum.uniq([50 | config.percentiles])
+
     scenarios_with_statistics =
       Parallel.map(scenarios, fn scenario ->
-        run_time_stats = scenario.run_times |> job_statistics() |> add_ips
-        memory_stats = job_statistics(scenario.memory_usages)
+        run_time_stats = scenario.run_times |> job_statistics(percentiles) |> add_ips
+        memory_stats = job_statistics(scenario.memory_usages, percentiles)
 
         %Scenario{
           scenario
@@ -173,7 +177,7 @@ defmodule Benchee.Statistics do
   ## Examples
 
       iex> run_times = [200, 400, 400, 400, 500, 500, 500, 700, 900]
-      iex> Benchee.Statistics.job_statistics(run_times)
+      iex> Benchee.Statistics.job_statistics(run_times, [50, 99])
       %Benchee.Statistics{
         average:       500.0,
         ips:           nil,
@@ -188,7 +192,7 @@ defmodule Benchee.Statistics do
         sample_size:   9
       }
 
-      iex> Benchee.Statistics.job_statistics([100])
+      iex> Benchee.Statistics.job_statistics([100], [50, 99])
       %Benchee.Statistics{
         average:       100.0,
         ips:           nil,
@@ -203,7 +207,7 @@ defmodule Benchee.Statistics do
         sample_size:   1
       }
 
-      iex> Benchee.Statistics.job_statistics([])
+      iex> Benchee.Statistics.job_statistics([], [])
       %Benchee.Statistics{
         average:       nil,
         ips:           nil,
@@ -219,18 +223,18 @@ defmodule Benchee.Statistics do
       }
 
   """
-  @spec job_statistics(samples) :: __MODULE__.t()
-  def job_statistics([]) do
+  @spec job_statistics(samples, list) :: __MODULE__.t()
+  def job_statistics([], _) do
     %__MODULE__{sample_size: 0}
   end
 
-  def job_statistics(measurements) do
+  def job_statistics(measurements, percentiles) do
     total = Enum.sum(measurements)
     num_iterations = length(measurements)
     average = total / num_iterations
     deviation = standard_deviation(measurements, average, num_iterations)
     standard_dev_ratio = if average == 0, do: 0, else: deviation / average
-    percentiles = Percentile.percentiles(measurements, [50, 99])
+    percentiles = Percentile.percentiles(measurements, percentiles)
     median = Map.fetch!(percentiles, 50)
     mode = Mode.mode(measurements)
     minimum = Enum.min(measurements)

lib/benchee/suite.ex

@@ -11,8 +11,8 @@ defmodule Benchee.Suite do
   configuration.
   """
   defstruct [
-    :configuration,
     :system,
+    configuration: %Benchee.Configuration{},
     scenarios: []
   ]
 

test/benchee/configuration_test.exs

@@ -71,11 +71,14 @@ defmodule Benchee.ConfigurationTest do
         time: 10,
         formatter_options: %{
           custom: %{option: true},
-          console: %{comparison: true, extended_statistics: true}
+          console: %{
+            comparison: true,
+            extended_statistics: true
+          }
         }
       }
 
-      assert ^expected = result
+      assert expected == result
     end
 
     test "it just replaces when given another configuration" do

test/benchee/statistics_test.exs

@@ -1,6 +1,6 @@
 defmodule Benchee.StatistcsTest do
   use ExUnit.Case, async: true
-  alias Benchee.{Statistics, Suite, Benchmark.Scenario}
+  alias Benchee.{Statistics, Suite, Benchmark.Scenario, Configuration}
   doctest Benchee.Statistics
 
   @sample_1 [600, 470, 170, 430, 300]
@@ -87,15 +87,71 @@ defmodule Benchee.StatistcsTest do
       assert_in_delta stats.std_dev_ratio, 0.41, 0.01
     end
 
-    test "preserves all other keys in the map handed to it" do
+    test "preserves all other keys in the suite handed to it" do
       suite = %Suite{
         scenarios: [],
-        configuration: %{formatters: []}
+        configuration: %Configuration{formatters: []}
       }
 
       assert %Suite{configuration: %{formatters: []}} = Statistics.statistics(suite)
     end
 
+    test "calculates percentiles configured by the user" do
+      suite = %Suite{
+        configuration: %Configuration{
+          percentiles: [25, 50, 75]
+        },
+        scenarios: [
+          %Scenario{
+            run_times: [1, 2],
+            memory_usages: [1, 2]
+          }
+        ]
+      }
+
+      %Suite{
+        scenarios: [
+          %Scenario{
+            run_time_statistics: %Statistics{
+              percentiles: %{
+                25 => _,
+                50 => _,
+                75 => _
+              }
+            }
+          }
+        ]
+      } = Statistics.statistics(suite)
+    end
+
+    test "always calculates the 50th percentile, even if not set in the config" do
+      suite = %Suite{
+        configuration: %Configuration{
+          percentiles: [25, 75]
+        },
+        scenarios: [
+          %Scenario{
+            run_times: [1, 2],
+            memory_usages: [1, 2]
+          }
+        ]
+      }
+
+      %Suite{
+        scenarios: [
+          %Scenario{
+            run_time_statistics: %Statistics{
+              percentiles: %{
+                25 => _,
+                50 => _,
+                75 => _
+              }
+            }
+          }
+        ]
+      } = Statistics.statistics(suite)
+    end
+
     @all_zeros [0, 0, 0, 0, 0]
     test "doesn't blow up when all measurements are zeros (mostly memory measurement)" do
       scenarios = [%Scenario{run_times: @all_zeros, memory_usages: @all_zeros}]