Skip to content

Commit

Permalink
Provide a sorted order of scenarios for all formatters
Browse files Browse the repository at this point in the history 
Before every formatter had to call `Statistics.sort` by themselves,
which is unnecessary work for the formatters. I think it comes
from a time where we had maps instead of lists. But now we can
provide an order.

This should also go towards some consistency in which scenarios
are displayed. Moreover we can make `Statistics.sort` private
limiting our possible interaction surface.
  • Loading branch information
@PragTob
PragTob committed Jan 13, 2019
1 parent 7c7bad1 commit 6972c08
Show file tree
Hide file tree
Showing 4 changed files with 82 additions and 73 deletions.
14 changes: 7 additions & 7 deletions lib/benchee/formatters/console.ex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ defmodule Benchee.Formatters.Console do

@behaviour Benchee.Formatter

alias Benchee.{Statistics, Suite}
alias Benchee.Suite
alias Benchee.Formatters.Console.{Memory, RunTime}

def format(suite), do: format(suite, %{})
Expand Down Expand Up @@ -71,22 +71,22 @@ defmodule Benchee.Formatters.Console do

scenarios
|> Enum.reduce([], &update_grouped_list/2)
|> Enum.reverse()
|> Enum.map(fn {input, scenarios} ->
scenarios
|> Statistics.sort()
|> generate_output(config, input)
generate_output(scenarios, config, input)
end)
end

# Normally one would prepend to lists and not append. In this case this lead to 2
# `Enum.reverse` scattered around. As these lists are usually very small (mostly less
# than 10 elements) I opted for `++` here.
defp update_grouped_list(scenario, grouped_scenarios) do
case List.keyfind(grouped_scenarios, scenario.input_name, 0) do
{_, group} ->
new_tuple = {scenario.input_name, [scenario | group]}
new_tuple = {scenario.input_name, group ++ [scenario]}
List.keyreplace(grouped_scenarios, scenario.input_name, 0, new_tuple)

_ ->
[{scenario.input_name, [scenario]} | grouped_scenarios]
grouped_scenarios ++ [{scenario.input_name, [scenario]}]
end
end

Expand Down
32 changes: 9 additions & 23 deletions lib/benchee/statistics.ex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ defmodule Benchee.Statistics do
times and then compute statistics like the average and the standard deviation.
"""

alias Benchee.{Benchmark.Scenario, Conversion.Duration, Statistics, Suite, Utility.Parallel}
alias Benchee.{Benchmark.Scenario, Conversion.Duration, Suite, Utility.Parallel}

alias Benchee.Statistics.Mode
alias Benchee.Statistics.Percentile
Expand Down Expand Up @@ -42,27 +42,6 @@ defmodule Benchee.Statistics do

@type samples :: [number]

@doc """
Sorts the given scenarios fastest to slowest by run_time average.
## Examples
iex> scenario_1 = %Benchee.Benchmark.Scenario{run_time_statistics: %Statistics{average: 100.0}}
iex> scenario_2 = %Benchee.Benchmark.Scenario{run_time_statistics: %Statistics{average: 200.0}}
iex> scenario_3 = %Benchee.Benchmark.Scenario{run_time_statistics: %Statistics{average: 400.0}}
iex> scenarios = [scenario_2, scenario_3, scenario_1]
iex> Benchee.Statistics.sort(scenarios)
[%Benchee.Benchmark.Scenario{run_time_statistics: %Statistics{average: 100.0}},
%Benchee.Benchmark.Scenario{run_time_statistics: %Statistics{average: 200.0}},
%Benchee.Benchmark.Scenario{run_time_statistics: %Statistics{average: 400.0}}]
"""
@spec sort([%Scenario{}]) :: [%Scenario{}]
def sort(scenarios) do
Enum.sort_by(scenarios, fn %Scenario{run_time_statistics: %Statistics{average: average}} ->
average
end)
end

@doc """
Takes a job suite with job run times, returns a map representing the
statistics of the job suite as follows:
Expand Down Expand Up @@ -167,7 +146,7 @@ defmodule Benchee.Statistics do
}
end)

%Suite{suite | scenarios: scenarios_with_statistics}
%Suite{suite | scenarios: sort(scenarios_with_statistics)}
end

@doc """
Expand Down Expand Up @@ -347,4 +326,11 @@ defmodule Benchee.Statistics do

%Suite{suite | scenarios: new_scenarios}
end

@spec sort([Scenario.t()]) :: [Scenario.t()]
defp sort(scenarios) do
Enum.sort_by(scenarios, fn scenario ->
{scenario.run_time_statistics.average, scenario.memory_usage_statistics.average}
end)
end
end
62 changes: 31 additions & 31 deletions test/benchee/formatters/console_test.exs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,20 +163,6 @@ defmodule Benchee.Formatters.ConsoleTest do

test "with multiple inputs and two jobs" do
scenarios = [
%Scenario{
name: "Job",
input_name: "My Arg",
input: "My Arg",
run_time_statistics: %Statistics{
average: 200.0,
ips: 5_000.0,
std_dev_ratio: 0.1,
median: 195.5,
percentiles: %{99 => 300.1},
sample_size: 200
},
memory_usage_statistics: %Statistics{}
},
%Scenario{
name: "Other Job",
input_name: "My Arg",
Expand All @@ -193,14 +179,14 @@ defmodule Benchee.Formatters.ConsoleTest do
},
%Scenario{
name: "Job",
input_name: "Other Arg",
input: "Other Arg",
input_name: "My Arg",
input: "My Arg",
run_time_statistics: %Statistics{
average: 400.0,
ips: 2_500.0,
std_dev_ratio: 0.15,
median: 395.0,
percentiles: %{99 => 500.1},
average: 200.0,
ips: 5_000.0,
std_dev_ratio: 0.1,
median: 195.5,
percentiles: %{99 => 300.1},
sample_size: 200
},
memory_usage_statistics: %Statistics{}
Expand All @@ -218,6 +204,20 @@ defmodule Benchee.Formatters.ConsoleTest do
sample_size: 200
},
memory_usage_statistics: %Statistics{}
},
%Scenario{
name: "Job",
input_name: "Other Arg",
input: "Other Arg",
run_time_statistics: %Statistics{
average: 400.0,
ips: 2_500.0,
std_dev_ratio: 0.15,
median: 395.0,
percentiles: %{99 => 500.1},
sample_size: 200
},
memory_usage_statistics: %Statistics{}
}
]

Expand All @@ -242,29 +242,29 @@ defmodule Benchee.Formatters.ConsoleTest do
test "with and without a tag" do
scenarios = [
%Scenario{
name: "job",
name: "job (improved)",
input_name: @no_input,
input: @no_input,
run_time_statistics: %Statistics{
average: 200.0,
ips: 5_000.0,
average: 100.0,
ips: 10_000.0,
std_dev_ratio: 0.1,
median: 195.5,
percentiles: %{99 => 300.1},
median: 90.0,
percentiles: %{99 => 200.1},
sample_size: 200
},
memory_usage_statistics: %Statistics{}
},
%Scenario{
name: "job (improved)",
name: "job",
input_name: @no_input,
input: @no_input,
run_time_statistics: %Statistics{
average: 100.0,
ips: 10_000.0,
average: 200.0,
ips: 5_000.0,
std_dev_ratio: 0.1,
median: 90.0,
percentiles: %{99 => 200.1},
median: 195.5,
percentiles: %{99 => 300.1},
sample_size: 200
},
memory_usage_statistics: %Statistics{}
Expand Down
47 changes: 35 additions & 12 deletions test/benchee/statistics_test.exs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,41 @@ defmodule Benchee.StatistcsTest do
assert memory_stats.sample_size == 5
end

test "sorts them by their average run time fastest to slowest" do
fourth = %Scenario{name: "4", run_times: [400.1]}
second = %Scenario{name: "2", run_times: [200.0]}
third = %Scenario{name: "3", run_times: [400.0]}
first = %Scenario{name: "1", run_times: [100.0]}
scenarios = [fourth, third, second, first]

sorted = Statistics.statistics(%Suite{scenarios: scenarios}).scenarios

assert Enum.map(sorted, fn scenario -> scenario.name end) == ["1", "2", "3", "4"]
end

test "sorts them by their average memory usage least to most" do
fourth = %Scenario{name: "4", memory_usages: [400.1]}
second = %Scenario{name: "2", memory_usages: [200.0]}
third = %Scenario{name: "3", memory_usages: [400.0]}
first = %Scenario{name: "1", memory_usages: [100.0]}
scenarios = [fourth, third, second, first]

sorted = Statistics.statistics(%Suite{scenarios: scenarios}).scenarios

assert Enum.map(sorted, fn scenario -> scenario.name end) == ["1", "2", "3", "4"]
end

test "sorts them by their average run time using memory as a tie breaker" do
second = %Scenario{name: "2", run_times: [100.0], memory_usages: [100.0]}
third = %Scenario{name: "3", run_times: [100.0], memory_usages: [100.1]}
first = %Scenario{name: "1", run_times: [100.0], memory_usages: [99.9]}
scenarios = [third, second, first]

sorted = Statistics.statistics(%Suite{scenarios: scenarios}).scenarios

assert Enum.map(sorted, fn scenario -> scenario.name end) == ["1", "2", "3"]
end

defp stats_for(suite, job_name, input_name) do
%Scenario{run_time_statistics: stats} =
Enum.find(suite.scenarios, fn scenario ->
Expand Down Expand Up @@ -197,16 +232,4 @@ defmodule Benchee.StatistcsTest do
assert stats.mode == nil
end
end

describe ".sort" do
test "sorts the benchmarks correctly and retains all data" do
fourth = %Scenario{run_time_statistics: %Statistics{average: 400.1}}
second = %Scenario{run_time_statistics: %Statistics{average: 200.0}}
third = %Scenario{run_time_statistics: %Statistics{average: 400.0}}
first = %Scenario{run_time_statistics: %Statistics{average: 100.0}}
scenarios = [fourth, second, third, first]

assert Statistics.sort(scenarios) == [first, second, third, fourth]
end
end
end

0 comments on commit 6972c08

Please sign in to comment.