Skip to content

Commit

Permalink
Merge pull request #260 from PragTob/sort-scenarios
Browse files Browse the repository at this point in the history 
Provide a sorted order of scenarios for all formatters
  • Loading branch information
@devonestes
devonestes committed Jan 15, 2019
2 parents 1a8a05e + 6972c08 commit cb5b399
Show file tree
Hide file tree
Showing 4 changed files with 82 additions and 73 deletions.
14 changes: 7 additions & 7 deletions lib/benchee/formatters/console.ex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ defmodule Benchee.Formatters.Console do

@behaviour Benchee.Formatter

alias Benchee.{Statistics, Suite}
alias Benchee.Suite
alias Benchee.Formatters.Console.{Memory, RunTime}

def format(suite), do: format(suite, %{})
Expand Down Expand Up @@ -71,22 +71,22 @@ defmodule Benchee.Formatters.Console do

scenarios
|> Enum.reduce([], &update_grouped_list/2)
|> Enum.reverse()
|> Enum.map(fn {input, scenarios} ->
scenarios
|> Statistics.sort()
|> generate_output(config, input)
generate_output(scenarios, config, input)
end)
end

# Normally one would prepend to lists and not append. In this case this lead to 2
# `Enum.reverse` scattered around. As these lists are usually very small (mostly less
# than 10 elements) I opted for `++` here.
defp update_grouped_list(scenario, grouped_scenarios) do
case List.keyfind(grouped_scenarios, scenario.input_name, 0) do
{_, group} ->
new_tuple = {scenario.input_name, [scenario | group]}
new_tuple = {scenario.input_name, group ++ [scenario]}
List.keyreplace(grouped_scenarios, scenario.input_name, 0, new_tuple)

_ ->
[{scenario.input_name, [scenario]} | grouped_scenarios]
grouped_scenarios ++ [{scenario.input_name, [scenario]}]
end
end

Expand Down
32 changes: 9 additions & 23 deletions lib/benchee/statistics.ex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ defmodule Benchee.Statistics do
times and then compute statistics like the average and the standard deviation.
"""

alias Benchee.{Benchmark.Scenario, Conversion.Duration, Statistics, Suite, Utility.Parallel}
alias Benchee.{Benchmark.Scenario, Conversion.Duration, Suite, Utility.Parallel}

alias Benchee.Statistics.Mode
alias Benchee.Statistics.Percentile
Expand Down Expand Up @@ -42,27 +42,6 @@ defmodule Benchee.Statistics do

@type samples :: [number]

@doc """
Sorts the given scenarios fastest to slowest by run_time average.
## Examples
iex> scenario_1 = %Benchee.Benchmark.Scenario{run_time_statistics: %Statistics{average: 100.0}}
iex> scenario_2 = %Benchee.Benchmark.Scenario{run_time_statistics: %Statistics{average: 200.0}}
iex> scenario_3 = %Benchee.Benchmark.Scenario{run_time_statistics: %Statistics{average: 400.0}}
iex> scenarios = [scenario_2, scenario_3, scenario_1]
iex> Benchee.Statistics.sort(scenarios)
[%Benchee.Benchmark.Scenario{run_time_statistics: %Statistics{average: 100.0}},
%Benchee.Benchmark.Scenario{run_time_statistics: %Statistics{average: 200.0}},
%Benchee.Benchmark.Scenario{run_time_statistics: %Statistics{average: 400.0}}]
"""
@spec sort([%Scenario{}]) :: [%Scenario{}]
def sort(scenarios) do
Enum.sort_by(scenarios, fn %Scenario{run_time_statistics: %Statistics{average: average}} ->
average
end)
end

@doc """
Takes a job suite with job run times, returns a map representing the
statistics of the job suite as follows:
Expand Down Expand Up @@ -167,7 +146,7 @@ defmodule Benchee.Statistics do
}
end)

%Suite{suite | scenarios: scenarios_with_statistics}
%Suite{suite | scenarios: sort(scenarios_with_statistics)}
end

@doc """
Expand Down Expand Up @@ -347,4 +326,11 @@ defmodule Benchee.Statistics do

%Suite{suite | scenarios: new_scenarios}
end

@spec sort([Scenario.t()]) :: [Scenario.t()]
defp sort(scenarios) do
Enum.sort_by(scenarios, fn scenario ->
{scenario.run_time_statistics.average, scenario.memory_usage_statistics.average}
end)
end
end
62 changes: 31 additions & 31 deletions test/benchee/formatters/console_test.exs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,20 +163,6 @@ defmodule Benchee.Formatters.ConsoleTest do

test "with multiple inputs and two jobs" do
scenarios = [
%Scenario{
name: "Job",
input_name: "My Arg",
input: "My Arg",
run_time_statistics: %Statistics{
average: 200.0,
ips: 5_000.0,
std_dev_ratio: 0.1,
median: 195.5,
percentiles: %{99 => 300.1},
sample_size: 200
},
memory_usage_statistics: %Statistics{}
},
%Scenario{
name: "Other Job",
input_name: "My Arg",
Expand All @@ -193,14 +179,14 @@ defmodule Benchee.Formatters.ConsoleTest do
},
%Scenario{
name: "Job",
input_name: "Other Arg",
input: "Other Arg",
input_name: "My Arg",
input: "My Arg",
run_time_statistics: %Statistics{
average: 400.0,
ips: 2_500.0,
std_dev_ratio: 0.15,
median: 395.0,
percentiles: %{99 => 500.1},
average: 200.0,
ips: 5_000.0,
std_dev_ratio: 0.1,
median: 195.5,
percentiles: %{99 => 300.1},
sample_size: 200
},
memory_usage_statistics: %Statistics{}
Expand All @@ -218,6 +204,20 @@ defmodule Benchee.Formatters.ConsoleTest do
sample_size: 200
},
memory_usage_statistics: %Statistics{}
},
%Scenario{
name: "Job",
input_name: "Other Arg",
input: "Other Arg",
run_time_statistics: %Statistics{
average: 400.0,
ips: 2_500.0,
std_dev_ratio: 0.15,
median: 395.0,
percentiles: %{99 => 500.1},
sample_size: 200
},
memory_usage_statistics: %Statistics{}
}
]

Expand All @@ -242,29 +242,29 @@ defmodule Benchee.Formatters.ConsoleTest do
test "with and without a tag" do
scenarios = [
%Scenario{
name: "job",
name: "job (improved)",
input_name: @no_input,
input: @no_input,
run_time_statistics: %Statistics{
average: 200.0,
ips: 5_000.0,
average: 100.0,
ips: 10_000.0,
std_dev_ratio: 0.1,
median: 195.5,
percentiles: %{99 => 300.1},
median: 90.0,
percentiles: %{99 => 200.1},
sample_size: 200
},
memory_usage_statistics: %Statistics{}
},
%Scenario{
name: "job (improved)",
name: "job",
input_name: @no_input,
input: @no_input,
run_time_statistics: %Statistics{
average: 100.0,
ips: 10_000.0,
average: 200.0,
ips: 5_000.0,
std_dev_ratio: 0.1,
median: 90.0,
percentiles: %{99 => 200.1},
median: 195.5,
percentiles: %{99 => 300.1},
sample_size: 200
},
memory_usage_statistics: %Statistics{}
Expand Down
47 changes: 35 additions & 12 deletions test/benchee/statistics_test.exs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,41 @@ defmodule Benchee.StatistcsTest do
assert memory_stats.sample_size == 5
end

test "sorts them by their average run time fastest to slowest" do
fourth = %Scenario{name: "4", run_times: [400.1]}
second = %Scenario{name: "2", run_times: [200.0]}
third = %Scenario{name: "3", run_times: [400.0]}
first = %Scenario{name: "1", run_times: [100.0]}
scenarios = [fourth, third, second, first]

sorted = Statistics.statistics(%Suite{scenarios: scenarios}).scenarios

assert Enum.map(sorted, fn scenario -> scenario.name end) == ["1", "2", "3", "4"]
end

test "sorts them by their average memory usage least to most" do
fourth = %Scenario{name: "4", memory_usages: [400.1]}
second = %Scenario{name: "2", memory_usages: [200.0]}
third = %Scenario{name: "3", memory_usages: [400.0]}
first = %Scenario{name: "1", memory_usages: [100.0]}
scenarios = [fourth, third, second, first]

sorted = Statistics.statistics(%Suite{scenarios: scenarios}).scenarios

assert Enum.map(sorted, fn scenario -> scenario.name end) == ["1", "2", "3", "4"]
end

test "sorts them by their average run time using memory as a tie breaker" do
second = %Scenario{name: "2", run_times: [100.0], memory_usages: [100.0]}
third = %Scenario{name: "3", run_times: [100.0], memory_usages: [100.1]}
first = %Scenario{name: "1", run_times: [100.0], memory_usages: [99.9]}
scenarios = [third, second, first]

sorted = Statistics.statistics(%Suite{scenarios: scenarios}).scenarios

assert Enum.map(sorted, fn scenario -> scenario.name end) == ["1", "2", "3"]
end

defp stats_for(suite, job_name, input_name) do
%Scenario{run_time_statistics: stats} =
Enum.find(suite.scenarios, fn scenario ->
Expand Down Expand Up @@ -197,16 +232,4 @@ defmodule Benchee.StatistcsTest do
assert stats.mode == nil
end
end

describe ".sort" do
test "sorts the benchmarks correctly and retains all data" do
fourth = %Scenario{run_time_statistics: %Statistics{average: 400.1}}
second = %Scenario{run_time_statistics: %Statistics{average: 200.0}}
third = %Scenario{run_time_statistics: %Statistics{average: 400.0}}
first = %Scenario{run_time_statistics: %Statistics{average: 100.0}}
scenarios = [fourth, second, third, first]

assert Statistics.sort(scenarios) == [first, second, third, fourth]
end
end
end

0 comments on commit cb5b399

Please sign in to comment.