Skip to content

Commit

Permalink
Add ips manually in the statistics code
Browse files Browse the repository at this point in the history
  • Loading branch information
@PragTob
PragTob committed May 1, 2018
1 parent 781e05b commit 71cfd80
Showing 1 changed file with 47 additions and 42 deletions.
89 changes: 47 additions & 42 deletions lib/benchee/statistics.ex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,10 @@ defmodule Benchee.Statistics do

@type t :: %__MODULE__{
average: float,
ips: float,
ips: float | nil,
std_dev: float,
std_dev_ratio: float,
std_dev_ips: float,
std_dev_ips: float | nil,
median: number,
percentiles: %{number => float},
mode: mode,
Expand Down Expand Up @@ -132,10 +132,10 @@ defmodule Benchee.Statistics do
},
memory_usage_statistics: %Benchee.Statistics{
average: 500.0,
ips: 0.0,
ips: nil,
std_dev: 200.0,
std_dev_ratio: 0.4,
std_dev_ips: 0.0,
std_dev_ips: nil,
median: 500.0,
percentiles: %{50 => 500.0, 99 => 900.0},
mode: [500, 400],
Expand All @@ -153,8 +153,8 @@ defmodule Benchee.Statistics do
def statistics(suite = %Suite{scenarios: scenarios}) do
scenarios_with_statistics =
Parallel.map(scenarios, fn scenario ->
run_time_stats = job_statistics(scenario.run_times, :run_time)
memory_stats = job_statistics(scenario.memory_usages, :memory)
run_time_stats = scenario.run_times |> job_statistics() |> add_ips
memory_stats = job_statistics(scenario.memory_usages)

%Scenario{
scenario
Expand All @@ -173,13 +173,13 @@ defmodule Benchee.Statistics do
## Examples
iex> run_times = [200, 400, 400, 400, 500, 500, 500, 700, 900]
iex> Benchee.Statistics.job_statistics(run_times, :run_time)
iex> Benchee.Statistics.job_statistics(run_times)
%Benchee.Statistics{
average: 500.0,
ips: 2000.0,
ips: nil,
std_dev: 200.0,
std_dev_ratio: 0.4,
std_dev_ips: 800.0,
std_dev_ips: nil,
median: 500.0,
percentiles: %{50 => 500.0, 99 => 900.0},
mode: [500, 400],
Expand All @@ -188,13 +188,13 @@ defmodule Benchee.Statistics do
sample_size: 9
}
iex> Benchee.Statistics.job_statistics([100], :run_time)
iex> Benchee.Statistics.job_statistics([100])
%Benchee.Statistics{
average: 100.0,
ips: 10_000.0,
ips: nil,
std_dev: 0,
std_dev_ratio: 0.0,
std_dev_ips: 0.0,
std_dev_ips: nil,
median: 100.0,
percentiles: %{50 => 100.0, 99 => 100.0},
mode: nil,
Expand All @@ -203,7 +203,7 @@ defmodule Benchee.Statistics do
sample_size: 1
}
iex> Benchee.Statistics.job_statistics([], :run_time)
iex> Benchee.Statistics.job_statistics([])
%Benchee.Statistics{
average: nil,
ips: nil,
Expand All @@ -219,19 +219,17 @@ defmodule Benchee.Statistics do
}
"""
@spec job_statistics(samples, atom) :: __MODULE__.t()
def job_statistics([], _) do
@spec job_statistics(samples) :: __MODULE__.t()
def job_statistics([]) do
%__MODULE__{sample_size: 0}
end

def job_statistics(measurements, run_time_or_memory) do
def job_statistics(measurements) do
total = Enum.sum(measurements)
num_iterations = length(measurements)
average = total / num_iterations
ips = iterations_per_second(average, run_time_or_memory)
deviation = standard_deviation(measurements, average, num_iterations)
standard_dev_ratio = if average == 0, do: 0, else: deviation / average
standard_dev_ips = ips * standard_dev_ratio
percentiles = Percentile.percentiles(measurements, [50, 99])
median = Map.fetch!(percentiles, 50)
mode = Mode.mode(measurements)
Expand All @@ -240,10 +238,8 @@ defmodule Benchee.Statistics do

%__MODULE__{
average: average,
ips: ips,
std_dev: deviation,
std_dev_ratio: standard_dev_ratio,
std_dev_ips: standard_dev_ips,
median: median,
percentiles: percentiles,
mode: mode,
Expand All @@ -253,6 +249,35 @@ defmodule Benchee.Statistics do
}
end

defp standard_deviation(_samples, _average, 1), do: 0

defp standard_deviation(samples, average, sample_size) do
total_variance =
Enum.reduce(samples, 0, fn sample, total ->
total + :math.pow(sample - average, 2)
end)

variance = total_variance / (sample_size - 1)
:math.sqrt(variance)
end

defp add_ips(statistics = %__MODULE{sample_size: 0}), do: statistics
defp add_ips(statistics) do
ips = iterations_per_second(statistics.average)
standard_dev_ips = ips * statistics.std_dev_ratio

%__MODULE__{
statistics |
ips: ips,
std_dev_ips: standard_dev_ips
}
end

defp iterations_per_second(0.0), do: 0.0
defp iterations_per_second(average_microseconds) do
Duration.microseconds({1, :second}) / average_microseconds
end

@doc """
Calculate additional percentiles and add them to the `run_time_statistics`.
Should only be used after `statistics/1`, to calculate extra values that
Expand Down Expand Up @@ -295,10 +320,10 @@ defmodule Benchee.Statistics do
},
memory_usage_statistics: %Benchee.Statistics{
average: 500.0,
ips: 0.0,
ips: nil,
std_dev: 200.0,
std_dev_ratio: 0.4,
std_dev_ips: 0.0,
std_dev_ips: nil,
median: 500.0,
percentiles: %{50 => 500.0, 99 => 900.0},
mode: [500, 400],
Expand All @@ -321,24 +346,4 @@ defmodule Benchee.Statistics do

%Suite{suite | scenarios: new_scenarios}
end

defp iterations_per_second(0.0, _), do: 0.0

defp iterations_per_second(_, :memory), do: 0.0

defp iterations_per_second(average_microseconds, :run_time) do
Duration.microseconds({1, :second}) / average_microseconds
end

defp standard_deviation(_samples, _average, 1), do: 0

defp standard_deviation(samples, average, sample_size) do
total_variance =
Enum.reduce(samples, 0, fn sample, total ->
total + :math.pow(sample - average, 2)
end)

variance = total_variance / (sample_size - 1)
:math.sqrt(variance)
end
end

0 comments on commit 71cfd80

Please sign in to comment.