Skip to content

Commit

Permalink
Finish the tests and move on the the lib folder
Browse files Browse the repository at this point in the history
  • Loading branch information
devonestes committed May 30, 2018
1 parent cbe9fc0 commit 165f16b
Show file tree
Hide file tree
Showing 5 changed files with 95 additions and 62 deletions.
36 changes: 20 additions & 16 deletions lib/benchee.ex
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,11 @@ for {module, moduledoc} <- [{Benchee, elixir_doc}, {:benchee, erlang_doc}] do
"""
def run(jobs, config \\ [])

def run(jobs, config) when is_list(config) do
do_run(jobs, config)
end

def run(config, jobs) when is_map(jobs) do
# pre 0.6.0 way of passing in the config first and as a map
do_run(jobs, config)
Expand All @@ -54,22 +56,23 @@ for {module, moduledoc} <- [{Benchee, elixir_doc}, {:benchee, erlang_doc}] do

defp run_benchmarks(jobs, config) do
config
|> Benchee.init
|> Benchee.system
|> Benchee.init()
|> Benchee.system()
|> add_benchmarking_jobs(jobs)
|> Benchee.measure
|> Benchee.statistics
|> Benchee.load
|> Benchee.measure()
|> Benchee.statistics()
|> Benchee.load()
end

defp output_results(suite = %{configuration: %{formatters: formatters}}) do
{parallelizable, serial} = Enum.split_with(formatters, &is_formatter_module?/1)

# why do we ignore this suite? It shouldn't be changed anyway.
_suite = Formatter.parallel_output(suite, parallelizable)
Enum.each serial, fn(output_function) ->

Enum.each(serial, fn output_function ->
output_function.(suite)
end
end)

suite
end
Expand All @@ -81,22 +84,23 @@ for {module, moduledoc} <- [{Benchee, elixir_doc}, {:benchee, erlang_doc}] do
|> Keyword.get(:behaviour, [])
|> Enum.member?(Benchee.Formatter)
end

defp is_formatter_module?(_), do: false

defp add_benchmarking_jobs(suite, jobs) do
Enum.reduce jobs, suite, fn({key, function}, suite_acc) ->
Enum.reduce(jobs, suite, fn {key, function}, suite_acc ->
Benchee.benchmark(suite_acc, key, function)
end
end)
end

defdelegate init(), to: Benchee.Configuration
defdelegate init(config), to: Benchee.Configuration
defdelegate system(suite), to: Benchee.System
defdelegate measure(suite), to: Benchee.Benchmark
defdelegate measure(suite, printer), to: Benchee.Benchmark
defdelegate init(), to: Benchee.Configuration
defdelegate init(config), to: Benchee.Configuration
defdelegate system(suite), to: Benchee.System
defdelegate measure(suite), to: Benchee.Benchmark
defdelegate measure(suite, printer), to: Benchee.Benchmark
defdelegate benchmark(suite, name, function), to: Benchee.Benchmark
defdelegate statistics(suite), to: Benchee.Statistics
defdelegate load(suite), to: Benchee.ScenarioLoader
defdelegate statistics(suite), to: Benchee.Statistics
defdelegate load(suite), to: Benchee.ScenarioLoader
defdelegate benchmark(suite, name, function, printer), to: Benchee.Benchmark
end
end
20 changes: 11 additions & 9 deletions lib/benchee/suite.ex
Original file line number Diff line number Diff line change
Expand Up @@ -16,24 +16,26 @@ defmodule Benchee.Suite do
scenarios: []
]

@type key :: atom | String.t
@type key :: atom | String.t()
@type optional_map :: map | nil
@type t :: %__MODULE__{
configuration: Benchee.Configuration.t | nil,
system: optional_map,
scenarios: [] | [Benchee.Benchmark.Scenario.t]
}
configuration: Benchee.Configuration.t() | nil,
system: optional_map,
scenarios: [] | [Benchee.Benchmark.Scenario.t()]
}
end

defimpl DeepMerge.Resolver, for: Benchee.Suite do
def resolve(original, override = %Benchee.Suite{}, resolver) do
cleaned_override = override
|> Map.from_struct
|> Enum.reject(fn({_key, value}) -> is_nil(value) end)
|> Map.new
cleaned_override =
override
|> Map.from_struct()
|> Enum.reject(fn {_key, value} -> is_nil(value) end)
|> Map.new()

Map.merge(original, cleaned_override, resolver)
end

def resolve(original, override, resolver) when is_map(override) do
Map.merge(original, override, resolver)
end
Expand Down
68 changes: 44 additions & 24 deletions lib/benchee/system.ex
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,17 @@ defmodule Benchee.System do
@doc """
Adds system information to the suite (currently elixir and erlang versions).
"""
@spec system(Suite.t) :: Suite.t
@spec system(Suite.t()) :: Suite.t()
def system(suite = %Suite{}) do
system_info = %{elixir: elixir(),
erlang: erlang(),
num_cores: num_cores(),
os: os(),
available_memory: available_memory(),
cpu_speed: cpu_speed()}
system_info = %{
elixir: elixir(),
erlang: erlang(),
num_cores: num_cores(),
os: os(),
available_memory: available_memory(),
cpu_speed: cpu_speed()
}

%Suite{suite | system: system_info}
end

Expand All @@ -30,11 +33,14 @@ defmodule Benchee.System do
"""
def erlang do
otp_release = :erlang.system_info(:otp_release)
file = Path.join([:code.root_dir, "releases", otp_release , "OTP_VERSION"])
file = Path.join([:code.root_dir(), "releases", otp_release, "OTP_VERSION"])

case File.read(file) do
{:ok, version} -> String.trim(version)
{:error, reason} ->
IO.puts "Error trying to dermine erlang version #{reason}"
{:ok, version} ->
String.trim(version)

{:error, reason} ->
IO.puts("Error trying to dermine erlang version #{reason}")
end
end

Expand All @@ -52,6 +58,7 @@ defmodule Benchee.System do
{_, name} = :os.type()
os(name)
end

defp os(:darwin), do: :macOS
defp os(:nt), do: :Windows
defp os(_), do: :Linux
Expand All @@ -65,28 +72,32 @@ defmodule Benchee.System do
defp cpu_speed(:Windows) do
parse_cpu_for(:Windows, system_cmd("WMIC", ["CPU", "GET", "NAME"]))
end

defp cpu_speed(:macOS) do
parse_cpu_for(:macOS, system_cmd("sysctl", ["-n", "machdep.cpu.brand_string"]))
end

defp cpu_speed(:Linux) do
parse_cpu_for(:Linux, system_cmd("cat", ["/proc/cpuinfo"]))
end

@linux_cpuinfo_regex ~r/model name.*:([\w \(\)\-\@\.]*)/i

def parse_cpu_for(_, "N/A"), do: "N/A"

def parse_cpu_for(:Windows, raw_output) do
"Name" <> cpu_info = raw_output
String.trim(cpu_info)
end

def parse_cpu_for(:macOS, raw_output), do: String.trim(raw_output)

def parse_cpu_for(:Linux, raw_output) do
match_info = Regex.run(@linux_cpuinfo_regex,
raw_output,
capture: :all_but_first)
match_info = Regex.run(@linux_cpuinfo_regex, raw_output, capture: :all_but_first)

case match_info do
[cpu_info] -> String.trim(cpu_info)
_ -> "Unrecognized processor"
_ -> "Unrecognized processor"
end
end

Expand All @@ -102,40 +113,49 @@ defmodule Benchee.System do
system_cmd("WMIC", ["COMPUTERSYSTEM", "GET", "TOTALPHYSICALMEMORY"])
)
end

defp available_memory(:macOS) do
parse_memory_for(:macOS, system_cmd("sysctl", ["-n", "hw.memsize"]))
end

defp available_memory(:Linux) do
parse_memory_for(:Linux, system_cmd("cat", ["/proc/meminfo"]))
end

defp parse_memory_for(_, "N/A"), do: "N/A"

defp parse_memory_for(:Windows, raw_output) do
[memory] = Regex.run(~r/\d+/, raw_output)
{memory, _} = Integer.parse(memory)
Memory.format(memory)
end

defp parse_memory_for(:macOS, raw_output) do
{memory, _} = Integer.parse(raw_output)
Memory.format(memory)
end

defp parse_memory_for(:Linux, raw_output) do
["MemTotal:" <> memory_info] = Regex.run(~r/MemTotal.*kB/, raw_output)
{memory_in_kilobytes, _} = memory_info
|> String.trim()
|> String.trim_trailing(" kB")
|> Integer.parse

{memory_in_bytes, _} = Memory.convert(
{memory_in_kilobytes, :kilobyte},
:byte
)

{memory_in_kilobytes, _} =
memory_info
|> String.trim()
|> String.trim_trailing(" kB")
|> Integer.parse()

{memory_in_bytes, _} =
Memory.convert(
{memory_in_kilobytes, :kilobyte},
:byte
)

Memory.format(memory_in_bytes)
end

def system_cmd(cmd, args, system_func \\ &System.cmd/2) do
{output, exit_code} = system_func.(cmd, args)

if exit_code > 0 do
IO.puts("Something went wrong trying to get system information:")
IO.puts(output)
Expand Down
24 changes: 16 additions & 8 deletions test/benchee/benchmark/repeated_measurement_test.exs
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,11 @@ defmodule Bencheee.Benchmark.RepeatedMeasurementTest do
{num_iterations, time} = determine_n_times(scenario, @scenario_context, false, FakeMeasurer)

assert num_iterations == 10
assert time == 5 # 50 adjusted by the 10 iteration factor
# 50 adjusted by the 10 iteration factor
assert time == 5

assert_received_exactly_n_times(:called, 11) # 1 initial + 10 more after repeat
# 1 initial + 10 more after repeat
assert_received_exactly_n_times(:called, 11)
end

test "doesn't do repetitions if the time is small enough from the get go" do
Expand All @@ -50,20 +52,23 @@ defmodule Bencheee.Benchmark.RepeatedMeasurementTest do
assert num_iterations == 1
assert time == 10

assert_received_exactly_n_times(:called, 1) # 1 initial + 10 more after repeat
# 1 initial + 10 more after repeat
assert_received_exactly_n_times(:called, 1)
end
end

describe ".measure" do
test "scales reported times approproately" do
scenario_context = %ScenarioContext{
@scenario_context
| num_iterations: 10,
| num_iterations: 10
}

scenario = %Scenario{
input: @no_input,
function: fn -> 42 end
}

Process.put(:test_measurement_time, 50)

time = measure(scenario, scenario_context, FakeMeasurer)
Expand All @@ -73,26 +78,29 @@ defmodule Bencheee.Benchmark.RepeatedMeasurementTest do

test "calls hooks appropriately even with multiple iterations" do
num_iterations = 10

scenario_context = %ScenarioContext{
@scenario_context
| num_iterations: num_iterations,
config: %Benchee.Configuration{
before_each: fn _ ->
send self(), :global_before
send(self(), :global_before)
@no_input
end,
after_each: fn _ -> send self(), :global_after end
after_each: fn _ -> send(self(), :global_after) end
}
}

scenario = %Scenario{
input: @no_input,
function: fn -> send(self(), :called) end,
before_each: fn _ ->
send self(), :local_before
send(self(), :local_before)
@no_input
end,
after_each: fn _ -> send self(), :local_after end
after_each: fn _ -> send(self(), :local_after) end
}

Process.put(:test_measurement_time, 50)

time = measure(scenario, scenario_context, FakeMeasurer)
Expand Down
9 changes: 4 additions & 5 deletions test/benchee/benchmark/runner_test.exs
Original file line number Diff line number Diff line change
Expand Up @@ -119,8 +119,7 @@ defmodule Benchee.Benchmark.RunnerTest do

@tag :memory_measure
test "records memory when the function only runs once" do
suite =
test_suite(%Suite{configuration: %{time: 0.0, warmup: 0.0, memory_time: 1_000_000}})
suite = test_suite(%Suite{configuration: %{time: 0.0, warmup: 0.0, memory_time: 1_000_000}})

new_suite =
suite
Expand All @@ -136,8 +135,7 @@ defmodule Benchee.Benchmark.RunnerTest do

@tag :memory_measure
test "correctly scales down memory usage of very fast functions" do
suite =
test_suite(%Suite{configuration: %{time: 0.0, warmup: 1, memory_time: 1_000_000}})
suite = test_suite(%Suite{configuration: %{time: 0.0, warmup: 1, memory_time: 1_000_000}})

new_suite =
suite
Expand Down Expand Up @@ -178,7 +176,8 @@ defmodule Benchee.Benchmark.RunnerTest do

[%{run_time_statistics: %{median: median}}] = suite.scenarios

assert median < 1500 # around ~78 on my machine, CI is awful for performance
# around ~78 on my machine, CI is awful for performance
assert median < 1500
end

@tag :performance
Expand Down

0 comments on commit 165f16b

Please sign in to comment.