Skip to content

Commit

Permalink
before/after_scenario implementation
Browse files Browse the repository at this point in the history 
before/after_scenario run exactly like they say before/after that
scenario - meaning before the warmup and after running the
measurements of all the iterations.
  • Loading branch information
@PragTob
PragTob committed Sep 18, 2017
1 parent 908b6d3 commit 6e55aaa
Show file tree
Hide file tree
Showing 6 changed files with 158 additions and 55 deletions.
34 changes: 30 additions & 4 deletions lib/benchee/benchmark/runner.ex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,16 +106,32 @@ defmodule Benchee.Benchmark.Runner do
printer.input_information(input_name, config)
printer.benchmarking(job_name, config)
Parallel.map(1..config.parallel, fn(_task_number) ->
run_warmup(scenario, scenario_context)
run_benchmark(scenario, scenario_context)
run_scenario(scenario, scenario_context)
end)
end

defp run_scenario(scenario, scenario_context) do
run_before_scenario(scenario, scenario_context)
_ = run_warmup(scenario, scenario_context)
measurements = run_benchmark(scenario, scenario_context)
run_after_scenario(scenario, scenario_context)
measurements
end

defp run_before_scenario(%{
before_scenario: local_before_scenario
},
%{
config: %{before_scenario: global_before_scenario}
}) do
if global_before_scenario, do: global_before_scenario.()
if local_before_scenario, do: local_before_scenario.()
end

defp run_warmup(scenario, scenario_context = %ScenarioContext{
config: %Configuration{warmup: warmup}
}) do
_ = measure_runtimes(scenario, scenario_context, warmup, false)
nil
measure_runtimes(scenario, scenario_context, warmup, false)
end

defp run_benchmark(scenario, scenario_context = %ScenarioContext{
Expand All @@ -127,6 +143,16 @@ defmodule Benchee.Benchmark.Runner do
measure_runtimes(scenario, scenario_context, run_time, fast_warning)
end

defp run_after_scenario(%{
after_scenario: local_after_scenario
},
%{
config: %{after_scenario: global_after_scenario}
}) do
if local_after_scenario, do: local_after_scenario.()
if global_after_scenario, do: global_after_scenario.()
end

defp measure_runtimes(scenario, context, run_time, fast_warning)
defp measure_runtimes(scenario, _, 0, _), do: scenario
defp measure_runtimes(scenario, scenario_context, run_time, fast_warning) do
Expand Down
8 changes: 6 additions & 2 deletions lib/benchee/benchmark/scenario.ex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,9 @@ defmodule Benchee.Benchmark.Scenario do
run_times: [],
memory_usages: [],
before_each: nil,
after_each: nil
after_each: nil,
before_scenario: nil,
after_scenario: nil
]

@type t :: %__MODULE__{
Expand All @@ -32,6 +34,8 @@ defmodule Benchee.Benchmark.Scenario do
memory_usages: [non_neg_integer] | [],
memory_usage_statistics: Benchee.Statistics.t | nil,
before_each: fun | nil,
after_each: fun | nil
after_each: fun | nil,
before_scenario: fun | nil,
after_scenario: fun | nil
}
end
24 changes: 18 additions & 6 deletions lib/benchee/configuration.ex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,9 @@ defmodule Benchee.Configuration do
# If you/your plugin/whatever needs it your data can go here
assigns: %{},
before_each: nil,
after_each: nil
after_each: nil,
before_scenario: nil,
after_scenario: nil
]

@type t :: %__MODULE__{
Expand All @@ -46,7 +48,9 @@ defmodule Benchee.Configuration do
formatter_options: map,
assigns: map,
before_each: fun | nil,
after_each: fun | nil
after_each: fun | nil,
before_scenario: fun | nil,
after_scenario: fun | nil
}

@type user_configuration :: map | keyword
Expand Down Expand Up @@ -135,7 +139,9 @@ defmodule Benchee.Configuration do
},
assigns: %{},
before_each: nil,
after_each: nil
after_each: nil,
before_scenario: nil,
after_scenario: nil
},
system: nil,
scenarios: []
Expand All @@ -160,7 +166,9 @@ defmodule Benchee.Configuration do
},
assigns: %{},
before_each: nil,
after_each: nil
after_each: nil,
before_scenario: nil,
after_scenario: nil
},
system: nil,
scenarios: []
Expand All @@ -185,7 +193,9 @@ defmodule Benchee.Configuration do
},
assigns: %{},
before_each: nil,
after_each: nil
after_each: nil,
before_scenario: nil,
after_scenario: nil
},
system: nil,
scenarios: []
Expand Down Expand Up @@ -219,7 +229,9 @@ defmodule Benchee.Configuration do
},
assigns: %{},
before_each: nil,
after_each: nil
after_each: nil,
before_scenario: nil,
after_scenario: nil
},
system: nil,
scenarios: []
Expand Down
111 changes: 79 additions & 32 deletions test/benchee/benchmark/runner_test.exs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -270,15 +270,20 @@ defmodule Benchee.Benchmark.RunnerTest do
}
}
|> test_suite
|> Benchmark.benchmark("job", {fn -> :timer.sleep 1 end,
before_each: fn -> send(me, :before) end,
after_each: fn -> send(me, :after) end})
|> Benchmark.benchmark("job", {
fn -> :timer.sleep 1 end,
before_each: fn -> send(me, :before) end,
after_each: fn -> send(me, :after) end,
before_scenario: fn -> send(me, :before_scenario) end,
after_scenario: fn -> send(me, :after_scenario) end})
|> Benchmark.measure(TestPrinter)

assert_received_exactly [:before, :after]
assert_received_exactly [
:before_scenario, :before, :after, :after_scenario
]
end

test "hooks trigger during warmup and runtime" do
test "hooks trigger during warmup and runtime but scenarios once" do
me = self()
%Suite{
configuration: %{
Expand All @@ -287,12 +292,17 @@ defmodule Benchee.Benchmark.RunnerTest do
}
}
|> test_suite
|> Benchmark.benchmark("job", {fn -> :timer.sleep 1 end,
before_each: fn -> send(me, :before) end,
after_each: fn -> send(me, :after) end})
|> Benchmark.benchmark("job", {
fn -> :timer.sleep 1 end,
before_each: fn -> send(me, :before) end,
after_each: fn -> send(me, :after) end,
before_scenario: fn -> send(me, :before_scenario) end,
after_scenario: fn -> send(me, :after_scenario) end})
|> Benchmark.measure(TestPrinter)

assert_received_exactly [:before, :after, :before, :after]
assert_received_exactly [
:before_scenario, :before, :after, :before, :after, :after_scenario
]
end

test "hooks trigger for each input" do
Expand All @@ -303,18 +313,25 @@ defmodule Benchee.Benchmark.RunnerTest do
time: 100,
before_each: fn -> send me, :global_before end,
after_each: fn -> send me, :global_after end,
before_scenario: fn -> send me, :global_before_scenario end,
after_scenario: fn -> send me, :global_after_scenario end,
inputs: %{"one" => 1, "two" => 2}
}
}
|> test_suite
|> Benchmark.benchmark("job", {fn(_) -> :timer.sleep 1 end,
before_each: fn -> send me, :local_before end,
after_each: fn -> send me, :local_after end})
|> Benchmark.benchmark("job", {
fn(_) -> :timer.sleep 1 end,
before_each: fn -> send(me, :local_before) end,
after_each: fn -> send(me, :local_after) end,
before_scenario: fn -> send(me, :local_before_scenario) end,
after_scenario: fn -> send(me, :local_after_scenario) end})
|> Benchmark.measure(TestPrinter)

assert_received_exactly [
:global_before, :local_before, :local_after, :global_after,
:global_before, :local_before, :local_after, :global_after
:global_before_scenario, :local_before_scenario, :global_before, :local_before, :local_after, :global_after, :local_after_scenario,
:global_after_scenario,
:global_before_scenario, :local_before_scenario, :global_before, :local_before, :local_after, :global_after, :local_after_scenario,
:global_after_scenario,
]
end

Expand All @@ -325,18 +342,22 @@ defmodule Benchee.Benchmark.RunnerTest do
warmup: 0,
time: 100,
before_each: fn -> send me, :global_before end,
after_each: fn -> send me, :global_after end
after_each: fn -> send me, :global_after end,
after_scenario: fn -> send me, :global_after_scenario end
}
}
|> test_suite
|> Benchmark.benchmark("job", {fn -> :timer.sleep 1 end,
before_each: fn -> send me, :local_1_before end})
|> Benchmark.benchmark("job", {
fn -> :timer.sleep 1 end,
before_each: fn -> send me, :local_1_before end,
before_scenario: fn -> send me, :local_scenario_before end})
|> Benchmark.benchmark("job 2", fn -> :timer.sleep 1 end)
|> Benchmark.measure(TestPrinter)

assert_received_exactly [
:global_before, :local_1_before, :global_after,
:global_before, :global_after
:global_before, :local_scenario_before, :local_1_before, :global_after,
:global_after_scenario,
:global_before, :global_after, :global_after_scenario
]
end

Expand All @@ -351,39 +372,65 @@ defmodule Benchee.Benchmark.RunnerTest do
}
}
|> test_suite
|> Benchmark.benchmark("job", {fn -> :timer.sleep 1 end,
before_each: fn -> send me, :local_before end,
after_each: fn -> send me, :local_after end})
|> Benchmark.benchmark("job 2", {fn -> :timer.sleep 1 end,
before_each: fn -> send me, :local_2_before end,
after_each: fn -> send me, :local_2_after end})
|> Benchmark.benchmark("job", {
fn -> :timer.sleep 1 end,
before_each: fn -> send me, :local_before end,
after_each: fn -> send me, :local_after end,
before_scenario: fn -> send me, :local_before_scenario end})
|> Benchmark.benchmark("job 2", {
fn -> :timer.sleep 1 end,
before_each: fn -> send me, :local_2_before end,
after_each: fn -> send me, :local_2_after end,
after_scenario: fn -> send me, :local_2_after_scenario end})
|> Benchmark.measure(TestPrinter)

assert_received_exactly [
:global_before, :local_before, :local_after, :global_after,
:global_before, :local_2_before, :local_2_after, :global_after
:local_before_scenario, :global_before, :local_before, :local_after,
:global_after,
:global_before, :local_2_before, :local_2_after, :global_after,
:local_2_after_scenario
]
end

test "before_each triggers for every invocation" do
test "each triggers for every invocation, scenario once" do
me = self()
suite = %Suite{
configuration: %{
warmup: 0,
time: 10_000,
before_each: fn -> send me, :global_before end,
after_each: fn -> send me, :global_after end
after_each: fn -> send me, :global_after end,
before_scenario: fn -> send me, :global_before_scenario end,
after_scenario: fn -> send me, :global_after_scenario end,
}
}
result =
suite
|> test_suite
|> Benchmark.benchmark("job", {fn -> :timer.sleep 1 end,
before_each: fn -> send me, :local_before end,
after_each: fn -> send me, :local_after end})
|> Benchmark.benchmark("job", {
fn -> :timer.sleep 1 end,
before_each: fn -> send me, :local_before end,
after_each: fn -> send me, :local_after end,
before_scenario: fn -> send(me, :local_before_scenario) end,
after_scenario: fn -> send(me, :local_after_scenario) end})
|> Benchmark.measure(TestPrinter)

{:messages, messages} = Process.info self(), :messages

global_before_sceneario_count =
Enum.count messages, fn(msg) -> msg == :global_before_scenario end
local_before_sceneario_count =
Enum.count messages, fn(msg) -> msg == :local_before_scenario end
local_after_sceneario_count =
Enum.count messages, fn(msg) -> msg == :local_after_scenario end
global_after_sceneario_count =
Enum.count messages, fn(msg) -> msg == :global_after_scenario end

assert global_before_sceneario_count == 1
assert local_before_sceneario_count == 1
assert local_after_sceneario_count == 1
assert global_after_sceneario_count == 1

global_before_count =
Enum.count messages, fn(message) -> message == :global_before end
local_before_count =
Expand Down
11 changes: 7 additions & 4 deletions test/benchee/benchmark_test.exs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,17 +54,20 @@ defmodule Benchee.BenchmarkTest do
end

test "can deal with the options tuple" do
function = fn -> 1 end
before = fn -> 2 end
function = fn -> 1 end
before = fn -> 2 end
after_scenario = fn -> 3 end
suite =
%Suite{}
|> Benchmark.benchmark("job", {function, before_each: before})
|> Benchmark.benchmark("job", {
function, before_each: before, after_scenario: after_scenario})

[scenario] = suite.scenarios
assert %{
job_name: "job",
function: ^function,
before_each: ^before } = scenario
before_each: ^before,
after_scenario: ^after_scenario } = scenario
end
end

Expand Down
25 changes: 18 additions & 7 deletions test/benchee_test.exs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -328,18 +328,29 @@ defmodule BencheeTest do
capture_io fn ->
myself = self()
Benchee.run %{
"sleeper" => {fn -> :timer.sleep 1 end,
before_each: fn -> send myself, :local_before end,
after_each: fn -> send myself, :local_after end},
"sleeper" => {
fn -> :timer.sleep 1 end,
before_each: fn -> send myself, :local_before end,
after_each: fn -> send myself, :local_after end,
before_scenario: fn -> send myself, :local_before_scenario end,
after_scenario: fn -> send myself, :local_after_scenario end},
"sleeper 2" => fn -> :timer.sleep 1 end
}, time: 0.0001, warmup: 0,
}, time: 0.0001,
warmup: 0,
before_each: fn -> send myself, :global_before end,
after_each: fn -> send myself, :global_after end
after_each: fn -> send myself, :global_after end,
before_scenario: fn -> send myself, :global_before_scenario end,
after_scenario: fn -> send myself, :global_after_scenario end
end

assert_received_exactly [
:global_before, :local_before, :local_after, :global_after,
:global_before, :global_after
# first job with all those local hooks
:global_before_scenario, :local_before_scenario, :global_before,
:local_before, :local_after, :global_after, :local_after_scenario,
:global_after_scenario,
# second job that only runs global hooks
:global_before_scenario, :global_before, :global_after,
:global_after_scenario
]
end
end
Expand Down

0 comments on commit 6e55aaa

Please sign in to comment.