Fix bug running tests in parallel benchmarks

If you were running benchmarks in parallel, you would see results for
each parallel process you were running. So, if you were running two
jobs, and setting your configuration to `parallel: 2`, you would see
four results in the formatter. This is now correctly showing only the
two jobs.

CHANGELOG.md

@@ -1,3 +1,15 @@
+## 0.12.0 (2018-01-11)
+
+A small release fixing a somewhat nasty bug involving running benchmarks in
+parallel.
+
+### Bugfixes (User Facing)
+
+* If you were running benchmarks in parallel, you would see results for each
+parallel process you were running. So, if you were running **two** jobs, and
+setting your configuration to `parallel: 2`, you would see **four** results in the
+formatter. This is now correctly showing only the **two** jobs.
+
 ## 0.11.0 (2017-11-30)
 
 A tiny little release with a bugfix and MOARE statistics for the console formatter.

lib/benchee/benchmark/runner.ex

@@ -37,9 +37,17 @@ defmodule Benchee.Benchmark.Runner do
            config: config
          }) do
     printer.benchmarking(job_name, input_name, config)
-    Parallel.map(1..config.parallel, fn(_task_number) ->
+    1..config.parallel
+    |> Parallel.map(fn(_task_number) ->
       run_scenario(scenario, scenario_context)
     end)
+    |> Enum.group_by(fn scenario -> {scenario.function, scenario.input} end)
+    |> Enum.map(fn {_, like_results} ->
+      consolidated_results =
+        Enum.flat_map(like_results, fn scenario -> scenario.run_times end)
+      [scenario | _] = like_results
+      %Benchee.Benchmark.Scenario{scenario | run_times: consolidated_results}
+    end)
   end
 
   defp run_scenario(scenario, scenario_context) do

mix.exs

@@ -1,7 +1,7 @@
 defmodule Benchee.Mixfile do
   use Mix.Project
 
-  @version "0.11.0"
+  @version "0.11.1"
 
   def project do
     [

test/benchee/benchmark/runner_test.exs

@@ -78,6 +78,17 @@ defmodule Benchee.Benchmark.RunnerTest do
       assert length(run_times_for(new_suite, "")) >= 12
     end
 
+    test "combines results for parallel benchmarks into a single scenario" do
+      suite = test_suite(%Suite{configuration: %{parallel: 4, time: 60_000}})
+
+      new_suite =
+        suite
+        |> Benchmark.benchmark("", fn -> :timer.sleep(10) end)
+        |> Benchmark.measure(TestPrinter)
+
+      assert length(new_suite.scenarios) == 1
+    end
+
     test "very fast functions print a warning" do
       output = ExUnit.CaptureIO.capture_io fn ->
         %Suite{configuration: %{print: %{fast_warning: true}}}