Merge 33a83c1 into de9e994

.exguard.exs

@@ -3,6 +3,12 @@ use ExGuard.Config
 project_files = ~r{\.(erl|ex|exs|eex|xrl|yrl)\z}i
 deps = ~r{deps}
 
+guard("compile and warn", run_on_start: true)
+|> command("MIX_ENV=test mix compile --warnings-as-errors")
+|> watch(project_files)
+|> ignore(deps)
+|> notification(:auto)
+
 guard("credo", run_on_start: true)
 |> command("mix credo --strict")
 |> watch(project_files)

lib/benchee/statistics.ex

@@ -21,6 +21,9 @@ defmodule Benchee.Statistics do
     :mode,
     :minimum,
     :maximum,
+    :relative_more,
+    :relative_less,
+    :absolute_difference,
     sample_size: 0
   ]
 
@@ -37,6 +40,9 @@ defmodule Benchee.Statistics do
           mode: mode,
           minimum: number,
           maximum: number,
+          relative_more: float | nil | :infinity,
+          relative_less: float | nil | :infinity,
+          absolute_difference: float | nil,
           sample_size: integer
         }
 
@@ -140,21 +146,28 @@ defmodule Benchee.Statistics do
   def statistics(suite = %Suite{scenarios: scenarios}) do
     config = suite.configuration
 
-    percentiles = Enum.uniq([50 | config.percentiles])
-
     scenarios_with_statistics =
-      Parallel.map(scenarios, fn scenario ->
-        run_time_stats = scenario.run_time_data.samples |> job_statistics(percentiles) |> add_ips
-        memory_stats = job_statistics(scenario.memory_usage_data.samples, percentiles)
+      scenarios
+      |> calculate_per_scenario_statistics(config)
+      |> sort()
+      |> calculate_relative_statistics(config.inputs)
 
-        %{
-          scenario
-          | run_time_data: %{scenario.run_time_data | statistics: run_time_stats},
-            memory_usage_data: %{scenario.memory_usage_data | statistics: memory_stats}
-        }
-      end)
+    %Suite{suite | scenarios: scenarios_with_statistics}
+  end
+
+  defp calculate_per_scenario_statistics(scenarios, config) do
+    percentiles = Enum.uniq([50 | config.percentiles])
+
+    Parallel.map(scenarios, fn scenario ->
+      run_time_stats = scenario.run_time_data.samples |> job_statistics(percentiles) |> add_ips
+      memory_stats = job_statistics(scenario.memory_usage_data.samples, percentiles)
 
-    %Suite{suite | scenarios: sort(scenarios_with_statistics)}
+      %{
+        scenario
+        | run_time_data: %{scenario.run_time_data | statistics: run_time_stats},
+          memory_usage_data: %{scenario.memory_usage_data | statistics: memory_stats}
+      }
+    end)
   end
 
   @doc """
@@ -266,6 +279,64 @@ defmodule Benchee.Statistics do
     }
   end
 
+  defp calculate_relative_statistics([], _inputs), do: []
+
+  defp calculate_relative_statistics(scenarios, inputs) do
+    scenarios
+    |> scenarios_by_input(inputs)
+    |> Enum.flat_map(fn scenarios_with_same_input ->
+      {reference, others} = split_reference_scenario(scenarios_with_same_input)
+      others_with_relative = statistics_relative_to(others, reference)
+      [reference | others_with_relative]
+    end)
+  end
+
+  defp scenarios_by_input(scenarios, nil), do: [scenarios]
+
+  # we can't just group_by `input_name` because that'd lose the order of inputs which might
+  # be important
+  defp scenarios_by_input(scenarios, inputs) do
+    Enum.map(inputs, fn {input_name, _} ->
+      Enum.filter(scenarios, fn scenario -> scenario.input_name == input_name end)
+    end)
+  end
+
+  # right now we take the first scenario as we sorted them and it is the fastest,
+  # whenever we implement #179 though this becomesd more involved
+  defp split_reference_scenario(scenarios) do
+    [reference | others] = scenarios
+    {reference, others}
+  end
+
+  defp statistics_relative_to(scenarios, reference) do
+    Enum.map(scenarios, fn scenario ->
+      scenario
+      |> update_in([Access.key!(:run_time_data), Access.key!(:statistics)], fn statistics ->
+        add_relative_statistics(statistics, reference.run_time_data.statistics)
+      end)
+      |> update_in([Access.key!(:memory_usage_data), Access.key!(:statistics)], fn statistics ->
+        add_relative_statistics(statistics, reference.memory_usage_data.statistics)
+      end)
+    end)
+  end
+
+  # we might not run time/memory --> we shouldn't crash then ;)
+  defp add_relative_statistics(statistics = %{average: nil}, _reference), do: statistics
+  defp add_relative_statistics(statistics, %{average: nil}), do: statistics
+
+  defp add_relative_statistics(statistics, reference_statistics) do
+    %__MODULE__{
+      statistics
+      | relative_more: zero_safe_division(statistics.average, reference_statistics.average),
+        relative_less: zero_safe_division(reference_statistics.average, statistics.average),
+        absolute_difference: statistics.average - reference_statistics.average
+    }
+  end
+
+  defp zero_safe_division(_, 0), do: :infinity
+  defp zero_safe_division(_, 0.0), do: :infinity
+  defp zero_safe_division(a, b), do: a / b
+
   @doc """
   Calculate additional percentiles and add them to the
   `run_time_data.statistics`. Should only be used after `statistics/1`, to

test/benchee/statistics_test.exs

@@ -39,27 +39,51 @@ defmodule Benchee.StatistcsTest do
         %Scenario{
           input: "Input 1",
           input_name: "Input 1",
-          job_name: "Job",
+          job_name: "Job 1",
+          run_time_data: %CollectionData{samples: @sample_1},
+          memory_usage_data: %CollectionData{samples: @sample_1}
+        },
+        %Scenario{
+          input: "Input 1",
+          input_name: "Input 1",
+          job_name: "Job 2",
+          run_time_data: %CollectionData{samples: @sample_2},
+          memory_usage_data: %CollectionData{samples: @sample_2}
+        },
+        %Scenario{
+          input: "Input 2",
+          input_name: "Input 2",
+          job_name: "Job 1",
           run_time_data: %CollectionData{samples: @sample_1},
           memory_usage_data: %CollectionData{samples: @sample_1}
         },
         %Scenario{
           input: "Input 2",
           input_name: "Input 2",
-          job_name: "Job",
+          job_name: "Job 2",
           run_time_data: %CollectionData{samples: @sample_2},
           memory_usage_data: %CollectionData{samples: @sample_2}
         }
       ]
 
-      suite = %Suite{scenarios: scenarios}
+      suite = %Suite{
+        scenarios: scenarios,
+        configuration: %Benchee.Configuration{
+          inputs: %{"Input 1" => "Input 1", "Input 2" => "Input2"}
+        }
+      }
+
       new_suite = Statistics.statistics(suite)
 
-      stats_1 = stats_for(new_suite, "Job", "Input 1")
-      stats_2 = stats_for(new_suite, "Job", "Input 2")
+      stats_1_1 = stats_for(new_suite, "Job 1", "Input 1")
+      stats_1_2 = stats_for(new_suite, "Job 2", "Input 1")
+      stats_2_1 = stats_for(new_suite, "Job 1", "Input 2")
+      stats_2_2 = stats_for(new_suite, "Job 2", "Input 2")
 
-      sample_1_asserts(stats_1)
-      sample_2_asserts(stats_2)
+      sample_1_asserts(stats_1_1)
+      sample_2_asserts(stats_1_2)
+      sample_1_asserts(stats_2_1)
+      sample_2_asserts(stats_2_2)
     end
 
     @mode_sample [55, 40, 67, 55, 44, 40, 10, 8, 55, 90, 67]
@@ -166,22 +190,30 @@ defmodule Benchee.StatistcsTest do
     test "doesn't blow up when all measurements are zeros (mostly memory measurement)" do
       scenarios = [
         %Scenario{
-          run_time_data: %CollectionData{samples: @all_zeros},
-          memory_usage_data: %CollectionData{samples: @all_zeros}
+          run_time_data: %CollectionData{samples: @all_zeros}
+        },
+        %Scenario{
+          run_time_data: %CollectionData{samples: @all_zeros}
         }
       ]
 
       suite = Statistics.statistics(%Suite{scenarios: scenarios})
 
       [
         %Scenario{
-          run_time_data: %CollectionData{statistics: run_time_stats},
-          memory_usage_data: %CollectionData{statistics: memory_stats}
+          run_time_data: %CollectionData{statistics: run_time_stats_1}
+        },
+        %Scenario{
+          run_time_data: %CollectionData{statistics: run_time_stats_2}
         }
       ] = suite.scenarios
 
-      assert run_time_stats.sample_size == 5
-      assert memory_stats.sample_size == 5
+      assert run_time_stats_1.average == 0.0
+      assert run_time_stats_2.sample_size == 5
+
+      assert run_time_stats_2.relative_more == :infinity
+      assert run_time_stats_2.relative_less == :infinity
+      assert run_time_stats_2.absolute_difference == 0.0
     end
 
     test "sorts them by their average run time fastest to slowest" do
@@ -253,6 +285,9 @@ defmodule Benchee.StatistcsTest do
       assert stats.maximum == 600
       assert stats.sample_size == 5
       assert stats.mode == nil
+      assert_in_delta stats.relative_more, 28.14, 0.01
+      assert_in_delta stats.relative_less, 0.0355, 0.001
+      assert stats.absolute_difference == 380.0
     end
 
     defp sample_2_asserts(stats) do
@@ -265,6 +300,9 @@ defmodule Benchee.StatistcsTest do
       assert stats.maximum == 23
       assert stats.sample_size == 6
       assert stats.mode == nil
+      assert stats.relative_more == nil
+      assert stats.relative_less == nil
+      assert stats.absolute_difference == nil
     end
   end
 end

test/benchee_test.exs

@@ -422,7 +422,7 @@ defmodule BencheeTest do
     assert length(occurences) == 3
   end
 
-  test "inputs can also be a list of 2-tuples" do
+  test "inputs can also be a list of 2-tuples and it then keeps the order" do
     output =
       capture_io(fn ->
         map_fun = fn i -> [i, i * i] end
@@ -592,20 +592,34 @@ defmodule BencheeTest do
     end)
   end
 
-  test "does not blow up setting all times to 0 and never executes a function" do
-    output =
-      capture_io(fn ->
-        Benchee.run(
-          %{
-            "never execute me" => fn -> raise "BOOOOM" end
-          },
-          time: 0,
-          warmup: 0,
-          memory_time: 0
-        )
-      end)
+  describe "edge cases" do
+    test "does not blow up setting all times to 0 and never executes a function" do
+      output =
+        capture_io(fn ->
+          Benchee.run(
+            %{
+              "never execute me" => fn -> raise "BOOOOM" end
+            },
+            time: 0,
+            warmup: 0,
+            memory_time: 0
+          )
+        end)
+
+      refute output =~ "never execute me"
+    end
+
+    test "does not blow up if nothing is specified" do
+      output =
+        capture_io(fn ->
+          Benchee.run(
+            %{},
+            @test_configuration
+          )
+        end)
 
-    refute output =~ "never execute me"
+      refute output =~ "Benchmarking"
+    end
   end
 
   describe "save & load" do