Display absolute difference for measurements

lib/benchee/formatters/console/helpers.ex

@@ -55,32 +55,43 @@ defmodule Benchee.Formatters.Console.Helpers do
         statistics,
         display_value,
         comparison_name,
+        display_unit,
         label_width,
         column_width
       ) do
-    "~*s~*s ~s"
+    "~*s~*s ~ts"
     |> :io_lib.format([
       -label_width,
       name,
       column_width,
       display_value,
-      comparison_display(statistics, comparison_name)
+      comparison_display(statistics, comparison_name, display_unit)
     ])
     |> to_string
   end
 
-  defp comparison_display(%Statistics{relative_more: nil, absolute_difference: nil}, _), do: ""
+  defp comparison_display(%Statistics{relative_more: nil, absolute_difference: nil}, _, _), do: ""
 
-  defp comparison_display(statistics, comparison_name) do
-    "- #{comparison_text(statistics, comparison_name)}\n"
+  defp comparison_display(statistics, comparison_name, unit) do
+    "- #{comparison_text(statistics, comparison_name)} - #{
+      absolute_difference_text(statistics, unit)
+    }\n"
   end
 
-  defp comparison_text(%Statistics{relative_more: :infinity}, name), do: "infinity x #{name}"
+  defp comparison_text(%Statistics{relative_more: :infinity}, name), do: " 	∞ x #{name}"
   defp comparison_text(%Statistics{relative_more: nil}, _), do: "N/A"
 
   defp comparison_text(statistics, comparison_name) do
     "~.2fx ~s"
     |> :io_lib.format([statistics.relative_more, comparison_name])
     |> to_string
   end
+
+  defp absolute_difference_text(statistics, unit) do
+    formatted_value = Format.format({Scale.scale(statistics.absolute_difference, unit), unit})
+
+    # currently the fastest/least consuming is always first so everything else eats more
+    # resources hence this is always +
+    "+#{formatted_value}"
+  end
 end

lib/benchee/formatters/console/memory.ex

@@ -106,7 +106,7 @@ defmodule Benchee.Formatters.Console.Memory do
   defp scenario_reports([scenario | other_scenarios], units, label_width, true) do
     [
       reference_report(scenario, units, label_width),
-      comparisons(scenario, units, label_width, other_scenarios),
+      comparisons(other_scenarios, units, label_width),
       "\n**All measurements for memory usage were the same**\n"
     ]
   end
@@ -209,7 +209,7 @@ defmodule Benchee.Formatters.Console.Memory do
     [
       Helpers.descriptor("Comparison"),
       reference_report(scenario, units, label_width)
-      | comparisons(scenario, units, label_width, other_scenarios)
+      | comparisons(other_scenarios, units, label_width)
     ]
   end
 
@@ -227,10 +227,8 @@ defmodule Benchee.Formatters.Console.Memory do
     |> to_string
   end
 
-  @spec comparisons(Scenario.t(), unit_per_statistic, integer, [Scenario.t()]) :: [String.t()]
-  defp comparisons(scenario, units, label_width, scenarios_to_compare) do
-    %Scenario{memory_usage_data: %{statistics: reference_stats}} = scenario
-
+  @spec comparisons([Scenario.t()], unit_per_statistic, integer) :: [String.t()]
+  defp comparisons(scenarios_to_compare, units, label_width) do
     Enum.map(
       scenarios_to_compare,
       fn scenario ->
@@ -242,6 +240,7 @@ defmodule Benchee.Formatters.Console.Memory do
           statistics,
           memory_format,
           "memory usage",
+          units.memory,
           label_width,
           @median_width
         )

lib/benchee/formatters/console/run_time.ex

@@ -264,6 +264,7 @@ defmodule Benchee.Formatters.Console.RunTime do
           statistics,
           ips_format,
           "slower",
+          units.run_time,
           label_width,
           @ips_width
         )

test/benchee/formatters/console/memory_test.exs

@@ -123,9 +123,9 @@ defmodule Benchee.Formatters.Console.MemoryTest do
       output = Memory.format_scenarios(scenarios, @console_config)
       [_, _, _, _, comp_header, reference, slower] = output
 
-      assert Regex.match?(~r/Comparison/, comp_header)
-      assert Regex.match?(~r/^First\s+90 B$/m, reference)
-      assert Regex.match?(~r/^Second\s+195.50 B\s+- 2.17x memory usage/, slower)
+      assert comp_header =~ ~r/Comparison/
+      assert reference =~ ~r/^First\s+90 B$/m
+      assert slower =~ ~r/^Second\s+195.50 B\s+- 2.17x memory usage - \+105\.50 B$/m
     end
 
     test "can omit the comparisons" do
@@ -169,9 +169,9 @@ defmodule Benchee.Formatters.Console.MemoryTest do
           })
         )
 
-      refute Regex.match?(~r/Comparison/i, output)
-      refute Regex.match?(~r/^First\s+90 B$/m, output)
-      refute Regex.match?(~r/^Second\s+195.50 B\s+- 2.17x memory usage/, output)
+      refute output =~ ~r/Comparison/i
+      refute output =~ ~r/^First\s+90 B$/m
+      refute output =~ ~r/^Second\s+195.50 B\s+- 2.17x memory usage/
     end
 
     test "adjusts the label width to longest name for comparisons" do
@@ -318,16 +318,16 @@ defmodule Benchee.Formatters.Console.MemoryTest do
       output = Memory.format_scenarios(scenarios, params)
       [_memory_title, _header1, _result1, title, header2, result2] = output
 
-      assert title =~ ~r/Extended statistics: /
-      assert header2 =~ ~r/minimum/
-      assert header2 =~ ~r/maximum/
-      assert header2 =~ ~r/sample size/
-      assert header2 =~ ~r/mode/
-      assert result2 =~ ~r/First job/
-      assert result2 =~ ~r/111.10/
-      assert result2 =~ ~r/333.30/
-      assert result2 =~ ~r/50 K/
-      assert result2 =~ ~r/201.20/
+      assert title =~ "Extended statistics: "
+      assert header2 =~ "minimum"
+      assert header2 =~ "maximum"
+      assert header2 =~ "sample size"
+      assert header2 =~ "mode"
+      assert result2 =~ "First job"
+      assert result2 =~ "111.10"
+      assert result2 =~ "333.30"
+      assert result2 =~ "50 K"
+      assert result2 =~ "201.20"
     end
 
     test "does nothing when there's no statistics to format" do
@@ -422,7 +422,7 @@ defmodule Benchee.Formatters.Console.MemoryTest do
 
       assert output =~ "First"
       assert output =~ "Second"
-      assert output =~ "infinity x memory usage"
+      assert output =~ "∞ x memory usage"
     end
 
     test "it doesn't blow up if some come back with a median et. al. of nil" do

test/benchee/formatters/console/run_time_test.exs

@@ -218,9 +218,9 @@ defmodule Benchee.Formatters.Console.RunTimeTest do
       [_, _, _, comp_header, reference, slower] =
         RunTime.format_scenarios(scenarios, @console_config)
 
-      assert Regex.match?(~r/Comparison/, comp_header)
-      assert Regex.match?(~r/^First\s+10 K$/m, reference)
-      assert Regex.match?(~r/^Second\s+5 K\s+- 2.00x slower/, slower)
+      assert comp_header =~ ~r/Comparison/
+      assert reference =~ ~r/^First\s+10 K$/m
+      assert slower =~ ~r/^Second\s+5 K\s+- 2.00x slower - \+100 ns$/m
     end
 
     test "can omit the comparisons" do
@@ -263,9 +263,9 @@ defmodule Benchee.Formatters.Console.RunTimeTest do
           })
         )
 
-      refute Regex.match?(~r/Comparison/i, output)
-      refute Regex.match?(~r/^First\s+10 K$/m, output)
-      refute Regex.match?(~r/^Second\s+5 K\s+- 2.00x slower/, output)
+      refute output =~ ~r/Comparison/i
+      refute output =~ ~r/^First\s+10 K$/m
+      refute output =~ ~r/^Second\s+5 K\s+- 2.00x slower/
     end
 
     test "adjusts the label width to longest name for comparisons" do

test/benchee/formatters/console_test.exs

@@ -70,14 +70,16 @@ defmodule Benchee.Formatters.ConsoleTest do
           )
         end)
 
-      assert output =~ ~r/First/
-      assert output =~ ~r/Second/
-      assert output =~ ~r/200/
-      assert output =~ ~r/5 K/
-      assert output =~ ~r/10.00%/
-      assert output =~ ~r/195.5/
-      assert output =~ ~r/300.1/
-      assert output =~ ~r/400.1/
+      assert output =~ "First"
+      assert output =~ "Second"
+      assert output =~ "200"
+      assert output =~ "5 K"
+      assert output =~ "10.00%"
+      assert output =~ "195.5"
+      assert output =~ "300.1"
+      assert output =~ "400.1"
+      assert output =~ "2.00x slower"
+      assert output =~ "+100 ns"
     end
   end
 
@@ -263,14 +265,14 @@ defmodule Benchee.Formatters.ConsoleTest do
       assert other_job =~ ~r/Other Job.+10.+100.+30\.00%.+98.+200\.1/
       assert job =~ ~r/Job.+5.+200.+10\.00%.+195\.5/
       ref =~ ~r/Other Job/
-      slower =~ ~r/Job.+slower/
+      slower =~ ~r/Job.+slower \+100/
 
       [input_header_2, _, other_job_2, job_2, _, ref_2, slower_2] = other_arg
       assert input_header_2 =~ "Other Arg"
       assert other_job_2 =~ ~r/Other Job.+4.+250.+31\.00%.+225\.5.+300\.1/
       assert job_2 =~ ~r/Job.+2\.5.+400.+15\.00%.+395/
       ref_2 =~ ~r/Other Job/
-      slower_2 =~ ~r/Job.+slower/
+      slower_2 =~ ~r/Job.+slower \+150/
     end
 
     test "with and without a tag" do

test/benchee_test.exs

@@ -800,11 +800,11 @@ defmodule BencheeTest do
       assert output =~ "B"
 
       assert output =~ "1.00x memory"
-      assert output =~ "infinity x memo"
+      assert output =~ "∞ x memo"
     end
   end
 
-  @slower_regex "\\s+- \\d+\\.\\d+x slower"
+  @slower_regex "\\s+- \\d+\\.\\d+x slower - \\+\\d+\\.\\d+.+"
   defp readme_sample_asserts(output, tag_string \\ "") do
     assert output =~ "warmup: 5 ms"
     assert output =~ "time: 10 ms"
@@ -820,7 +820,7 @@ defmodule BencheeTest do
 
     # In windows time resolution seems to be milliseconds, hence even
     # standard examples produce a fast warning.
-    # So we skip this basic is everything going fine test on windows
+    # So we skip this "basically everything is going fine" test on windows
     unless windows?(), do: refute(output =~ ~r/fast/i)
   end