Add methods to isinvariant, isregression, isimprovement, etc. (#85

)

Manifest.toml

@@ -4,10 +4,10 @@
 uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 
 [[BenchmarkTools]]
-deps = ["JSON", "Printf", "Statistics", "Test"]
-git-tree-sha1 = "5d1dd8577643ba9014574cd40d9c028cd5e4b85a"
+deps = ["JSON", "Printf", "Statistics"]
+git-tree-sha1 = "90b73db83791c5f83155016dd1cc1f684d4e1361"
 uuid = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
-version = "0.4.2"
+version = "0.4.3"
 
 [[Dates]]
 deps = ["Printf"]
@@ -22,10 +22,10 @@ deps = ["Markdown"]
 uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
 
 [[JSON]]
-deps = ["Dates", "Distributed", "Mmap", "Sockets", "Test", "Unicode"]
-git-tree-sha1 = "1f7a25b53ec67f5e9422f1f551ee216503f4a0fa"
+deps = ["Dates", "Mmap", "Parsers", "Unicode"]
+git-tree-sha1 = "b34d7cef7b337321e97d22242c3c2b91f476748e"
 uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
-version = "0.20.0"
+version = "0.21.0"
 
 [[LibGit2]]
 uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"
@@ -47,8 +47,14 @@ uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
 [[Mmap]]
 uuid = "a63ad114-7e13-5084-954f-fe012c677804"
 
+[[Parsers]]
+deps = ["Dates", "Test"]
+git-tree-sha1 = "ef0af6c8601db18c282d092ccbd2f01f3f0cd70b"
+uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
+version = "0.3.7"
+
 [[Pkg]]
-deps = ["Dates", "LibGit2", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"]
+deps = ["Dates", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"]
 uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 
 [[Printf]]

Project.toml

@@ -1,6 +1,6 @@
 name = "PkgBenchmark"
 uuid = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d"
-version = "0.2.3"
+version = "0.2.4"
 
 [deps]
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
@@ -13,7 +13,7 @@ Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca"
 
 [compat]
-BenchmarkTools = "0.4"
+BenchmarkTools = "^0.4.3"
 ProgressMeter = "1"
 julia = "1"
 

src/PkgBenchmark.jl

@@ -11,7 +11,7 @@ using Dates
 using InteractiveUtils
 using Printf
 
-export benchmarkpkg, judge, writeresults, readresults, export_markdown
+export benchmarkpkg, judge, writeresults, readresults, export_markdown, memory
 export BenchmarkConfig, BenchmarkResults, BenchmarkJudgement
 
 include("benchmarkconfig.jl")

src/benchmarkjudgement.jl

@@ -23,6 +23,24 @@ target_result(judgement::BenchmarkJudgement) = judgement.target_results
 baseline_result(judgement::BenchmarkJudgement) = judgement.baseline_results
 benchmarkgroup(judgement::BenchmarkJudgement) = judgement.benchmarkgroup
 
+BenchmarkTools.isinvariant(f, judgement::BenchmarkJudgement) = BenchmarkTools.isinvariant(f, benchmarkgroup(judgement))
+BenchmarkTools.isinvariant(judgement::BenchmarkJudgement) = BenchmarkTools.isinvariant(benchmarkgroup(judgement))
+
+BenchmarkTools.isregression(f, judgement::BenchmarkJudgement) = BenchmarkTools.isregression(f, benchmarkgroup(judgement))
+BenchmarkTools.isregression(judgement::BenchmarkJudgement) = BenchmarkTools.isregression(benchmarkgroup(judgement))
+
+BenchmarkTools.isimprovement(f, judgement::BenchmarkJudgement) = BenchmarkTools.isimprovement(f, benchmarkgroup(judgement))
+BenchmarkTools.isimprovement(judgement::BenchmarkJudgement) = BenchmarkTools.isimprovement(benchmarkgroup(judgement))
+
+BenchmarkTools.invariants(f, judgement::BenchmarkJudgement) = BenchmarkTools.invariants(f, benchmarkgroup(judgement))
+BenchmarkTools.invariants(judgement::BenchmarkJudgement) = BenchmarkTools.invariants(benchmarkgroup(judgement))
+
+BenchmarkTools.regressions(f, judgement::BenchmarkJudgement) = BenchmarkTools.regressions(f, benchmarkgroup(judgement))
+BenchmarkTools.regressions(judgement::BenchmarkJudgement) = BenchmarkTools.regressions(benchmarkgroup(judgement))
+
+BenchmarkTools.improvements(f, judgement::BenchmarkJudgement) = BenchmarkTools.improvements(f, benchmarkgroup(judgement))
+BenchmarkTools.improvements(judgement::BenchmarkJudgement) = BenchmarkTools.improvements(benchmarkgroup(judgement))
+
 function Base.show(io::IO, judgement::BenchmarkJudgement)
     target, base = judgement.target_results, judgement.baseline_results
     print(io, "Benchmarkjudgement (target / baseline):\n")

test/runtests.jl

@@ -167,5 +167,27 @@ temp_pkg_dir(;tmp_dir = tmp_dir) do
         export_markdown(stdout, judgement; export_invariants = true)
         judgement = judge(TEST_PACKAGE_NAME, "HEAD", custom_loadpath=old_pkgdir)
         test_structure(PkgBenchmark.benchmarkgroup(judgement))
+        judgement = judge(TEST_PACKAGE_NAME, "HEAD", "HEAD", custom_loadpath=old_pkgdir)
+        judgement = judge(TEST_PACKAGE_NAME, "HEAD", "HEAD"; custom_loadpath=old_pkgdir, retune=true)
+        @test PkgBenchmark.benchmarkgroup(judgement) == judgement.benchmarkgroup
+        @test PkgBenchmark.benchmarkgroup(judgement) === judgement.benchmarkgroup
+        @test isinvariant(judgement) == isinvariant(judgement.benchmarkgroup)
+        @test isinvariant(time, judgement) == isinvariant(time, judgement.benchmarkgroup)
+        @test isinvariant(memory, judgement) == isinvariant(memory, judgement.benchmarkgroup)
+        @test isregression(judgement) == isregression(judgement.benchmarkgroup)
+        @test isregression(time, judgement) == isregression(time, judgement.benchmarkgroup)
+        @test isregression(memory, judgement) == isregression(memory, judgement.benchmarkgroup)
+        @test isimprovement(judgement) == isimprovement(judgement.benchmarkgroup)
+        @test isimprovement(time, judgement) == isimprovement(time, judgement.benchmarkgroup)
+        @test isimprovement(memory, judgement) == isimprovement(memory, judgement.benchmarkgroup)
+        @test BenchmarkTools.invariants(judgement) == BenchmarkTools.invariants(judgement.benchmarkgroup)
+        @test BenchmarkTools.invariants(time, judgement) == BenchmarkTools.invariants(time, judgement.benchmarkgroup)
+        @test BenchmarkTools.invariants(memory, judgement) == BenchmarkTools.invariants(memory, judgement.benchmarkgroup)
+        @test BenchmarkTools.regressions(judgement) == BenchmarkTools.regressions(judgement.benchmarkgroup)
+        @test BenchmarkTools.regressions(time, judgement) == BenchmarkTools.regressions(time, judgement.benchmarkgroup)
+        @test BenchmarkTools.regressions(memory, judgement) == BenchmarkTools.regressions(memory, judgement.benchmarkgroup)
+        @test BenchmarkTools.improvements(judgement) == BenchmarkTools.improvements(judgement.benchmarkgroup)
+        @test BenchmarkTools.improvements(time, judgement) == BenchmarkTools.improvements(time, judgement.benchmarkgroup)
+        @test BenchmarkTools.improvements(memory, judgement) == BenchmarkTools.improvements(memory, judgement.benchmarkgroup)
     end
 end