From e57c41e3502c69b97867da81eb0bdb5804fa00be Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Mon, 5 Aug 2019 13:07:19 -0400 Subject: [PATCH 1/3] Show all results (not just significant results) in the exported Markdown --- src/benchmarkjudgement.jl | 43 ++++++++++++++++++++++++++------------- test/runtests.jl | 2 ++ 2 files changed, 31 insertions(+), 14 deletions(-) diff --git a/src/benchmarkjudgement.jl b/src/benchmarkjudgement.jl index bdaa46c..4514747 100644 --- a/src/benchmarkjudgement.jl +++ b/src/benchmarkjudgement.jl @@ -35,13 +35,13 @@ function Base.show(io::IO, judgement::BenchmarkJudgement) base.julia_commit[1:6]) end -function export_markdown(file::String, results::BenchmarkJudgement) +function export_markdown(file::String, results::BenchmarkJudgement; kwargs...) open(file, "w") do f - export_markdown(f, results) + export_markdown(f, results; kwargs...) end end -function export_markdown(io::IO, judgement::BenchmarkJudgement) +function export_markdown(io::IO, judgement::BenchmarkJudgement; allrows::Bool = false) target, baseline = judgement.target_results, judgement.baseline_results function env_strs(res) return if isempty(benchmarkconfig(res).env) @@ -90,21 +90,36 @@ function export_markdown(io::IO, judgement::BenchmarkJudgement) _update_col_widths!(cw, ids, t) end - print(io, """ - ## Results - A ratio greater than `1.0` denotes a possible regression (marked with $(_REGRESS_MARK)), while a ratio less - than `1.0` denotes a possible improvement (marked with $(_IMPROVE_MARK)). Only significant results - results - that indicate possible regressions or improvements - are shown below (thus, an empty table means that all - benchmark results remained invariant between builds). + if allrows + print(io, """ + ## Results + A ratio greater than `1.0` denotes a possible regression (marked with $(_REGRESS_MARK)), while a ratio less + than `1.0` denotes a possible improvement (marked with $(_IMPROVE_MARK)). All results are shown below. - | ID$(" "^(cw[1]-2)) | time ratio$(" "^(cw[2]-10)) | memory ratio$(" "^(cw[3]-12)) | - |---$("-"^(cw[1]-2))-|-----------$("-"^(cw[2]-10))-|-------------$("-"^(cw[3]-12))-| - """) + | ID$(" "^(cw[1]-2)) | time ratio$(" "^(cw[2]-10)) | memory ratio$(" "^(cw[3]-12)) | + |---$("-"^(cw[1]-2))-|-----------$("-"^(cw[2]-10))-|-------------$("-"^(cw[3]-12))-| + """) - for (ids, t) in entries - if BenchmarkTools.isregression(t) || BenchmarkTools.isimprovement(t) + for (ids, t) in entries println(io, _resultrow(ids, t, cw)) end + else + print(io, """ + ## Results + A ratio greater than `1.0` denotes a possible regression (marked with $(_REGRESS_MARK)), while a ratio less + than `1.0` denotes a possible improvement (marked with $(_IMPROVE_MARK)). Only significant results - results + that indicate possible regressions or improvements - are shown below (thus, an empty table means that all + benchmark results remained invariant between builds). + + | ID$(" "^(cw[1]-2)) | time ratio$(" "^(cw[2]-10)) | memory ratio$(" "^(cw[3]-12)) | + |---$("-"^(cw[1]-2))-|-----------$("-"^(cw[2]-10))-|-------------$("-"^(cw[3]-12))-| + """) + + for (ids, t) in entries + if BenchmarkTools.isregression(t) || BenchmarkTools.isimprovement(t) + println(io, _resultrow(ids, t, cw)) + end + end end println(io) diff --git a/test/runtests.jl b/test/runtests.jl index 63a7eff..b0b8371 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -163,6 +163,8 @@ temp_pkg_dir(;tmp_dir = tmp_dir) do judgement = judge(TEST_PACKAGE_NAME, "HEAD~", "HEAD", custom_loadpath=old_pkgdir) test_structure(PkgBenchmark.benchmarkgroup(judgement)) export_markdown(stdout, judgement) + export_markdown(stdout, judgement; allrows = false) + export_markdown(stdout, judgement; allrows = true) judgement = judge(TEST_PACKAGE_NAME, "HEAD", custom_loadpath=old_pkgdir) test_structure(PkgBenchmark.benchmarkgroup(judgement)) end From fca7642b482631f2a0ee037dff88e9e01530e6d5 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Mon, 19 Aug 2019 23:55:00 -0400 Subject: [PATCH 2/3] Rename `allrows` to `export_invariants` --- src/benchmarkjudgement.jl | 4 ++-- test/runtests.jl | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/benchmarkjudgement.jl b/src/benchmarkjudgement.jl index 4514747..721c2d0 100644 --- a/src/benchmarkjudgement.jl +++ b/src/benchmarkjudgement.jl @@ -41,7 +41,7 @@ function export_markdown(file::String, results::BenchmarkJudgement; kwargs...) end end -function export_markdown(io::IO, judgement::BenchmarkJudgement; allrows::Bool = false) +function export_markdown(io::IO, judgement::BenchmarkJudgement; export_invariants::Bool = false) target, baseline = judgement.target_results, judgement.baseline_results function env_strs(res) return if isempty(benchmarkconfig(res).env) @@ -90,7 +90,7 @@ function export_markdown(io::IO, judgement::BenchmarkJudgement; allrows::Bool = _update_col_widths!(cw, ids, t) end - if allrows + if export_invariants print(io, """ ## Results A ratio greater than `1.0` denotes a possible regression (marked with $(_REGRESS_MARK)), while a ratio less diff --git a/test/runtests.jl b/test/runtests.jl index b0b8371..00523a4 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -163,8 +163,8 @@ temp_pkg_dir(;tmp_dir = tmp_dir) do judgement = judge(TEST_PACKAGE_NAME, "HEAD~", "HEAD", custom_loadpath=old_pkgdir) test_structure(PkgBenchmark.benchmarkgroup(judgement)) export_markdown(stdout, judgement) - export_markdown(stdout, judgement; allrows = false) - export_markdown(stdout, judgement; allrows = true) + export_markdown(stdout, judgement; export_invariants = false) + export_markdown(stdout, judgement; export_invariants = true) judgement = judge(TEST_PACKAGE_NAME, "HEAD", custom_loadpath=old_pkgdir) test_structure(PkgBenchmark.benchmarkgroup(judgement)) end From 1f9a87eba01bae08cd34d83383b49e9354275a1c Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Tue, 20 Aug 2019 00:00:27 -0400 Subject: [PATCH 3/3] Update the docstring for `export_markdown` --- src/benchmarkresults.jl | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/benchmarkresults.jl b/src/benchmarkresults.jl index af6a411..b376115 100644 --- a/src/benchmarkresults.jl +++ b/src/benchmarkresults.jl @@ -85,11 +85,17 @@ function readresults(file::String) end """ - export_markdown(file::String, results::Union{BenchmarkResults, BenchmarkJudgement}) - export_markdown(io::IO, results::Union{BenchmarkResults, BenchmarkJudgement}) + export_markdown(file::String, results::BenchmarkResults) + export_markdown(io::IO, results::BenchmarkResults) + export_markdown(file::String, results::BenchmarkJudgement; export_invariants=false) + export_markdown(io::IO, results::BenchmarkJudgement; export_invariants=false) Writes the `results` to `file` or `io` in markdown format. +When exporting a `BenchmarkJudgement`, by default only the results corresponding to +possible regressions or improvements will be included. To also export the invariant +results, set `export_invariants=true`. + See also: [`BenchmarkResults`](@ref), [`BenchmarkJudgement`](@ref) """ function export_markdown(file::String, results::BenchmarkResults)