Skip to content

Commit

Permalink
use a naming convention instead of a function for dict based API (#44)
Browse files Browse the repository at this point in the history
  • Loading branch information
KristofferC authored Jul 28, 2017
1 parent 8b056cb commit 6bf0c90
Show file tree
Hide file tree
Showing 6 changed files with 13 additions and 33 deletions.
3 changes: 0 additions & 3 deletions benchmark/benchmarks_dict.jl
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
using PkgBenchmark
using BenchmarkTools
using UnicodePlots

Expand All @@ -25,5 +24,3 @@ for f in (sin, cos, tan)
SUITE["trigonometry"]["hyperbolic"][string(f), x] = @benchmarkable $(f)($x)
end
end

register_suite(SUITE)
4 changes: 3 additions & 1 deletion docs/src/man/define_benchmarks.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@

Benchmarks are to be written in `<PKGROOT>/benchmark/benchmarks.jl` and can be defined in two different ways:

* Using the standard dictionary based interface from BenchmarkTools, as documented [here](https://github.com/JuliaCI/BenchmarkTools.jl/blob/master/doc/manual.md#defining-benchmark-suites). Note that the suite need to be registered with PkgBenchmark by calling `register_suite(SUITE)`. An example file using the dictionary based interface can be found [here](https://github.com/JuliaCI/PkgBenchmark.jl/blob/master/benchmark/benchmarks_dict.jl).
* Using the standard dictionary based interface from BenchmarkTools, as documented [here](https://github.com/JuliaCI/BenchmarkTools.jl/blob/master/doc/manual.md#defining-benchmark-suites). The naming convention
that must be used is to name the benchmark suite variable `SUITE`. An example file using the dictionary based interface can be found [here](https://github.com/JuliaCI/PkgBenchmark.jl/blob/master/benchmark/benchmarks_dict.jl). Note that there is no need to have PkgBenchmark loaded
to define the benchmark suite if the dict based interface is used.
* Using the `@benchgroup` and `@bench` macros. These are analogous to `@testset` and `@test` macros, with slightly different syntax. An example file using the macro based interface can be found [here](https://github.com/JuliaCI/PkgBenchmark.jl/blob/master/benchmark/benchmarks.jl).

`<PKGROOT>/benchmark/REQUIRE` can contain dependencies needed to run the benchmark suite.
Expand Down
4 changes: 2 additions & 2 deletions src/PkgBenchmark.jl
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,10 @@ using BenchmarkTools
using FileIO
using JLD

export benchmarkpkg, judge, @benchgroup, @bench, register_suite
export benchmarkpkg, judge, @benchgroup, @bench, writeresults, readresults

include("util.jl")
include("define_benchmarks.jl")
include("macros.jl")
include("runbenchmark.jl")
include("judge.jl")

Expand Down
18 changes: 0 additions & 18 deletions src/define_benchmarks.jl → src/macros.jl
Original file line number Diff line number Diff line change
@@ -1,21 +1,3 @@
##################
# Dict based API #
##################
SUITE = nothing

"""
register_suite(suite::BenchmarkGroup)
Registers the benchmark suite `suite` with PkgBenchmark so that it is used
when running [`benchmarkpkg`](@ref).
"""
function register_suite(bg::BenchmarkGroup)
global SUITE = bg
end

_reset_suite() = global SUITE = nothing
_get_suite() = SUITE

###################
# Macro based API #
###################
Expand Down
7 changes: 2 additions & 5 deletions src/runbenchmark.jl
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,9 @@ end

function runbenchmark_local(file, output, tunefile, retune)
_reset_stack()
_reset_suite()

include(file)

suite = if _get_suite() != nothing
_get_suite()
suite = if isdefined(Main, :SUITE)
Main.SUITE
else
_root_group()
end
Expand Down
10 changes: 6 additions & 4 deletions test/runtests.jl
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ using PkgBenchmark
using BenchmarkTools
using Base.Test

const BENCHMARK_DIR = joinpath(dirname(@__FILE__), "..", "benchmark")

function temp_pkg_dir(fn::Function; tmp_dir=joinpath(tempdir(), randstring()),
remove_tmp_dir::Bool=true, initialize::Bool=true)
# Used in tests below to set up and tear down a sandboxed package directory
Expand Down Expand Up @@ -34,13 +36,13 @@ end

@testset "structure" begin
@testset "macro" begin
include(Pkg.dir("PkgBenchmark", "benchmark", "benchmarks.jl"))
include(joinpath(BENCHMARK_DIR, "benchmarks.jl"))
test_structure(PkgBenchmark._top_group())
end

@testset "dict" begin
include(Pkg.dir("PkgBenchmark", "benchmark", "benchmarks_dict.jl"))
test_structure(PkgBenchmark._get_suite())
results = benchmarkpkg("PkgBenchmark", script = joinpath(BENCHMARK_DIR, "benchmarks_dict.jl"))
test_structure(results)
end
end

Expand Down Expand Up @@ -108,7 +110,7 @@ temp_pkg_dir(;tmp_dir = tmp_dir) do
results = PkgBenchmark.benchmarkpkg(TEST_PACKAGE_NAME, "HEAD"; custom_loadpath=old_pkgdir, resultsdir=tmp)
test_structure(results)
@test isfile(resfile)
@test PkgBenchmark.readresults(resfile) == results
@test readresults(resfile) == results

# Make a dummy commit and test comparing HEAD and HEAD~
touch(joinpath(testpkg_path, "dummy"))
Expand Down

0 comments on commit 6bf0c90

Please sign in to comment.