many changes (#57)

* many changes

* use JSON to save stuff since that is what BenchmarkTools does
* simplify API be removing some kwargs
* remove macro based API, this should exist in BenchmarkTools if we want it.
* remove caching of results, too brittle

* add update warning

.gitignore

@@ -5,3 +5,4 @@
 .*.sw?
 docs/build
 benchmark/.tune.jld
+deps/already_showed

LICENSE.md

@@ -1,6 +1,6 @@
 The PkgBenchmark.jl package is licensed under the MIT "Expat" License:
 
-> Copyright (c) 2016: Shashi Gowda.
+> Copyright (c) 2016-2018: Shashi Gowda, Kristoffer Carlsson.
 >
 > Permission is hereby granted, free of charge, to any person obtaining
 > a copy of this software and associated documentation files (the

README.md

@@ -4,15 +4,14 @@
 
 | **Documentation**                                                               | **PackageEvaluator**                                                                            | **Build Status**                                                                                |
 |:-------------------------------------------------------------------------------:|:-----------------------------------------------------------------------------------------------:|:-----------------------------------------------------------------------------------------------:|
-| [![][docs-stable-img]][docs-stable-url] [![][docs-latest-img]][docs-latest-url] | [![][pkg-0.5-img]][pkg-0.5-url] [![][pkg-0.6-img]][pkg-0.6-url] | [![][travis-img]][travis-url] [![Coverage Status][coverage-img]][coverage-url] |
+| [![][docs-stable-img]][docs-stable-url] [![][docs-latest-img]][docs-latest-url] | [![][pkg-0.6-img]][pkg-0.6-url] | [![][travis-img]][travis-url] [![Coverage Status][coverage-img]][coverage-url] |
 
 ## Introduction
 
 PkgBenchmark provides an interface for Julia package developers to track performance changes of their packages.
 
 The package contains the following features
 
-* A macro based interface, similar to the `Base.Test` interface, to define a suite of benchmarks. It is, however, also possible to use the dictionary based interface defined in [BenchmarkTools](https://github.com/JuliaCI/BenchmarkTools.jl) in which case it isn't even needed to depend on this package to write the benchmar suite.
 * Running the benchmark suite at a specified commit, branch or tag. The path to the julia executable, the command line flags, and the environment variables can be customized.
 * Comparing performance of a package between different package commits, branches or tags.
 * Exporting results to markdown for benchmarks and comparisons, similar to how Nanosoldier reports results for the benchmarks on Base Julia.
@@ -32,7 +31,7 @@ julia> Pkg.add("PkgBenchmark")
 
 ## Project Status
 
-The package is tested against Julia `0.5` and `0.6` on Linux and macOS.
+The package is tested against Julia `0.6` on Linux and macOS.
 
 ## Contributing and Questions
 
@@ -48,8 +47,6 @@ Contributions are welcome, as are feature requests and suggestions. Please open
 [travis-img]: https://travis-ci.org/JuliaCI/PkgBenchmark.jl.svg?branch=master
 [travis-url]: https://travis-ci.org/JuliaCI/PkgBenchmark.jl
 
-[pkg-0.5-img]: http://pkg.julialang.org/badges/PkgBenchmark_0.5.svg
-[pkg-0.5-url]: http://pkg.julialang.org/?pkg=PkgBenchmark&ver=0.5
 [pkg-0.6-img]: http://pkg.julialang.org/badges/PkgBenchmark_0.6.svg
 [pkg-0.6-url]: http://pkg.julialang.org/?pkg=PkgBenchmark&ver=0.6
 

REQUIRE

@@ -1,5 +1,4 @@
-julia 0.5
-FileIO
-JLD
+julia 0.6
+JSON
 BenchmarkTools
-ProgressMeter
+ProgressMeter

appveyor.yml

@@ -1,7 +1,5 @@
 environment:
   matrix:
-  - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x86/0.5/julia-0.5-latest-win32.exe"
-  - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x64/0.5/julia-0.5-latest-win64.exe"
   - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x86/0.6/julia-0.6-latest-win32.exe"
   - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x64/0.6/julia-0.6-latest-win64.exe"
 

benchmark/benchmarks.jl

@@ -1,31 +1,26 @@
-using PkgBenchmark
+using BenchmarkTools
 using UnicodePlots
 
-@benchgroup "utf8" ["string", "unicode"] begin
-    teststr = String(join(rand(MersenneTwister(1), 'a':'d', 10^4)))
-    @bench "replace" replace($teststr, "a", "b")
-    @bench "join" join($teststr, $teststr)
+const SUITE = BenchmarkGroup()
 
-    @benchgroup "plots" begin
-        @bench "fnplot" lineplot([sin, cos], -π/2, 2pi)
-    end
-end
+SUITE["utf8"] = BenchmarkGroup(["string", "unicode"])
+teststr = String(join(rand(MersenneTwister(1), 'a':'d', 10^4)))
+SUITE["utf8"]["replace"] = @benchmarkable replace($teststr, "a", "b")
+SUITE["utf8"]["join"] = @benchmarkable join($teststr, $teststr)
+SUITE["utf8"]["plots"] = BenchmarkGroup()
+SUITE["utf8"]["plots"]["fnplot"] = @benchmarkable lineplot([sin, cos], -π/2, 2pi)
 
-@benchgroup "trigonometry" ["math", "triangles"] begin
-    # nested groups
-    @benchgroup "circular" begin
-        for f in (sin, cos, tan)
-            for x in (0.0, pi)
-                @bench string(f), x $(f)($x)
-            end
-        end
+SUITE["trigonometry"] = BenchmarkGroup(["math", "triangles"])
+SUITE["trigonometry"]["circular"] = BenchmarkGroup()
+for f in (sin, cos, tan)
+    for x in (0.0, pi)
+        SUITE["trigonometry"]["circular"][string(f), x] = @benchmarkable ($f)($x)
     end
+end
 
-    @benchgroup "hyperbolic" begin
-        for f in (sinh, cosh, tanh)
-            for x in (0.0, pi)
-                @bench string(f), x $(f)($x)
-            end
-        end
+SUITE["trigonometry"]["hyperbolic"] = BenchmarkGroup()
+for f in (sin, cos, tan)
+    for x in (0.0, pi)
+        SUITE["trigonometry"]["hyperbolic"][string(f), x] = @benchmarkable ($f)($x)
     end
 end

deps/build.jl

@@ -0,0 +1,9 @@
+if !isfile(joinpath(@__DIR__, "already_showed"))
+    print_with_color(Base.info_color(), STDERR,
+    """
+    PkgBenchmark has been completely rewritten. Please see https://github.com/JuliaCI/PkgBenchmark.jl/
+    for updated documentation and examples. Code written for previous versions of PkgBenchmark is
+    unlikely to still work.
+    """)
+    touch("already_showed")
+end

docs/src/define_benchmarks.md

@@ -1,76 +1,10 @@
 # Defining a benchmark suite
 
-
-Benchmarks are to be written in `<PKGROOT>/benchmark/benchmarks.jl` and can be defined in two different ways:
-
-* Using the standard dictionary based interface from BenchmarkTools, as documented [here](https://github.com/JuliaCI/BenchmarkTools.jl/blob/master/doc/manual.md#defining-benchmark-suites). The naming convention that must be used is to name the benchmark suite variable `SUITE`. An example file using the dictionary based interface can be found [here](https://github.com/JuliaCI/PkgBenchmark.jl/blob/master/benchmark/benchmarks_dict.jl). Note that there is no need to have PkgBenchmark loaded to define the benchmark suite if the dict based interface is used.
-* Using the `@benchgroup` and `@bench` macros. These are analogous to `@testset` and `@test` macros, with slightly different syntax. An example file using the macro based interface can be found [here](https://github.com/JuliaCI/PkgBenchmark.jl/blob/master/benchmark/benchmarks.jl).
-
+Benchmarks are to be written in `<PKGROOT>/benchmark/benchmarks.jl` and are defined using the standard dictionary based interface from BenchmarkTools, as documented [here](https://github.com/JuliaCI/BenchmarkTools.jl/blob/master/doc/manual.md#defining-benchmark-suites). The naming convention that must be used is to name the benchmark suite variable `SUITE`. An example file using the dictionary based interface can be found [here](https://github.com/JuliaCI/PkgBenchmark.jl/blob/master/benchmark/benchmarks.jl). Note that there is no need to have PkgBenchmark loaded to define the benchmark suite.
 
 !!! note
     Running this script directly does not actually run the benchmarks, this is the job of PkgBenchmark, see the next section.
 
-## Writing benchmarks using the macro based API
-
-### `@benchgroup`
-
-`@benchgroup` defines a benchmark group. It can contain nested `@benchgroup` and `@bench` expressions.
-
-**Syntax:**
-
-```julia
-@benchgroup <name> [<tags>] begin
-  <expr>
-end
-```
-
-`<name>` is a string naming the benchmark group. `<tags>` is a vector of strings, tags for the benchmark group, and is optional. `<expr>` are expressions that can contain `@benchgroup` or `@bench` calls.
-
-### `@bench`
-
-`@bench` creates a benchmark under the current `@benchgroup`.
-
-**Syntax:**
-
-```julia
-@bench <name>... <expr>
-```
-
-`<name>` is a name/id for the benchmark, the last argument to `@bench`, `<expr>`, is the expression to be benchmarked, and has the same [interpolation features](https://github.com/JuliaCI/BenchmarkTools.jl/blob/master/doc/manual.md#interpolating-values-into-benchmark-expressions) as the `@benchmarkable` macro from BenchmarkTools.
-
-### Example
-
-An example `benchmark/benchmarks.jl` script would look like:
-
-```julia
-using PkgBenchmark
-
-@benchgroup "utf8" ["string", "unicode"] begin
-    teststr = UTF8String(join(rand(MersenneTwister(1), 'a':'d', 10^4)))
-    @bench "replace" replace($teststr, "a", "b")
-    @bench "join" join($teststr, $teststr)
-end
-
-@benchgroup "trigonometry" ["math", "triangles"] begin
-    # nested groups
-    @benchgroup "circular" begin
-        for f in (sin, cos, tan)
-            for x in (0.0, pi)
-                @bench string(f), x $(f)($x)
-            end
-        end
-    end
-
-    @benchgroup "hyperbolic" begin
-        for f in (sinh, cosh, tanh)
-            for x in (0.0, pi)
-                @bench string(f), x $(f)($x)
-            end
-        end
-    end
-end
-```
-
 ## Custom requirements for benchmarks
 
 `<PKGROOT>/benchmark/REQUIRE` can contain dependencies needed to run the benchmark suite, similarly how `<PKGROOT>/benchmark/REQUIRE` can contain dependencies for the tests.

docs/src/export_markdown.md

@@ -14,7 +14,6 @@ Assuming that we have gotten a `BenchmarkResults` or `BenchmarkJudgement` from a
 julia> using GitHub, JSON, PkgBenchmark
 
 julia> results = benchmarkpkg("PkgBenchmark");
-[...]
 
 julia> gist_json = JSON.parse(
             """
@@ -34,7 +33,7 @@ julia> posted_gist = create_gist(params = gist_json);
 
 julia> url = get(posted_gist.html_url)
 URI(https://gist.github.com/317378b4fcf2fb4c5585b104c3b177a8)
-``` 
+```
 
 !!! note
     Consider using an extension to your browser to make the gist webpage use full width in order for the tables

docs/src/index.md

@@ -4,10 +4,9 @@ PkgBenchmark provides an interface for Julia package developers to track perform
 
 The package contains the following features
 
-* A macro based interface, similar to the `Base.Test` interface, to define a suite of benchmarks. It is, however, also possible to use the dictionary based interface defined in [BenchmarkTools](https://github.com/JuliaCI/BenchmarkTools.jl) in which case it isn't even needed to depend on this package to write the benchmar suite.
 * Running the benchmark suite at a specified commit, branch or tag. The path to the julia executable, the command line flags, and the environment variables can be customized.
 * Comparing performance of a package between different package commits, branches or tags.
-* Exporting results to markdown for benchmarks and comparisons, similar to how Nanosoldier reports results for the benchmarks on Base Julia.
+* Exporting results to markdown for benchmarks and comparisons, similar to how Nanosoldier reports results for the benchmarks in Base Julia.
 
 ## Installation
 

docs/src/ref.md

@@ -3,7 +3,7 @@ Pages = ["ref.md"]
 Modules = [PkgBenchmark]
 ```
 
-    ```@autodocs
+```@autodocs
 Modules = [PkgBenchmark]
 Private = false
 ```

src/PkgBenchmark.jl

@@ -3,15 +3,12 @@ __precompile__()
 module PkgBenchmark
 
 using BenchmarkTools
-using FileIO
-using JLD
-using Compat
+using JSON
 using ProgressMeter
 
-export benchmarkpkg, judge, @benchgroup, @bench, writeresults, readresults, export_markdown
+export benchmarkpkg, judge, writeresults, readresults, export_markdown
 export BenchmarkConfig, BenchmarkResults, BenchmarkJudgement
 
-include("macros.jl")
 include("benchmarkconfig.jl")
 include("benchmarkresults.jl")
 include("benchmarkjudgement.jl")

src/benchmarkconfig.jl

@@ -11,20 +11,12 @@ This includes the following:
   the command flags used (e.g. optimization level with `-O`).
 * Custom environment variables (e.g. `JULIA_NUM_THREADS`).
 """
-immutable BenchmarkConfig
+struct BenchmarkConfig
     id::Union{String,Void}
     juliacmd::Cmd
     env::Dict{String,Any}
 end
 
-function _hash(pkgname::String, pkgcommit::String, juliacommit, config::BenchmarkConfig)
-    return hash(pkgname,
-                hash(juliacommit,
-                    hash(length(config.juliacmd) > 1 ? config.juliacmd[2:end] : 0,
-                        hash(pkgcommit,
-                            hash(config.env)))))
-end
-
 """
     BenchmarkConfig(;id::Union{String, Void} = nothing,
                      juliacmd::Cmd = `$(joinpath(JULIA_HOME, Base.julia_exename()))`,
@@ -41,9 +33,15 @@ Creates a `BenchmarkConfig` from the following keyword arguments:
 
 # Examples
 ```julia
-BenchmarkConfig(id = "performance_improvements",
-                juliacmd = `julia -O3`,
-                env = Dict("JULIA_NUM_THREADS" => 4))
+julia> using Pkgbenchmark
+
+julia> BenchmarkConfig(id = "performance_improvements",
+                       juliacmd = `julia -O3`,
+                       env = Dict("JULIA_NUM_THREADS" => 4))
+BenchmarkConfig:
+    id: performance_improvements
+    juliacmd: `julia -O3`
+    env: JULIA_NUM_THREADS => 4
 ```
 """
 function BenchmarkConfig(;id::Union{String,Void} = nothing,
@@ -57,6 +55,24 @@ BenchmarkConfig(cfg::BenchmarkConfig) = cfg
 BenchmarkConfig(str::String) = BenchmarkConfig(id = str)
 BenchmarkConfig(::Void) = BenchmarkConfig()
 
+function BenchmarkConfig(d::Dict)
+    BenchmarkConfig(
+        [fieldtype(BenchmarkConfig, fname)(d[string(fname)]) for fname in fieldnames(BenchmarkConfig)]...
+    )
+end
+
+# Arr!...
+function Base.Cmd(d::Dict)
+    Cmd(
+        Cmd(convert(Vector{String}, d["exec"])),
+        d["ignorestatus"],
+        d["flags"],
+        d["env"],
+        d["dir"],
+    )
+end
+
+
 const _INDENT = "    "
 
 function Base.show(io::IO, bcfg::BenchmarkConfig)