Loosen nanosecond resolution restriction

README.md

@@ -1,3 +1,5 @@
+Note: Some type definitions were changed between BenchmarkTools v0.0.2 and v0.0.3. To deserialize old JLD-formatted data using a newer version of BenchmarkTools, use `BenchmarkTools.loadold(args...)` instead of `JLD.load(args...)`.
+
 # BenchmarkTools.jl
 
 [![Build Status](https://travis-ci.org/JuliaCI/BenchmarkTools.jl.svg?branch=master)](https://travis-ci.org/JuliaCI/BenchmarkTools.jl)

REQUIRE

@@ -1,2 +1,3 @@
 julia 0.4
 Compat 0.7.20
+JLD 0.6.1

doc/manual.md

@@ -841,7 +841,7 @@ Caching parameters in this manner leads to a far shorter turnaround time, and mo
 
 # Miscellaneous tips and info
 
-- Times reported by BenchmarkTools are limited to nanosecond resolution, though derived estimates might report fractions of nanoseconds.
+- BenchmarkTools restricts the minimum measurable benchmark execution time to one picosecond.
 - If you use `rand` or something similar to generate the values that are used in your benchmarks, you should seed the RNG (or provide a seeded RNG) so that the values are consistent between trials/samples/evaluations.
 - BenchmarkTools attempts to be robust against machine noise occurring between *samples*, but BenchmarkTools can't do very much about machine noise occurring between *trials*. To cut down on the latter kind of noise, it is advised that you dedicate CPUs and memory to the benchmarking Julia process by using a shielding tool such as [cset](http://manpages.ubuntu.com/manpages/precise/man1/cset.1.html).
 - On some machines, for some versions of BLAS and Julia, the number of BLAS worker threads can exceed the number of available cores. This can occasionally result in scheduling issues and inconsistent performance for BLAS-heavy benchmarks. To fix this issue, you can use `BLAS.set_num_threads(i::Int)` in the Julia REPL to ensure that the number of BLAS threads is equal to or less than the number of available cores.

src/BenchmarkTools.jl

@@ -1,6 +1,7 @@
 module BenchmarkTools
 
 using Compat
+import JLD
 
 # `show` compatibility for pre-JuliaLang/julia#16354 builds
 if VERSION < v"0.5.0-dev+4305"
@@ -59,6 +60,12 @@ export tune!,
        @benchmark,
        @benchmarkable
 
+###########################
+# Backwards Compatibility #
+###########################
+
+include("compat.jl")
+
 ##########################################
 # Plotting Facilities (loaded on demand) #
 ##########################################

src/compat.jl

@@ -0,0 +1,42 @@
+############################################
+# Backwards-compatible JLD Deserialization #
+############################################
+
+type OldParameters
+    seconds::Float64
+    samples::Int
+    evals::Int
+    overhead::Int
+    gctrial::Bool
+    gcsample::Bool
+    time_tolerance::Float64
+    memory_tolerance::Float64
+end
+
+type OldTrial
+    params::Parameters
+    times::Vector{Float64}
+    gctimes::Vector{Float64}
+    memory::Int
+    allocs::Int
+end
+
+function JLD.readas(p::OldParameters)
+    return Parameters(p.seconds, p.samples, p.evals, Float64(p.overhead), p.gctrial,
+                      p.gcsample, p.time_tolerance, p.memory_tolerance)
+end
+
+function JLD.readas(t::OldTrial)
+    new_times = convert(Vector{Float64}, t.times)
+    new_gctimes = convert(Vector{Float64}, t.gctimes)
+    return Trial(t.params, new_times, new_gctimes, t.memory, t.allocs)
+end
+
+function loadold(args...)
+    JLD.translate("BenchmarkTools.Parameters", "BenchmarkTools.OldParameters")
+    JLD.translate("BenchmarkTools.Trial", "BenchmarkTools.OldTrial")
+    result = JLD.load(args...)
+    JLD.translate("BenchmarkTools.Parameters", "BenchmarkTools.Parameters")
+    JLD.translate("BenchmarkTools.Trial", "BenchmarkTools.Trial")
+    return result
+end

src/execution.jl

@@ -41,7 +41,7 @@ end
 
 function _lineartrial(b::Benchmark, p::Parameters = b.params; maxevals = RESOLUTION, kwargs...)
     params = Parameters(p; kwargs...)
-    estimates = zeros(Int, maxevals)
+    estimates = zeros(maxevals)
     completed = 0
     params.gctrial && gc()
     start_time = time()
@@ -111,7 +111,7 @@ end
 
 function tune!(b::Benchmark, p::Parameters = b.params;
                verbose::Bool = false, pad = "", kwargs...)
-    estimate = minimum(lineartrial(b, p; kwargs...))
+    estimate = ceil(Int, minimum(lineartrial(b, p; kwargs...)))
     b.params.evals = guessevals(estimate)
     return b
 end
@@ -234,8 +234,8 @@ macro benchmarkable(args...)
                     __sample_time = time_ns() - __start_time
                     __gcdiff = Base.GC_Diff(Base.gc_num(), __gc_start)
                     $($(Expr(:quote, teardown)))
-                    __time = max(Int(cld(__sample_time, __evals)) - __params.overhead, 1)
-                    __gctime = max(Int(cld(__gcdiff.total_time, __evals)) - __params.overhead, 0)
+                    __time = max((__sample_time / __evals) - __params.overhead, 0.001)
+                    __gctime = max((__gcdiff.total_time / __evals) - __params.overhead, 0.0)
                     __memory = Int(fld(__gcdiff.allocd, __evals))
                     __allocs = Int(fld(__gcdiff.malloc + __gcdiff.realloc +
                                        __gcdiff.poolalloc + __gcdiff.bigalloc,

src/parameters.jl

@@ -9,7 +9,7 @@ type Parameters
     seconds::Float64
     samples::Int
     evals::Int
-    overhead::Int
+    overhead::Float64
     gctrial::Bool
     gcsample::Bool
     time_tolerance::Float64
@@ -80,16 +80,13 @@ end
         nullfunc()
     end
     sample_time = time_ns() - start_time
-    return Int(cld(sample_time, evals))
+    return sample_time / evals
 end
 
 function estimate_overhead()
     x = typemax(Int)
     for _ in 1:10000
-        y = overhead_sample(RESOLUTION)
-        if y < x
-            x = y
-        end
+        x = min(x, overhead_sample(RESOLUTION))
     end
     return x
 end

src/trials.jl

@@ -4,13 +4,13 @@
 
 type Trial
     params::Parameters
-    times::Vector{Int}
-    gctimes::Vector{Int}
+    times::Vector{Float64}
+    gctimes::Vector{Float64}
     memory::Int
     allocs::Int
 end
 
-Trial(params::Parameters) = Trial(params, Int[], Int[], typemax(Int), typemax(Int))
+Trial(params::Parameters) = Trial(params, Float64[], Float64[], typemax(Int), typemax(Int))
 
 @compat function Base.:(==)(a::Trial, b::Trial)
     return a.params == b.params &&
@@ -246,7 +246,7 @@ function prettytime(t)
     else
         value, units = t / 1e9, "s"
     end
-    return string(@sprintf("%.2f", value), " ", units)
+    return string(@sprintf("%.3f", value), " ", units)
 end
 
 function prettymemory(b)

test/CompatTests.jl

@@ -0,0 +1,16 @@
+module CompatTests
+
+using Base.Test
+using BenchmarkTools
+using JLD
+
+old_data = BenchmarkTools.loadold(joinpath(dirname(@__FILE__), "old_data.jld"))
+new_data = JLD.load(joinpath(dirname(@__FILE__), "new_data.jld"))
+
+@test old_data["params"] == old_data["trial"].params
+@test new_data["params"] == new_data["trial"].params
+
+@test old_data["params"] == new_data["params"]
+@test old_data["trial"] == new_data["trial"]
+
+end # module

test/TrialsTests.jl

@@ -154,12 +154,12 @@ tj_r_2 = judge(tr; time_tolerance = 2.0, memory_tolerance = 2.0)
 @test BenchmarkTools.prettydiff(1.0) == "+0.00%"
 @test BenchmarkTools.prettydiff(2.0) == "+100.00%"
 
-@test BenchmarkTools.prettytime(999) == "999.00 ns"
-@test BenchmarkTools.prettytime(1000) == "1.00 μs"
-@test BenchmarkTools.prettytime(999_999) == "1000.00 μs"
-@test BenchmarkTools.prettytime(1_000_000) == "1.00 ms"
-@test BenchmarkTools.prettytime(999_999_999) == "1000.00 ms"
-@test BenchmarkTools.prettytime(1_000_000_000) == "1.00 s"
+@test BenchmarkTools.prettytime(999) == "999.000 ns"
+@test BenchmarkTools.prettytime(1000) == "1.000 μs"
+@test BenchmarkTools.prettytime(999_999) == "999.999 μs"
+@test BenchmarkTools.prettytime(1_000_000) == "1.000 ms"
+@test BenchmarkTools.prettytime(999_999_999) == "1000.000 ms"
+@test BenchmarkTools.prettytime(1_000_000_000) == "1.000 s"
 
 @test BenchmarkTools.prettymemory(1023) == "1023.00 bytes"
 @test BenchmarkTools.prettymemory(1024) == "1.00 kb"

test/runtests.jl

@@ -13,3 +13,8 @@ println("done (took ", toq(), " seconds)")
 print("Testing execution..."); tic()
 include("ExecutionTests.jl")
 println("done (took ", toq(), " seconds)")
+
+
+print("Testing backwards compatibility..."); tic()
+include("CompatTests.jl")
+println("done (took ", toq(), " seconds)")