Skip to content

Commit

Permalink
initial implementation of (#7)
Browse files Browse the repository at this point in the history
  • Loading branch information
Andrey Oskin committed Feb 22, 2022
1 parent 6a6867f commit 376aeb8
Show file tree
Hide file tree
Showing 7 changed files with 405 additions and 174 deletions.
5 changes: 3 additions & 2 deletions Project.toml
View file
Original file line number Diff line number Diff line change
@@ -1,22 +1,23 @@
name = "BenchmarkExt"
uuid = "f97012b4-579f-4109-a65c-74c581afaadd"
version = "0.1.1"
version = "0.1.2"

[deps]
JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
Profile = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79"
Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"

[compat]
JSON = "0.18, 0.19, 0.20, 0.21"
julia = "1"

[extras]
StableRNGs = "860ef19b-820b-49d6-a774-d7a799459cd3"
ReTest = "e0db7c4e-2690-44b9-bad6-7687da720f89"
StableRNGs = "860ef19b-820b-49d6-a774-d7a799459cd3"
Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

Expand Down
57 changes: 41 additions & 16 deletions src/BenchmarkExt.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,21 +8,33 @@ using Statistics
using UUIDs: uuid4
using Printf
using Profile

using TOML

const BENCHMARKEXT_VERSION = v"0.1.0"

##############
# Parameters #
##############
########################################
# Preferences
########################################

include("preferences.jl")

# Public unexported API:
# set_preferences,
# save_preferences!,
# load_preferences!


########################################
# Parameters
########################################

include("parameters.jl")

export loadparams!

##############
# Trial Data #
##############
########################################
# Trial Data
########################################

include("trials.jl")

Expand All @@ -41,9 +53,9 @@ export gctime,
rmskew,
trim

##################
# Benchmark Data #
##################
########################################
# Benchmark Data
########################################

include("groups.jl")

Expand All @@ -57,9 +69,9 @@ export BenchmarkGroup,
@benchmarkset,
@case

######################
# Execution Strategy #
######################
########################################
# Execution Strategy
########################################

include("execution.jl")

Expand All @@ -72,10 +84,23 @@ export tune!,
@btime,
@bprofile

#################
# Serialization #
#################
########################################
# Serialization
########################################

include("serialization.jl")

########################################
# Pretty printing
########################################

include("pprints/trial_classical.jl")
include("pprints/trial_fancy.jl")

########################################
# Initialization
########################################

include("init.jl")

end # module BenchmarkExt
3 changes: 3 additions & 0 deletions src/init.jl
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
function __init__()
initialize_prefs()
end
41 changes: 41 additions & 0 deletions src/pprints/trial_classical.jl
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
abstract type AbstractOutput end

show_output(::AbstractOutput, io, ::MIME"text/plain", t) = nothing

struct TrialClassicalOutputPre <: AbstractOutput end
struct TrialClassicalHistogram <: AbstractOutput end
struct TrialClassicalOutputPost <: AbstractOutput end

function show_output(::TrialClassicalOutputPre, io, ::MIME"text/plain", t::Trial)
if length(t) > 0
min = minimum(t)
max = maximum(t)
med = median(t)
avg = mean(t)
memorystr = string(prettymemory(memory(min)))
allocsstr = string(iallocs(min))
minstr = string(prettytime(time(min)), " (", prettypercent(gcratio(min)), " GC)")
maxstr = string(prettytime(time(max)), " (", prettypercent(gcratio(max)), " GC)")
medstr = string(prettytime(time(med)), " (", prettypercent(gcratio(med)), " GC)")
meanstr = string(prettytime(time(avg)), " (", prettypercent(gcratio(avg)), " GC)")
else
memorystr = "N/A"
allocsstr = "N/A"
minstr = "N/A"
maxstr = "N/A"
medstr = "N/A"
meanstr = "N/A"
end
println(io, "BenchmarkTools.Trial: ")
pad = get(io, :pad, "")
println(io, pad, " memory estimate: ", memorystr)
println(io, pad, " allocs estimate: ", allocsstr)
println(io, pad, " --------------")
println(io, pad, " minimum time: ", minstr)
println(io, pad, " median time: ", medstr)
println(io, pad, " mean time: ", meanstr)
println(io, pad, " maximum time: ", maxstr)
println(io, pad, " --------------")
println(io, pad, " samples: ", length(t))
print(io, pad, " evals/sample: ", t.params.evals)
end
224 changes: 224 additions & 0 deletions src/pprints/trial_fancy.jl
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,224 @@
struct TrialFancyOutputPre end

function prep_trial_fancy_data(t::Trial)
length(t) > 1 || return nothing

perm = sortperm(t.times)
times = t.times[perm]
gctimes = t.gctimes[perm]

med = median(t)
avg = mean(t)
std = Statistics.std(t)
min = minimum(t)
max = maximum(t)

medtime, medgc = prettytime(time(med)), prettypercent(gcratio(med))
avgtime, avggc = prettytime(time(avg)), prettypercent(gcratio(avg))
stdtime, stdgc = prettytime(time(std)), prettypercent(Statistics.std(gctimes ./ times))
mintime, mingc = prettytime(time(min)), prettypercent(gcratio(min))
maxtime, maxgc = prettytime(time(max)), prettypercent(gcratio(max))

memorystr = string(prettymemory(memory(min)))
allocsstr = string(iallocs(min))

return (perm, times, gctimes,
med, avg, std, min, max,
medtime, medgc,
avgtime, avggc,
stdtime, stdgc,
mintime, mingc,
maxtime, maxgc,
memorystr, allocsstr)
end

function show_output(::TrialFancyOutputPre, io, ::MIME"text/plain", t::Trial)
pad = get(io, :pad, "")
print(io, "BenchmarkExt.Trial: ", length(t), " sample", if length(t) > 1 "s" else "" end,
" with ", t.params.evals, " evaluation", if t.params.evals > 1 "s" else "" end ,".\n")

if length(t) > 1
(perm, times, gctimes,
med, avg, std, min, max,
medtime, medgc,
avgtime, avggc,
stdtime, stdgc,
mintime, mingc,
maxtime, maxgc,
memorystr, allocsstr) = prep_trial_fancy_data(t)
elseif length(t) == 1
perm = sortperm(t.times)
times = t.times[perm]
gctimes = t.gctimes[perm]

print(io, pad, " Single result which took ")
printstyled(io, prettytime(times[1]); color=:blue)
print(io, " (", prettypercent(gctimes[1]/times[1]), " GC) ")
print(io, "to evaluate,\n")
print(io, pad, " with a memory estimate of ")
printstyled(io, prettymemory(t.memory[1]); color=:yellow)
print(io, ", over ")
printstyled(io, t.allocs[1]; color=:yellow)
print(io, " allocations.")
return
else
print(io, pad, " No results.")
return
end

lmaxtimewidth = maximum(length.((medtime, avgtime, mintime)))
rmaxtimewidth = maximum(length.((stdtime, maxtime)))
lmaxgcwidth = maximum(length.((medgc, avggc, mingc)))
rmaxgcwidth = maximum(length.((stdgc, maxgc)))

# Main stats

print(io, pad, " Range ")
printstyled(io, "("; color=:light_black)
printstyled(io, "min"; color=:cyan, bold=true)
print(io, " … ")
printstyled(io, "max"; color=:magenta)
printstyled(io, "): "; color=:light_black)
printstyled(io, lpad(mintime, lmaxtimewidth); color=:cyan, bold=true)
print(io, " … ")
printstyled(io, lpad(maxtime, rmaxtimewidth); color=:magenta)
print(io, " ")
printstyled(io, "┊"; color=:light_black)
print(io, " GC ")
printstyled(io, "("; color=:light_black)
print(io, "min … max")
printstyled(io, "): "; color=:light_black)
print(io, lpad(mingc, lmaxgcwidth), " … ", lpad(maxgc, rmaxgcwidth))

print(io, "\n", pad, " Time ")
printstyled(io, "("; color=:light_black)
printstyled(io, "median"; color=:blue, bold=true)
printstyled(io, "): "; color=:light_black)
printstyled(io, lpad(medtime, lmaxtimewidth), rpad(" ", rmaxtimewidth + 5); color=:blue, bold=true)
printstyled(io, "┊"; color=:light_black)
print(io, " GC ")
printstyled(io, "("; color=:light_black)
print(io, "median")
printstyled(io, "): "; color=:light_black)
print(io, lpad(medgc, lmaxgcwidth))

print(io, "\n", pad, " Time ")
printstyled(io, "("; color=:light_black)
printstyled(io, "mean"; color=:green, bold=true)
print(io, " ± ")
printstyled(io, "σ"; color=:green)
printstyled(io, "): "; color=:light_black)
printstyled(io, lpad(avgtime, lmaxtimewidth); color=:green, bold=true)
print(io, " ± ")
printstyled(io, lpad(stdtime, rmaxtimewidth); color=:green)
print(io, " ")
printstyled(io, "┊"; color=:light_black)
print(io, " GC ")
printstyled(io, "("; color=:light_black)
print(io, "mean ± σ")
printstyled(io, "): "; color=:light_black)
print(io, lpad(avggc, lmaxgcwidth), " ± ", lpad(stdgc, rmaxgcwidth))

end

struct TrialFancyHistogram end

function show_output(::TrialFancyHistogram, io, ::MIME"text/plain", t::Trial)
length(t) <= 1 && return
pad = get(io, :pad, "")
(perm, times, gctimes,
med, avg, std, min, max,
medtime, medgc,
avgtime, avggc,
stdtime, stdgc,
mintime, mingc,
maxtime, maxgc,
memorystr, allocsstr) = prep_trial_fancy_data(t)

lmaxtimewidth = maximum(length.((medtime, avgtime, mintime)))
rmaxtimewidth = maximum(length.((stdtime, maxtime)))
lmaxgcwidth = maximum(length.((medgc, avggc, mingc)))
rmaxgcwidth = maximum(length.((stdgc, maxgc)))

histquantile = 0.99
# The height and width of the printed histogram in characters.
histheight = 2
histwidth = 42 + lmaxtimewidth + rmaxtimewidth

histtimes = times[1:round(Int, histquantile*end)]
histmin = get(io, :histmin, first(histtimes))
histmax = get(io, :histmax, last(histtimes))
logbins = get(io, :logbins, nothing)
bins = bindata(histtimes, histwidth - 1, histmin, histmax)
append!(bins, [1, floor((1-histquantile) * length(times))])
# if median size of (bins with >10% average data/bin) is less than 5% of max bin size, log the bin sizes
if logbins === true || (logbins === nothing && median(filter(b -> b > 0.1 * length(times) / histwidth, bins)) / maximum(bins) < 0.05)
bins, logbins = log.(1 .+ bins), true
else
logbins = false
end
hist = asciihist(bins, histheight)
hist[:,end-1] .= ' '
maxbin = maximum(bins)

delta1 = (histmax - histmin) / (histwidth - 1)
if delta1 > 0
medpos = 1 + round(Int, (histtimes[length(times) ÷ 2] - histmin) / delta1)
avgpos = 1 + round(Int, (mean(times) - histmin) / delta1)
else
medpos, avgpos = 1, 1
end

print(io, "\n")
for r in axes(hist, 1)
print(io, "\n", pad, " ")
for (i, bar) in enumerate(view(hist, r, :))
color = :default
if i == avgpos color = :green end
if i == medpos color = :blue end
printstyled(io, bar; color=color)
end
end

remtrailingzeros(timestr) = replace(timestr, r"\.?0+ " => " ")
minhisttime, maxhisttime = remtrailingzeros.(prettytime.(round.([histmin, histmax], sigdigits=3)))

print(io, "\n", pad, " ", minhisttime)
caption = "Histogram: " * ( logbins ? "log(frequency)" : "frequency" ) * " by time"
if logbins
printstyled(io, " " ^ ((histwidth - length(caption)) ÷ 2 - length(minhisttime)); color=:light_black)
printstyled(io, "Histogram: "; color=:light_black)
printstyled(io, "log("; bold=true, color=:light_black)
printstyled(io, "frequency"; color=:light_black)
printstyled(io, ")"; bold=true, color=:light_black)
printstyled(io, " by time"; color=:light_black)
else
printstyled(io, " " ^ ((histwidth - length(caption)) ÷ 2 - length(minhisttime)), caption; color=:light_black)
end
print(io, lpad(maxhisttime, ceil(Int, (histwidth - length(caption)) / 2) - 1), " ")
printstyled(io, "<"; bold=true)
end

struct TrialFancyOutputPost end

function show_output(::TrialFancyOutputPost, io, ::MIME"text/plain", t::Trial)
# Memory info
length(t) <= 1 && return
pad = get(io, :pad, "")
(perm, times, gctimes,
med, avg, std, min, max,
medtime, medgc,
avgtime, avggc,
stdtime, stdgc,
mintime, mingc,
maxtime, maxgc,
memorystr, allocsstr) = prep_trial_fancy_data(t)

print(io, "\n\n", pad, " Memory estimate")
printstyled(io, ": "; color=:light_black)
printstyled(io, memorystr; color=:yellow)
print(io, ", allocs estimate")
printstyled(io, ": "; color=:light_black)
printstyled(io, allocsstr; color=:yellow)
print(io, ".")
end
Loading

0 comments on commit 376aeb8

Please sign in to comment.