-
Notifications
You must be signed in to change notification settings - Fork 2
/
results.jl
80 lines (63 loc) · 2.5 KB
/
results.jl
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
using DataFrames, Plots, StatsPlots, Latexify, CSV, LaTeXStrings
using ArgParse, Printf, Statistics
pgfplotsx()
s = ArgParseSettings()
@add_arg_table! s begin
"--plots"
action = :store_true
"--tables"
action = :store_true
end
args = parse_args(s)
resultsdir = joinpath(pwd(), "benchmarks", "results")
resultsfiles = filter(fname -> startswith(fname, "result-") && endswith(fname, ".csv") && !endswith(fname, "stan-spec0.csv"), readdir(resultsdir))
typed = mapreduce(f -> begin
@info f
CSV.read(joinpath(resultsdir, f))
end, vcat, filter(fname -> endswith(fname, "spec1.csv"), resultsfiles));
untyped = mapreduce(f -> CSV.read(joinpath(resultsdir, f)), vcat, filter(fname -> endswith(fname, "spec0.csv"), resultsfiles));
untyped.ppl .= "turing(untyped)"
data = vcat(typed, untyped);
models = unique(data.model)
ppls = unique(data.ppl)
if args["plots"]
# make plots folder
mkpath(joinpath(pwd(), "plots"))
for model in models
p = @df data[(data.type .== "time") .& (data.model .== model),:] boxplot(:ppl, :value, ylabel = "runtime (seconds)", legend=nothing)
savefig(p, joinpath(pwd(), "plots", string(model, "-boxplot.pdf")))
end
end
if args["tables"]
# create latex table
rtable = DataFrame(Array{Any}(undef,0,length(ppls)+1), vcat("", map(m -> LaTeXString(m), ppls)))
for model in models
pplrow = map(ppl -> begin
drow = data[(data.type .== "time") .& (data.model .== model) .& (data.ppl .== ppl), :value]
μ = @sprintf("%.3f",mean(drow))
σ = @sprintf("%.3f",std(drow))
s = string(μ, " \\pm ", σ)
LaTeXString(s)
end, ppls)
push!(rtable, vcat(LaTeXString(model), pplrow))
end
ltabel = latexify(rtable, env=:table, booktabs=true)
open("results_table.tex", "w") do io
write(io, ltabel)
end
# create latex table
rtable = DataFrame(Array{Any}(undef,0,length(ppls)+1), vcat("", map(m -> LaTeXString(m), ppls)))
for model in models
pplrow = map(ppl -> begin
drow = data[(data.type .== "time_forward") .& (data.model .== model) .& (data.ppl .== ppl), :value]
μ = @sprintf("%.3E",mean(drow))
s = string(μ)
LaTeXString(s)
end, ppls)
push!(rtable, vcat(LaTeXString(model), pplrow))
end
ltabel = latexify(rtable, env=:table, booktabs=true)
open("forward_results_table.tex", "w") do io
write(io, ltabel)
end
end