add benchmark utilities

Project.toml

@@ -4,6 +4,7 @@ version = "0.1.11"
 
 [deps]
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
+JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 LinearOperators = "5c8ed15e-5a4c-59e4-a42b-c7e8811fb125"
 Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
@@ -12,6 +13,7 @@ Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 
 [compat]
 DataFrames = "^0.20"
+JLD2 = "0.1.12"
 LinearOperators = "0.4.0, 0.5.0, 0.6.0, 0.7.0, 1.0"
 NLPModels = "0.7.0, 0.8.0, 0.9, 0.10.0, 0.11.0, 0.12"
 julia = "^1.0.0"

docs/src/api.md

@@ -17,11 +17,22 @@ project_step!
 
 ## Benchmarking
 
+### Main Benchmarking Functions
+
 ```@docs
 bmark_solvers
 solve_problems
 ```
 
+### Utilities
+
+```@docs
+save_stats
+load_stats
+count_unique
+quick_summary
+```
+
 ## Line-Search
 
 ```@docs

src/SolverTools.jl

@@ -7,7 +7,7 @@ using LinearAlgebra, Logging, Printf
 using LinearOperators, NLPModels
 
 # auxiliary packages
-using DataFrames
+using DataFrames, JLD2
 
 # Auxiliary.
 include("auxiliary/blas.jl")
@@ -21,6 +21,7 @@ include("trust-region/trust-region.jl")
 
 # Utilities.
 include("bmark/bmark_solvers.jl")
+include("bmark/bmark_utils.jl")
 include("bmark/run_solver.jl")
 
 end

src/bmark/bmark_utils.jl

@@ -0,0 +1,130 @@
+export save_stats, load_stats, count_unique, quick_summary
+
+
+"""
+    save_stats(stats, filename; kwargs...)
+
+Write the benchmark statistics `stats` to a file named `filename`.
+
+#### Arguments
+
+* `stats::Dict{Symbol,DataFrame}`: benchmark statistics such as returned by `bmark_solvers()`
+* `filename::AbstractString`: the output file name.
+
+#### Keyword arguments
+
+* `force::Bool=false`: whether to overwrite `filename` if it already exists
+* `key::String="stats"`: the key under which the data can be read from `filename` later.
+
+#### Return value
+
+This method returns an error if `filename` exists and `force==false`.
+On success, it returns the value of `jldopen(filename, "w")`.
+"""
+function save_stats(stats::Dict{Symbol,DataFrame}, filename::AbstractString; force::Bool=false, key::String="stats")
+  isfile(filename) && !force && error("$filename already exists; use `force=true` to overwrite")
+  jldopen(filename, "w") do file
+    file[key] = stats
+  end
+end
+
+"""
+    stats = load_stats(filename; kwargs...)
+
+#### Arguments
+
+* `filename::AbstractString`: the input file name.
+
+#### Keyword arguments
+
+* `key::String="stats"`: the key under which the data can be read in `filename`.
+  The key should be the same as the one used when `save_stats()` was called.
+
+#### Return value
+
+A `Dict{Symbol,DataFrame}` containing the statistics stored in file `filename`.
+The user should `import DataFrames` before calling `load_stats()`.
+"""
+function load_stats(filename::AbstractString; key::String="stats")
+  jldopen(filename) do file
+    file[key]
+  end
+end
+
+@doc raw"""
+    vals = count_unique(X)
+
+Count the number of occurrences of each value in `X`.
+
+#### Arguments
+
+* `X`: an iterable.
+
+#### Return value
+
+A `Dict{eltype(X),Int}` whose keys are the unique elements in `X` and
+values are their number of occurrences.
+
+Example: the snippet
+
+    stats = load_stats("mystats.jld2")
+    for solver ∈ keys(stats)
+      @info "$solver statuses" count_unique(stats[solver].status)
+    end
+
+displays the number of occurrences of each final status for each solver in `stats`.
+"""
+function count_unique(X)
+  vals = Dict{eltype(X),Int}()
+  for x ∈ X
+    vals[x] = x ∈ keys(vals) ? (vals[x] + 1) : 1
+  end
+  vals
+end
+
+"""
+    statuses, avgs = quick_summary(stats; kwargs...)
+
+Call `count_unique()` and compute a few average measures for each solver in `stats`.
+
+#### Arguments
+
+* `stats::Dict{Symbol,DataFrame}`: benchmark statistics such as returned by `bmark_solvers()`.
+
+#### Keyword arguments
+
+* `cols::Vector{Symbol}`: symbols indicating `DataFrame` columns in solver statistics for which we
+  compute averages. Default: `[:iter, :neval_obj, :neval_grad, :neval_hess, :neval_hprod, :elapsed_time]`.
+
+#### Return value
+
+* `statuses::Dict{Symbol,Dict{Symbol,Int}}`: a dictionary of number of occurrences of each final
+  status for each solver in `stats`. Each value in this dictionary is returned by `count_unique()`
+* `avgs::Dict{Symbol,Dict{Symbol,Float64}}`: a dictionary that contains averages of performance measures
+  across all problems for each solver. Each `avgs[solver]` is a `Dict{Symbol,Float64}` where the measures
+  are those given in the keyword argument `cols` and values are averages of those measures across all problems.
+
+Example: the snippet
+
+    statuses, avgs = quick_summary(stats)
+    for solver ∈ keys(stats)
+      @info "statistics for" solver statuses[solver] avgs[solver]
+    end
+
+displays quick summary and averages for each solver.
+"""
+function quick_summary(stats::Dict{Symbol,DataFrame};
+                       cols::Vector{Symbol}=[:iter, :neval_obj, :neval_grad, :neval_hess, :neval_hprod, :elapsed_time])
+  nproblems = size(stats[first(keys(stats))], 1)
+  statuses = Dict{Symbol,Dict{Symbol,Int}}()
+  avgs = Dict{Symbol,Dict{Symbol,Float64}}()
+  for solver ∈ keys(stats)
+    statuses[solver] = count_unique(stats[solver].status)
+    avgs[solver] = Dict{Symbol,Float64}()
+    for col ∈ cols
+      avgs[solver][col] = sum(stats[solver][!, col]) / nproblems
+    end
+  end
+  statuses, avgs
+end
+

test/test_bmark.jl

@@ -24,6 +24,21 @@ function test_bmark()
     for k in keys(solvers)
       @test haskey(stats, k)
     end
+
+    # write stats to file
+    filename = tempname()
+    save_stats(stats, filename)
+
+    # read stats from file
+    stats2 = load_stats(filename)
+
+    # check that they are the same
+    for k ∈ keys(stats)
+      @test k ∈ keys(stats2)
+      @test stats[k] == stats2[k]
+    end
+
+    statuses, avgs = quick_summary(stats)
   end
 end