-
Notifications
You must be signed in to change notification settings - Fork 87
/
Benchmarks.jl
75 lines (67 loc) · 2.07 KB
/
Benchmarks.jl
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
# Copyright (c) 2017: Miles Lubin and contributors
# Copyright (c) 2017: Google Inc.
#
# Use of this source code is governed by an MIT-style license that can be found
# in the LICENSE.md file or at https://opensource.org/licenses/MIT.
module TestBenchmarks
using Test
import MathOptInterface as MOI
function runtests()
for name in names(@__MODULE__; all = true)
if startswith("$(name)", "test_")
@testset "$(name)" begin
getfield(@__MODULE__, name)()
end
end
end
return
end
function test_suite()
suite = MOI.Benchmarks.suite() do
return MOI.Utilities.MockOptimizer(MOI.Utilities.Model{Float64}())
end
@test length(suite.data) == length(MOI.Benchmarks.BENCHMARKS)
suite = MOI.Benchmarks.suite(exclude = [r"delete_"]) do
return MOI.Utilities.MockOptimizer(MOI.Utilities.Model{Float64}())
end
# Note: update this value whenever more benchmarks are added to
# `src/Benchmarks/Benchmarks.jl`.
@test 6 <= length(suite.data) <= length(MOI.Benchmarks.BENCHMARKS) - 3
return
end
function test_baseline()
params = joinpath(@__DIR__, "baseline_params.json")
baseline = joinpath(@__DIR__, "baseline_baseline.json")
@test !isfile(params)
@test !isfile(baseline)
suite = MOI.Benchmarks.suite() do
return MOI.Utilities.MockOptimizer(MOI.Utilities.Model{Float64}())
end
MOI.Benchmarks.create_baseline(
suite,
"baseline";
directory = @__DIR__,
samples = 1,
verbose = true,
)
@test isfile(params)
@test isfile(baseline)
suite = MOI.Benchmarks.suite() do
return MOI.Utilities.MockOptimizer(MOI.Utilities.Model{Float64}())
end
MOI.Benchmarks.compare_against_baseline(
suite,
"baseline";
directory = @__DIR__,
samples = 1,
verbose = true,
)
rm(params)
rm(baseline)
report = read(joinpath(@__DIR__, "report.txt"), String)
@test occursin("=> invariant", report)
rm(joinpath(@__DIR__, "report.txt"))
return
end
end
TestBenchmarks.runtests()