From acb9e6b906352a35b2f2513df826d15223e61ae1 Mon Sep 17 00:00:00 2001 From: odow Date: Tue, 9 Mar 2021 16:21:46 +1300 Subject: [PATCH] Document Test submodule --- docs/src/manual/implementing.md | 96 -------- docs/src/submodules/Bridges/reference.md | 3 +- docs/src/submodules/Test/overview.md | 284 ++++++++++++++++++----- 3 files changed, 221 insertions(+), 162 deletions(-) diff --git a/docs/src/manual/implementing.md b/docs/src/manual/implementing.md index 27b271239a..873671d979 100644 --- a/docs/src/manual/implementing.md +++ b/docs/src/manual/implementing.md @@ -215,99 +215,3 @@ should be stored in a `src/MOI_wrapper` folder and included by a By convention, optimizers should not be exported and should be named `PackageName.Optimizer`. For example, `CPLEX.Optimizer`, `Gurobi.Optimizer`, and `Xpress.Optimizer`. - -## Testing guideline - -The skeleton below can be used for the wrapper test file of a solver named -`FooBar`. -```julia -using Test - -using MathOptInterface -const MOI = MathOptInterface -const MOIT = MOI.Test -const MOIU = MOI.Utilities -const MOIB = MOI.Bridges - -import FooBar -const OPTIMIZER_CONSTRUCTOR = MOI.OptimizerWithAttributes( - FooBar.Optimizer, MOI.Silent() => true -) -const OPTIMIZER = MOI.instantiate(OPTIMIZER_CONSTRUCTOR) - -@testset "SolverName" begin - @test MOI.get(OPTIMIZER, MOI.SolverName()) == "FooBar" -end - -@testset "supports_default_copy_to" begin - @test MOIU.supports_default_copy_to(OPTIMIZER, false) - # Use `@test !...` if names are not supported - @test MOIU.supports_default_copy_to(OPTIMIZER, true) -end - -const BRIDGED = MOI.instantiate( - OPTIMIZER_CONSTRUCTOR, with_bridge_type = Float64 -) -const CONFIG = MOIT.TestConfig(atol=1e-6, rtol=1e-6) - -@testset "Unit" begin - # Test all the functions included in dictionary `MOI.Test.unittests`, - # except functions "number_threads" and "solve_qcp_edge_cases." - MOIT.unittest( - BRIDGED, - CONFIG, - ["number_threads", "solve_qcp_edge_cases"] - ) -end - -@testset "Modification" begin - MOIT.modificationtest(BRIDGED, CONFIG) -end - -@testset "Continuous Linear" begin - MOIT.contlineartest(BRIDGED, CONFIG) -end - -@testset "Continuous Conic" begin - MOIT.contlineartest(BRIDGED, CONFIG) -end - -@testset "Integer Conic" begin - MOIT.intconictest(BRIDGED, CONFIG) -end -``` - -Test functions like `MOI.Test.unittest` and `MOI.Test.modificationtest` are -wrappers around corresponding dictionaries `MOI.Test.unittests` and -`MOI.Test.modificationtests`. The keys of each dictionary (strings describing -the test) map to functions that take two arguments: an optimizer and a -`MOI.Test.TestConfig` object. Exclude tests by passing a vector of strings -corresponding to the test keys you want to exclude as the third positional -argument to the test function (e.g., `MOI.Test.unittest`). - -Print a list of all keys using `println.(keys(MOI.Test.unittests))` - -The optimizer `BRIDGED` constructed with [`instantiate`](@ref) -automatically bridges constraints that are not supported by `OPTIMIZER` -using the bridges listed in [Bridges](@ref). It is recommended for an -implementation of MOI to only support constraints that are natively supported -by the solver and let bridges transform the constraint to the appropriate form. -For this reason it is expected that tests may not pass if `OPTIMIZER` is used -instead of `BRIDGED`. - -To test that a specific problem can be solved without bridges, a specific test -can be run with `OPTIMIZER` instead of `BRIDGED`. For instance -```julia -@testset "Interval constraints" begin - MOIT.linear10test(OPTIMIZER, CONFIG) -end -``` -checks that `OPTIMIZER` implements support for -[`ScalarAffineFunction`](@ref)-in-[`Interval`](@ref). - -If the wrapper does not support building the model incrementally (i.e. with -[`add_variable`](@ref) and [`add_constraint`](@ref)), -then [`Utilities.supports_default_copy_to`](@ref) can be replaced by -[`Utilities.supports_allocate_load`](@ref) if appropriate (see -[Implementing copy](@ref)). - diff --git a/docs/src/submodules/Bridges/reference.md b/docs/src/submodules/Bridges/reference.md index 2063245f0b..714a29717a 100644 --- a/docs/src/submodules/Bridges/reference.md +++ b/docs/src/submodules/Bridges/reference.md @@ -33,8 +33,7 @@ constraints of different types. There are two important concepts to distinguish: by using the list of bridges that were added to it by [`Bridges.add_bridge`](@ref). [`Bridges.full_bridge_optimizer`](@ref) wraps a model in a [`Bridges.LazyBridgeOptimizer`](@ref) where all the bridges defined - in MOI are added. This is the recommended way to use bridges in the - [Testing guideline](@ref), and JuMP automatically calls + in MOI are added. JuMP automatically calls [`Bridges.full_bridge_optimizer`](@ref) when attaching an optimizer. [`Bridges.debug_supports_constraint`](@ref) and [`Bridges.debug_supports`](@ref) allow introspection into the bridge selection rationale of diff --git a/docs/src/submodules/Test/overview.md b/docs/src/submodules/Test/overview.md index 4513561168..83dbd31fef 100644 --- a/docs/src/submodules/Test/overview.md +++ b/docs/src/submodules/Test/overview.md @@ -7,88 +7,244 @@ end DocTestFilters = [r"MathOptInterface|MOI"] ``` -# The `Test` submodule +# The Test submodule -All solvers use the tests in this repository as extra correctness tests for themselves. -If we find a bug in one solver, instead of adding a test to that particular repository, we add it here so that all solvers can benefit. -All supported solvers are tested on travis with https://github.com/blegat/SolverTests. +The `Test` submodule provides tools to help solvers implement unit tests in +order to ensure they implement the MathOptInterface API correctly, and to check +for solver-correctness. -## Example of adding a test +We use a centralized repository of tests, so that if we find a bug in one +solver, instead of adding a test to that particular repository, we add it here +so that all solvers can benefit. -To give an example, ECOS errored calling `optimize!(model); optimize!(model)` ([ECOS.jl PR #72](https://github.com/jump-dev/ECOS.jl/pull/72)). -We could add a test to ECOS.jl, but that would only stop us from re-introducing the bug to ECOS.jl in the future. -Instead if we add a test here, then all solvers (e.g., SCS.jl, Gurobi.jl, Mosek.jl, ...) will also check that they handle a double optimize call! +## How to test a solver -For this test, we care about correctness, rather than performance. -We don't expect solvers to efficiently decide that they have already solved the problem, -only that calling `optimize!` twice doesn't throw an error or give the wrong answer. +The skeleton below can be used for the wrapper test file of a solver named +`FooBar`. Remove unnecessary tests as appropriate. -To resolve this issue, follow these steps (tested on Julia v1.5): +```julia +# ============================ /test/MOI_wrapper.jl ============================ +module TestFoobar + +import FooBar +using MathOptInterface +using Test + +const MOI = MathOptInterface + +const OPTIMIZER_CONSTRUCTOR = MOI.OptimizerWithAttributes( + FooBar.Optimizer, + MOI.Silent() => true +) +const OPTIMIZER = MOI.instantiate(OPTIMIZER_CONSTRUCTOR) + +const BRIDGED = MOI.instantiate( + OPTIMIZER_CONSTRUCTOR, with_bridge_type = Float64 +) +const CONFIG = MOI.Test.TestConfig( + # Modify tolerances as necessary. + atol = 1e-6, + rtol = 1e-6, + # Set false if dual solutions are not generated + duals = true, + # Set false if infeasibility certificates are not generated + infeas_certificates = true, + # Use MOI.LOCALLY_SOLVED for local solvers. + optimal_status = MOI.OPTIMAL, + # Set true if basis information is available + basis = false, +) + +function test_SolverName() + @test MOI.get(OPTIMIZER, MOI.SolverName()) == "FooBar" +end -1. Install the `MathOptInterface` julia package in `dev` mode ([ref](https://julialang.github.io/Pkg.jl/v1/managing-packages/#developing-1)): +function test_supports_default_copy_to() + @test MOI.Utilities.supports_default_copy_to(OPTIMIZER, false) + # Use `@test !...` if names are not supported + @test MOI.Utilities.supports_default_copy_to(OPTIMIZER, true) +end -```julia -julia> ] -(@v1.5) pkg> dev ECOS -(@v1.5) pkg> dev MathOptInterface -``` +function test_unittest() + # Test all the functions included in dictionary `MOI.Test.unittests`, + # except functions "number_threads" and "solve_qcp_edge_cases." + MOI.Test.unittest( + BRIDGED, + CONFIG, + ["number_threads", "solve_qcp_edge_cases"] + ) +end -2. From here on, proceed with making the following changes in the `~/.julia/dev/MathOptInterface` folder (or equivalent `dev` path on your machine) -3. Since the double-optimize error involves solving an optimization problem, -add a new test to [src/Test/UnitTests/solve.jl](https://github.com/jump-dev/MathOptInterface.jl/blob/master/src/Test/UnitTests/solve.jl). -The test should be something like +function test_modification() + MOI.Test.modificationtest(BRIDGED, CONFIG) +end -```julia -function solve_twice(model::MOI.ModelLike, config::TestConfig) - MOI.empty!(model) - x = MOI.add_variable(model) - c = MOI.add_constraint(model, MOI.SingleVariable(x), MOI.GreaterThan(1.0)) - MOI.set(model, MOI.ObjectiveSense(), MOI.MIN_SENSE) - MOI.set(model, MOI.ObjectiveFunction{MOI.SingleVariable}(), MOI.SingleVariable(x)) - if config.solve - MOI.optimize!(model) - MOI.optimize!(model) - MOI.get(model, MOI.TerminationStatus()) == MOI.OPTIMAL - MOI.get(model, MOI.VariablePrimal(), x) == 1.0 - end +function test_contlinear() + MOI.Test.contlineartest(BRIDGED, CONFIG) end -unittests["solve_twice"] = solve_twice -``` -2. Add a test for the test you just wrote (We test the tests!) +function test_contquadratictest() + MOI.Test.contquadratictest(OPTIMIZER, CONFIG) +end - a. Add the name of the test ("solve_twice") to the end of the array in `MOIT.unittest(...)` ([link](https://github.com/jump-dev/MathOptInterface.jl/blob/7543afe4b5151cf36bbd18181c1bb5c83266ae2f/test/Test/unit.jl#L51-L52)). +function test_contconic() + MOI.Test.contlineartest(BRIDGED, CONFIG) +end - b. Add a test for the test towards the end of the "Unit Tests" test set ([link](https://github.com/jump-dev/MathOptInterface.jl/blob/7543afe4b5151cf36bbd18181c1bb5c83266ae2f/test/Test/unit.jl#L394)). The test should look something like +function test_intconic() + MOI.Test.intconictest(BRIDGED, CONFIG) +end -```julia -@testset "solve_twice" begin - MOIU.set_mock_optimize!(mock, - (mock::MOIU.MockOptimizer) -> MOIU.mock_optimize!( - mock, - MOI.OPTIMAL, - (MOI.FEASIBLE_POINT, [1.0]), - ), - (mock::MOIU.MockOptimizer) -> MOIU.mock_optimize!( - mock, - MOI.OPTIMAL, - (MOI.FEASIBLE_POINT, [1.0]), - ) - ) - MOIT.solve_twice(mock, config) +function test_default_objective_test() + MOI.Test.default_objective_test(OPTIMIZER) end -``` -In the above `mock` is a `MOI.Utilities.MockOptimizer` that is defined earlier in the file. -In this test, `MOIU.set_mock_optimize!` loads `mock` with two results. Each says -that the `MOI.[TerminationStatus](@ref)` is `MOI.OPTIMAL`, that the -`MOI.[PrimalStatus](@ref)` is `MOI.FEASIBLE_POINT`, and that there is one -variable with a `MOI.VariableValue` or `1.0`. +function test_default_status_test() + MOI.Test.default_status_test(OPTIMIZER) +end -3. Run the tests: +function test_nametest() + MOI.Test.nametest(OPTIMIZER) +end + +function test_validtest() + MOI.Test.validtest(OPTIMIZER) +end + +function test_emptytest() + MOI.Test.emptytest(OPTIMIZER) +end + +function test_orderedindicestest() + MOI.Test.orderedindicestest(OPTIMIZER) +end + +function test_scalar_function_constant_not_zero() + MOI.Test.scalar_function_constant_not_zero(OPTIMIZER) +end + +# This function runs all functions in this module starting with `test_`. +function runtests() + for name in names(@__MODULE__; all = true) + if startswith("$(name)", "test_") + @testset "$(name)" begin + getfield(@__MODULE__, name)() + end + end + end +end + +end # module TestFooBar +TestFooBar.runtests() ``` -(@v1.5) pkg> test ECOS + +Test functions like `MOI.Test.unittest` and `MOI.Test.modificationtest` are +wrappers around corresponding dictionaries `MOI.Test.unittests` and +`MOI.Test.modificationtests`. Exclude tests by passing a vector of strings +corresponding to the test keys you want to exclude as the third positional +argument to the test function. + +!!! tip + Print a list of all keys using `println.(keys(MOI.Test.unittests))` + +The optimizer `BRIDGED` constructed with [`instantiate`](@ref) +automatically bridges constraints that are not supported by `OPTIMIZER` +using the bridges listed in [Bridges](@ref). It is recommended for an +implementation of MOI to only support constraints that are natively supported +by the solver and let bridges transform the constraint to the appropriate form. +For this reason it is expected that tests may not pass if `OPTIMIZER` is used +instead of `BRIDGED`. + +To test that a specific problem can be solved without bridges, a specific test +can be added with `OPTIMIZER` instead of `BRIDGED`. For example: +```julia +function test_interval_constraints() + MOI.Test.linear10test(OPTIMIZER, CONFIG) +end ``` +checks that `OPTIMIZER` implements support for +[`ScalarAffineFunction`](@ref)-in-[`Interval`](@ref). + +## How to add a test + +To give an example, ECOS errored calling [`optimize!`](@ref) twice in a row. +(See [ECOS.jl PR #72](https://github.com/jump-dev/ECOS.jl/pull/72).) + +We could add a test to ECOS.jl, but that would only stop us from re-introducing +the bug to ECOS.jl in the future. + +Instead, if we add a test to `MOI.Test`, then all solvers will also check that +they handle a double optimize call! -4. Finally, commit the changes to git from `~/.julia/dev/MathOptInterface`. Use a branch name of the format `initials/issueNumber_issueShortTitle`, and submit the PR for review. +For this test, we care about correctness, rather than performance. therefore, we +don't expect solvers to efficiently decide that they have already solved the +problem, only that calling [`optimize!`](@ref) twice doesn't throw an error or +give the wrong answer. + +To resolve this issue, follow these steps (tested on Julia v1.5): + +1. Install the `MathOptInterface` julia package in `dev` mode + ([ref](https://julialang.github.io/Pkg.jl/v1/managing-packages/#developing-1)): + ```julia + julia> ] + (@v1.5) pkg> dev ECOS + (@v1.5) pkg> dev MathOptInterface + ``` +2. From here on, proceed with making the following changes in the + `~/.julia/dev/MathOptInterface` folder (or equivalent `dev` path on your + machine). +3. Since the double-optimize error involves solving an optimization problem, + add a new test to [src/Test/UnitTests/solve.jl](https://github.com/jump-dev/MathOptInterface.jl/blob/master/src/Test/UnitTests/solve.jl). + The test should be something like + ```julia + function solve_twice(model::MOI.ModelLike, config::TestConfig) + MOI.empty!(model) + x = MOI.add_variable(model) + c = MOI.add_constraint(model, MOI.SingleVariable(x), MOI.GreaterThan(1.0)) + MOI.set(model, MOI.ObjectiveSense(), MOI.MIN_SENSE) + MOI.set(model, MOI.ObjectiveFunction{MOI.SingleVariable}(), MOI.SingleVariable(x)) + if config.solve + MOI.optimize!(model) + MOI.optimize!(model) + MOI.get(model, MOI.TerminationStatus()) == MOI.OPTIMAL + MOI.get(model, MOI.VariablePrimal(), x) == 1.0 + end + end + unittests["solve_twice"] = solve_twice + ``` +2. Add a test for the test you just wrote. (We test the tests!) + a. Add the name of the test (`"solve_twice"`) to the end of the array in + `MOI.Test.unittest(...)` ([link](https://github.com/jump-dev/MathOptInterface.jl/blob/7543afe4b5151cf36bbd18181c1bb5c83266ae2f/test/Test/unit.jl#L51-L52)). + b. Add a test for the test towards the end of the "Unit Tests" test set + ([link](https://github.com/jump-dev/MathOptInterface.jl/blob/7543afe4b5151cf36bbd18181c1bb5c83266ae2f/test/Test/unit.jl#L394)). + The test should look something like + ```julia + @testset "solve_twice" begin + MOI.Utilities.set_mock_optimize!(mock, + (mock::MOI.Utilities.MockOptimizer) -> MOI.Utilities.mock_optimize!( + mock, + MOI.OPTIMAL, + (MOI.FEASIBLE_POINT, [1.0]), + ), + (mock::MOI.Utilities.MockOptimizer) -> MOI.Utilities.mock_optimize!( + mock, + MOI.OPTIMAL, + (MOI.FEASIBLE_POINT, [1.0]), + ) + ) + MOI.Test.solve_twice(mock, config) + end + ``` + In the above `mock` is a `MOI.Utilities.MockOptimizer` that is defined + tesearlier in the file. In this test, `MOI.Utilities.set_mock_optimize!` loads + `mock` with two results. Each says that the + [`TerminationStatus`](@ref) is `MOI.OPTIMAL`, that the + [`PrimalStatus`](@ref) is `MOI.FEASIBLE_POINT`, and that there is one + variable with a `MOI.VariableValue` or `1.0` +3. Run the tests: + ```julia + (@v1.5) pkg> test ECOS + ``` +4. Finally, commit the changes to git from `~/.julia/dev/MathOptInterface` and + submit the PR for review.