Merge pull request #682 from carrascomj/feat/solver-timeout

feat: accept solver timeout as parameter on CLI

src/memote/suite/api.py

@@ -64,8 +64,16 @@ def validate_model(path):
     return model, sbml_ver, notifications
 
 
-def test_model(model, sbml_version=None, results=False, pytest_args=None,
-               exclusive=None, skip=None, experimental=None):
+def test_model(
+    model,
+    sbml_version=None,
+    results=False,
+    pytest_args=None,
+    exclusive=None,
+    skip=None,
+    experimental=None,
+    solver_timeout=None
+):
     """
     Test a model and optionally store results as JSON.
 
@@ -84,6 +92,8 @@ def test_model(model, sbml_version=None, results=False, pytest_args=None,
         precedence over ``skip``.
     skip : iterable, optional
         Names of test cases or modules to skip.
+    solver_timeout: int, optional
+        Timeout in seconds to set on the mathematical optimization solver.
 
     Returns
     -------
@@ -103,6 +113,7 @@ def test_model(model, sbml_version=None, results=False, pytest_args=None,
         # Disable pytest capturing so that the solver log output can be seen
         # immediately.
         pytest_args.insert(0, "-s")
+    model.solver.configuration.timeout = solver_timeout
     plugin = ResultCollectionPlugin(model, sbml_version=sbml_version,
                                     exclusive=exclusive, skip=skip,
                                     experimental_config=experimental)

src/memote/suite/cli/config.py

@@ -51,6 +51,7 @@ class Memote(SectionSchema):
             type=click.Choice(["cplex", "glpk", "gurobi", "glpk_exact"]),
             default="glpk"
         )
+        solver_timeout = Param(type=int, default=None)
         experimental = Param(type=click.Path(exists=False, dir_okay=False),
                              default=None)
 

src/memote/suite/cli/reports.py

@@ -69,6 +69,10 @@ def report():
               type=click.Choice(["cplex", "glpk", "gurobi", "glpk_exact"]),
               default="glpk", show_default=True,
               help="Set the solver to be used.")
+@click.option("--solver-timeout",
+              type=int, default=None,
+              help="Timeout in seconds to set on the mathematical "
+                   "optimization solver.")
 @click.option("--experimental", type=click.Path(exists=True, dir_okay=False),
               default=None, callback=callbacks.validate_experimental,
               help="Define additional tests using experimental data.")
@@ -87,7 +91,7 @@ def report():
                    "the documentation for the expected YAML format used. This "
                    "option can be specified multiple times.")
 def snapshot(model, filename, pytest_args, exclusive, skip, solver,
-             experimental, custom_tests, custom_config):
+             solver_timeout, experimental, custom_tests, custom_config):
     """
     Take a snapshot of a model's state and generate a report.
 
@@ -113,7 +117,8 @@ def snapshot(model, filename, pytest_args, exclusive, skip, solver,
     model_obj.solver = solver
     _, results = api.test_model(model_obj, sbml_version=sbml_ver, results=True,
                                 pytest_args=pytest_args, skip=skip,
-                                exclusive=exclusive, experimental=experimental)
+                                exclusive=exclusive, experimental=experimental,
+                                solver_timeout=solver_timeout)
     with open(filename, "w", encoding="utf-8") as file_handle:
         LOGGER.info("Writing snapshot report to '%s'.", filename)
         file_handle.write(api.snapshot_report(results, config))
@@ -176,11 +181,12 @@ def history(location, model, filename, deployment, custom_config):
 
 
 def _test_diff(model_and_model_ver_tuple, pytest_args, skip,
-               exclusive, experimental):
+               exclusive, experimental, solver_timeout):
     model, sbml_ver = model_and_model_ver_tuple
     _, diff_results = api.test_model(
         model, sbml_version=sbml_ver, results=True, pytest_args=pytest_args,
-        skip=skip, exclusive=exclusive, experimental=experimental)
+        skip=skip, exclusive=exclusive, experimental=experimental,
+        solver_timeout=solver_timeout)
     return diff_results
 
 
@@ -204,6 +210,10 @@ def _test_diff(model_and_model_ver_tuple, pytest_args, skip,
 @click.option("--solver", type=click.Choice(["cplex", "glpk", "gurobi"]),
               default="glpk", show_default=True,
               help="Set the solver to be used.")
+@click.option("--solver-timeout",
+              type=int, default=None,
+              help="Timeout in seconds to set on the mathematical "
+                   "optimization solver.")
 @click.option("--experimental", type=click.Path(exists=True, dir_okay=False),
               default=None, callback=callbacks.validate_experimental,
               help="Define additional tests using experimental data.")
@@ -224,7 +234,7 @@ def _test_diff(model_and_model_ver_tuple, pytest_args, skip,
                    "(memote.readthedocs.io). This option can be specified "
                    "multiple times.")
 def diff(models, filename, pytest_args, exclusive, skip, solver,
-         experimental, custom_tests, custom_config):
+         solver_timeout, experimental, custom_tests, custom_config):
     """
     Take a snapshot of all the supplied models and generate a diff report.
 
@@ -274,7 +284,8 @@ def diff(models, filename, pytest_args, exclusive, skip, solver,
     # Running pytest in individual processes to avoid interference
     partial_test_diff = partial(_test_diff, pytest_args=pytest_args,
                                 skip=skip, exclusive=exclusive,
-                                experimental=experimental)
+                                experimental=experimental,
+                                solver_timeout=solver_timeout)
     pool = Pool(min(len(models), cpu_count()))
     results = pool.map(partial_test_diff, model_and_model_ver_tuple)
 

src/memote/suite/cli/runner.py

@@ -111,6 +111,10 @@ def cli():
               type=click.Choice(["cplex", "glpk", "gurobi", "glpk_exact"]),
               default="glpk", show_default=True,
               help="Set the solver to be used.")
+@click.option("--solver-timeout",
+              type=int, default=None,
+              help="Timeout in seconds to set on the mathematical "
+                   "optimization solver.")
 @click.option("--experimental", type=click.Path(exists=True, dir_okay=False),
               default=None, callback=callbacks.validate_experimental,
               help="Define additional tests using experimental data.")
@@ -130,7 +134,7 @@ def cli():
 @click.argument("model", type=click.Path(exists=True, dir_okay=False),
                 envvar="MEMOTE_MODEL")
 def run(model, collect, filename, location, ignore_git, pytest_args, exclusive,
-        skip, solver, experimental, custom_tests, deployment,
+        skip, solver, solver_timeout, experimental, custom_tests, deployment,
         skip_unchanged):
     """
     Run the test suite on a single model and collect results.
@@ -183,7 +187,8 @@ def is_verbose(arg):
     code, result = api.test_model(
         model=model, sbml_version=sbml_ver, results=True,
         pytest_args=pytest_args, skip=skip,
-        exclusive=exclusive, experimental=experimental)
+        exclusive=exclusive, experimental=experimental,
+        solver_timeout=solver_timeout)
     if collect:
         if repo is None:
             manager = ResultManager()
@@ -264,15 +269,16 @@ def _model_from_stream(stream, filename):
     return model, sbml_ver, notifications
 
 
-def _test_history(model, sbml_ver, solver, manager, commit, pytest_args, skip,
-                  exclusive, experimental):
+def _test_history(model, sbml_ver, solver, solver_timeout, manager, commit,
+                  pytest_args, skip, exclusive, experimental):
     model.solver = solver
     # Load the experimental configuration using model information.
     if experimental is not None:
         experimental.load(model)
     _, result = api.test_model(
         model, sbml_version=sbml_ver, results=True, pytest_args=pytest_args,
-        skip=skip, exclusive=exclusive, experimental=experimental)
+        skip=skip, exclusive=exclusive, experimental=experimental,
+        solver_timeout=solver_timeout)
     manager.store(result, commit=commit)
 
 
@@ -292,6 +298,10 @@ def _test_history(model, sbml_ver, solver, manager, commit, pytest_args, skip,
 @click.option("--solver", type=click.Choice(["cplex", "glpk", "gurobi"]),
               default="glpk", show_default=True,
               help="Set the solver to be used.")
+@click.option("--solver-timeout",
+              type=int, default=None,
+              help="Timeout in seconds to set on the mathematical "
+                   "optimization solver.")
 @click.option("--exclusive", multiple=True, metavar="TEST",
               help="The name of a test or test module to be run exclusively. "
                    "All other tests are skipped. This option can be used "
@@ -303,8 +313,9 @@ def _test_history(model, sbml_ver, solver, manager, commit, pytest_args, skip,
                 envvar="MEMOTE_MODEL")
 @click.argument("message")
 @click.argument("commits", metavar="[COMMIT] ...", nargs=-1)
-def history(model, message, rewrite, solver, location, pytest_args, deployment,
-            commits, skip, exclusive, experimental=None):  # noqa: D301
+def history(model, message, rewrite, solver, solver_timeout, location,
+            pytest_args, deployment, commits, skip, exclusive,
+            experimental=None):  # noqa: D301
     """
     Re-compute test results for the git branch history.
 
@@ -406,7 +417,7 @@ def history(model, message, rewrite, solver, location, pytest_args, deployment,
             continue
         proc = Process(
             target=_test_history,
-            args=(model_obj, sbml_ver, solver, manager, commit,
+            args=(model_obj, sbml_ver, solver, solver_timeout, manager, commit,
                   pytest_args, skip, exclusive, experimental))
         proc.start()
         proc.join()

src/memote/support/consistency.py

@@ -85,6 +85,12 @@ def check_stoichiometric_consistency(model):
     problem = model.problem
     # The transpose of the stoichiometric matrix N.T in the paper.
     stoich_trans = problem.Model()
+    # We clone the existing configuration in order to apply non-default
+    # settings, for example, for solver verbosity or timeout.
+    stoich_trans.configuration = problem.Configuration.clone(
+        config=model.solver.configuration,
+        problem=stoich_trans
+    )
     internal_rxns = con_helpers.get_internals(model)
     metabolites = set(met for rxn in internal_rxns for met in rxn.metabolites)
     LOGGER.info("model '%s' has %d internal reactions", model.id,
@@ -135,6 +141,12 @@ def find_unconserved_metabolites(model):
     """
     problem = model.problem
     stoich_trans = problem.Model()
+    # We clone the existing configuration in order to apply non-default
+    # settings, for example, for solver verbosity or timeout.
+    stoich_trans.configuration = problem.Configuration.clone(
+        config=model.solver.configuration,
+        problem=stoich_trans
+    )
     internal_rxns = con_helpers.get_internals(model)
     metabolites = set(met for rxn in internal_rxns for met in rxn.metabolites)
     # The binary variables k[i] in the paper.
@@ -214,6 +226,12 @@ def find_inconsistent_min_stoichiometry(model, atol=1e-13):
     inc_minimal = set()
     (problem, indicators) = con_helpers.create_milp_problem(
         left_ns, metabolites, Model, Variable, Constraint, Objective)
+    # We clone the existing configuration in order to apply non-default
+    # settings, for example, for solver verbosity or timeout.
+    problem.configuration = model.problem.Configuration.clone(
+        config=model.solver.configuration,
+        problem=problem
+    )
     LOGGER.debug(str(problem))
     cuts = list()
     for met in unconserved_mets:

tests/test_for_suite/test_for_api.py

@@ -66,6 +66,16 @@ def test_test_model_result(model):
     assert len(result) > 0
 
 
+@pytest.mark.parametrize("model", ["textbook"], indirect=["model"])
+def test_test_model_timeout(model):
+    api.test_model(
+        model,
+        solver_timeout=1,
+        exclusive="test_find_reactions_unbounded_flux_default_condition",
+    )
+    assert model.solver.configuration.timeout == 1
+
+
 @pytest.mark.parametrize("model", [
     "complete_failure",
 ], indirect=["model"])