Mq/pretesting (#57)

Add pretesting for glyph-remote

.gitignore

@@ -1,3 +1,4 @@
+src/
 *.sqlite
 dask-worker-space
 doc/source/dev/api

README.md

@@ -34,7 +34,7 @@ to their optimization tasks.
 Installation
 ------------
 
-Glyph is a **python 3.5+** only package.
+Glyph is a **python 3.6+** only package.
 
 You can install the latest stable version from PyPI with pip
 

doc/source/index.rst

@@ -30,7 +30,6 @@ Content
    usr/getting_started.rst
    usr/concepts.rst
    usr/glyph_remote.rst
-   usr/gui.rst
 
    Publications <usr/publications.rst>
    About <usr/about.rst>

doc/source/usr/glyph_remote.rst

@@ -50,6 +50,8 @@ The possible action values are:
 +-------------------+--------------------+----------------------------+
 | *EXPERIMENT*      | list of expressions| list of fitness value(s)   |
 +-------------------+--------------------+----------------------------+
+| *METADATA*        | –                  | any                        |
++-------------------+--------------------+----------------------------+
 | *SHUTDOWN*        | –                  | –                          |
 +-------------------+--------------------+----------------------------+
 
@@ -123,3 +125,91 @@ to one are variables and -1 is reserved for symbolic constants.
         "x": 0,
       },
     }
+
+
+GUI
+---
+
+
+Install
+~~~~~~~
+
+Glyph comes with an optional GUI to use the ``glyph-remote`` script with more convenience.
+
+The GUI uses the package ``wxPython``. The installation manual can be found `here <https://github.com/wxWidgets/Phoenix/blob/master/README.rst#prerequisites>`_
+and `Website <https://wxpython.org/>`_.
+
+
+**Manual Gooey installtion**
+
+
+Since up-to-date (28.08.2018) the necessary changes to the used graphic library Gooey are not part of the master branch,
+it might be necessary to install Gooey by hand from the repo `https://github.com/Magnati/Gooey <https://github.com/Magnati/Gooey>`_ in three steps.
+
+- ``pip install -e "git+git@github.com:Magnati/Gooey.git#egg=gooey"``
+
+
+**Installation with pip installtion**
+
+
+To install glyph including the gui option use the following command:
+
+.. code-block::
+
+    python pip install pyglyph[gui]``
+
+To start the script with the gui just use the ``--gui`` parameter:
+
+.. code-block::
+
+    glyph-remote --gui
+
+Usage
+~~~~~~
+
+Within the GUI there is a tab for each group of parameters.
+If all parameters are set, click the start-button to start the experiment.
+
+
+Pretesting & Constraints
+------------------------
+
+In glyph-remote, genetic operations can be constrained. A genetic operation (i.e. every operation that create or modifies the genotype of an individual).
+If a constraint is violated, the genetic operation is rejected. If out of time, the last candidate is used.
+
+Currently, two different types of constraints are implemented:
+- algebraic constraints using sympy
+- pretesting constraints
+
+Algebraic constraints
+~~~~~~~~~~~~~~~~~~~~~
+
+Sympy is used to check whether expressions are:
+
+- zero
+- constant
+- infinite
+
+The three options can be individually activated.
+
+
+Pretesting
+~~~~~~~~~~
+
+You can invoke file-based pretesting with the `--constraints_pretest filename.py` flag.
+The flag `--constraints_pretest_function` lets you pass the function name which will be invoked to pretest individuals.
+
+The function is expected to return a boolean, depending on the individual is rejected (False) or accepted (True).
+
+An example file could look like this:
+
+.. code-block::
+
+    import time
+
+
+    def chi(ind):
+        time.sleep(1)
+        print(f"Hello World, this is {ind}")
+        return True
+

examples/remote/experiment.py

@@ -49,6 +49,9 @@ def work(self, request):
         action = request['action']
         if action == "CONFIG":
             return self.config
+        elif action == "PRETEST":
+            logger.info("Pretest OK")
+            return dict(ok=True)
         elif action == "SHUTDOWN":
             return self.shutdown()
         elif action == "EXPERIMENT":

glyph/application.py

@@ -3,6 +3,7 @@
 
 """Convenience classes and functions that allow you to quickly build gp apps."""
 
+import abc
 import os
 import sys
 import time
@@ -34,24 +35,24 @@ def update_logbook_record(runner):
     if not runner.mstats:
         runner.mstats = create_stats(len(runner.population[0].fitness.values))
     record = runner.mstats.compile(runner.population)
-    runner.logbook.record(gen=runner.step_count, evals=runner._evals, **record)
+    runner.logbook.record(gen=runner.step_count,
+                          evals=runner._evals,
+                          **record
+    )
 
 
 DEFAULT_CALLBACKS_GP_RUNNER = (update_pareto_front, update_logbook_record)
 
 
 class GPRunner(object):
-    """Runner for gp problem sets.
-
-    Takes care of propper initialization, execution, and accounting of a gp run
-    (i.e. population creation, random state, generation count, hall of fame, and
-    logbook). The method init() has to be called once before stepping through
-    the evolution process with method step(); init() and step() invoke the
-    assessment runner.
-    """
-
     def __init__(self, IndividualClass, algorithm_factory, assessment_runner, callbacks=DEFAULT_CALLBACKS_GP_RUNNER):
-        """Init GPRunner.
+        """Runner for gp problem sets.
+
+        Takes care of propper initialization, execution, and accounting of a gp run
+        (i.e. population creation, random state, generation count, hall of fame, and
+        logbook). The method init() has to be called once before stepping through
+        the evolution process with method step(); init() and step() invoke the
+        assessment runner.
 
         :param IndividualClass: Class inherited from gp.AExpressionTree.
         :param algorithm_factory: callable() -> gp algorithm, as defined in
@@ -137,18 +138,16 @@ def log(app):
 
 
 class Application(object):
-    """An application based on `GPRunner`.
+    def __init__(self, config, gp_runner, checkpoint_file=None, callbacks=DEFAULT_CALLBACKS):
+        """An application based on `GPRunner`.
 
-    Controls execution of the runner and adds checkpointing and logging
-    functionality; also defines a set of available command line options and
-    their default values.
+        Controls execution of the runner and adds checkpointing and logging
+        functionality; also defines a set of available command line options and
+        their default values.
 
-    To create a full console application one can use the factory function
-    create_console_app().
-    """
+        To create a full console application one can use the factory function
+        default_console_app().
 
-    def __init__(self, config, gp_runner, checkpoint_file=None, callbacks=DEFAULT_CALLBACKS):
-        """
         :param config: Container holding all configs
         :type config: dict or argparse.Namespace
         :param gp_runner: Instance of `GPRunner`
@@ -196,7 +195,12 @@ def run(self, break_condition=None):
 
     def _update(self):
         for cb in self.callbacks:
-            cb(self)
+            try:
+                logger.debug(f"Running callback {cb}.")
+                cb(self)
+            except Exception as e:
+                logger.error(f"Error during execution of {cb}")
+                logger.warning(e)
 
     def checkpoint(self):
         """Checkpoint current state of evolution."""
@@ -290,6 +294,7 @@ def create(cls, config, *args, **kwargs):
         return cls._create(config, *args, **kwargs)
 
     @staticmethod
+    @abc.abstractmethod
     def add_options(parser):
         """Add available parser options."""
         raise NotImplementedError
@@ -299,7 +304,7 @@ def get_from_mapping(cls, key):
         try:
             func = cls._mapping[key]
         except KeyError:
-            raise RuntimeError('Option {} not supported'.format(key))
+            raise RuntimeError(f"Option {key} not supported")
         return func
 
 
@@ -452,6 +457,78 @@ def add_options(parser):
         # todo
 
 
+class ConstraintsFactory(AFactory):
+    @staticmethod
+    def add_options(parser):
+        parser.add_argument(
+            "--constraints_timeout",
+            type=utils.argparse.non_negative_int,
+            default=60,
+            help="Seconds before giving up and using a new random individual (default: 60)"
+        )
+        parser.add_argument(
+            "--constraints_n_retries",
+            type=utils.argparse.non_negative_int,
+            default=30,
+            help="Number of genetic operation before giving up and using a new random individual (default: 30)"
+        )
+        parser.add_argument(
+            "--constraints_zero",
+            action="store_false",
+            default=True,
+            help="Discard zero individuals (default: True)",
+        )
+        parser.add_argument(
+            "--constraints_constant",
+            action="store_false",
+            default=True,
+            help="Discard constant individuals (default: True)",
+        )
+        parser.add_argument(
+            "--constraints_infty",
+            action="store_false",
+            default=True,
+            help="Discard individuals with infinities (default: True)",
+        )
+        parser.add_argument(
+            "--constraints_pretest",
+            default=False,
+            help="Path to pretest file."
+        )
+        parser.add_argument(
+            "--constraints_pretest_function",
+            type=str,
+            default="chi",
+            help="Path to pretest file."
+        )
+        parser.add_argument(
+            "--constraints_pretest_service",
+            action="store_true",
+            help="Use service for pretesting."
+        )
+
+    @staticmethod
+    def _create(config, com=None):
+        constraints = []
+        if config.constraints_zero or config.constraints_infty or config.constraints_constant:
+            constraints.append(
+                gp.NonFiniteExpression(
+                    zero=config.constraints_zero,
+                    infty=config.constraints_infty,
+                    constant=config.constraints_constant,
+                )
+            )
+        if config.constraints_pretest:
+            constraints.append(
+                gp.PreTest(config.constraints_pretest,
+                           fun=config.constraints_pretest_function
+                )
+            )
+        # if config.constraints_pretest_service: # todo (enable after com refactor)
+        # constraints.append(gp.PreTestService(com))
+        return gp.Constraint(constraints)
+
+
 def safe(file_name, **kwargs):
     """Dump kwargs to file."""
     with open(file_name, "wb") as file: