Merge branch 'master' into inverse_bayes_example

.gitignore

@@ -7,3 +7,4 @@ cov_*
 code/tests/regression_tests/output/
 .cache
 libensemble.egg-info
+docs/_build

.travis.yml

@@ -4,6 +4,7 @@ python:
   - 3.4
   - 3.5
   - 3.6
+  #- 3.7
 
 os: linux
 dist: trusty
@@ -58,6 +59,7 @@ install:
   - pip install petsc petsc4py
   - pip install pytest
   - pip install pytest-cov
+  - pip install pytest-timeout 
   - pip install coveralls
   - conda install --no-deps nlopt  
   # For confirmation of MPI library being used.

README.rst

@@ -79,15 +79,17 @@ regularly on:
 
 * `Travis CI <https://travis-ci.org/Libensemble/libensemble>`_
 
-The test suite requires the pytest and pytest-cov packages to be installed and
-can be run from the libensemble/tests directory of the source distribution
-using the following methods::
+The test suite requires the pytest, pytest-cov and pytest-timeout packages to be installed and can be run from the libensemble/tests directory of the source distribution by running::
 
-    ./run-tests.sh (optionally specify eg. -p 3 for Python3)
+    ./run-tests.sh
 
 To clean the test repositories run::
 
     ./run-tests.sh -c
+    
+Further options are available. To see a complete list of options run::
+
+    ./run-tests.sh -h
 
 Coverage reports are produced separately for unit tests and regression tests
 under the relevant directories. For parallel tests, the union of all processors

docs/conf.py

@@ -20,8 +20,10 @@
 import os
 import sys
 
-from unittest.mock import MagicMock
-#from mock import MagicMock
+if sys.version_info >= (3, 3):
+    from unittest.mock import MagicMock
+else:
+    from mock import MagicMock
 
 class Mock(MagicMock):
     @classmethod
@@ -56,6 +58,7 @@ def __getattr__(cls, name):
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = ['sphinx.ext.autodoc', 'sphinx.ext.napoleon']
+#extensions = ['sphinx.ext.autodoc', 'sphinx.ext.napoleon', 'numpydoc']
 #extensions = ['sphinx.ext.autodoc', 'breathe']
 #breathe_projects = { "libEnsemble": "../code/src/xml/" }
 #breathe_default_project = "libEnsemble"
@@ -78,7 +81,7 @@ def __getattr__(cls, name):
 
 # General information about the project.
 project = 'libEnsemble'
-copyright = '2017, Jeffrey Larson'
+copyright = '2018, Jeffrey Larson'
 author = 'Jeffrey Larson and Stephen Hudson'
 
 # The version info for the project you're documenting, acts as replacement for

docs/data_structures/alloc_specs.rst

@@ -0,0 +1,23 @@
+alloc_specs
+===========
+
+Allocation function specifications to be set in user calling script and passed to libE.libE()::
+
+    alloc_specs: [dict, optional] :
+        'alloc_f' [func] :
+            Default: give_sim_work_first
+        'out' [list of tuples] :
+            Default: [('allocated',bool)]
+        'batch_mode' [bool] :
+            Default: []
+        'num_inst' [int] :
+            Default: []
+            
+        The 'batch_mode' and 'num_inst' are specific arguments for the allocation function give_sim_work_first
+
+:Examples:
+
+From: ::
+
+:See Also:
+

docs/data_structures/data_structures.rst

@@ -0,0 +1,19 @@
+Data Structures
+===============
+
+libEnsemble has the following data structures.
+
+.. toctree::
+   :maxdepth: 1
+   :caption: libEnsemble Data Structures:
+
+   history_array
+   worker_array
+   work_dict
+   libE_specs
+   sim_specs
+   gen_specs
+   exit_criteria   
+   alloc_specs
+   persis_info
+

docs/data_structures/exit_criteria.rst

@@ -0,0 +1,23 @@
+exit_criteria
+=============
+
+Exit criteria for libEnsemble::
+
+    exit_criteria: [dict]: 
+        
+        Optional keys (At least one must be given) :
+        
+        'sim_max' [int] : 
+            Stop after this many sim_f evaluations have been completed
+        'gen_max' [int] : 
+            Stop after this many points have been generated by gen_f
+        'elapsed_wallclock_time' [float] : 
+            Stop after this amount of seconds have elapsed (since the libEnsemble manager has been initialized)
+        'stop_val' [(str,float)] : 
+            Stop when H[str] (for some field str returned from sim_out or gen_out) has been observed with a value less than the float given
+            
+:Examples:
+
+From: ::
+
+:See Also:

docs/data_structures/gen_specs.rst

@@ -0,0 +1,38 @@
+gen_specs
+=========
+
+Generation function specifications to be set in user calling script and passed to libE.libE()::
+
+    gen_specs: [dict]:
+
+        Required keys :     
+        
+        'gen_f' [func] : 
+            generates inputs to sim_f
+        'in' [list] : 
+            field names (as strings) that will be given to gen_f
+        'out' [list of tuples (field name, data type, [size])] :
+            gen_f outputs that will be stored in the libEnsemble history
+            
+        Optional keys :
+    
+        'save_every_k' [int] :
+            Save history array every k steps
+
+:Examples:
+
+From: libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py::
+
+    gen_specs = {'gen_f': uniform_random_sample,
+                 'in': ['sim_id'],
+                 'out': [('x',float,2),
+                        ],
+                 'lb': np.array([-3,-2]),
+                 'ub': np.array([ 3, 2]),
+                 'gen_batch_size': 500,
+                 'batch_mode': True,
+                 'num_inst':1,
+                 'save_every_k': 300
+                 }
+ 
+:See Also:

docs/data_structures/history_array.rst

@@ -0,0 +1,13 @@
+history array
+=============
+
+Stores the history of the output from gen_f and sim_f:: 
+
+    H: numpy structured array
+        History array storing rows for each point. Field names are in
+        libensemble/libE_fields.py
+
+
+:Examples:
+
+:See Also:

docs/data_structures/libE_specs.rst

@@ -0,0 +1,23 @@
+libE_specs
+==========
+
+Specifications for libEnsemble::
+
+    libE_specs: [dict] :
+        'comm' [MPI communicator] :
+            libEnsemble communicator. Default: MPI.COMM_WORLD
+        'color' [int] :
+            Communicator color. Default: 0
+        'manager_ranks' [set] :
+            Default: [0]
+        'worker_ranks' [set] :
+            Default: [1 to comm.Get_size()-1]
+        'queue_update_function' [func] :
+            Default: []
+            
+
+:Examples:
+
+From: ::
+
+:See Also:

docs/data_structures/persis_info.rst

@@ -0,0 +1,14 @@
+persis_info
+===========
+
+Supply peristent information to libEnsemble::
+
+    persis_info: [dict] :
+        Dictionary containing persistent info
+
+:Examples:
+
+From: ::
+
+
+:See Also:

docs/data_structures/sim_specs.rst

@@ -0,0 +1,46 @@
+sim_specs
+=========
+
+
+Simulation function specifications to be set in user calling script and passed to libE.libE()::
+
+
+    sim_specs: [dict]:
+
+        Required keys :    
+        
+        'sim_f' [func] : 
+            the simulation function being evaluated
+        'in' [list] :
+            field names (as strings) that will be given to sim_f
+        'out' [list of tuples (field name, data type, [size])] :
+            sim_f outputs that will be stored in the libEnsemble history
+            
+
+        Optional keys :
+        
+        'save_every_k' [int] :
+            Save history array every k steps
+        'sim_dir' [str] :
+            Name of simulation directory which will be copied for each worker
+        'sim_dir_prefix' [str] :
+            A prefix path specifying where to create sim directories
+        
+        Additional entires in sim_specs will be given to sim_f
+        
+        
+:Examples:
+
+From: libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling.py::
+
+    sim_specs = {'sim_f': six_hump_camel, # This is the function whose output is being minimized
+                 'in': ['x'],             # These keys will be given to the above function
+                 'out': [('f',float),     # This is the output from the function being minimized
+                        ],
+                 'save_every_k': 400  
+                 }
+
+:See Also:
+
+
+

docs/data_structures/work_dict.rst

@@ -0,0 +1,41 @@
+work dictionary
+===============
+
+Dictionary with integer keys ``i`` and dictionary values to be given to worker ``i``. 
+``Work[i]`` has the following form::
+
+
+    Work[i]: [dict]:
+
+        Required keys :    
+        'persis_info' [dict]: Any persistent info to be sent to worker 'i' 
+
+        'H_fields' [list]: The field names of the history 'H' to be sent to worker 'i' 
+
+        'tag' [int]: 'EVAL_SIM_TAG' (resp. 'EVAL_GEN_TAG') if worker 'i' is to call sim_func (resp. gen_func) 
+
+        'libE_info' [dict]: This information is sent to and returned from the worker to help libEnsemble quickly update the 'H' and 'W'. 
+            Available keys are:
+
+            H_rows' [list of ints]: History rows to send to worker 'i'
+
+            blocking' [list of ints]: Workers to be blocked by the calculation given to worker 'i'
+
+            persistent' [bool]: True if worker 'i' will enter persistent mode 
+        
+        
+:Examples:
+
+.. How to link directly to the file?
+
+| For allocation functions using persistent workers, see 
+| ``libensemble/tests/regression_tests/test_6-hump_camel_persistent_uniform_sampling.py`` 
+| or 
+| ``libensemble/tests/regression_tests/test_6-hump_camel_uniform_sampling_with_persistent_localopt_gens.py``
+|
+| For allocation functions giving work that blocks other workers, see 
+| ``libensemble/tests/regression_tests/test_6-hump_camel_with_different_nodes_uniform_sample.py``
+
+:See Also:
+
+

docs/data_structures/worker_array.rst

@@ -0,0 +1,28 @@
+worker array
+=============
+
+Stores information to inform the allocation function about the current state of
+the workers. Workers can be in a variety of states. We take the following
+convention:
+
+=========================================   =======  ============  =======
+Worker state                                 active  persis_state  blocked
+=========================================   =======  ============  =======
+idle worker                                    0          0           0   
+active, nonpersistent sim                      1          0           0   
+active, nonpersistent gen                      2          0           0   
+active, persistent sim                         1          1           0   
+active, persistent gen                         2          2           0   
+waiting, persistent sim                        0          1           0   
+waiting, persistent gen                        0          2           0   
+worker blocked by some other calculation       1          0           1   
+=========================================   =======  ============  =======
+
+:Note:
+
+* libE only receives from workers with 'active' nonzero
+* libE only calls the alloc_f if some worker has 'active' zero
+
+:Examples:
+
+:See Also:

docs/index.rst

@@ -15,14 +15,15 @@ Library for managing ensemble-like collections of computations.
    readme
    contributing
    release_notes
-   user_guide
 
 .. toctree::
    :maxdepth: 2
-   :caption: libEnsemble API:
+   :caption: libEnsemble Reference:
 
-   libe_module
-   sim_gen_funcs
+   user_guide
+   data_structures/data_structures
+   libE_module
+   sim_gen_alloc_funcs
    manager_modules
    worker_modules