Merge branch 'master' into fix-pylint-errors

.gitattributes

@@ -0,0 +1,7 @@
+.github/ export-ignore
+.gitignore export-ignore
+.gitattributes export-ignore
+.landscape.yml export-ignore
+.readthedocs.yml export-ignore
+.travis.yml  export-ignore
+.zenodo.json export-ignore

.travis.yml

@@ -4,11 +4,6 @@ python:
     - "3.6"
 install: pip install tox-travis
 script:
-  - tox -e test
-  - tox -e checkstyle
-  - tox -e testcov
-  - tox -e lint
-  - tox -e checktypes
-  - tox -e documentation
+  - tox -e travis
 env:
   OMP_NUM_THREADS: 4

doc/source/development.rst

@@ -141,7 +141,6 @@ in the repository's root folder (``pyndl``) or by executing
 in the documentation folder (``pyndl/doc/``).
 
 
-
 Continuous Integration
 ----------------------
 
@@ -185,6 +184,7 @@ Release Process
 1. Merge Pull Requests with new features or bugfixes into *pyndl*'s' ``master``
    branch. Ensure, that the version is adequately increased (``X.Y+1.Z`` for new
    features and ``X.Y.Z+1`` for a bugfix).
+
 2. Create a new release on Github of the `master` branch of the form ``vX.Y.Z``.
    Add a description of the new feature or bugfix
 
@@ -209,3 +209,5 @@ Release Process
 .. code:: bash
 
     twine upload -s dist/*
+
+6. Check if the new version is on pypi (https://pypi.python.org/pypi/pyndl/).

doc/source/index.rst

@@ -16,4 +16,5 @@
    examples
    pyndl
    development
+   misc
    about

doc/source/misc.rst

@@ -0,0 +1,101 @@
+Miscellaneous
+=============
+
+This is collection of more or less unrelated tips and tricks that can be helpful
+during development and maintanance.
+
+Running ``pyndl`` within ``R`` code
+-----------------------------------
+
+In order to run ``pyndl`` within ``R`` code first install Python and ``pyndl``
+as described in the install instructions. Make sure ``pyndl`` runs for your
+user within Python.
+
+Now we can switch to ``R`` and install the ``reticulate`` package
+(https://cran.r-project.org/web/packages/reticulate/vignettes/introduction.html)
+After having the ``reticulate`` package installed we can run within R the following code:
+
+.. code:: r
+
+    library(reticulate)
+
+    learn_weights <- function(event_file) {
+        py_env <- py_run_string(
+            paste(
+                  "from pyndl import ndl",
+                  paste0("weights = ndl.ndl('", event_file, "', alpha=0.01, betas=(1.0, 1.0), remove_duplicates=True)"),
+                  "weight_matrix = weights.data",
+                  "outcome_names = weights.coords['outcomes'].values",
+                  "cue_names = weights.coords['cues'].values",
+                  sep = "\n"
+            ),
+            convert = FALSE
+        )
+        wm <- py_to_r(py_env$weight_matrix)
+        rownames(wm) <- py_to_r(py_env$outcome_names)
+        colnames(wm) <- py_to_r(py_env$cue_names)
+        py_run_string(
+            paste(
+                  "del cue_names",
+                  "del outcome_names",
+                  "del weight_matrix",
+                  "del weights",
+                  sep = "\n"
+            ),
+            convert = FALSE
+        )
+        wm
+    }
+
+After having defined this funtion a gzipped tab seperated event file can be learned using:
+
+.. code:: r
+
+    wm <- learn_weights('event_file.tab.gz')
+
+Note that this code needs at the moment slightly more than two times the size
+of the weights matrix.
+
+There might be a way to learn the weight matrix without any copying between R and Python, but this needs to be elaborated a bit further. The basic idea is
+
+1. to create the the matrix in R (in Fortran mode),
+2. borrow / make the matrix available in Python,
+3. transpose the matrix in Python to get it into C mode
+4. learn the weights in place,
+5. Check that the matrix in R has the weights learned as a side effect of the
+   Python code.
+
+Further reading:
+
+- https://cran.r-project.org/web/packages/reticulate/vignettes/introduction.html
+- https://cran.r-project.org/web/packages/reticulate/vignettes/arrays.html
+- https://stackoverflow.com/questions/44379525/r-reticulate-how-do-i-clear-a-python-object-from-memory
+
+
+Local testing with conda
+------------------------
+
+Sometimes it might be useful to test if ``pyndl`` works in a clean python
+environment. Besides ``tox`` this is possible with ``conda`` as well. The
+commands are as follows:
+
+.. code:: bash
+
+    conda create -n testpyndl
+    conda activate testpyndl
+    conda install python
+    python -c 'from pyndl import ndl; print("success")'  # this should fail
+    git clone https://github.com/quantling/pyndl.git
+    pip install pyndl
+    python -c 'from pyndl import ndl; print("success")'  # this should succeed
+    conda deactivate
+    conda env remove -n testpyndl
+
+
+Memory profiling
+----------------
+
+Sometimes it is useful to monitory the memory footprint of the python process.
+This can be achieved by using ``memory_profiler``
+(https://pypi.python.org/pypi/memory_profiler).
+

pyndl/__init__.py

@@ -17,7 +17,7 @@
 __author__ = ('Konstantin Sering, Marc Weitz, '
               'David-Elias Künstle, Lennard Schneider')
 __author_email__ = 'konstantin.sering@uni-tuebingen.de'
-__version__ = '0.4.1'
+__version__ = '0.5.0'
 __license__ = 'MIT'
 __description__ = ('Naive discriminative learning implements learning and '
                    'classification models based on the Rescorla-Wagner '
@@ -43,7 +43,7 @@ def sysinfo():
     Prints system the dependency information
     """
     pyndl = pkg_resources.working_set.by_key["pyndl"]
-    dependencies = [str(r) for r in pyndl.requires()]
+    dependencies = [r.project_name for r in pyndl.requires()]
 
     header = ("Pyndl Information\n"
               "=================\n\n")

pyndl/ndl.py

@@ -56,7 +56,8 @@ def events_from_file(event_path):
 def ndl(events, alpha, betas, lambda_=1.0, *,
         method='openmp', weights=None,
         number_of_threads=8, len_sublists=10, remove_duplicates=None,
-        verbose=False):
+        verbose=False, temporary_directory=None,
+        events_per_temporary_file=10000000):
     """
     Calculate the weights for all_outcomes over all events in event_file
     given by the files path.
@@ -89,6 +90,12 @@ def ndl(events, alpha, betas, lambda_=1.0, *,
         preferred!)
     verbose : bool
         print some output if True.
+    temporary_directory : str
+        path to directory to use for storing temporary files created;
+        if none is provided, the operating system's default will
+        be used (/tmp on unix)
+    events_per_temporary_file: int
+        Number of events in each temporary binary file. Has to be larger than 1
 
     Returns
     -------
@@ -155,16 +162,19 @@ def ndl(events, alpha, betas, lambda_=1.0, *,
 
     beta1, beta2 = betas
 
-    with tempfile.TemporaryDirectory(prefix="pyndl") as binary_path:
+    with tempfile.TemporaryDirectory(prefix="pyndl", dir=temporary_directory) as binary_path:
         number_events = preprocess.create_binary_event_files(events, binary_path, cue_map,
                                                              outcome_map, overwrite=True,
                                                              number_of_processes=number_of_threads,
+                                                             events_per_file=events_per_temporary_file,
                                                              remove_duplicates=remove_duplicates,
                                                              verbose=verbose)
         assert n_events == number_events, (str(n_events) + ' ' + str(number_events))
         binary_files = [os.path.join(binary_path, binary_file)
                         for binary_file in os.listdir(binary_path)
                         if os.path.isfile(os.path.join(binary_path, binary_file))]
+        # sort binary files as they were created
+        binary_files.sort(key=lambda filename: int(os.path.basename(filename)[9:-4]))
         if verbose:
             print('start learning...')
         # learning