[MRG] Caching and small run preparation improvements

.travis.yml

@@ -122,9 +122,9 @@ install:
        conda install --yes --quiet anaconda-client conda-build jinja2 pip setuptools;
     fi
   - if [[ $MINIMAL_VERSIONS == 'yes' ]]; then
-    travis_retry conda create -n travis_conda --yes --quiet pip python=$PYTHON numpy==1.9 scipy==0.14 libgfortran==1 nose sphinx ipython sympy==0.7.6 jinja2==2.7 pyparsing setuptools coverage;
+    travis_retry conda create -n travis_conda --yes --quiet pip python=$PYTHON numpy==1.9 scipy==0.14 libgfortran==1 nose "sphinx<1.6.3" ipython sympy==0.7.6 jinja2==2.7 pyparsing setuptools coverage;
     else
-    travis_retry conda create -n travis_conda --yes --quiet pip python=$PYTHON numpy nose sphinx ipython "sympy!=1.1.0" pyparsing jinja2 setuptools coverage;
+    travis_retry conda create -n travis_conda --yes --quiet pip python=$PYTHON numpy nose "sphinx<1.6.3" ipython "sympy!=1.1.0" pyparsing jinja2 setuptools coverage;
     fi
   - source activate travis_conda
   # On Python 2: Install an older version of scipy that still has weave

appveyor.yml

@@ -101,7 +101,7 @@ install:
   - 'python -c "import struct; print(struct.calcsize(''P'') * 8)"'
 
   # Install the build dependencies of the project via conda
-  - 'appveyor-retry conda install --yes --quiet numpy nose sphinx "sympy!=1.1.0" pyparsing jinja2 ipython setuptools cython'
+  - 'appveyor-retry conda install --yes --quiet numpy nose "sphinx<1.6.3" "sympy!=1.1.0" pyparsing jinja2 ipython setuptools cython'
   # Install an older version of scipy (which still has weave) on Python 2
   - 'if "%PYTHON_VERSION:~0,1%" == "2" appveyor-retry conda install --yes --quiet -c brian-team weave scipy'
   - 'if "%PYTHON_VERSION:~0,1%" == "3" appveyor-retry conda install --yes --quiet scipy'

brian2/codegen/generators/base.py

@@ -249,6 +249,7 @@ def translate(self, code, dtype):
                              'you are sure that the order of operations does not '
                              'matter. ' + error_msg))
 
-        return self.translate_statement_sequence(scalar_statements,
-                                                 vector_statements)
+        translated = self.translate_statement_sequence(scalar_statements,
+                                                       vector_statements)
 
+        return translated

brian2/codegen/generators/cpp_generator.py

@@ -390,16 +390,6 @@ def determine_keywords(self):
                 support_code.extend(sc)
                 pointers.extend(ps)
                 hash_defines.extend(hd)
-
-
-        # delete the user-defined functions from the namespace and add the
-        # function namespaces (if any)
-        for funcname, func in user_functions:
-            del self.variables[funcname]
-            func_namespace = func.implementations[self.codeobj_class].get_namespace(self.owner)
-            if func_namespace is not None:
-                self.variables.update(func_namespace)
-
         support_code.append(self.universal_support_code)
 
 

brian2/codegen/generators/cython_generator.py

@@ -285,14 +285,6 @@ def determine_keywords(self):
                 # fallback to Python object
                 load_namespace.append('{0} = _namespace["{1}"]'.format(varname, varname))
 
-        # delete the user-defined functions from the namespace and add the
-        # function namespaces (if any)
-        for funcname, func in user_functions:
-            del self.variables[funcname]
-            func_namespace = func.implementations[self.codeobj_class].get_namespace(self.owner)
-            if func_namespace is not None:
-                self.variables.update(func_namespace)
-
         return {'load_namespace': '\n'.join(load_namespace),
                 'support_code_lines': support_code}
 

brian2/codegen/generators/numpy_generator.py

@@ -273,13 +273,6 @@ def translate_one_statement_sequence(self, statements, scalar=False):
             lines.extend(self.vectorise_code(statements, variables,
                                              variable_indices))
 
-        # Make sure we do not use the __call__ function of Function objects but
-        # rather the Python function stored internally. The __call__ function
-        # would otherwise return values with units
-        for varname, var in variables.iteritems():
-            if isinstance(var, Function):
-                variables[varname] = var.implementations[self.codeobj_class].get_code(self.owner)
-
         return lines
 
     def determine_keywords(self):

brian2/codegen/runtime/cython_rt/cython_rt.py

@@ -5,6 +5,7 @@
 
 from brian2.core.variables import (DynamicArrayVariable, ArrayVariable,
                                    AuxiliaryVariable, Subexpression)
+from brian2.core.functions import Function
 from brian2.core.preferences import prefs, BrianPreference
 from brian2.utils.logger import get_logger
 from brian2.utils.stringtools import get_identifiers
@@ -114,6 +115,15 @@ def run(self):
 
     # the following are copied from WeaveCodeObject
 
+    def _insert_func_namespace(self, func):
+        impl = func.implementations[self]
+        func_namespace = impl.get_namespace(self.owner)
+        if func_namespace is not None:
+            self.namespace.update(func_namespace)
+        if impl.dependencies is not None:
+            for dep in impl.dependencies.itervalues():
+                self._insert_func_namespace(dep)
+
     def variables_to_namespace(self):
 
         # Variables can refer to values that are either constant (e.g. dt)
@@ -125,6 +135,8 @@ def variables_to_namespace(self):
         self.nonconstant_values = []
 
         for name, var in self.variables.iteritems():
+            if isinstance(var, Function):
+                self._insert_func_namespace(var)
             if isinstance(var, (AuxiliaryVariable, Subexpression)):
                 continue
             try:

brian2/codegen/runtime/numpy_rt/numpy_rt.py

@@ -10,6 +10,7 @@
 from brian2.core.preferences import prefs, BrianPreference
 from brian2.core.variables import (DynamicArrayVariable, ArrayVariable,
                                    AuxiliaryVariable, Subexpression)
+from brian2.core.functions import Function
 
 from ...codeobject import CodeObject, constant_or_scalar
 
@@ -188,8 +189,12 @@ def variables_to_namespace(self):
                     raise TypeError()
                 value = var.get_value()
             except TypeError:
-                # A dummy Variable without value or a function
-                self.namespace[name] = var
+                # Either a dummy Variable without a value or a Function object
+                if isinstance(var, Function):
+                    impl = var.implementations[self.__class__].get_code(self.owner)
+                    self.namespace[name] = impl
+                else:
+                    self.namespace[name] = var
                 continue
 
             if isinstance(var, ArrayVariable):

brian2/codegen/runtime/weave_rt/weave_rt.py

@@ -162,6 +162,15 @@ def is_available(cls):
                         'failed_compile_test')
             return False
 
+    def _insert_func_namespace(self, func):
+        impl = func.implementations[self]
+        func_namespace = impl.get_namespace(self.owner)
+        if func_namespace is not None:
+            self.namespace.update(func_namespace)
+        if impl.dependencies is not None:
+            for dep in impl.dependencies.itervalues():
+                self._insert_func_namespace(dep)
+
     def variables_to_namespace(self):
 
         # Variables can refer to values that are either constant (e.g. dt)
@@ -173,7 +182,11 @@ def variables_to_namespace(self):
         self.nonconstant_values = []
 
         for name, var in self.variables.iteritems():
-            if isinstance(var, (AuxiliaryVariable, Subexpression, Function)):
+            if isinstance(var, Function):
+                self._insert_func_namespace(var)
+                continue  # Everything else has already been dealt with in the
+                          # CodeGenerator (support code, renaming, etc.)
+            elif isinstance(var, (AuxiliaryVariable, Subexpression)):
                 continue
             try:
                 value = var.get_value()

brian2/codegen/translation.py

@@ -22,14 +22,13 @@
 
 from brian2.core.preferences import prefs
 from brian2.core.variables import Variable, Subexpression, AuxiliaryVariable
+from brian2.utils.caching import cached
 from brian2.core.functions import Function
 from brian2.utils.stringtools import (deindent, strip_empty_lines,
                                       get_identifiers)
 from brian2.utils.topsort import topsort
-from brian2.units.fundamentalunits import Unit, DIMENSIONLESS
 from brian2.parsing.statements import parse_statement
-from brian2.parsing.sympytools import (str_to_sympy, sympy_to_str,
-                                       check_expression_for_multiple_stateful_functions)
+from brian2.parsing.sympytools import str_to_sympy, sympy_to_str
 
 from .statements import Statement
 from .optimisation import optimise_statements
@@ -166,9 +165,11 @@ def is_scalar_expression(expr, variables):
                (isinstance(variables[name], Function) and variables[name].stateless)
                for name in identifiers)
 
-
+@cached
 def make_statements(code, variables, dtype, optimise=True, blockname=''):
     '''
+    make_statements(code, variables, dtype, optimise=True, blockname='')
+
     Turn a series of abstract code statements into Statement objects, inferring
     whether each line is a set/declare operation, whether the variables are
     constant or not, and handling the cacheing of subexpressions.

brian2/core/variables.py

@@ -6,14 +6,15 @@
 import functools
 import numbers
 
-import sympy
 import numpy as np
+import sympy
 
-from brian2.utils.stringtools import get_identifiers, word_substitute
 from brian2.units.fundamentalunits import (Quantity, get_unit, DIMENSIONLESS,
                                            fail_for_dimension_mismatch,
                                            Dimension)
 from brian2.utils.logger import get_logger
+from brian2.utils.stringtools import get_identifiers, word_substitute
+from brian2.utils.caching import CacheKey
 
 from .base import weakproxy_with_fallback, device_override
 from .preferences import prefs
@@ -89,7 +90,7 @@ def variables_by_owner(variables, owner):
                  if getattr(var.owner, 'name', None) is owner_name])
 
 
-class Variable(object):
+class Variable(CacheKey):
     '''
     An object providing information about model variables (including implicit
     variables such as ``t`` or ``xi``). This class should never be
@@ -127,6 +128,9 @@ class Variable(object):
         Whether this variable is an array. Allows for simpler check than testing
         ``isinstance(var, ArrayVariable)``. Defaults to ``False``.
     '''
+
+    _cache_irrelevant_attributes = {'owner'}
+
     def __init__(self, name, dimensions=DIMENSIONLESS, owner=None, dtype=None,
                  scalar=False, constant=False, read_only=False, dynamic=False,
                  array=False):
@@ -275,6 +279,7 @@ def __repr__(self):
                                   read_only=repr(self.read_only))
 
 
+
 # ------------------------------------------------------------------------------
 # Concrete classes derived from `Variable` -- these are the only ones ever
 # instantiated.
@@ -514,6 +519,10 @@ class DynamicArrayVariable(ArrayVariable):
         corresponding pre- and post-synaptic indices are not). Defaults to
         ``False``.
     '''
+    # The size of a dynamic variable can of course change and changes in
+    # size should not invalidate the cache
+    cache_irrelevant_attributes = (ArrayVariable._cache_irrelevant_attributes |
+                                   {'size'})
 
     def __init__(self, name, owner, size, device, dimensions=DIMENSIONLESS,
                  dtype=None, constant=False, needs_reference_update=False,
@@ -550,6 +559,7 @@ def __init__(self, name, owner, size, device, dimensions=DIMENSIONLESS,
                                                    read_only=read_only,
                                                    unique=unique)
 
+
     @property
     def dimensions(self):
         logger.warn('The DynamicArrayVariable.dimensions attribute is '
@@ -653,7 +663,6 @@ def __repr__(self):
                                   expr=repr(self.expr),
                                   owner=self.owner.name)
 
-
 # ------------------------------------------------------------------------------
 # Classes providing views on variables and storing variables information
 # ------------------------------------------------------------------------------
@@ -1360,7 +1369,6 @@ def dtype(self):
         return self.get_item(slice(None), level=1).dtype
 
 
-
 class Variables(collections.Mapping):
     '''
     A container class for storing `Variable` objects. Instances of this class

brian2/devices/cpp_standalone/device.py

@@ -19,6 +19,7 @@
 from brian2.codegen.cpp_prefs import get_compiler_and_args
 from brian2.core.network import Network
 from brian2.devices.device import Device, all_devices, set_device, reset_device
+from brian2.core.functions import Function
 from brian2.core.variables import *
 from brian2.core.namespace import get_local_namespace
 from brian2.groups.group import Group
@@ -766,18 +767,23 @@ def copy_source_files(self, writer, directory):
                                   'random', 'randomkit', 'randomkit.h'),
                      os.path.join(directory, 'brianlib', 'randomkit', 'randomkit.h'))
 
+    def _insert_func_namespace(self, func, code_object, namespace):
+        impl = func.implementations[CPPStandaloneCodeObject]
+        func_namespace = impl.get_namespace(code_object.owner)
+        if func_namespace is not None:
+            namespace.update(func_namespace)
+        if impl.dependencies is not None:
+            for dep in impl.dependencies.itervalues():
+                self._insert_func_namespace(dep, code_object, namespace)
+
     def write_static_arrays(self, directory):
-        # # Find np arrays in the namespaces and convert them into static
-        # # arrays. Hopefully they are correctly used in the code: For example,
-        # # this works for the namespaces for functions with C++ (e.g. TimedArray
-        # # treats it as a C array) but does not work in places that are
-        # # implicitly vectorized (state updaters, resets, etc.). But arrays
-        # # shouldn't be used there anyway.
+        # Write Function namespaces as static arrays
         for code_object in self.code_objects.itervalues():
-            for name, value in code_object.variables.iteritems():
-                if isinstance(value, np.ndarray):
-                    self.static_arrays[name] = value
-
+            for var in code_object.variables.itervalues():
+                if isinstance(var, Function):
+                    self._insert_func_namespace(var, code_object,
+                                                self.static_arrays)
+
         logger.diagnostic("static arrays: "+str(sorted(self.static_arrays.keys())))
 
         static_array_specs = []
@@ -834,7 +840,7 @@ def compile_source(self, directory, compiler, debug, clean):
                     os.remove('winmake.log')
                 with std_silent(debug):
                     if clean:
-                        os.system('%s >>winmake.log 2>&1 && %s clean >>winmake.log 2>&1' % (vcvars_cmd, make_cmd))
+                        os.system('%s >>winmake.log 2>&1 && %s clean > NUL 2>&1' % (vcvars_cmd, make_cmd))
                     x = os.system('%s >>winmake.log 2>&1 && %s %s>>winmake.log 2>&1' % (vcvars_cmd,
                                                                                         make_cmd,
                                                                                         make_args))
@@ -851,7 +857,7 @@ def compile_source(self, directory, compiler, debug, clean):
             else:
                 with std_silent(debug):
                     if clean:
-                        os.system('make clean')
+                        os.system('make clean >/dev/null 2>&1')
                     if debug:
                         x = os.system('make debug')
                     else:

brian2/devices/device.py

@@ -253,7 +253,7 @@ def code_object_class(self, codeobj_class=None):
     def code_object(self, owner, name, abstract_code, variables, template_name,
                     variable_indices, codeobj_class=None,
                     template_kwds=None, override_conditional_write=None):
-
+        name = find_name(name)
         codeobj_class = self.code_object_class(codeobj_class)
         template = getattr(codeobj_class.templater, template_name)
         iterate_all = template.iterate_all
@@ -305,8 +305,6 @@ def code_object(self, owner, name, abstract_code, variables, template_name,
         logger.diagnostic('%s snippet (scalar):\n%s' % (name, indent(code_representation(scalar_code))))
         logger.diagnostic('%s snippet (vector):\n%s' % (name, indent(code_representation(vector_code))))
 
-        name = find_name(name)
-
         code = template(scalar_code, vector_code,
                         owner=owner, variables=variables, codeobj_name=name,
                         variable_indices=variable_indices,