From ddb25ef450d20c6f2315cd41c5f86c163d97d565 Mon Sep 17 00:00:00 2001 From: thesamovar Date: Thu, 4 Jul 2013 17:31:50 -0400 Subject: [PATCH 01/15] Moved ast_parser to new brian2.parsing package, and removed BitAnd and BitOr The reason for removing BitAnd and BitOr is that we want them to be equivalent to and and or in Python, but they have different precedence. Previously we were hacking this by replacing '&' with 'and' but this is confusing. Better to just simply use 'and' and 'or' I think. --- brian2/codegen/languages/cpp/cpp.py | 2 +- brian2/codegen/languages/python/python.py | 2 +- brian2/codegen/parsing.py | 2 +- brian2/parsing/__init__.py | 0 .../ast_parser.py => parsing/rendering.py} | 17 +---------------- examples/I-F_curve_LIF.py | 6 +++--- 6 files changed, 7 insertions(+), 22 deletions(-) create mode 100644 brian2/parsing/__init__.py rename brian2/{codegen/ast_parser.py => parsing/rendering.py} (90%) diff --git a/brian2/codegen/languages/cpp/cpp.py b/brian2/codegen/languages/cpp/cpp.py index 62b9ccb26..e2dad0eb2 100644 --- a/brian2/codegen/languages/cpp/cpp.py +++ b/brian2/codegen/languages/cpp/cpp.py @@ -12,7 +12,7 @@ from ..base import Language, CodeObject from ..templates import LanguageTemplater -from ...ast_parser import CPPNodeRenderer +from brian2.parsing.rendering import CPPNodeRenderer logger = get_logger(__name__) try: diff --git a/brian2/codegen/languages/python/python.py b/brian2/codegen/languages/python/python.py index ceebe807e..ecbd5b3c2 100644 --- a/brian2/codegen/languages/python/python.py +++ b/brian2/codegen/languages/python/python.py @@ -4,7 +4,7 @@ from ..base import Language, CodeObject from ..templates import LanguageTemplater -from ...ast_parser import NumpyNodeRenderer +from brian2.parsing.rendering import NumpyNodeRenderer __all__ = ['PythonLanguage', 'PythonCodeObject'] diff --git a/brian2/codegen/parsing.py b/brian2/codegen/parsing.py index da32c770d..12e8fcd5c 100644 --- a/brian2/codegen/parsing.py +++ b/brian2/codegen/parsing.py @@ -7,7 +7,7 @@ from sympy.printing.str import StrPrinter from .functions.numpyfunctions import DEFAULT_FUNCTIONS, log10 -from .ast_parser import SympyNodeRenderer +from brian2.parsing.rendering import SympyNodeRenderer def parse_statement(code): diff --git a/brian2/parsing/__init__.py b/brian2/parsing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/brian2/codegen/ast_parser.py b/brian2/parsing/rendering.py similarity index 90% rename from brian2/codegen/ast_parser.py rename to brian2/parsing/rendering.py index f568e376c..c925c14bd 100644 --- a/brian2/codegen/ast_parser.py +++ b/brian2/parsing/rendering.py @@ -2,7 +2,7 @@ import sympy -from .functions.numpyfunctions import DEFAULT_FUNCTIONS +from brian2.codegen.functions.numpyfunctions import DEFAULT_FUNCTIONS __all__ = ['NodeRenderer', 'NumpyNodeRenderer', @@ -20,8 +20,6 @@ class NodeRenderer(object): 'Div': '/', 'Pow': '**', 'Mod': '%', - 'BitAnd': 'and', - 'BitOr': 'or', # Compare 'Lt': '<', 'LtE': '<=', @@ -42,14 +40,10 @@ class NodeRenderer(object): def render_expr(self, expr, strip=True): if strip: expr = expr.strip() - expr = expr.replace('&', ' and ') - expr = expr.replace('|', ' or ') node = ast.parse(expr, mode='eval') return self.render_node(node.body) def render_code(self, code): - code = code.replace('&', ' and ') - code = code.replace('|', ' or ') lines = [] for node in ast.parse(code).body: lines.append(self.render_node(node)) @@ -136,9 +130,6 @@ def render_Assign(self, node): class NumpyNodeRenderer(NodeRenderer): expression_ops = NodeRenderer.expression_ops.copy() expression_ops.update({ - # BinOps - 'BitAnd': '*', - 'BitOr': '+', # Unary ops 'Not': 'logical_not', 'Invert': 'logical_not', @@ -151,9 +142,6 @@ class NumpyNodeRenderer(NodeRenderer): class SympyNodeRenderer(NodeRenderer): expression_ops = NodeRenderer.expression_ops.copy() expression_ops.update({ - # BinOps - 'BitAnd': '&', - 'BitOr': '|', # Compare 'Eq': 'Eq', 'NotEq': 'Ne', @@ -194,9 +182,6 @@ def render_Num(self, node): class CPPNodeRenderer(NodeRenderer): expression_ops = NodeRenderer.expression_ops.copy() expression_ops.update({ - # BinOps - 'BitAnd': '&&', - 'BitOr': '||', # Unary ops 'Not': '!', 'Invert': '!', diff --git a/examples/I-F_curve_LIF.py b/examples/I-F_curve_LIF.py index c932a7103..a2fde70d3 100644 --- a/examples/I-F_curve_LIF.py +++ b/examples/I-F_curve_LIF.py @@ -11,11 +11,11 @@ N = 1000 tau = 10 * ms eqs = ''' -dv/dt=(v0-v)/tau : volt (active) +dv/dt=(v0-v)/tau : volt # (unless-refractory) v0 : volt ''' -group = NeuronGroup(N, equations=eqs, threshold='v>10 * mV', - reset='v = 0 * mV') +group = NeuronGroup(N, eqs, threshold='v>10*mV', + reset='v = 0*mV') group.refractory = 5 * ms group.v = 0 * mV group.v0 = linspace(0 * mV, 20 * mV, N) From 7956c03f570788d33385c195db40a89641365f28 Mon Sep 17 00:00:00 2001 From: thesamovar Date: Thu, 4 Jul 2013 18:58:28 -0400 Subject: [PATCH 02/15] Added new abstract_code_dependencies in parsing.dependencies and created tests --- brian2/parsing/dependencies.py | 130 ++++++++++++++++++ ..._syntax_translation.py => test_parsing.py} | 42 +++++- 2 files changed, 166 insertions(+), 6 deletions(-) create mode 100644 brian2/parsing/dependencies.py rename brian2/tests/{test_syntax_translation.py => test_parsing.py} (72%) diff --git a/brian2/parsing/dependencies.py b/brian2/parsing/dependencies.py new file mode 100644 index 000000000..4167057e7 --- /dev/null +++ b/brian2/parsing/dependencies.py @@ -0,0 +1,130 @@ +import ast + +from brian2.utils.stringtools import deindent +from collections import namedtuple + +__all__ = ['abstract_code_dependencies'] + +def get_read_write_funcs(parsed_code): + allids = set([]) + read = set([]) + write = set([]) + funcs = set([]) + for node in ast.walk(parsed_code): + if node.__class__ is ast.Name: + allids.add(node.id) + if node.ctx.__class__ is ast.Store: + write.add(node.id) + elif node.ctx.__class__ is ast.Load: + read.add(node.id) + else: + raise SyntaxError + elif node.__class__ is ast.Call: + funcs.add(node.func.id) + + read = read-funcs + + # check that there's no funky stuff going on with functions + if funcs.intersection(write): + raise SyntaxError("Cannot assign to functions in abstract code") + + return allids, read, write, funcs + + +def abstract_code_dependencies(code, known_vars=None, known_funcs=None): + ''' + Analyses identifiers used in abstract code blocks + + Parameters + ---------- + + code : str + The abstract code block. + known_vars : set + The set of known variable names. + known_funcs : set + The set of known function names. + + Returns + ------- + + results : namedtuple with the following fields + ``all`` + The set of all identifiers that appear in this code block, + including functions. + ``read`` + The set of values that are read, excluding functions. + ``write`` + The set of all values that are written to. + ``funcs`` + The set of all function names. + ``unknown_read`` + The set of all unknown variables whose values are read. Equal + to ``read-known_vars``. + ``unknown_write`` + The set of all unknown variables written to. Equal to + ``write-known_vars``. + ``unknown_funcs`` + The set of all unknown function names, equal to + ``funcs-known_funcs``. + ``undefined_read`` + The set of all unknown variables whose values are read before they + are written to. If this set is nonempty it usually indicates an + error, since a variable that is read should either have been + defined in the code block (in which case it will appear in + ``newly_defined``) or already be known. + ``newly_defined`` + The set of all variable names which are newly defined in this + abstract code block. + ''' + if known_vars is None: + known_vars = set([]) + if known_funcs is None: + known_funcs = set([]) + + code = deindent(code) + parsed_code = ast.parse(code, mode='exec') + + # Get the list of all variables that are read from and written to, + # ignoring the order + allids, read, write, funcs = get_read_write_funcs(parsed_code) + + # Now check if there are any values that are unknown and read before + # they are written to + defined = known_vars.copy() + newly_defined = set([]) + undefined_read = set([]) + for line in parsed_code.body: + _, cur_read, cur_write, _ = get_read_write_funcs(line) + undef = cur_read-defined + undefined_read |= undef + newly_defined |= (cur_write-defined)-undefined_read + defined |= cur_write + + # Return the results as a named tuple + results = dict( + all=allids, + read=read, + write=write, + funcs=funcs, + unknown_read=read-known_vars, + unknown_write=write-known_vars, + unknown_funcs=funcs-known_funcs, + undefined_read=undefined_read, + newly_defined=newly_defined, + ) + return namedtuple('AbstractCodeDependencies', results.keys())(**results) + + +if __name__=='__main__': + code = ''' + x = y+z + a = f(b) + ''' + known_vars = set(['y', 'z']) + print deindent(code) + print 'known_vars:', known_vars + print + r = abstract_code_dependencies(code, known_vars) + for k, v in r.__dict__.items(): + print k+':', ', '.join(list(v)) diff --git a/brian2/tests/test_syntax_translation.py b/brian2/tests/test_parsing.py similarity index 72% rename from brian2/tests/test_syntax_translation.py rename to brian2/tests/test_parsing.py index 4b1b74996..248399fd8 100644 --- a/brian2/tests/test_syntax_translation.py +++ b/brian2/tests/test_parsing.py @@ -1,10 +1,11 @@ ''' -Tests the brian2.codegen.syntax package +Tests the brian2.parsing package ''' from brian2.utils.stringtools import get_identifiers -from brian2.codegen.ast_parser import (NodeRenderer, NumpyNodeRenderer, - CPPNodeRenderer, - ) +from brian2.parsing.rendering import (NodeRenderer, NumpyNodeRenderer, + CPPNodeRenderer, + ) +from brian2.parsing.dependencies import abstract_code_dependencies from numpy.testing import assert_allclose @@ -35,8 +36,6 @@ 1+a 1+3 a>0.5 and b>0.5 - a>0.5&b>0.5&c>0.5 - (a>0.5) & (b>0.5) & (c>0.5) a>0.5 and b>0.5 or c>0.5 a>0.5 and b>0.5 or not c>0.5 2%4 @@ -126,8 +125,39 @@ def evaluator(expr, ns): parse_expressions(SympyRenderer(), evaluator) +def test_abstract_code_dependencies(): + code = ''' + a = b+c + d = b+c + a = func_a() + a = func_b() + a = e+d + ''' + known_vars = set(['a', 'b', 'c']) + known_funcs = set(['func_a']) + res = abstract_code_dependencies(code, known_vars, known_funcs) + expected_res = dict( + all=['a', 'b', 'c', 'd', 'e', + 'func_a', 'func_b', + ], + read=['b', 'c', 'd', 'e'], + write=['a', 'd'], + funcs=['func_a', 'func_b'], + unknown_read=['d', 'e'], + unknown_write=['d'], + unknown_funcs=['func_b'], + undefined_read=['e'], + newly_defined=['d'], + ) + for k, v in expected_res.items(): + if not getattr(res, k)==set(v): + raise AssertionError("For '%s' result is %s expected %s" % ( + k, getattr(res, k), set(v))) + if __name__=='__main__': test_parse_expressions_python() test_parse_expressions_numpy() test_parse_expressions_cpp() test_parse_expressions_sympy() + test_abstract_code_dependencies() + From bd5bad3fa8b122cda5c28e379931f4b3bc3e701b Mon Sep 17 00:00:00 2001 From: thesamovar Date: Fri, 5 Jul 2013 00:50:01 -0400 Subject: [PATCH 03/15] Removed ~ from ast parsing for the same reason as & and | --- brian2/parsing/rendering.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/brian2/parsing/rendering.py b/brian2/parsing/rendering.py index c925c14bd..4ab522cc9 100644 --- a/brian2/parsing/rendering.py +++ b/brian2/parsing/rendering.py @@ -29,7 +29,6 @@ class NodeRenderer(object): 'NotEq': '!=', # Unary ops 'Not': 'not', - 'Invert': '~', 'UAdd': '+', 'USub': '-', # Bool ops @@ -132,7 +131,6 @@ class NumpyNodeRenderer(NodeRenderer): expression_ops.update({ # Unary ops 'Not': 'logical_not', - 'Invert': 'logical_not', # Bool ops 'And': '*', 'Or': '+', @@ -147,7 +145,6 @@ class SympyNodeRenderer(NodeRenderer): 'NotEq': 'Ne', # Unary ops 'Not': '~', - 'Invert': '~', # Bool ops 'And': '&', 'Or': '|', @@ -184,7 +181,6 @@ class CPPNodeRenderer(NodeRenderer): expression_ops.update({ # Unary ops 'Not': '!', - 'Invert': '!', # Bool ops 'And': '&&', 'Or': '||', From 70c08087115492b5c76cb9eccf38d7393098c3dd Mon Sep 17 00:00:00 2001 From: thesamovar Date: Fri, 5 Jul 2013 01:14:36 -0400 Subject: [PATCH 04/15] Added is_boolean_expression using ast parsing --- brian2/parsing/expressions.py | 99 +++++++++++++++++++++++++++++++++++ brian2/tests/test_parsing.py | 22 +++++++- 2 files changed, 119 insertions(+), 2 deletions(-) create mode 100644 brian2/parsing/expressions.py diff --git a/brian2/parsing/expressions.py b/brian2/parsing/expressions.py new file mode 100644 index 000000000..74f5cc049 --- /dev/null +++ b/brian2/parsing/expressions.py @@ -0,0 +1,99 @@ +import ast + +def is_boolean_expression(expr, boolvars=None, boolfuncs=None): + ''' + Determines if an expression is of boolean type or not + + Parameters + ---------- + + expr : str + The expression to test + boolvars : set + The set of variables of boolean type. + boolfuncs : set + The set of functions which return booleans. + + Returns + ------- + + isbool : bool + Whether or not the expression is boolean. + + Raises + ------ + + SyntaxError + If the expression ought to be boolean but is not, + for example ``x Date: Fri, 5 Jul 2013 01:15:03 -0400 Subject: [PATCH 05/15] Forget to include __all__ in previous commit --- brian2/parsing/expressions.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/brian2/parsing/expressions.py b/brian2/parsing/expressions.py index 74f5cc049..af20ba8c4 100644 --- a/brian2/parsing/expressions.py +++ b/brian2/parsing/expressions.py @@ -1,5 +1,7 @@ import ast +__all__ = ['is_boolean_expression'] + def is_boolean_expression(expr, boolvars=None, boolfuncs=None): ''' Determines if an expression is of boolean type or not From 191cb6c12adc97863d16bc5865f2ae6d29166308 Mon Sep 17 00:00:00 2001 From: thesamovar Date: Fri, 5 Jul 2013 02:23:49 -0400 Subject: [PATCH 06/15] Added unit parsing to parsing.expressions, didn't include support for functions yet --- brian2/parsing/expressions.py | 178 +++++++++++++++++++++++++++++----- brian2/tests/test_parsing.py | 29 +++++- 2 files changed, 183 insertions(+), 24 deletions(-) diff --git a/brian2/parsing/expressions.py b/brian2/parsing/expressions.py index af20ba8c4..cb4e0559a 100644 --- a/brian2/parsing/expressions.py +++ b/brian2/parsing/expressions.py @@ -1,6 +1,18 @@ +''' +AST parsing based analysis of expressions +''' + import ast -__all__ = ['is_boolean_expression'] +from brian2.units.fundamentalunits import (get_unit_fast, + DimensionMismatchError, + have_same_dimensions, + ) +from brian2.units import allunits +from brian2.units import stdunits + +__all__ = ['is_boolean_expression', + 'parse_expression_unit',] def is_boolean_expression(expr, boolvars=None, boolfuncs=None): ''' @@ -76,26 +88,146 @@ def is_boolean_expression(expr, boolvars=None, boolfuncs=None): return False -if __name__=='__main__': - EVF = [ - (True, 'a or b', ['a', 'b'], []), - (True, 'True', [], []), - (True, 'a Date: Fri, 5 Jul 2013 10:14:05 -0400 Subject: [PATCH 07/15] Allow unitless_expr**unitless_expr even if RHS is variable power --- brian2/parsing/expressions.py | 2 ++ brian2/tests/test_parsing.py | 1 + 2 files changed, 3 insertions(+) diff --git a/brian2/parsing/expressions.py b/brian2/parsing/expressions.py index cb4e0559a..9d1f7dddc 100644 --- a/brian2/parsing/expressions.py +++ b/brian2/parsing/expressions.py @@ -186,6 +186,8 @@ def parse_expression_unit(expr, varunits, funcunits, use_standard_units=True): elif op=='Div': u = left/right elif op=='Pow': + if have_same_dimensions(left, 1) and have_same_dimensions(right, 1): + return get_unit_fast(1) if expr.right.__class__ is not ast.Num: raise SyntaxError("Cannot parse unit expression with variable power") u = left**expr.right.n diff --git a/brian2/tests/test_parsing.py b/brian2/tests/test_parsing.py index 3e5bd50da..ef301777c 100644 --- a/brian2/tests/test_parsing.py +++ b/brian2/tests/test_parsing.py @@ -189,6 +189,7 @@ def test_parse_expression_unit(): (volt**2, 'b**2'), (volt*amp, 'a%(b*c)'), (volt, '-b'), + (1, '(a/a)**(a/a)'), ] for expect, expr in EE: if expect is DimensionMismatchError: From 0dee7ccf7b2681568aec2b361c012bfea3112436 Mon Sep 17 00:00:00 2001 From: thesamovar Date: Fri, 5 Jul 2013 10:59:22 -0400 Subject: [PATCH 08/15] Check for arguments being sets in abstract_code_dependencies --- brian2/parsing/dependencies.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/brian2/parsing/dependencies.py b/brian2/parsing/dependencies.py index 4167057e7..c00cbc63a 100644 --- a/brian2/parsing/dependencies.py +++ b/brian2/parsing/dependencies.py @@ -58,6 +58,15 @@ def abstract_code_dependencies(code, known_vars=None, known_funcs=None): The set of all values that are written to. ``funcs`` The set of all function names. + ``known_all`` + The set of all identifiers that appear in this code block and + are known. + ``known_read`` + The set of known values that are read, excluding functions. + ``known_write`` + The set of known values that are written to. + ``known_funcs`` + The set of known functions that are used. ``unknown_read`` The set of all unknown variables whose values are read. Equal to ``read-known_vars``. @@ -81,8 +90,12 @@ def abstract_code_dependencies(code, known_vars=None, known_funcs=None): known_vars = set([]) if known_funcs is None: known_funcs = set([]) + if not isinstance(known_vars, set): + known_vars = set(known_vars) + if not isinstance(known_funcs, set): + known_funcs = set(known_funcs) - code = deindent(code) + code = deindent(code, docstring=True) parsed_code = ast.parse(code, mode='exec') # Get the list of all variables that are read from and written to, @@ -107,6 +120,10 @@ def abstract_code_dependencies(code, known_vars=None, known_funcs=None): read=read, write=write, funcs=funcs, + known_all=allids.intersection(known_vars.union(known_funcs)), + known_read=read.intersection(known_vars), + known_write=write.intersection(known_vars), + known_funcs=funcs.intersection(known_funcs), unknown_read=read-known_vars, unknown_write=write-known_vars, unknown_funcs=funcs-known_funcs, From c5c9950698b3b84659ece447d5467bf8bd6d1016 Mon Sep 17 00:00:00 2001 From: thesamovar Date: Fri, 5 Jul 2013 12:18:09 -0400 Subject: [PATCH 09/15] Updated test for abstract_code_dependencies --- brian2/tests/test_parsing.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/brian2/tests/test_parsing.py b/brian2/tests/test_parsing.py index ef301777c..916f09ee4 100644 --- a/brian2/tests/test_parsing.py +++ b/brian2/tests/test_parsing.py @@ -146,6 +146,10 @@ def test_abstract_code_dependencies(): read=['b', 'c', 'd', 'e'], write=['a', 'd'], funcs=['func_a', 'func_b'], + known_all=['a', 'b', 'c', 'func_a'], + known_read=['b', 'c'], + known_write=['a'], + known_funcs=['func_a'], unknown_read=['d', 'e'], unknown_write=['d'], unknown_funcs=['func_b'], From 0381348480ef1bcfe724762316da96606ca494e0 Mon Sep 17 00:00:00 2001 From: thesamovar Date: Fri, 5 Jul 2013 12:40:23 -0400 Subject: [PATCH 10/15] Added support for augmented assign to rendering --- brian2/parsing/rendering.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/brian2/parsing/rendering.py b/brian2/parsing/rendering.py index 4ab522cc9..44d876a3d 100644 --- a/brian2/parsing/rendering.py +++ b/brian2/parsing/rendering.py @@ -34,6 +34,13 @@ class NodeRenderer(object): # Bool ops 'And': 'and', 'Or': 'or', + # Augmented assign + 'AugAdd': '+=', + 'AugSub': '-=', + 'AugMult': '*=', + 'AugDiv': '/=', + 'AugPow': '**=', + 'AugMod': '%=', } def render_expr(self, expr, strip=True): @@ -124,6 +131,12 @@ def render_Assign(self, node): raise SyntaxError("Only support syntax like a=b not a=b=c") return '%s = %s' % (self.render_node(node.targets[0]), self.render_node(node.value)) + + def render_AugAssign(self, node): + target = node.target.id + rhs = self.render_node(node.value) + op = self.expression_ops['Aug'+node.op.__class__.__name__] + return '%s %s %s' % (target, op, rhs) class NumpyNodeRenderer(NodeRenderer): From b64cad1238575fb125be033fb2cfffe3b4067a48 Mon Sep 17 00:00:00 2001 From: thesamovar Date: Fri, 5 Jul 2013 15:26:38 -0400 Subject: [PATCH 11/15] Added parsing.functions --- brian2/parsing/functions.py | 305 +++++++++++++++++++++++++++++++++++ brian2/tests/test_parsing.py | 65 +++++++- 2 files changed, 369 insertions(+), 1 deletion(-) create mode 100644 brian2/parsing/functions.py diff --git a/brian2/parsing/functions.py b/brian2/parsing/functions.py new file mode 100644 index 000000000..5e558f036 --- /dev/null +++ b/brian2/parsing/functions.py @@ -0,0 +1,305 @@ +import ast +import inspect + +from brian2.utils.stringtools import deindent, indent, get_identifiers + +from rendering import NodeRenderer + +__all__ = ['AbstractCodeFunction', + 'abstract_code_from_function', + 'extract_abstract_code_functions', + 'substitute_abstract_code_functions', + ] + + +class AbstractCodeFunction(object): + ''' + The information defining an abstract code function + + Has attributes corresponding to initialisation parameters + + Parameters + ---------- + + name : str + The function name. + args : list of str + The arguments to the function. + code : str + The abstract code string consisting of the body of the function less + the return statement. + return_expr : str or None + The expression returned, or None if there is nothing returned. + ''' + def __init__(self, name, args, code, return_expr): + self.name = name + self.args = args + self.code = code + self.return_expr = return_expr + def __str__(self): + s = 'def %s(%s):\n%s\n return %s\n' % (self.name, + ', '.join(self.args), + indent(self.code), + self.return_expr) + return s + __repr__ = __str__ + + +def abstract_code_from_function(func): + ''' + Converts the body of the function to abstract code + + Parameters + ---------- + func : function, str or ast.FunctionDef + The function object to convert. Note that the arguments to the + function are ignored. + + Returns + ------- + func : AbstractCodeFunction + The corresponding abstract code function + + Raises + ------ + SyntaxError + If unsupported features are used such as if statements or indexing. + ''' + if callable(func): + code = deindent(inspect.getsource(func)) + funcnode = ast.parse(code, mode='exec').body[0] + elif isinstance(func, str): + funcnode = ast.parse(func, mode='exec').body[0] + elif func.__class__ is ast.FunctionDef: + funcnode = func + else: + raise TypeError("Unsupported function type") + + if funcnode.args.vararg is not None: + raise SyntaxError("No support for variable number of arguments") + if funcnode.args.kwarg is not None: + raise SyntaxError("No support for arbitrary keyword arguments") + if len(funcnode.args.defaults): + raise SyntaxError("No support for default values in functions") + + nodes = funcnode.body + nr = NodeRenderer() + lines = [] + return_expr = None + for node in nodes: + if node.__class__ is ast.Return: + return_expr = nr.render_node(node.value) + break + else: + lines.append(nr.render_node(node)) + abstract_code = '\n'.join(lines) + args = [arg.id for arg in funcnode.args.args] + name = funcnode.name + return AbstractCodeFunction(name, args, abstract_code, return_expr) + + +def extract_abstract_code_functions(code): + ''' + Returns a set of abstract code functions from function definitions. + + Returns all functions defined at the top level and ignores any other + code in the string. + + Parameters + ---------- + code : str + The code string defining some functions. + + Returns + ------- + funcs : dict + A mapping ``(name, func)`` for ``func`` an `AbstractCodeFunction`. + ''' + code = deindent(code) + nodes = ast.parse(code, mode='exec').body + funcs = {} + for node in nodes: + if node.__class__ is ast.FunctionDef: + func = abstract_code_from_function(node) + funcs[func.name] = func + return funcs + + +class VarRewriter(ast.NodeTransformer): + ''' + Rewrites all variable names in names by prepending pre + ''' + def __init__(self, pre): + self.pre = pre + def visit_Name(self, node): + return ast.Name(id=self.pre+node.id, ctx=node.ctx) + def visit_Call(self, node): + args = [self.visit(arg) for arg in node.args] + return ast.Call(func=ast.Name(id=node.func.id, ctx=ast.Load()), + args=args, keywords=[], starargs=None, kwargs=None) + + +class FunctionRewriter(ast.NodeTransformer): + ''' + Inlines a function call using temporary variables + + numcalls is the number of times the function rewriter has been called so + far, this is used to make sure that when recursively inlining there is no + name aliasing. The substitute_abstract_code_functions ensures that this is + kept up to date between recursive runs. + + The pre attribute is the set of lines to be inserted above the currently + being processed line, i.e. the inline code. + + The visit method returns the current line processed so that the function + call is replaced with the output of the inlining. + ''' + def __init__(self, func, numcalls=0): + self.func = func + self.numcalls = numcalls + self.pre = [] + self.suspend = False + def visit_Call(self, node): + # we suspend operations during an inlining operation, then resume + # afterwards, see below, so we only ever try to expand one inline + # function call at a time, i.e. no f(f(x)). This case is handled + # by the recursion. + if self.suspend: + return node + # We only work with the function we're provided + if node.func.id!=self.func.name: + return node + # Suspend while processing arguments (no recursion) + self.suspend = True + args = [self.visit(arg) for arg in node.args] + self.suspend = False + # The basename is used for function-local variables + basename = '_inline_'+self.func.name+'_'+str(self.numcalls) + # Assign all the function-local variables + for argname, arg in zip(self.func.args, args): + newpre = ast.Assign(targets=[ast.Name(id='%s_%s'%(basename, argname), + ctx=ast.Store())], + value=arg) + self.pre.append(newpre) + # Rewrite the lines of code of the function using the names defined + # above + vr = VarRewriter(basename+'_') + for funcline in ast.parse(self.func.code).body: + self.pre.append(vr.visit(funcline)) + # And rewrite the return expression + return_expr = vr.visit(ast.parse(self.func.return_expr, mode='eval').body) + self.pre.append(ast.Assign(targets=[ast.Name(id=basename, + ctx=ast.Store())], + value=return_expr)) + # Finally we replace the function call with the output of the inlining + newnode = ast.Name(id=basename) + self.numcalls += 1 + return newnode + + +def substitute_abstract_code_functions(code, funcs): + ''' + Performs inline substitution of all the functions in the code + + Parameters + ---------- + code : str + The abstract code to make inline substitutions into. + funcs : list, dict or set of AbstractCodeFunction + The function substitutions to use, note in the case of a dict, the + keys are ignored and the function name is used. + + Returns + ------- + code : str + The code with inline substitutions performed. + ''' + if isinstance(funcs, (list, set)): + newfuncs = dict() + for f in funcs: + newfuncs[f.name] = f + funcs = newfuncs + + code = deindent(code) + lines = ast.parse(code, mode='exec').body + + # This is a slightly nasty hack, but basically we just check by looking at + # the existing identifiers how many inline operations have already been + # performed by previous calls to this function + ids = get_identifiers(code) + funcstarts = {} + for func in funcs.values(): + subids = set([id for id in ids if id.startswith('_inline_'+func.name+'_')]) + subids = set([id.replace('_inline_'+func.name+'_', '') for id in subids]) + alli = [] + for subid in subids: + p = subid.find('_') + if p>0: + subid = subid[:p] + i = int(subid) + alli.append(i) + if len(alli)==0: + i = 0 + else: + i = max(alli)+1 + funcstarts[func.name] = i + + # Now we rewrite all the lines, replacing each line with a sequence of + # lines performing the inlining + newlines = [] + for line in lines: + for func in funcs.values(): + rw = FunctionRewriter(func, funcstarts[func.name]) + line = rw.visit(line) + newlines.extend(rw.pre) + funcstarts[func.name] = rw.numcalls + newlines.append(line) + + # Now we render to a code string + nr = NodeRenderer() + newcode = '\n'.join(nr.render_node(line) for line in newlines) + + # We recurse until no changes in the code to ensure that all functions + # are expanded if one function refers to another, etc. + if newcode==code: + return newcode + else: + return substitute_abstract_code_functions(newcode, funcs) + + +if __name__=='__main__': + if 1: + def f(x): + y = x*x + return y + def g(x): + return f(x)+1 + code = ''' + z = f(x) + z = f(x)+f(y) + w = f(z) + h = f(f(w)) + p = g(g(x)) + ''' + funcs = [abstract_code_from_function(f), + abstract_code_from_function(g), + ] + print substitute_abstract_code_functions(code, funcs) + if 0: + code = ''' + def f(x): + return x*x + def g(V): + V += 1 + ''' + funcs = extract_abstract_code_functions(code) + for k, v in funcs.items(): + print v + if 0: + def f(V, w): + V = w + V += x + x = y*z + return x+y + print abstract_code_from_function(f) + \ No newline at end of file diff --git a/brian2/tests/test_parsing.py b/brian2/tests/test_parsing.py index 916f09ee4..b5134162e 100644 --- a/brian2/tests/test_parsing.py +++ b/brian2/tests/test_parsing.py @@ -1,13 +1,16 @@ ''' Tests the brian2.parsing package ''' -from brian2.utils.stringtools import get_identifiers +from brian2.utils.stringtools import get_identifiers, deindent from brian2.parsing.rendering import (NodeRenderer, NumpyNodeRenderer, CPPNodeRenderer, ) from brian2.parsing.dependencies import abstract_code_dependencies from brian2.parsing.expressions import (is_boolean_expression, parse_expression_unit) +from brian2.parsing.functions import (abstract_code_from_function, + extract_abstract_code_functions, + substitute_abstract_code_functions) from brian2.units import volt, amp, DimensionMismatchError, have_same_dimensions from numpy.testing import assert_allclose, assert_raises @@ -202,7 +205,63 @@ def test_parse_expression_unit(): u = parse_expression_unit(expr, varunits, {}) assert have_same_dimensions(u, expect) +def test_abstract_code_from_function(): + # test basic functioning + def f(x): + y = x+1 + return y*y + ac = abstract_code_from_function(f) + assert ac.name=='f' + assert ac.args==['x'] + assert ac.code.strip()=='y = x + 1' + assert ac.return_expr=='y * y' + # Check that unsupported features raise an error + def f(x): + return x[:] + assert_raises(SyntaxError, abstract_code_from_function, f) + +def test_extract_abstract_code_functions(): + code = ''' + def f(x): + return x*x + + def g(V): + V += 1 + + irrelevant_code_here() + ''' + funcs = extract_abstract_code_functions(code) + assert funcs['f'].return_expr == 'x * x' + assert funcs['g'].args == ['V'] + + +def test_substitute_abstract_code_functions(): + def f(x): + y = x*x + return y + def g(x): + return f(x)+1 + code = ''' + z = f(x) + z = f(x)+f(y) + w = f(z) + h = f(f(w)) + p = g(g(x)) + ''' + funcs = [abstract_code_from_function(f), + abstract_code_from_function(g), + ] + subcode = substitute_abstract_code_functions(code, funcs) + for x, y in [(0, 1), (1, 0), (0.124323, 0.4549483)]: + ns1 = {'x':x, 'y':y, 'f':f, 'g':g} + ns2 = {'x':x, 'y':y} + exec deindent(code) in ns1 + exec subcode in ns2 + for k in ['z', 'w', 'h', 'p']: + assert ns1[k]==ns2[k] + + if __name__=='__main__': test_parse_expressions_python() test_parse_expressions_numpy() @@ -211,3 +270,7 @@ def test_parse_expression_unit(): test_abstract_code_dependencies() test_is_boolean_expression() test_parse_expression_unit() + test_abstract_code_from_function() + test_extract_abstract_code_functions() + test_substitute_abstract_code_functions() + \ No newline at end of file From f98044a86f90ed89e2359aa3abb8aea4d0c14c83 Mon Sep 17 00:00:00 2001 From: thesamovar Date: Fri, 5 Jul 2013 15:37:06 -0400 Subject: [PATCH 12/15] Merged master into examples/I-F_curve_LIF --- examples/I-F_curve_LIF.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/examples/I-F_curve_LIF.py b/examples/I-F_curve_LIF.py index a2fde70d3..3297bc7cc 100644 --- a/examples/I-F_curve_LIF.py +++ b/examples/I-F_curve_LIF.py @@ -11,12 +11,11 @@ N = 1000 tau = 10 * ms eqs = ''' -dv/dt=(v0-v)/tau : volt # (unless-refractory) +dv/dt=(v0-v)/tau : volt (unless-refractory) v0 : volt ''' -group = NeuronGroup(N, eqs, threshold='v>10*mV', - reset='v = 0*mV') -group.refractory = 5 * ms +group = NeuronGroup(N, equations=eqs, threshold='v>10 * mV', + reset='v = 0 * mV', refractory=5*ms) group.v = 0 * mV group.v0 = linspace(0 * mV, 20 * mV, N) From a2b5c9f4f8dc67a73aeef685f41645727fc12238 Mon Sep 17 00:00:00 2001 From: mstimberg Date: Sun, 7 Jul 2013 00:13:51 +0200 Subject: [PATCH 13/15] Add brian2.parsing to packages in setup.py --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 0e95f50dc..9d7d189ad 100644 --- a/setup.py +++ b/setup.py @@ -43,6 +43,7 @@ def run(self): 'brian2.groups', 'brian2.memory', 'brian2.monitors', + 'brain2.parsing', 'brian2.sphinxext', 'brian2.stateupdaters', 'brian2.tests', From 99d478cfc093a0b0ff60241ddc6bc675c8b4fdaa Mon Sep 17 00:00:00 2001 From: mstimberg Date: Sun, 7 Jul 2013 00:49:14 +0200 Subject: [PATCH 14/15] Add brian2.parsing to packages in setup.py: fixed typo --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9d7d189ad..912ec4ef5 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ def run(self): 'brian2.groups', 'brian2.memory', 'brian2.monitors', - 'brain2.parsing', + 'brian2.parsing', 'brian2.sphinxext', 'brian2.stateupdaters', 'brian2.tests', From c55c1228e36d4325abdac2dd0e259dc354507cc3 Mon Sep 17 00:00:00 2001 From: Marcel Stimberg Date: Mon, 8 Jul 2013 11:37:52 +0200 Subject: [PATCH 15/15] Make brian2.parsing.functions Python3 compatible --- brian2/parsing/functions.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/brian2/parsing/functions.py b/brian2/parsing/functions.py index 5e558f036..840783775 100644 --- a/brian2/parsing/functions.py +++ b/brian2/parsing/functions.py @@ -93,7 +93,12 @@ def abstract_code_from_function(func): else: lines.append(nr.render_node(node)) abstract_code = '\n'.join(lines) - args = [arg.id for arg in funcnode.args.args] + try: + # Python 2 + args = [arg.id for arg in funcnode.args.args] + except AttributeError: + # Python 3 + args = [arg.arg for arg in funcnode.args.args] name = funcnode.name return AbstractCodeFunction(name, args, abstract_code, return_expr)