Pythran file

doc/papers/sc2013/bench/pythran/setup.py

@@ -27,7 +27,7 @@ def run(self):
                 module = pythran.cxx_generator(
                         extension.name,
                         content,
-                        pythran.spec_parser(content),
+                        pythran.pyspec_parser(content),
                         []
                         )
 

pythran/__init__.py

@@ -18,7 +18,7 @@
 >>> cxx = generate_cxx('my_module', code, {'foo':([int],)})
 
 Eventually, the type information can be translated from a string:
->>> spec = spec_parser('#pythran export foo(int)')
+>>> spec = pyspec_parser('#pythran export foo(int)')
 >>> cxx = generate_cxx('my_module', code, spec)
 
 Higher level entry points include:
@@ -39,7 +39,7 @@
 from pythran.toolchain import (generate_cxx, compile_cxxfile, compile_cxxcode,
                                compile_pythrancode, compile_pythranfile,
                                test_compile)
-from pythran.spec import spec_parser
+from pythran.spec import spec_parser, pyspec_parser
 from pythran.dist import PythranExtension
 from pythran.version import __version__
 

pythran/spec.py

@@ -27,7 +27,6 @@ class SpecParser:
 
     # lex part
     reserved = {
-        '#pythran': 'PYTHRAN',
         'export': 'EXPORT',
         'list': 'LIST',
         'set': 'SET',
@@ -64,11 +63,7 @@ class SpecParser:
     t_RARRAY = r'\]'
     t_LARRAY = r'\['
 
-    # regexp to extract pythran specs from comments
-    # the first part matches lines with a comment and the pythran keyword
-    # the second part matches lines with comments following the pythran ones
-    FILTER = re.compile(r'^\s*#\s*pythran[^\n\r]*[\n\r]+'
-                        r'^(?:\s*#[^\n\r]*[\n\r]+)*', re.MULTILINE)
+    COMMENT_REGEX = re.compile(r'^\s*(#|//).*\n?', re.MULTILINE)
 
     def t_IDENTIFER(self, t):
         r'\#?[a-zA-Z_][a-zA-Z_0-9]*'
@@ -91,11 +86,11 @@ def t_newline(self, t):
 
     def p_exports(self, p):
         '''exports :
-                   | PYTHRAN EXPORT export_list opt_craps exports'''
+                   | EXPORT export_list opt_craps exports'''
 
     def p_export_list(self, p):
         '''export_list : export
-                  | export COMMA export_list'''
+                       | export COMMA export_list'''
 
     def p_export(self, p):
         '''export : IDENTIFIER LPAREN opt_types RPAREN
@@ -115,7 +110,6 @@ def p_opt_craps(self, p):
     def p_crap(self, p):
         '''crap : CRAP
                 | IDENTIFIER
-                | EXPORT
                 | LPAREN
                 | RPAREN
                 | LARRAY
@@ -245,7 +239,6 @@ def __init__(self):
                                 write_tables=False)
 
     def __call__(self, path_or_text):
-        self.exports = dict()
         self.input_file = None
         if os.path.isfile(path_or_text):
             self.input_file = path_or_text
@@ -254,18 +247,47 @@ def __call__(self, path_or_text):
         else:
             data = path_or_text
 
-        raw = "\n".join(SpecParser.FILTER.findall(data))
-        pythran_data = (re.sub(r'#\s*pythran', '\_o< pythran >o_/', raw)
-                        .replace('#', '')
-                        .replace('\_o< pythran >o_/', '#pythran'))
-        self.parser.parse(pythran_data, lexer=self.lexer)
+        return self._parse(data)
+
+    def _parse(self, data):
+        data = SpecParser.COMMENT_REGEX.sub(r'\n', data)
+
+        self.exports = dict()
+        self.parser.parse(data, lexer=self.lexer)
         if not self.exports:
             import logging
             logging.warn("No pythran specification, "
                          "no function will be exported")
         return self.exports
 
 
+class PySpecParser(SpecParser):
+
+    # regexp to extract pythran specs from comments
+    # the first part matches lines with a comment and the pythran keyword
+    # the second part matches lines with comments following the pythran ones
+    FILTER = re.compile(r'^\s*#\s*pythran[^\n\r]*[\n\r]+'
+                        r'^(?:\s*#[^\n\r]*[\n\r]+)*', re.MULTILINE)
+    def __init__(self):
+        SpecParser.__init__(self)
+
+    def _parse(self, data):
+        # Keep only pythran comment directives
+        i = 0
+        pythran_data = ""
+        for match in PySpecParser.FILTER.finditer(data):
+            begin, end = match.span()
+            pythran_data += '\n' * data[i:begin].count('\n') + data[begin:end]
+            i = end
+        pythran_data += '\n' * data[i:len(data)].count('\n')
+
+        # Remove #pythran python prefix
+        pythran_data = (re.sub(r'#\s*pythran', '', pythran_data)
+                        .replace('#', ''))
+
+        return SpecParser._parse(self, pythran_data)
+
+
 def expand_specs(specs):
     '''
     Expand a spec in its various variant
@@ -318,3 +340,7 @@ def specs_to_docstrings(specs, docstrings):
 
 def spec_parser(path):
     return SpecParser()(path)
+
+
+def pyspec_parser(path):
+    return PySpecParser()(path)

pythran/tests/cases/zero_pythran.py

@@ -0,0 +1,4 @@
+#runas zero(10,20)
+#bench zero(6000,6000)
+def zero(n,m): return [[0 for row in xrange(n)] for col in xrange(m)]
+

pythran/tests/cases/zero_pythran.pythran

@@ -0,0 +1,2 @@
+# This is a pythran file
+export zero(int, int)

pythran/tests/test_env.py

@@ -16,7 +16,7 @@
 
 import pytest
 
-from pythran import compile_pythrancode, spec_parser, frontend
+from pythran import compile_pythrancode, pyspec_parser, frontend
 from pythran.config import have_gmp_support
 from pythran.backend import Python
 from pythran.middlend import refine
@@ -375,7 +375,7 @@ class TestFromDir(TestEnv):
     def interface(name=None, file_=None):
         """ Return Pythran specs."""
         default_value = {name: []}
-        return spec_parser(open(file_).read()) if file_ else default_value
+        return pyspec_parser(file_) if file_ else default_value
 
     def __init__(self, *args, **kwargs):
         """ Dynamically add methods for unittests, second stage. """

pythran/tests/test_spec_parser.py

@@ -38,7 +38,7 @@ class TestSpecParser(unittest.TestCase):
 
     def test_parser(self):
         real_path = os.path.splitext(os.path.realpath(__file__))[0]+".py"
-        print(pythran.spec_parser(real_path))
+        print(pythran.pyspec_parser(real_path))
 
     def test_invalid_specs0(self):
         code = '#pythran export foo()\ndef foo(n): return n'
@@ -61,7 +61,7 @@ def test_multiline_spec0(self):
               # )
 def foo(): return
             '''
-        self.assertTrue(pythran.spec_parser(code))
+        self.assertTrue(pythran.pyspec_parser(code))
 
     def test_multiline_spec1(self):
         code = '''
@@ -70,7 +70,7 @@ def test_multiline_spec1(self):
               # )
 def foo(i,j): return
             '''
-        self.assertTrue(pythran.spec_parser(code))
+        self.assertTrue(pythran.pyspec_parser(code))
 
     def test_multiline_spec2(self):
         code = '''
@@ -81,22 +81,22 @@ def test_multiline_spec2(self):
               # )
 def foo(i,j,k): return
             '''
-        self.assertTrue(pythran.spec_parser(code))
+        self.assertTrue(pythran.pyspec_parser(code))
 
     def test_crappy_spec0(self):
         code = '''
 #      pythran export foo(int) this is an int test
 def foo(i): return
             '''
-        self.assertTrue(pythran.spec_parser(code))
+        self.assertTrue(pythran.pyspec_parser(code))
 
     def test_crappy_spec1(self):
         code = '''
 #      pythran export foo(int)
 #this is a pythran export test
 def foo(i): return
             '''
-        self.assertTrue(pythran.spec_parser(code))
+        self.assertTrue(pythran.pyspec_parser(code))
 
     def test_middle_spec0(self):
         code = '''
@@ -105,7 +105,7 @@ def foo(i): return
 #this is a pythran export test
 def bar(i): return
             '''
-        self.assertTrue(pythran.spec_parser(code))
+        self.assertTrue(pythran.pyspec_parser(code))
 
 
     def test_middle_spec1(self):
@@ -117,12 +117,12 @@ def foo(i): return
 #      pythran export foo(float)
 def bar(i): return
             '''
-        self.assertEquals(len(pythran.spec_parser(code)), 1)
-        self.assertEquals(len(pythran.spec_parser(code)['foo']), 2)
+        self.assertEquals(len(pythran.pyspec_parser(code)), 1)
+        self.assertEquals(len(pythran.pyspec_parser(code)['foo']), 2)
 
     def test_var_export0(self):
         code = '''
 #      pythran export foo
 foo = 1
             '''
-        self.assertTrue(pythran.spec_parser(code))
+        self.assertTrue(pythran.pyspec_parser(code))

pythran/toolchain.py

@@ -300,7 +300,7 @@ def compile_cxxcode(module_name, cxxcode, output_binary=None, keep_temp=False,
     return output_binary
 
 
-def compile_pythrancode(module_name, pythrancode, specs=None,
+def compile_pythrancode(module_name, path_or_text, specs=None,
                         opts=None, cpponly=False, output_file=None,
                         **kwargs):
     '''Pythran code (string) -> c++ code -> native module
@@ -309,9 +309,15 @@ def compile_pythrancode(module_name, pythrancode, specs=None,
     '''
 
     # Autodetect the Pythran spec if not given as parameter
-    from pythran.spec import spec_parser
+    from pythran.spec import pyspec_parser
     if specs is None:
-        specs = spec_parser(pythrancode)
+        specs = pyspec_parser(path_or_text)
+
+    if os.path.isfile(path_or_text):
+        with open(path_or_text) as fd:
+            pythrancode = fd.read()
+    else:
+        pythrancode = path_or_text
 
     # Generate C++, get a PythonModule object
     module = generate_cxx(module_name, pythrancode, specs, opts)
@@ -354,7 +360,13 @@ def compile_pythranfile(file_path, output_file=None, module_name=None,
     # Add compiled module path to search for imported modules
     sys.path.append(os.path.dirname(file_path))
 
-    output_file = compile_pythrancode(module_name, open(file_path).read(),
+    specs = None
+    spec_file = os.path.join(os.path.dirname(file_path), os.path.splitext(file_path)[0] + ".pythran")
+    if os.path.isfile(spec_file):
+        from pythran.spec import spec_parser
+        specs = spec_parser(spec_file)
+
+    output_file = compile_pythrancode(module_name, file_path, specs,
                                       output_file=output_file,
                                       cpponly=cpponly,
                                       **kwargs)