Merge pull request #3662 from minrk/coding_transform

add strip_encoding_cookie transformer

strips # coding: utf-8 comments from input, which can cause SyntaxErrors

closes #3245

IPython/core/inputsplitter.py

@@ -75,6 +75,7 @@
 from IPython.core.inputtransformer import (leading_indent,
                                            classic_prompt,
                                            ipy_prompt,
+                                           strip_encoding_cookie,
                                            cellmagic,
                                            assemble_logical_lines,
                                            help_end,
@@ -488,6 +489,7 @@ def __init__(self, line_input_checker=True, physical_line_transforms=None,
             self.physical_line_transforms = [leading_indent(),
                                              classic_prompt(),
                                              ipy_prompt(),
+                                             strip_encoding_cookie(),
                                             ]
 
         self.assemble_logical_lines = assemble_logical_lines()

IPython/core/inputtransformer.py

@@ -5,6 +5,7 @@
 
 from IPython.core.splitinput import LineInfo
 from IPython.utils import tokenize2
+from IPython.utils.openpy import cookie_comment_re
 from IPython.utils.tokenize2 import generate_tokens, untokenize, TokenError
 
 #-----------------------------------------------------------------------------
@@ -422,6 +423,30 @@ def leading_indent():
                 line = (yield line)
 
 
+@CoroutineInputTransformer.wrap
+def strip_encoding_cookie():
+    """Remove encoding comment if found in first two lines
+    
+    If the first or second line has the `# coding: utf-8` comment,
+    it will be removed.
+    """
+    line = ''
+    while True:
+        line = (yield line)
+        # check comment on first two lines
+        for i in range(2):
+            if line is None:
+                break
+            if cookie_comment_re.match(line):
+                line = (yield "")
+            else:
+                line = (yield line)
+
+        # no-op on the rest of the cell
+        while line is not None:
+            line = (yield line)
+
+
 assign_system_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))'
                               r'\s*=\s*!\s*(?P<cmd>.*)')
 assign_system_template = '%s = get_ipython().getoutput(%r)'

IPython/core/magics/code.py

@@ -157,7 +157,7 @@ def pastebin(self, parameter_s=''):
     def loadpy(self, arg_s):
         """Alias of `%load`
 
-        `%loadpy` has gained some flexibility and droped the requirement of a `.py`
+        `%loadpy` has gained some flexibility and dropped the requirement of a `.py`
         extension. So it has been renamed simply into %load. You can look at
         `%load`'s docstring for more info.
         """

IPython/core/tests/test_inputtransformer.py

@@ -65,6 +65,13 @@ def transform_checker(tests, transformer, **kwargs):
         ('    ','    '),  # blank lines are kept intact
         ],
 
+       strip_encoding_cookie =
+       [
+        ('# -*- encoding: utf-8 -*-', ''),
+        ('# coding: latin-1', ''),
+       ],
+
+
        # Tests for the escape transformer to leave normal code alone
        escaped_noesc =
        [ ('    ', '    '),
@@ -203,6 +210,20 @@ def transform_checker(tests, transformer, **kwargs):
           ],
          ],
 
+       strip_encoding_cookie =
+       [
+        [
+            ('# -*- coding: utf-8 -*-', ''),
+            ('foo', 'foo'),
+        ],
+        [
+            ('#!/usr/bin/env python', '#!/usr/bin/env python'),
+            ('# -*- coding: latin-1 -*-', ''),
+            # only the first-two lines
+            ('# -*- coding: latin-1 -*-', '# -*- coding: latin-1 -*-'),
+        ],
+       ],
+
        multiline_datastructure_prompt =
        [ [('>>> a = [1,','a = [1,'),
           ('... 2]','2]'),
@@ -291,6 +312,11 @@ def test_ipy_prompt():
     for example in syntax_ml['ipy_prompt']:
         transform_checker(example, ipt.ipy_prompt)
 
+def test_coding_cookie():
+    tt.check_pairs(transform_and_reset(ipt.strip_encoding_cookie), syntax['strip_encoding_cookie'])
+    for example in syntax_ml['strip_encoding_cookie']:
+        transform_checker(example, ipt.strip_encoding_cookie)
+
 def test_assemble_logical_lines():
     tests = \
     [ [(u"a = \\", None),