Use ast to handle PEP 328 multi-line imports

Closes #1422
Closes #2019

This fixes #1422 and fixes #2019.

CONTRIBUTORS.txt

@@ -237,3 +237,5 @@ contributors:
 * Tomer Chachamu, Richard Goodman: simplifiable-if-expression
 
 * Benjamin Drung: contributing Debian Developer
+
+* Scott Worley: contributor

ChangeLog

@@ -140,6 +140,10 @@ Release date: TBA
 
      Close #2430
 
+   * Fix --ignore-imports to understand multi-line imports
+
+     Close #1422
+     Close #2019
 
 What's New in Pylint 2.1.1?
 ===========================

pylint/checkers/similar.py

@@ -20,6 +20,9 @@
 from __future__ import print_function
 import sys
 from collections import defaultdict
+from itertools import groupby
+
+import astroid
 
 from pylint.utils import decoding_stream
 from pylint.interfaces import IRawChecker
@@ -153,10 +156,20 @@ def stripped_lines(lines, ignore_comments, ignore_docstrings, ignore_imports):
     """return lines with leading/trailing whitespace and any ignored code
     features removed
     """
+    if ignore_imports:
+        tree = astroid.parse(''.join(lines))
+        node_is_import_by_lineno = (
+            (node.lineno, isinstance(node, (astroid.Import, astroid.ImportFrom)))
+            for node in tree.body)
+        line_begins_import = {
+            lineno: all(is_import for _, is_import in node_is_import_group)
+            for lineno, node_is_import_group
+            in groupby(node_is_import_by_lineno, key=lambda x: x[0])}
+        current_line_is_import = False
 
     strippedlines = []
     docstring = None
-    for line in lines:
+    for lineno, line in enumerate(lines, start=1):
         line = line.strip()
         if ignore_docstrings:
             if not docstring and (line.startswith('"""') or line.startswith("'''")):
@@ -167,8 +180,10 @@ def stripped_lines(lines, ignore_comments, ignore_docstrings, ignore_imports):
                     docstring = None
                 line = ""
         if ignore_imports:
-            if line.startswith("import ") or line.startswith("from "):
-                line = ""
+            current_line_is_import = line_begins_import.get(
+                lineno, current_line_is_import)
+            if current_line_is_import:
+                line = ''
         if ignore_comments:
             # XXX should use regex in checkers/format to avoid cutting
             # at a "#" in a string

pylint/test/input/hide_code_with_imports.py

@@ -0,0 +1,16 @@
+quicksort = lambda a: qs1(a,0,len(a)-1)                               ; import a
+qs1 = lambda a,lo,hi: qs2(a,lo,hi) if lo<hi else a                    ; import b
+qs2 = lambda a,lo,hi: qs3(lo,hi,*qsp(a,lo,hi))                        ; import c
+qs3 = lambda lo,hi,a,p: qs1(qs1(a,p+1,hi),lo,p-1)                     ; import d
+qsp = lambda a,lo,hi: qsp1(a,lo,hi,a[hi],lo,lo)                       ; import e
+qsp1 = lambda a,lo,hi,p,i,j: qsp2(a,lo,hi,p,i,j,j<hi)                 ; import f
+qsp2 = lambda a,lo,hi,p,i,j,c: qspt(a,lo,hi,p,i,j,qsp3 if c else qsp7); import g
+qspt = lambda a,lo,hi,p,i,j,n: n(a,lo,hi,p,i,j)                       ; import h
+qsp3 = lambda a,lo,hi,p,i,j: qsp4(a,lo,hi,p,i,j,a[j]<p)               ; import i
+qsp4 = lambda a,lo,hi,p,i,j,c: qspt(a,lo,hi,p,i,j,qsp5 if c else qsp6); import j
+qsp5 = lambda a,lo,hi,p,i,j: qsp1(sw(a,i,j),lo,hi,p,i+1,j+1)          ; import k
+qsp6 = lambda a,lo,hi,p,i,j: qsp1(a,lo,hi,p,i,j+1)                    ; import l
+qsp7 = lambda a,lo,hi,p,i,j: (sw(a,i,hi), i)                          ; import m
+sw = lambda a,i,j: sw1(enumerate(a),i,j,a[i],(-1,a[j]))               ; import n
+sw1 = lambda a,i,j,ai,aj: sw2([aj if x[0]==i else x for x in a],j,ai) ; import o
+sw2 = lambda a,j,ai: [ai if x[0]==j else x[1] for x in a]             ; import p

pylint/test/input/multiline-import

@@ -0,0 +1,8 @@
+from foo import (
+  bar,
+  baz,
+  quux,
+  quuux,
+  quuuux,
+  quuuuux,
+)

pylint/test/unittest_checker_similar.py

@@ -18,8 +18,12 @@
 
 from pylint.checkers import similar
 
-SIMILAR1 = join(dirname(abspath(__file__)), "input", "similar1")
-SIMILAR2 = join(dirname(abspath(__file__)), "input", "similar2")
+SIMILAR1 = join(dirname(abspath(__file__)), 'input', 'similar1')
+SIMILAR2 = join(dirname(abspath(__file__)), 'input', 'similar2')
+MULTILINE = join(dirname(abspath(__file__)), 'input', 'multiline-import')
+HIDE_CODE_WITH_IMPORTS = join(
+    dirname(abspath(__file__)), 'input', 'hide_code_with_imports.py')
+
 
 
 def test_ignore_comments():
@@ -100,6 +104,45 @@ def test_ignore_imports():
     )
 
 
+def test_multiline_imports():
+    output = StringIO()
+    with redirect_stdout(output), pytest.raises(SystemExit) as ex:
+        similar.Run([MULTILINE, MULTILINE])
+    assert ex.value.code == 0
+    assert output.getvalue().strip() == ("""
+8 similar lines in 2 files
+==%s:0
+==%s:0
+   from foo import (
+     bar,
+     baz,
+     quux,
+     quuux,
+     quuuux,
+     quuuuux,
+   )
+TOTAL lines=16 duplicates=8 percent=50.00
+""" % (MULTILINE, MULTILINE)).strip()
+
+
+def test_ignore_multiline_imports():
+    output = StringIO()
+    with redirect_stdout(output), pytest.raises(SystemExit) as ex:
+        similar.Run(['--ignore-imports', MULTILINE, MULTILINE])
+    assert ex.value.code == 0
+    assert output.getvalue().strip() == """
+TOTAL lines=16 duplicates=0 percent=0.00
+""".strip()
+
+
+def test_no_hide_code_with_imports():
+    output = StringIO()
+    with redirect_stdout(output), pytest.raises(SystemExit) as ex:
+        similar.Run(['--ignore-imports'] + 2 * [HIDE_CODE_WITH_IMPORTS])
+    assert ex.value.code == 0
+    assert "TOTAL lines=32 duplicates=16 percent=50.00" in output.getvalue()
+
+
 def test_ignore_nothing():
     output = StringIO()
     with redirect_stdout(output), pytest.raises(SystemExit) as ex: