Fix AstroidError in similarity checker with imports/signatures igno…

…red (#6357)

Co-authored-by: Daniël van Noord <13665637+DanielNoord@users.noreply.github.com>

ChangeLog

@@ -26,6 +26,12 @@ Release date: TBA
   Closes #5815
   Closes #5406
 
+* Fixed an ``AstroidError`` in 2.13.0 raised by the ``duplicate-code`` checker with
+  ``ignore-imports`` or ``ignore-signatures`` enabled.
+
+  Closes #6301
+
+
 What's New in Pylint 2.13.5?
 ============================
 Release date: 2022-04-06

doc/whatsnew/2.13.rst

@@ -205,6 +205,11 @@ Other Changes
   Closes #5815
   Closes #5406
 
+* Fixed an ``AstroidError`` in 2.13.0 raised by the ```duplicate-code``` checker with
+  ``ignore-imports`` or ``ignore-signatures`` enabled.
+
+  Closes #6301
+
 * Use the ``tomli`` package instead of ``toml`` to parse ``.toml`` files.
 
  Closes #5885

pylint/checkers/similar.py

@@ -30,6 +30,7 @@
 from typing import (
     TYPE_CHECKING,
     Any,
+    Callable,
     Dict,
     FrozenSet,
     Generator,
@@ -359,28 +360,25 @@ def append_stream(
             readlines = decoding_stream(stream, encoding).readlines
         else:
             readlines = stream.readlines  # type: ignore[assignment] # hint parameter is incorrectly typed as non-optional
+
         try:
-            active_lines: List[str] = []
-            if hasattr(self, "linter"):
-                # Remove those lines that should be ignored because of disables
-                for index, line in enumerate(readlines()):
-                    if self.linter._is_one_message_enabled("R0801", index + 1):  # type: ignore[attr-defined]
-                        active_lines.append(line)
-            else:
-                active_lines = readlines()
-
-            self.linesets.append(
-                LineSet(
-                    streamid,
-                    active_lines,
-                    self.ignore_comments,
-                    self.ignore_docstrings,
-                    self.ignore_imports,
-                    self.ignore_signatures,
-                )
-            )
+            lines = readlines()
         except UnicodeDecodeError:
-            pass
+            lines = []
+
+        self.linesets.append(
+            LineSet(
+                streamid,
+                lines,
+                self.ignore_comments,
+                self.ignore_docstrings,
+                self.ignore_imports,
+                self.ignore_signatures,
+                line_enabled_callback=self.linter._is_one_message_enabled  # type: ignore[attr-defined]
+                if hasattr(self, "linter")
+                else None,
+            )
+        )
 
     def run(self) -> None:
         """Start looking for similarities and display results on stdout."""
@@ -552,6 +550,7 @@ def stripped_lines(
     ignore_docstrings: bool,
     ignore_imports: bool,
     ignore_signatures: bool,
+    line_enabled_callback: Union[Callable[[str, int], bool], None] = None,
 ) -> List[LineSpecifs]:
     """Return tuples of line/line number/line type with leading/trailing whitespace and any ignored code features removed.
 
@@ -560,6 +559,7 @@ def stripped_lines(
     :param ignore_docstrings: if true, any line that is a docstring is removed from the result
     :param ignore_imports: if true, any line that is an import is removed from the result
     :param ignore_signatures: if true, any line that is part of a function signature is removed from the result
+    :param line_enabled_callback: If called with "R0801" and a line number, a return value of False will disregard the line
     :return: the collection of line/line number/line type tuples
     """
     if ignore_imports or ignore_signatures:
@@ -611,6 +611,10 @@ def _get_functions(
     strippedlines = []
     docstring = None
     for lineno, line in enumerate(lines, start=1):
+        if line_enabled_callback is not None and not line_enabled_callback(
+            "R0801", lineno
+        ):
+            continue
         line = line.strip()
         if ignore_docstrings:
             if not docstring:
@@ -657,11 +661,17 @@ def __init__(
         ignore_docstrings: bool = False,
         ignore_imports: bool = False,
         ignore_signatures: bool = False,
+        line_enabled_callback: Union[Callable[[str, int], bool], None] = None,
     ) -> None:
         self.name = name
         self._real_lines = lines
         self._stripped_lines = stripped_lines(
-            lines, ignore_comments, ignore_docstrings, ignore_imports, ignore_signatures
+            lines,
+            ignore_comments,
+            ignore_docstrings,
+            ignore_imports,
+            ignore_signatures,
+            line_enabled_callback=line_enabled_callback,
         )
 
     def __str__(self):

tests/test_similar.py

@@ -44,7 +44,13 @@ def _runtest(self, args: List[str], code: int) -> None:
     @staticmethod
     def _run_pylint(args: List[str], out: TextIO) -> int:
         """Runs pylint with a patched output."""
-        args = args + ["--persistent=no"]
+        args = args + [
+            "--persistent=no",
+            "--enable=astroid-error",
+            # Enable functionality that will build another ast
+            "--ignore-imports=y",
+            "--ignore-signatures=y",
+        ]
         with _patch_streams(out):
             with pytest.raises(SystemExit) as cm:
                 with warnings.catch_warnings():
@@ -63,8 +69,10 @@ def _test_output(self, args: List[str], expected_output: str) -> None:
         out = StringIO()
         self._run_pylint(args, out=out)
         actual_output = self._clean_paths(out.getvalue())
+        actual_output_stripped = actual_output.strip()
         expected_output = self._clean_paths(expected_output)
-        assert expected_output.strip() in actual_output.strip()
+        assert expected_output.strip() in actual_output_stripped
+        assert "Fatal error" not in actual_output_stripped
 
     def test_duplicate_code_raw_strings_all(self) -> None:
         """Test similar lines in 3 similar files."""