Fix AstroidError in similarity checker with imports/signatures igno…

…red (#6357)

Co-authored-by: Daniël van Noord <13665637+DanielNoord@users.noreply.github.com>

ChangeLog

@@ -214,6 +214,11 @@ Release date: TBA
   Closes #5815
   Closes #5406
 
+* Fixed an ``AstroidError`` in 2.13.0 raised by the ``duplicate-code`` checker with
+  ``ignore-imports`` or ``ignore-signatures`` enabled.
+
+  Closes #6301
+
 
 What's New in Pylint 2.13.5?
 ============================

doc/whatsnew/2.13.rst

@@ -205,6 +205,11 @@ Other Changes
   Closes #5815
   Closes #5406
 
+* Fixed an ``AstroidError`` in 2.13.0 raised by the ```duplicate-code``` checker with
+  ``ignore-imports`` or ``ignore-signatures`` enabled.
+
+  Closes #6301
+
 * Use the ``tomli`` package instead of ``toml`` to parse ``.toml`` files.
 
  Closes #5885

pylint/checkers/similar.py

@@ -28,7 +28,7 @@
 import sys
 import warnings
 from collections import defaultdict
-from collections.abc import Generator, Iterable
+from collections.abc import Callable, Generator, Iterable
 from getopt import getopt
 from io import BufferedIOBase, BufferedReader, BytesIO
 from itertools import chain, groupby
@@ -366,28 +366,25 @@ def append_stream(
             readlines = decoding_stream(stream, encoding).readlines
         else:
             readlines = stream.readlines  # type: ignore[assignment] # hint parameter is incorrectly typed as non-optional
+
         try:
-            active_lines: list[str] = []
-            if hasattr(self, "linter"):
-                # Remove those lines that should be ignored because of disables
-                for index, line in enumerate(readlines()):
-                    if self.linter._is_one_message_enabled("R0801", index + 1):  # type: ignore[attr-defined]
-                        active_lines.append(line)
-            else:
-                active_lines = readlines()
-
-            self.linesets.append(
-                LineSet(
-                    streamid,
-                    active_lines,
-                    self.namespace.ignore_comments,
-                    self.namespace.ignore_docstrings,
-                    self.namespace.ignore_imports,
-                    self.namespace.ignore_signatures,
-                )
-            )
+            lines = readlines()
         except UnicodeDecodeError:
-            pass
+            lines = []
+
+        self.linesets.append(
+            LineSet(
+                streamid,
+                lines,
+                self.namespace.ignore_comments,
+                self.namespace.ignore_docstrings,
+                self.namespace.ignore_imports,
+                self.namespace.ignore_signatures,
+                line_enabled_callback=self.linter._is_one_message_enabled  # type: ignore[attr-defined]
+                if hasattr(self, "linter")
+                else None,
+            )
+        )
 
     def run(self) -> None:
         """Start looking for similarities and display results on stdout."""
@@ -563,6 +560,7 @@ def stripped_lines(
     ignore_docstrings: bool,
     ignore_imports: bool,
     ignore_signatures: bool,
+    line_enabled_callback: Callable[[str, int], bool] | None = None,
 ) -> list[LineSpecifs]:
     """Return tuples of line/line number/line type with leading/trailing whitespace and any ignored code features removed.
 
@@ -571,6 +569,7 @@ def stripped_lines(
     :param ignore_docstrings: if true, any line that is a docstring is removed from the result
     :param ignore_imports: if true, any line that is an import is removed from the result
     :param ignore_signatures: if true, any line that is part of a function signature is removed from the result
+    :param line_enabled_callback: If called with "R0801" and a line number, a return value of False will disregard the line
     :return: the collection of line/line number/line type tuples
     """
     if ignore_imports or ignore_signatures:
@@ -622,6 +621,10 @@ def _get_functions(
     strippedlines = []
     docstring = None
     for lineno, line in enumerate(lines, start=1):
+        if line_enabled_callback is not None and not line_enabled_callback(
+            "R0801", lineno
+        ):
+            continue
         line = line.strip()
         if ignore_docstrings:
             if not docstring:
@@ -668,11 +671,17 @@ def __init__(
         ignore_docstrings: bool = False,
         ignore_imports: bool = False,
         ignore_signatures: bool = False,
+        line_enabled_callback: Callable[[str, int], bool] | None = None,
     ) -> None:
         self.name = name
         self._real_lines = lines
         self._stripped_lines = stripped_lines(
-            lines, ignore_comments, ignore_docstrings, ignore_imports, ignore_signatures
+            lines,
+            ignore_comments,
+            ignore_docstrings,
+            ignore_imports,
+            ignore_signatures,
+            line_enabled_callback=line_enabled_callback,
         )
 
     def __str__(self):

tests/test_similar.py

@@ -35,7 +35,13 @@ def _runtest(self, args: list[str], code: int) -> None:
     @staticmethod
     def _run_pylint(args: list[str], out: TextIO) -> int:
         """Runs pylint with a patched output."""
-        args = args + ["--persistent=no"]
+        args = args + [
+            "--persistent=no",
+            "--enable=astroid-error",
+            # Enable functionality that will build another ast
+            "--ignore-imports=y",
+            "--ignore-signatures=y",
+        ]
         with _patch_streams(out):
             with pytest.raises(SystemExit) as cm:
                 with warnings.catch_warnings():
@@ -54,8 +60,10 @@ def _test_output(self, args: list[str], expected_output: str) -> None:
         out = StringIO()
         self._run_pylint(args, out=out)
         actual_output = self._clean_paths(out.getvalue())
+        actual_output_stripped = actual_output.strip()
         expected_output = self._clean_paths(expected_output)
-        assert expected_output.strip() in actual_output.strip()
+        assert expected_output.strip() in actual_output_stripped
+        assert "Fatal error" not in actual_output_stripped
 
     def test_duplicate_code_raw_strings_all(self) -> None:
         """Test similar lines in 3 similar files."""