Merge pull request #7744 from stuartarchibald/fix/7730

Fix issues with locating/parsing source during DebugInfo emission.

numba/core/errors.py

@@ -97,6 +97,12 @@ class NumbaIRAssumptionWarning(NumbaPedanticWarning):
     Warning category for reporting an IR assumption violation.
     """
 
+
+class NumbaDebugInfoWarning(NumbaWarning):
+    """
+    Warning category for an issue with the emission of debug information.
+    """
+
 # These are needed in the color formatting of errors setup
 
 

numba/core/lowering.py

@@ -13,7 +13,7 @@
                         targetconfig)
 from numba.core.errors import (LoweringError, new_error_context, TypingError,
                                LiteralTypingError, UnsupportedError,
-                               NumbaWarning)
+                               NumbaDebugInfoWarning)
 from numba.core.funcdesc import default_mangler
 from numba.core.environment import Environment
 from numba.core.analysis import compute_use_defs
@@ -100,19 +100,32 @@ def _compute_def_location(self):
                 raw_source_str, _ = inspect.getsourcelines(fn)
             except OSError:
                 msg = ("Could not find source for function: "
-                       f"{self.func_ir.func_id.func}")
-                warnings.warn(NumbaWarning(msg))
+                       f"{self.func_ir.func_id.func}. Debug line information "
+                       "may be inaccurate.")
+                warnings.warn(NumbaDebugInfoWarning(msg))
             else:
                 # Parse the source and find the line with `def <func>` in it, it
                 # is assumed that if the compilation has made it this far that
                 # the source is at least legal and has valid syntax.
 
-                # join the source as a block and dedent it
+                # Join the source as a block and dedent it.
                 source_str = textwrap.dedent(''.join(raw_source_str))
-                src_ast = ast.parse(source_str)
+                # Deal with unparsable source (see #7730), this can be caused
+                # by continuation lines/comments at indent levels that are
+                # invalid when the just function source is parsed in isolation.
+                src_ast = None
+                try:
+                    src_ast = ast.parse(source_str)
+                except IndentationError:
+                    msg = ("Could not parse the source for function: "
+                           f"{self.func_ir.func_id.func}. Debug line "
+                           "information may be inaccurate. This is often "
+                           "caused by comments/docstrings/line continuation "
+                           "that is at a lesser indent level than the source.")
+                    warnings.warn(NumbaDebugInfoWarning(msg))
                 # pull the definition out of the AST, only if it seems valid
                 # i.e. one thing in the body
-                if len(src_ast.body) == 1:
+                if src_ast is not None and len(src_ast.body) == 1:
                     pydef = src_ast.body.pop()
                     # -1 as lines start at 1 and this is an offset.
                     pydef_offset = pydef.lineno - 1

numba/tests/test_debuginfo.py

@@ -3,12 +3,16 @@
 import re
 import numpy as np
 import math
+from textwrap import dedent
+import unittest
+import warnings
 
-from numba.tests.support import TestCase, override_config, needs_subprocess
+from numba.tests.support import (TestCase, override_config, needs_subprocess,
+                                 ignore_internal_warnings)
 from numba import jit, njit
 from numba.core import types, utils
 from numba.core.datamodel import default_manager
-import unittest
+from numba.core.errors import NumbaDebugInfoWarning
 import llvmlite.binding as llvm
 
 #NOTE: These tests are potentially sensitive to changes in SSA or lowering
@@ -637,6 +641,50 @@ def foo(missing=None):
         # expect ir.LiteralStructType([])
         self.assertEqual(base_ty, ('!{}'))
 
+    def test_missing_source(self):
+        strsrc = """
+        def foo():
+            return 1
+        """
+        l = dict()
+        exec(dedent(strsrc), {}, l)
+        foo = njit(debug=True)(l['foo'])
+
+        with warnings.catch_warnings(record=True) as w:
+            warnings.simplefilter('always', NumbaDebugInfoWarning)
+            ignore_internal_warnings()
+            foo()
+
+        self.assertEqual(len(w), 1)
+        found = w[0]
+        self.assertEqual(found.category, NumbaDebugInfoWarning)
+        msg = str(found.message)
+        # make sure the warning contains the right message
+        self.assertIn('Could not find source for function', msg)
+        # and refers to the offending function
+        self.assertIn(str(foo.py_func), msg)
+
+    def test_unparsable_indented_source(self):
+
+        @njit(debug=True)
+        def foo():
+# NOTE: THIS COMMENT MUST START AT COLUMN 0 FOR THIS SAMPLE CODE TO BE VALID # noqa: E115, E501
+            return 1
+
+        with warnings.catch_warnings(record=True) as w:
+            warnings.simplefilter('always', NumbaDebugInfoWarning)
+            ignore_internal_warnings()
+            foo()
+
+        self.assertEqual(len(w), 1)
+        found = w[0]
+        self.assertEqual(found.category, NumbaDebugInfoWarning)
+        msg = str(found.message)
+        # make sure the warning contains the right message
+        self.assertIn('Could not parse the source for function', msg)
+        # and refers to the offending function
+        self.assertIn(str(foo.py_func), msg)
+
 
 if __name__ == '__main__':
     unittest.main()