Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

Loading…

construct 'lnotab' byte string for code objects #93

Open
wants to merge 1 commit into from

2 participants

@scoder
Owner

The intention of this change is to eventually be able to support runtime mapping of line numbers for code objects. What CPython does is to pack a byte code offset to line number offset map into a byte string and store it in the code object. This is what this patch does as well, so that the line number could be computed by CPython by simply using current source line - first source line of function as a fake byte code offset for Cython functions when tracing or raising an exception. This will allow reuse of the code object over the whole body of a function.

A current problem is the syntax tree layout for Cython functions that have a code object. Here, the code object lives outside of the function, specifically in the fused functions object. It would be better to move it inside of the function itself, where the function code could use it. This is not entirely trivial because the fused function code currently reuses one code object for all specialisations, which doesn't work well with the idea of letting each separate function own it.

By itself, this patch isn't very interesting because Cython does not currently use byte code offsets in frame objects. But it's the first non-trivial step in that direction and should be applied at some point before refactoring the syntax tree.

@scoder scoder construct 'lnotab' byte string for code objects to eventually support…
… runtime mapping of line numbers

--HG--
extra : transplant_source : %B8%83%8A%1E%0B3%BE%9D%03%C0%E6%B8%EBE%24B%F8%AA%103
a6d2d73
@thedrow

Can you please rebase this PR?
It has some merge conflicts.

@scoder
Owner

updated to latest master

@thedrow

woot and everything works.
Why isn't this merged yet?

@scoder
Owner

because there is no use case that I know of - what do you use it for?

@scoder
Owner

Or, well, let's put it differently - it isn't useful all by itself for existing tools. I guess custom user code could make use of it, though.

@thedrow

To my understanding this PR will allow coverage.py to report coverage on cython modules which is useful.

@scoder
Owner

did you try it?

@thedrow

Nope not yet.

@scoder
Owner

Tried it (it's been a while...) and it doesn't help. In fact, tracing works just fine, it's just that coverage.py gets it wrong somehow (it fails to keep track of when it's in a function or not). Might be something simple to improve in Cython, but it might also just be a bug (or missing feature) in coverage.py. But that's not related to this pull request. The users mailing list is a better place to discuss this if you're interested.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Commits on Mar 14, 2012
  1. @scoder

    construct 'lnotab' byte string for code objects to eventually support…

    scoder authored
    … runtime mapping of line numbers
    
    --HG--
    extra : transplant_source : %B8%83%8A%1E%0B3%BE%9D%03%C0%E6%B8%EBE%24B%F8%AA%103
This page is out of date. Refresh to see the latest.
View
16 Cython/Compiler/ExprNodes.py
@@ -8035,10 +8035,13 @@ class CodeObjectNode(ExprNode):
#
# def_node DefNode the Python function node
# varnames TupleNode a tuple with all local variable names
+ # lno_tab BytesNode a bytes string containing compressed line number offsets
+ # (see CPython's Objects/lnotab_notes.txt)
- subexprs = ['varnames']
+ subexprs = ['varnames', 'lno_tab']
is_temp = False
result_code = None
+ lno_tab = None
def __init__(self, def_node):
ExprNode.__init__(self, def_node.pos, def_node=def_node)
@@ -8064,6 +8067,12 @@ def generate_result_code(self, code):
if self.result_code is None:
self.result_code = code.get_py_const(py_object_type, 'codeobj', cleanup_level=2)
+ if self.lno_tab:
+ self.lno_tab.generate_evaluation_code(code)
+ lno_tab = self.lno_tab.result()
+ else:
+ lno_tab = Naming.empty_bytes
+
code = code.get_cached_constants_writer()
code.mark_pos(self.pos)
func = self.def_node
@@ -8094,10 +8103,13 @@ def generate_result_code(self, code):
file_path_const, # filename
func_name, # name
self.pos[1], # firstlineno
- Naming.empty_bytes, # lnotab
+ lno_tab, # lnotab
code.error_goto_if_null(self.result_code, self.pos),
))
+ if self.lno_tab:
+ self.lno_tab.generate_disposal_code(code)
+
class DefaultLiteralArgNode(ExprNode):
# CyFunction's literal argument default value
View
56 Cython/Compiler/ParseTreeTransforms.py
@@ -19,7 +19,7 @@
from .Visitor import CythonTransform, EnvTransform, ScopeTrackingTransform
from .UtilNodes import LetNode, LetRefNode, ResultRefNode
from .TreeFragment import TreeFragment
-from .StringEncoding import EncodedString
+from .StringEncoding import EncodedString, BytesLiteral, _unicode
from .Errors import error, warning, CompileError, InternalError
from .Code import UtilityCode
@@ -2418,6 +2418,60 @@ def visit_CFuncDefNode(self, node):
return node
+class CreateLineNumberMaps(EnvTransform):
+ """
+ Create line number maps for code objects as specified
+ by CPython's Objects/lnotab_notes.txt. This is a simplified
+ implementation because we don't have byte code, so we use
+ the line numbers also as byte code offsets, thus making the
+ runtime lookup straight forward as well.
+ """
+ line_numbers = None
+
+ def visit_CodeObjectNode(self, node):
+ outer_context = self.line_numbers
+ def_node = node.def_node
+ self.line_numbers = set()
+ self.visitchildren(def_node)
+ if self.line_numbers:
+ node.lno_tab = self.build_lnotab(def_node.pos, self.line_numbers)
+ self.line_numbers = outer_context
+ return node
+
+ def visit_Node(self, node):
+ """
+ Collect line numbers of all non-nogil nodes within functions
+ (where Python code line numbers are relevant).
+ """
+ if self.line_numbers is not None and not self.current_env().nogil:
+ self.line_numbers.add(node.pos[1])
+ self.visitchildren(node)
+ return node
+
+ def build_lnotab(self, node_pos, line_numbers):
+ line_numbers = sorted(line_numbers)
+
+ chr255 = chr(255)
+ lnotab = []
+ last_line = node_pos[1]
+ for line in line_numbers:
+ offset = line - last_line
+ if offset > 255:
+ lnotab.append(chr255 * ((offset // 255) * 2))
+ offset %= 255
+ if not offset:
+ continue
+ lnotab.append(chr(offset)*2)
+ last_line = line
+
+ lnotab_string = ''.join(lnotab)
+ if isinstance(lnotab_string, _unicode): # Py3
+ lnotab_string = lnotab_string.encode('iso8859-1')
+ string_literal = BytesLiteral(lnotab_string)
+ string_literal.encoding = 'iso8859-1'
+ return ExprNodes.BytesNode(node_pos, value=string_literal)
+
+
class GilCheck(VisitorTransform):
"""
Call `node.gil_check(env)` on each node to make sure we hold the
View
3  Cython/Compiler/Pipeline.py
@@ -137,7 +137,7 @@ def create_pipeline(context, mode, exclude_classes=()):
from .ParseTreeTransforms import CalculateQualifiedNamesTransform
from .TypeInference import MarkParallelAssignments, MarkOverflowingArithmetic
from .ParseTreeTransforms import AdjustDefByDirectives, AlignFunctionDefinitions
- from .ParseTreeTransforms import RemoveUnreachableCode, GilCheck
+ from .ParseTreeTransforms import RemoveUnreachableCode, CreateLineNumberMaps, GilCheck
from .FlowControl import ControlFlowAnalysis
from .AnalysedTreeTransforms import AutoTestDictTransform
from .AutoDocTransforms import EmbedSignature
@@ -207,6 +207,7 @@ def create_pipeline(context, mode, exclude_classes=()):
FinalOptimizePhase(context),
GilCheck(),
UseUtilityCodeDefinitions(context),
+ CreateLineNumberMaps(context),
]
filtered_stages = []
for s in stages:
Something went wrong with that request. Please try again.