Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
bpo-42739: Don't use sentinels to mark end of line table. (GH-25657)
* Add length parameter to PyLineTable_InitAddressRange and doen't use sentinel values at end of table. Makes the line number table more robust.

* Update PyCodeAddressRange to match PEP 626.
  • Loading branch information
markshannon committed Apr 29, 2021
1 parent 53dd6c9 commit c76da79
Show file tree
Hide file tree
Showing 13 changed files with 4,966 additions and 4,984 deletions.
11 changes: 8 additions & 3 deletions Include/cpython/code.h
Expand Up @@ -135,12 +135,17 @@ PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno);
PyAPI_FUNC(int) PyCode_Addr2Line(PyCodeObject *, int);

/* for internal use only */
struct _opaque {
int computed_line;
char *lo_next;
char *limit;
};

typedef struct _line_offsets {
int ar_start;
int ar_end;
int ar_line;
int ar_computed_line;
char *lo_next;
struct _opaque opaque;
} PyCodeAddressRange;

/* Update *bounds to describe the first and one-past-the-last instructions in the
Expand Down Expand Up @@ -170,7 +175,7 @@ PyAPI_FUNC(int) _PyCode_SetExtra(PyObject *code, Py_ssize_t index,
int _PyCode_InitAddressRange(PyCodeObject* co, PyCodeAddressRange *bounds);

/** Out of process API for initializing the line number table. */
void PyLineTable_InitAddressRange(char *linetable, int firstlineno, PyCodeAddressRange *range);
void PyLineTable_InitAddressRange(char *linetable, Py_ssize_t length, int firstlineno, PyCodeAddressRange *range);

/** API for traversing the line number table. */
int PyLineTable_NextAddressRange(PyCodeAddressRange *range);
Expand Down
6 changes: 3 additions & 3 deletions Lib/ctypes/test/test_values.py
Expand Up @@ -80,9 +80,9 @@ class struct_frozen(Structure):
continue
items.append((entry.name.decode("ascii"), entry.size))

expected = [("__hello__", 139),
("__phello__", -139),
("__phello__.spam", 139),
expected = [("__hello__", 137),
("__phello__", -137),
("__phello__.spam", 137),
]
self.assertEqual(items, expected, "PyImport_FrozenModules example "
"in Doc/library/ctypes.rst may be out of date")
Expand Down
3 changes: 2 additions & 1 deletion Lib/importlib/_bootstrap_external.py
Expand Up @@ -350,6 +350,7 @@ def _write_atomic(path, data, mode=0o666):
# Python 3.10a7 3435 Use instruction offsets (as opposed to byte offsets).
# Python 3.10b1 3436 (Add GEN_START bytecode #43683)
# Python 3.10b1 3437 (Undo making 'annotations' future by default - We like to dance among core devs!)
# Python 3.10b1 3438 Safer line number table handling.

#
# MAGIC must change whenever the bytecode emitted by the compiler may no
Expand All @@ -359,7 +360,7 @@ def _write_atomic(path, data, mode=0o666):
# Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array
# in PC/launcher.c must also be updated.

MAGIC_NUMBER = (3437).to_bytes(2, 'little') + b'\r\n'
MAGIC_NUMBER = (3438).to_bytes(2, 'little') + b'\r\n'
_RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c

_PYCACHE = '__pycache__'
Expand Down
6 changes: 6 additions & 0 deletions Lib/test/test_code.py
Expand Up @@ -264,6 +264,12 @@ def func2():
new_code = code.replace(**{attr: value})
self.assertEqual(getattr(new_code, attr), value)

def test_empty_linetable(self):
def func():
pass
new_code = code = func.__code__.replace(co_linetable=b'')
self.assertEqual(list(new_code.co_lines()), [])


def isinterned(s):
return s is sys.intern(('_' + s + '_')[1:-1])
Expand Down
2 changes: 1 addition & 1 deletion Lib/test/test_dis.py
Expand Up @@ -172,7 +172,7 @@ def bug42562():


# Set line number for 'pass' to None
bug42562.__code__ = bug42562.__code__.replace(co_linetable=b'\x04\x80\xff\x80')
bug42562.__code__ = bug42562.__code__.replace(co_linetable=b'\x04\x80')


dis_bug42562 = """\
Expand Down
@@ -0,0 +1,4 @@
The internal representation of line number tables is changed to not use
sentinels, and an explicit length parameter is added to the out of process
API function ``PyLineTable_InitAddressRange``. This makes the handling of
line number tables more robust in some circumstances.
41 changes: 21 additions & 20 deletions Objects/codeobject.c
Expand Up @@ -456,15 +456,15 @@ code_getlnotab(PyCodeObject *code, void *closure)
}
_PyCode_InitAddressRange(code, &bounds);
while (PyLineTable_NextAddressRange(&bounds)) {
if (bounds.ar_computed_line != line) {
if (bounds.opaque.computed_line != line) {
int bdelta = bounds.ar_start - code_offset;
int ldelta = bounds.ar_computed_line - line;
int ldelta = bounds.opaque.computed_line - line;
if (!emit_delta(&bytes, bdelta, ldelta, &table_offset)) {
Py_DECREF(bytes);
return NULL;
}
code_offset = bounds.ar_start;
line = bounds.ar_computed_line;
line = bounds.opaque.computed_line;
}
}
_PyBytes_Resize(&bytes, table_offset);
Expand Down Expand Up @@ -1120,44 +1120,43 @@ code_linesiterator(PyCodeObject *code, PyObject *Py_UNUSED(args))
static void
retreat(PyCodeAddressRange *bounds)
{
int ldelta = ((signed char *)bounds->lo_next)[-1];
int ldelta = ((signed char *)bounds->opaque.lo_next)[-1];
if (ldelta == -128) {
ldelta = 0;
}
bounds->ar_computed_line -= ldelta;
bounds->lo_next -= 2;
bounds->opaque.computed_line -= ldelta;
bounds->opaque.lo_next -= 2;
bounds->ar_end = bounds->ar_start;
bounds->ar_start -= ((unsigned char *)bounds->lo_next)[-2];
ldelta = ((signed char *)bounds->lo_next)[-1];
bounds->ar_start -= ((unsigned char *)bounds->opaque.lo_next)[-2];
ldelta = ((signed char *)bounds->opaque.lo_next)[-1];
if (ldelta == -128) {
bounds->ar_line = -1;
}
else {
bounds->ar_line = bounds->ar_computed_line;
bounds->ar_line = bounds->opaque.computed_line;
}
}

static void
advance(PyCodeAddressRange *bounds)
{
bounds->ar_start = bounds->ar_end;
int delta = ((unsigned char *)bounds->lo_next)[0];
assert (delta < 255);
int delta = ((unsigned char *)bounds->opaque.lo_next)[0];
bounds->ar_end += delta;
int ldelta = ((signed char *)bounds->lo_next)[1];
bounds->lo_next += 2;
int ldelta = ((signed char *)bounds->opaque.lo_next)[1];
bounds->opaque.lo_next += 2;
if (ldelta == -128) {
bounds->ar_line = -1;
}
else {
bounds->ar_computed_line += ldelta;
bounds->ar_line = bounds->ar_computed_line;
bounds->opaque.computed_line += ldelta;
bounds->ar_line = bounds->opaque.computed_line;
}
}

static inline int
at_end(PyCodeAddressRange *bounds) {
return ((unsigned char *)bounds->lo_next)[0] == 255;
return bounds->opaque.lo_next >= bounds->opaque.limit;
}

int
Expand Down Expand Up @@ -1256,20 +1255,22 @@ PyCode_Addr2Line(PyCodeObject *co, int addrq)
}

void
PyLineTable_InitAddressRange(char *linetable, int firstlineno, PyCodeAddressRange *range)
PyLineTable_InitAddressRange(char *linetable, Py_ssize_t length, int firstlineno, PyCodeAddressRange *range)
{
range->lo_next = linetable;
range->opaque.lo_next = linetable;
range->opaque.limit = range->opaque.lo_next + length;
range->ar_start = -1;
range->ar_end = 0;
range->ar_computed_line = firstlineno;
range->opaque.computed_line = firstlineno;
range->ar_line = -1;
}

int
_PyCode_InitAddressRange(PyCodeObject* co, PyCodeAddressRange *bounds)
{
char *linetable = PyBytes_AS_STRING(co->co_linetable);
PyLineTable_InitAddressRange(linetable, co->co_firstlineno, bounds);
Py_ssize_t length = PyBytes_GET_SIZE(co->co_linetable);
PyLineTable_InitAddressRange(linetable, length, co->co_firstlineno, bounds);
return bounds->ar_line;
}

Expand Down
4 changes: 0 additions & 4 deletions Objects/lnotab_notes.txt
Expand Up @@ -39,7 +39,6 @@ Note that the end - start value is always positive.
Finally, in order to fit into a single byte we need to convert start deltas to the range 0 <= delta <= 254,
and line deltas to the range -127 <= delta <= 127.
A line delta of -128 is used to indicate no line number.
A start delta of 255 is used as a sentinel to mark the end of the table.
Also note that a delta of zero indicates that there are no bytecodes in the given range,
which means we can use an invalid line number for that range.

Expand All @@ -54,7 +53,6 @@ Final form:
16 +1
0 +127 (line 135, but the range is empty as no bytecodes are at line 135)
4 +73
255 (end mark) ---

Iterating over the table.
-------------------------
Expand All @@ -68,8 +66,6 @@ def co_lines(code):
end = 0
table_iter = iter(code.internal_line_table):
for sdelta, ldelta in table_iter:
if sdelta == 255:
break
if ldelta == 0: # No change to line number, just accumulate changes to end
end += odelta
continue
Expand Down
4 changes: 0 additions & 4 deletions Python/compile.c
Expand Up @@ -6959,10 +6959,6 @@ assemble(struct compiler *c, int addNone)
if (!assemble_line_range(&a)) {
return 0;
}
/* Emit sentinel at end of line number table */
if (!assemble_emit_linetable_pair(&a, 255, -128)) {
goto error;
}

if (_PyBytes_Resize(&a.a_lnotab, a.a_lnotab_off) < 0) {
goto error;
Expand Down
2 changes: 1 addition & 1 deletion Python/frozen_hello.h
Expand Up @@ -8,5 +8,5 @@ const unsigned char _Py_M__hello[] = {
5,112,114,105,110,116,169,0,114,1,0,0,0,114,1,0,
0,0,122,14,60,102,114,111,122,101,110,32,104,101,108,108,
111,62,218,8,60,109,111,100,117,108,101,62,1,0,0,0,
115,6,0,0,0,4,0,12,1,255,128,
115,4,0,0,0,4,0,12,1,
};

0 comments on commit c76da79

Please sign in to comment.