Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

bpo-43693: Add the MAKE_CELL opcode and interleave fast locals offsets. #26396

Merged
merged 14 commits into from
Jun 7, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions Doc/library/dis.rst
Original file line number Diff line number Diff line change
Expand Up @@ -1056,6 +1056,14 @@ All of the following opcodes use their arguments.
Deletes local ``co_varnames[var_num]``.


.. opcode:: MAKE_CELL (i)

Creates a new cell in slot ``i``. If that slot is empty then
ericsnowcurrently marked this conversation as resolved.
Show resolved Hide resolved
that value is stored into the new cell.

.. versionadded:: 3.11


.. opcode:: LOAD_CLOSURE (i)

Pushes a reference to the cell contained in slot ``i`` of the "fast locals"
Expand Down
11 changes: 9 additions & 2 deletions Include/cpython/code.h
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,16 @@
# error "this header file must not be included directly"
#endif

/* Each instruction in a code object is a fixed-width value,
ericsnowcurrently marked this conversation as resolved.
Show resolved Hide resolved
* currently 2 bytes: 1-byte opcode + 1-byte oparg. The EXTENDED_ARG
* opcode allows for larger values but the current limit is 3 uses
* of EXTENDED_ARG (see Python/wordcode_helpers.h), for a maximum
* 32-bit value. This aligns with the note in Python/compile.c
* (compiler_addop_i_line) indicating that the max oparg value is
* 2**32 - 1, rather than INT_MAX.
ericsnowcurrently marked this conversation as resolved.
Show resolved Hide resolved
*/

typedef uint16_t _Py_CODEUNIT;
// Each oparg must fit in the second half of _Py_CODEUNIT, hence 8 bits.
#define _Py_MAX_OPARG 255

#ifdef WORDS_BIGENDIAN
# define _Py_OPCODE(word) ((word) >> 8)
Expand Down
2 changes: 2 additions & 0 deletions Include/internal/pycore_frame.h
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ _PyFrame_GetBuiltins(PyFrameObject *f)

int _PyFrame_TakeLocals(PyFrameObject *f);

PyAPI_FUNC(int) _PyFrame_OpAlreadyRan(PyFrameObject *f, int opcode, int oparg);

#ifdef __cplusplus
}
#endif
Expand Down
9 changes: 5 additions & 4 deletions Include/opcode.h

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion Lib/importlib/_bootstrap_external.py
Original file line number Diff line number Diff line change
Expand Up @@ -357,6 +357,7 @@ def _write_atomic(path, data, mode=0o666):
# Python 3.11a1 3452 (drop nlocals from marshaled code objects)
# Python 3.11a1 3453 (add co_fastlocalnames and co_fastlocalkinds)
# Python 3.11a1 3454 (compute cell offsets relative to locals bpo-43693)
# Python 3.11a1 3455 (add MAKE_CELL bpo-43693)

#
# MAGIC must change whenever the bytecode emitted by the compiler may no
Expand All @@ -366,7 +367,7 @@ def _write_atomic(path, data, mode=0o666):
# Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array
# in PC/launcher.c must also be updated.

MAGIC_NUMBER = (3454).to_bytes(2, 'little') + b'\r\n'
MAGIC_NUMBER = (3455).to_bytes(2, 'little') + b'\r\n'
_RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c

_PYCACHE = '__pycache__'
Expand Down
10 changes: 6 additions & 4 deletions Lib/opcode.py
Original file line number Diff line number Diff line change
Expand Up @@ -181,14 +181,16 @@ def jabs_op(name, op):
def_op('MAKE_FUNCTION', 132) # Flags
def_op('BUILD_SLICE', 133) # Number of items

def_op('LOAD_CLOSURE', 135)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why the opcode shuffle?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My initial impression was that MAKE_CELL should be at the front of the cell-related list and I figured it would be better to preserve the gap at 134. I figured it wouldn't matter but if it's a problem I can put MAKE_CELL at 139.

def_op('MAKE_CELL', 135)
hasfree.append(135)
def_op('LOAD_DEREF', 136)
def_op('LOAD_CLOSURE', 136)
hasfree.append(136)
def_op('STORE_DEREF', 137)
def_op('LOAD_DEREF', 137)
hasfree.append(137)
def_op('DELETE_DEREF', 138)
def_op('STORE_DEREF', 138)
hasfree.append(138)
def_op('DELETE_DEREF', 139)
hasfree.append(139)

def_op('CALL_FUNCTION_KW', 141) # #args + #kwargs
def_op('CALL_FUNCTION_EX', 142) # Flags
Expand Down
131 changes: 70 additions & 61 deletions Lib/test/test_dis.py
Original file line number Diff line number Diff line change
Expand Up @@ -427,15 +427,17 @@ def foo(x):
return foo

dis_nested_0 = """\
%3d 0 LOAD_CLOSURE 2 (y)
2 BUILD_TUPLE 1
4 LOAD_CONST 1 (<code object foo at 0x..., file "%s", line %d>)
6 LOAD_CONST 2 ('_h.<locals>.foo')
8 MAKE_FUNCTION 8 (closure)
10 STORE_FAST 1 (foo)

%3d 12 LOAD_FAST 1 (foo)
14 RETURN_VALUE
0 MAKE_CELL 2 (y)

%3d 2 LOAD_CLOSURE 2 (y)
4 BUILD_TUPLE 1
6 LOAD_CONST 1 (<code object foo at 0x..., file "%s", line %d>)
8 LOAD_CONST 2 ('_h.<locals>.foo')
10 MAKE_FUNCTION 8 (closure)
12 STORE_FAST 1 (foo)

%3d 14 LOAD_FAST 1 (foo)
16 RETURN_VALUE
""" % (_h.__code__.co_firstlineno + 1,
__file__,
_h.__code__.co_firstlineno + 1,
Expand All @@ -444,15 +446,17 @@ def foo(x):

dis_nested_1 = """%s
Disassembly of <code object foo at 0x..., file "%s", line %d>:
%3d 0 LOAD_CLOSURE 1 (x)
2 BUILD_TUPLE 1
4 LOAD_CONST 1 (<code object <listcomp> at 0x..., file "%s", line %d>)
6 LOAD_CONST 2 ('_h.<locals>.foo.<locals>.<listcomp>')
8 MAKE_FUNCTION 8 (closure)
10 LOAD_DEREF 2 (y)
12 GET_ITER
14 CALL_FUNCTION 1
16 RETURN_VALUE
0 MAKE_CELL 1 (x)

%3d 2 LOAD_CLOSURE 1 (x)
4 BUILD_TUPLE 1
6 LOAD_CONST 1 (<code object <listcomp> at 0x..., file "%s", line %d>)
8 LOAD_CONST 2 ('_h.<locals>.foo.<locals>.<listcomp>')
10 MAKE_FUNCTION 8 (closure)
12 LOAD_DEREF 2 (y)
14 GET_ITER
16 CALL_FUNCTION 1
18 RETURN_VALUE
""" % (dis_nested_0,
__file__,
_h.__code__.co_firstlineno + 1,
Expand Down Expand Up @@ -958,59 +962,64 @@ def jumpy():
#print('expected_opinfo_jumpy = [\n ',
#',\n '.join(map(str, _instructions)), ',\n]', sep='')

#dis.dis(outer)

Instruction = dis.Instruction
expected_opinfo_outer = [
Instruction(opname='LOAD_CONST', opcode=100, arg=8, argval=(3, 4), argrepr='(3, 4)', offset=0, starts_line=2, is_jump_target=False),
Instruction(opname='LOAD_CLOSURE', opcode=135, arg=3, argval='a', argrepr='a', offset=2, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CLOSURE', opcode=135, arg=4, argval='b', argrepr='b', offset=4, starts_line=None, is_jump_target=False),
Instruction(opname='BUILD_TUPLE', opcode=102, arg=2, argval=2, argrepr='', offset=6, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=code_object_f, argrepr=repr(code_object_f), offset=8, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='outer.<locals>.f', argrepr="'outer.<locals>.f'", offset=10, starts_line=None, is_jump_target=False),
Instruction(opname='MAKE_FUNCTION', opcode=132, arg=9, argval=9, argrepr='defaults, closure', offset=12, starts_line=None, is_jump_target=False),
Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='f', argrepr='f', offset=14, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_GLOBAL', opcode=116, arg=0, argval='print', argrepr='print', offset=16, starts_line=7, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=136, arg=3, argval='a', argrepr='a', offset=18, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=136, arg=4, argval='b', argrepr='b', offset=20, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval='', argrepr="''", offset=22, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=6, argval=1, argrepr='1', offset=24, starts_line=None, is_jump_target=False),
Instruction(opname='BUILD_LIST', opcode=103, arg=0, argval=0, argrepr='', offset=26, starts_line=None, is_jump_target=False),
Instruction(opname='BUILD_MAP', opcode=105, arg=0, argval=0, argrepr='', offset=28, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=7, argval='Hello world!', argrepr="'Hello world!'", offset=30, starts_line=None, is_jump_target=False),
Instruction(opname='CALL_FUNCTION', opcode=131, arg=7, argval=7, argrepr='', offset=32, starts_line=None, is_jump_target=False),
Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=34, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_FAST', opcode=124, arg=2, argval='f', argrepr='f', offset=36, starts_line=8, is_jump_target=False),
Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=38, starts_line=None, is_jump_target=False),
Instruction(opname='MAKE_CELL', opcode=135, arg=3, argval='a', argrepr='a', offset=0, starts_line=None, is_jump_target=False),
Instruction(opname='MAKE_CELL', opcode=135, arg=4, argval='b', argrepr='b', offset=2, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=8, argval=(3, 4), argrepr='(3, 4)', offset=4, starts_line=2, is_jump_target=False),
Instruction(opname='LOAD_CLOSURE', opcode=136, arg=3, argval='a', argrepr='a', offset=6, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CLOSURE', opcode=136, arg=4, argval='b', argrepr='b', offset=8, starts_line=None, is_jump_target=False),
Instruction(opname='BUILD_TUPLE', opcode=102, arg=2, argval=2, argrepr='', offset=10, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=code_object_f, argrepr=repr(code_object_f), offset=12, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='outer.<locals>.f', argrepr="'outer.<locals>.f'", offset=14, starts_line=None, is_jump_target=False),
Instruction(opname='MAKE_FUNCTION', opcode=132, arg=9, argval=9, argrepr='defaults, closure', offset=16, starts_line=None, is_jump_target=False),
Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='f', argrepr='f', offset=18, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_GLOBAL', opcode=116, arg=0, argval='print', argrepr='print', offset=20, starts_line=7, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=137, arg=3, argval='a', argrepr='a', offset=22, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=137, arg=4, argval='b', argrepr='b', offset=24, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval='', argrepr="''", offset=26, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=6, argval=1, argrepr='1', offset=28, starts_line=None, is_jump_target=False),
Instruction(opname='BUILD_LIST', opcode=103, arg=0, argval=0, argrepr='', offset=30, starts_line=None, is_jump_target=False),
Instruction(opname='BUILD_MAP', opcode=105, arg=0, argval=0, argrepr='', offset=32, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=7, argval='Hello world!', argrepr="'Hello world!'", offset=34, starts_line=None, is_jump_target=False),
Instruction(opname='CALL_FUNCTION', opcode=131, arg=7, argval=7, argrepr='', offset=36, starts_line=None, is_jump_target=False),
Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=38, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_FAST', opcode=124, arg=2, argval='f', argrepr='f', offset=40, starts_line=8, is_jump_target=False),
Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=42, starts_line=None, is_jump_target=False),
]

expected_opinfo_f = [
Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=(5, 6), argrepr='(5, 6)', offset=0, starts_line=3, is_jump_target=False),
Instruction(opname='LOAD_CLOSURE', opcode=135, arg=5, argval='a', argrepr='a', offset=2, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CLOSURE', opcode=135, arg=6, argval='b', argrepr='b', offset=4, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CLOSURE', opcode=135, arg=3, argval='c', argrepr='c', offset=6, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CLOSURE', opcode=135, arg=4, argval='d', argrepr='d', offset=8, starts_line=None, is_jump_target=False),
Instruction(opname='BUILD_TUPLE', opcode=102, arg=4, argval=4, argrepr='', offset=10, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=code_object_inner, argrepr=repr(code_object_inner), offset=12, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='outer.<locals>.f.<locals>.inner', argrepr="'outer.<locals>.f.<locals>.inner'", offset=14, starts_line=None, is_jump_target=False),
Instruction(opname='MAKE_FUNCTION', opcode=132, arg=9, argval=9, argrepr='defaults, closure', offset=16, starts_line=None, is_jump_target=False),
Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='inner', argrepr='inner', offset=18, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_GLOBAL', opcode=116, arg=0, argval='print', argrepr='print', offset=20, starts_line=5, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=136, arg=5, argval='a', argrepr='a', offset=22, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=136, arg=6, argval='b', argrepr='b', offset=24, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=136, arg=3, argval='c', argrepr='c', offset=26, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=136, arg=4, argval='d', argrepr='d', offset=28, starts_line=None, is_jump_target=False),
Instruction(opname='CALL_FUNCTION', opcode=131, arg=4, argval=4, argrepr='', offset=30, starts_line=None, is_jump_target=False),
Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=32, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_FAST', opcode=124, arg=2, argval='inner', argrepr='inner', offset=34, starts_line=6, is_jump_target=False),
Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=36, starts_line=None, is_jump_target=False),
Instruction(opname='MAKE_CELL', opcode=135, arg=3, argval='c', argrepr='c', offset=0, starts_line=None, is_jump_target=False),
Instruction(opname='MAKE_CELL', opcode=135, arg=4, argval='d', argrepr='d', offset=2, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=(5, 6), argrepr='(5, 6)', offset=4, starts_line=3, is_jump_target=False),
Instruction(opname='LOAD_CLOSURE', opcode=136, arg=5, argval='a', argrepr='a', offset=6, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CLOSURE', opcode=136, arg=6, argval='b', argrepr='b', offset=8, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CLOSURE', opcode=136, arg=3, argval='c', argrepr='c', offset=10, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CLOSURE', opcode=136, arg=4, argval='d', argrepr='d', offset=12, starts_line=None, is_jump_target=False),
Instruction(opname='BUILD_TUPLE', opcode=102, arg=4, argval=4, argrepr='', offset=14, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=code_object_inner, argrepr=repr(code_object_inner), offset=16, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='outer.<locals>.f.<locals>.inner', argrepr="'outer.<locals>.f.<locals>.inner'", offset=18, starts_line=None, is_jump_target=False),
Instruction(opname='MAKE_FUNCTION', opcode=132, arg=9, argval=9, argrepr='defaults, closure', offset=20, starts_line=None, is_jump_target=False),
Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='inner', argrepr='inner', offset=22, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_GLOBAL', opcode=116, arg=0, argval='print', argrepr='print', offset=24, starts_line=5, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=137, arg=5, argval='a', argrepr='a', offset=26, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=137, arg=6, argval='b', argrepr='b', offset=28, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=137, arg=3, argval='c', argrepr='c', offset=30, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=137, arg=4, argval='d', argrepr='d', offset=32, starts_line=None, is_jump_target=False),
Instruction(opname='CALL_FUNCTION', opcode=131, arg=4, argval=4, argrepr='', offset=34, starts_line=None, is_jump_target=False),
Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=36, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_FAST', opcode=124, arg=2, argval='inner', argrepr='inner', offset=38, starts_line=6, is_jump_target=False),
Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=40, starts_line=None, is_jump_target=False),
]

expected_opinfo_inner = [
Instruction(opname='LOAD_GLOBAL', opcode=116, arg=0, argval='print', argrepr='print', offset=0, starts_line=4, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=136, arg=2, argval='a', argrepr='a', offset=2, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=136, arg=3, argval='b', argrepr='b', offset=4, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=136, arg=4, argval='c', argrepr='c', offset=6, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=136, arg=5, argval='d', argrepr='d', offset=8, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=137, arg=2, argval='a', argrepr='a', offset=2, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=137, arg=3, argval='b', argrepr='b', offset=4, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=137, arg=4, argval='c', argrepr='c', offset=6, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_DEREF', opcode=137, arg=5, argval='d', argrepr='d', offset=8, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='e', argrepr='e', offset=10, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_FAST', opcode=124, arg=1, argval='f', argrepr='f', offset=12, starts_line=None, is_jump_target=False),
Instruction(opname='CALL_FUNCTION', opcode=131, arg=6, argval=6, argrepr='', offset=14, starts_line=None, is_jump_target=False),
Expand Down
51 changes: 51 additions & 0 deletions Lib/test/test_scope.py
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,57 @@ def bar():
self.assertEqual(foo(a=42), 50)
self.assertEqual(foo(), 25)

def testCellIsArgAndEscapes(self):
# We need to be sure that a cell passed in as an arg still
# gets wrapped in a new cell if the arg escapes into an
# inner function (closure).

def external():
value = 42
def inner():
return value
cell, = inner.__closure__
return cell
cell_ext = external()

def spam(arg):
def eggs():
return arg
return eggs

eggs = spam(cell_ext)
cell_closure, = eggs.__closure__
cell_eggs = eggs()

self.assertIs(cell_eggs, cell_ext)
self.assertIsNot(cell_eggs, cell_closure)

def testCellIsLocalAndEscapes(self):
# We need to be sure that a cell bound to a local still
# gets wrapped in a new cell if the local escapes into an
# inner function (closure).

def external():
value = 42
def inner():
return value
cell, = inner.__closure__
return cell
cell_ext = external()

def spam(arg):
cell = arg
def eggs():
return cell
return eggs

eggs = spam(cell_ext)
cell_closure, = eggs.__closure__
cell_eggs = eggs()

self.assertIs(cell_eggs, cell_ext)
self.assertIsNot(cell_eggs, cell_closure)

def testRecursion(self):

def f(x):
Expand Down
Loading