Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

bpo-32949: Better bytecodes for "with" statement. #5112

Closed
wants to merge 6 commits into from
Closed

bpo-32949: Better bytecodes for "with" statement. #5112

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
ece8d8f
Produce cleaner bytecode for 'with' and 'async with' by generating se…
markshannon Jan 6, 2018
55c32d2
Update importlib
markshannon Jan 6, 2018
9ffaa2d
Remove unused bytecodes.
markshannon Jan 6, 2018
e9f7689
Update expected results for test_dis.
markshannon Jan 6, 2018
7d6ae92
Fix frame.set_lineno to account for new SETUP_WITH and deleete obsolu…
markshannon Mar 11, 2018
5039d5d
New bytecodes for 'with': Add news item and bump magic number.
markshannon Mar 13, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 2 additions & 2 deletions Include/opcode.h
View file
Open in desktop
Expand Up @@ -30,10 +30,12 @@ extern "C" {
#define BINARY_TRUE_DIVIDE 27
#define INPLACE_FLOOR_DIVIDE 28
#define INPLACE_TRUE_DIVIDE 29
#define RERAISE 48
#define GET_AITER 50
#define GET_ANEXT 51
#define BEFORE_ASYNC_WITH 52
#define BEGIN_FINALLY 53
#define WITH_EXCEPT_START 54
#define INPLACE_ADD 55
#define INPLACE_SUBTRACT 56
#define INPLACE_MULTIPLY 57
Expand All @@ -57,8 +59,6 @@ extern "C" {
#define INPLACE_AND 77
#define INPLACE_XOR 78
#define INPLACE_OR 79
#define WITH_CLEANUP_START 81
#define WITH_CLEANUP_FINISH 82
#define RETURN_VALUE 83
#define IMPORT_STAR 84
#define SETUP_ANNOTATIONS 85
Expand Down
3 changes: 2 additions & 1 deletion Lib/importlib/_bootstrap_external.py
View file
Open in desktop
Expand Up @@ -247,6 +247,7 @@ def _write_atomic(path, data, mode=0o666):
# Python 3.7a4 3392 (PEP 552: Deterministic pycs #31650)
# Python 3.7b1 3393 (remove STORE_ANNOTATION opcode #32550)
# Python 3.8a1 3400 (move frame block handling to compiler #17611)
# Python 3.8a1 3401 (simplified bytecode for with blocks #32949)
#
# MAGIC must change whenever the bytecode emitted by the compiler may no
# longer be understood by older implementations of the eval loop (usually
Expand All @@ -255,7 +256,7 @@ def _write_atomic(path, data, mode=0o666):
# Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array
# in PC/launcher.c must also be updated.

MAGIC_NUMBER = (3400).to_bytes(2, 'little') + b'\r\n'
MAGIC_NUMBER = (3401).to_bytes(2, 'little') + b'\r\n'
_RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c

_PYCACHE = '__pycache__'
Expand Down
6 changes: 4 additions & 2 deletions Lib/opcode.py
View file
Open in desktop
Expand Up @@ -84,10 +84,13 @@ def jabs_op(name, op):
def_op('INPLACE_FLOOR_DIVIDE', 28)
def_op('INPLACE_TRUE_DIVIDE', 29)

def_op('RERAISE', 48)

def_op('GET_AITER', 50)
def_op('GET_ANEXT', 51)
def_op('BEFORE_ASYNC_WITH', 52)
def_op('BEGIN_FINALLY', 53)
def_op('WITH_EXCEPT_START', 54)

def_op('INPLACE_ADD', 55)
def_op('INPLACE_SUBTRACT', 56)
Expand Down Expand Up @@ -115,8 +118,7 @@ def jabs_op(name, op):
def_op('INPLACE_AND', 77)
def_op('INPLACE_XOR', 78)
def_op('INPLACE_OR', 79)
def_op('WITH_CLEANUP_START', 81)
def_op('WITH_CLEANUP_FINISH', 82)

def_op('RETURN_VALUE', 83)
def_op('IMPORT_STAR', 84)
def_op('SETUP_ANNOTATIONS', 85)
Expand Down
49 changes: 32 additions & 17 deletions Lib/test/test_dis.py
View file
Open in desktop
Expand Up @@ -740,7 +740,7 @@ async def async_def():
Argument count: 0
Kw-only arguments: 0
Number of locals: 2
Stack size: 10
Stack size: 9
Flags: OPTIMIZED, NEWLOCALS, NOFREE, COROUTINE
Constants:
0: None
Expand Down Expand Up @@ -966,7 +966,7 @@ def jumpy():
Instruction(opname='LOAD_CONST', opcode=100, arg=6, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=96, starts_line=None, is_jump_target=False),
Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='', offset=98, starts_line=None, is_jump_target=False),
Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=100, starts_line=None, is_jump_target=False),
Instruction(opname='SETUP_FINALLY', opcode=122, arg=70, argval=174, argrepr='to 174', offset=102, starts_line=20, is_jump_target=True),
Instruction(opname='SETUP_FINALLY', opcode=122, arg=90, argval=194, argrepr='to 194', offset=102, starts_line=20, is_jump_target=True),
Instruction(opname='SETUP_FINALLY', opcode=122, arg=12, argval=118, argrepr='to 118', offset=104, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=1, argrepr='1', offset=106, starts_line=21, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=7, argval=0, argrepr='0', offset=108, starts_line=None, is_jump_target=False),
Expand All @@ -986,29 +986,39 @@ def jumpy():
Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='', offset=136, starts_line=None, is_jump_target=False),
Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=138, starts_line=None, is_jump_target=False),
Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=140, starts_line=None, is_jump_target=False),
Instruction(opname='JUMP_FORWARD', opcode=110, arg=26, argval=170, argrepr='to 170', offset=142, starts_line=None, is_jump_target=False),
Instruction(opname='JUMP_FORWARD', opcode=110, arg=46, argval=190, argrepr='to 190', offset=142, starts_line=None, is_jump_target=False),
Instruction(opname='END_FINALLY', opcode=88, arg=None, argval=None, argrepr='', offset=144, starts_line=None, is_jump_target=True),
Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=146, starts_line=25, is_jump_target=True),
Instruction(opname='SETUP_WITH', opcode=143, arg=14, argval=164, argrepr='to 164', offset=148, starts_line=None, is_jump_target=False),
Instruction(opname='SETUP_WITH', opcode=143, arg=24, argval=174, argrepr='to 174', offset=148, starts_line=None, is_jump_target=False),
Instruction(opname='STORE_FAST', opcode=125, arg=1, argval='dodgy', argrepr='dodgy', offset=150, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=152, starts_line=26, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=9, argval='Never reach this', argrepr="'Never reach this'", offset=154, starts_line=None, is_jump_target=False),
Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='', offset=156, starts_line=None, is_jump_target=False),
Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=158, starts_line=None, is_jump_target=False),
Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=160, starts_line=None, is_jump_target=False),
Instruction(opname='BEGIN_FINALLY', opcode=53, arg=None, argval=None, argrepr='', offset=162, starts_line=None, is_jump_target=False),
Instruction(opname='WITH_CLEANUP_START', opcode=81, arg=None, argval=None, argrepr='', offset=164, starts_line=None, is_jump_target=True),
Instruction(opname='WITH_CLEANUP_FINISH', opcode=82, arg=None, argval=None, argrepr='', offset=166, starts_line=None, is_jump_target=False),
Instruction(opname='END_FINALLY', opcode=88, arg=None, argval=None, argrepr='', offset=168, starts_line=None, is_jump_target=False),
Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=170, starts_line=None, is_jump_target=True),
Instruction(opname='BEGIN_FINALLY', opcode=53, arg=None, argval=None, argrepr='', offset=172, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=174, starts_line=28, is_jump_target=True),
Instruction(opname='LOAD_CONST', opcode=100, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=176, starts_line=None, is_jump_target=False),
Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='', offset=178, starts_line=None, is_jump_target=False),
Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=180, starts_line=None, is_jump_target=False),
Instruction(opname='END_FINALLY', opcode=88, arg=None, argval=None, argrepr='', offset=182, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=184, starts_line=None, is_jump_target=False),
Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=186, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=162, starts_line=None, is_jump_target=False),
Instruction(opname='DUP_TOP', opcode=4, arg=None, argval=None, argrepr='', offset=164, starts_line=None, is_jump_target=False),
Instruction(opname='DUP_TOP', opcode=4, arg=None, argval=None, argrepr='', offset=166, starts_line=None, is_jump_target=False),
Instruction(opname='CALL_FUNCTION', opcode=131, arg=3, argval=3, argrepr='', offset=168, starts_line=None, is_jump_target=False),
Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=170, starts_line=None, is_jump_target=False),
Instruction(opname='JUMP_FORWARD', opcode=110, arg=16, argval=190, argrepr='to 190', offset=172, starts_line=None, is_jump_target=False),
Instruction(opname='WITH_EXCEPT_START', opcode=54, arg=None, argval=None, argrepr='', offset=174, starts_line=None, is_jump_target=True),
Instruction(opname='POP_JUMP_IF_TRUE', opcode=115, arg=180, argval=180, argrepr='', offset=176, starts_line=None, is_jump_target=False),
Instruction(opname='RERAISE', opcode=48, arg=None, argval=None, argrepr='', offset=178, starts_line=None, is_jump_target=False),
Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=180, starts_line=None, is_jump_target=True),
Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=182, starts_line=None, is_jump_target=False),
Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=184, starts_line=None, is_jump_target=False),
Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=186, starts_line=None, is_jump_target=False),
Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=188, starts_line=None, is_jump_target=False),
Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=190, starts_line=None, is_jump_target=True),
Instruction(opname='BEGIN_FINALLY', opcode=53, arg=None, argval=None, argrepr='', offset=192, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=194, starts_line=28, is_jump_target=True),
Instruction(opname='LOAD_CONST', opcode=100, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=196, starts_line=None, is_jump_target=False),
Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='', offset=198, starts_line=None, is_jump_target=False),
Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=200, starts_line=None, is_jump_target=False),
Instruction(opname='END_FINALLY', opcode=88, arg=None, argval=None, argrepr='', offset=202, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=204, starts_line=None, is_jump_target=False),
Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=206, starts_line=None, is_jump_target=False)
]

# One last piece of inspect fodder to check the default line number handling
Expand All @@ -1021,6 +1031,10 @@ def simple(): pass

class InstructionTests(BytecodeTestCase):

def __init__(self, *args):
super().__init__(*args)
self.maxDiff = None

def test_default_first_line(self):
actual = dis.get_instructions(simple)
self.assertEqual(list(actual), expected_opinfo_simple)
Expand Down Expand Up @@ -1052,6 +1066,7 @@ def test_jumpy(self):
# get_instructions has its own tests above, so can rely on it to validate
# the object oriented API
class BytecodeTests(unittest.TestCase):

def test_instantiation(self):
# Test with function, method, code string and code object
for obj in [_f, _C(1).__init__, "a=1", _f.__code__]:
Expand Down
4 changes: 4 additions & 0 deletions Misc/NEWS.d/next/Core and Builtins/2018-03-13-14-46-03.bpo-32949.v821M7.rst
View file
Open in desktop
@@ -0,0 +1,4 @@
Removes overly complex WITH_CLEANUP_START and WITH_CLEANUP_FINISH byteocodes.
Adds much simpler RERAISE and WITH_EXCEPT_START bytecodes.
Differentiates between exceptional and non-exceptional flow in the compiler.

4 changes: 2 additions & 2 deletions Objects/frameobject.c
View file
Open in desktop
Expand Up @@ -232,7 +232,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno)
delta_iblock++;
}
}
if (op != FOR_ITER) {
if (op == SETUP_FINALLY) {
blockstack[blockstack_top++] = target_addr;
}
break;
Expand Down Expand Up @@ -264,7 +264,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno)
PyTryBlock *b = &f->f_blockstack[f->f_iblock];
delta = (f->f_stacktop - f->f_valuestack) - b->b_level;
if (b->b_type == SETUP_FINALLY &&
code[b->b_handler] == WITH_CLEANUP_START)
code[b->b_handler] == WITH_EXCEPT_START)
{
/* Pop the exit function. */
delta++;
Expand Down
127 changes: 28 additions & 99 deletions Python/ceval.c
View file
Open in desktop
Expand Up @@ -1844,6 +1844,15 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag)
DISPATCH();
}

TARGET(RERAISE) {
PyObject *exc = POP();
PyObject *val = POP();
PyObject *tb = POP();
assert(PyExceptionClass_Check(exc));
PyErr_Restore(exc, val, tb);
goto exception_unwind;
}

TARGET(POP_FINALLY) {
/* If oparg is 0 at the top of the stack are 1 or 6 values:
Either:
Expand Down Expand Up @@ -1907,13 +1916,12 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag)

TARGET(BEGIN_FINALLY) {
/* Push NULL onto the stack for using it in END_FINALLY,
POP_FINALLY, WITH_CLEANUP_START and WITH_CLEANUP_FINISH.
POP_FINALLY.
*/
PUSH(NULL);
FAST_DISPATCH();
}

PREDICTED(END_FINALLY);
TARGET(END_FINALLY) {
/* At the top of the stack are 1 or 6 values:
Either:
Expand Down Expand Up @@ -2954,113 +2962,34 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag)
DISPATCH();
}

TARGET(WITH_CLEANUP_START) {
/* At the top of the stack are 1 or 6 values indicating
how/why we entered the finally clause:
- TOP = NULL
TARGET(WITH_EXCEPT_START) {
/* At the top of the stack are 7 values:
- (TOP, SECOND, THIRD) = exc_info()
(FOURTH, FITH, SIXTH) = previous exception for EXCEPT_HANDLER
Below them is EXIT, the context.__exit__ or context.__aexit__
bound method.
In the first case, we must call
EXIT(None, None, None)
otherwise we must call
EXIT(TOP, SECOND, THIRD)

In the first case, we remove EXIT from the
stack, leaving TOP, and push TOP on the stack.
Otherwise we shift the bottom 3 values of the
stack down, replace the empty spot with NULL, and push
None on the stack.

Finally we push the result of the call.
- (FOURTH, FITH, SIXTH) = previous exception for EXCEPT_HANDLER
- SEVENTH: the context.__exit__ bound method
We call SEVENTH(TOP, SECOND, THIRD).
Then we push again the TOP exception and the __exit__
return value.
*/
PyObject *stack[3];
PyObject *exit_func;
PyObject *exit_stack[3];
PyObject *exc, *val, *tb, *res;

val = tb = Py_None;
exc = TOP();
if (exc == NULL) {
STACKADJ(-1);
exit_func = TOP();
SET_TOP(exc);
exc = Py_None;
}
else {
assert(PyExceptionClass_Check(exc));
PyObject *tp2, *exc2, *tb2;
PyTryBlock *block;
val = SECOND();
tb = THIRD();
tp2 = FOURTH();
exc2 = PEEK(5);
tb2 = PEEK(6);
exit_func = PEEK(7);
SET_VALUE(7, tb2);
SET_VALUE(6, exc2);
SET_VALUE(5, tp2);
/* UNWIND_EXCEPT_HANDLER will pop this off. */
SET_FOURTH(NULL);
/* We just shifted the stack down, so we have
to tell the except handler block that the
values are lower than it expects. */
assert(f->f_iblock > 0);
block = &f->f_blockstack[f->f_iblock - 1];
assert(block->b_type == EXCEPT_HANDLER);
assert(block->b_level > 0);
block->b_level--;
}

stack[0] = exc;
stack[1] = val;
stack[2] = tb;
res = _PyObject_FastCall(exit_func, stack, 3);
Py_DECREF(exit_func);
val = SECOND();
tb = THIRD();
assert(exc != Py_None);
assert(!PyLong_Check(exc));

exit_func = PEEK(7);
exit_stack[0] = exc;
exit_stack[1] = val;
exit_stack[2] = tb;
res = _PyObject_FastCall(exit_func, exit_stack, 3);
if (res == NULL)
goto error;

Py_INCREF(exc); /* Duplicating the exception on the stack */
PUSH(exc);
PUSH(res);
PREDICT(WITH_CLEANUP_FINISH);
DISPATCH();
}

PREDICTED(WITH_CLEANUP_FINISH);
TARGET(WITH_CLEANUP_FINISH) {
/* TOP = the result of calling the context.__exit__ bound method
SECOND = either None or exception type

If SECOND is None below is NULL or the return address,
otherwise below are 7 values representing an exception.
*/
PyObject *res = POP();
PyObject *exc = POP();
int err;

if (exc != Py_None)
err = PyObject_IsTrue(res);
else
err = 0;

Py_DECREF(res);
Py_DECREF(exc);

if (err < 0)
goto error;
else if (err > 0) {
/* There was an exception and a True return.
* We must manually unwind the EXCEPT_HANDLER block
* which was created when the exception was caught,
* otherwise the stack will be in an inconsisten state.
*/
PyTryBlock *b = PyFrame_BlockPop(f);
assert(b->b_type == EXCEPT_HANDLER);
UNWIND_EXCEPT_HANDLER(b);
PUSH(NULL);
}
PREDICT(END_FINALLY);
DISPATCH();
}

Expand Down