diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 636d2d88cb2819..8469f7f01f2c03 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -43,6 +43,7 @@ void _PyUnicode_ExactDealloc(PyObject *); #define SET_TOP(v) (stack_pointer[-1] = (v)) #define SET_SECOND(v) (stack_pointer[-2] = (v)) #define PEEK(n) (stack_pointer[-(n)]) +#define POKE(n, v) (stack_pointer[-(n)] = (v)) #define PUSH(val) (*(stack_pointer++) = (val)) #define POP() (*(--stack_pointer)) #define TOP() PEEK(1) @@ -63,12 +64,13 @@ do { \ /* Flow control macros */ #define DEOPT_IF(cond, instname) ((void)0) +#define ERROR_IF(cond, labelname) ((void)0) #define JUMPBY(offset) ((void)0) #define GO_TO_INSTRUCTION(instname) ((void)0) #define DISPATCH_SAME_OPARG() ((void)0) #define DISPATCH() ((void)0) -#define inst(name) case name: +#define inst(name, ...) case name: #define super(name) static int SUPER_##name #define family(name) static int family_##name @@ -79,6 +81,10 @@ typedef struct { PyObject *kwnames; } CallShape; +// Dummy variables for stack effects. +static PyObject *value, *value1, *value2, *left, *right, *res, *sum, *prod, *sub; +static PyObject *container, *start, *stop, *v; + static PyObject * dummy_func( PyThreadState *tstate, @@ -104,12 +110,10 @@ dummy_func( and that all operation that succeed call DISPATCH() ! */ // BEGIN BYTECODES // - // stack effect: ( -- ) - inst(NOP) { + inst(NOP, (--)) { } - // stack effect: ( -- ) - inst(RESUME) { + inst(RESUME, (--)) { assert(tstate->cframe == &cframe); assert(frame == cframe.current_frame); if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { @@ -117,45 +121,31 @@ dummy_func( } } - // stack effect: ( -- __0) - inst(LOAD_CLOSURE) { + inst(LOAD_CLOSURE, (-- value)) { /* We keep LOAD_CLOSURE so that the bytecode stays more readable. */ - PyObject *value = GETLOCAL(oparg); - if (value == NULL) { - goto unbound_local_error; - } + value = GETLOCAL(oparg); + ERROR_IF(value == NULL, unbound_local_error); Py_INCREF(value); - PUSH(value); } - // stack effect: ( -- __0) - inst(LOAD_FAST_CHECK) { - PyObject *value = GETLOCAL(oparg); - if (value == NULL) { - goto unbound_local_error; - } + inst(LOAD_FAST_CHECK, (-- value)) { + value = GETLOCAL(oparg); + ERROR_IF(value == NULL, unbound_local_error); Py_INCREF(value); - PUSH(value); } - // stack effect: ( -- __0) - inst(LOAD_FAST) { - PyObject *value = GETLOCAL(oparg); + inst(LOAD_FAST, (-- value)) { + value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - PUSH(value); } - // stack effect: ( -- __0) - inst(LOAD_CONST) { - PyObject *value = GETITEM(consts, oparg); + inst(LOAD_CONST, (-- value)) { + value = GETITEM(consts, oparg); Py_INCREF(value); - PUSH(value); } - // stack effect: (__0 -- ) - inst(STORE_FAST) { - PyObject *value = POP(); + inst(STORE_FAST, (value --)) { SETLOCAL(oparg, value); } @@ -165,9 +155,7 @@ dummy_func( super(STORE_FAST__STORE_FAST) = STORE_FAST + STORE_FAST; super (LOAD_CONST__LOAD_FAST) = LOAD_CONST + LOAD_FAST; - // stack effect: (__0 -- ) - inst(POP_TOP) { - PyObject *value = POP(); + inst(POP_TOP, (value --)) { Py_DECREF(value); } @@ -177,166 +165,113 @@ dummy_func( BASIC_PUSH(NULL); } - // stack effect: (__0, __1 -- ) - inst(END_FOR) { - PyObject *value = POP(); - Py_DECREF(value); - value = POP(); - Py_DECREF(value); + inst(END_FOR, (value1, value2 --)) { + Py_DECREF(value1); + Py_DECREF(value2); } - // stack effect: ( -- ) - inst(UNARY_POSITIVE) { - PyObject *value = TOP(); - PyObject *res = PyNumber_Positive(value); + inst(UNARY_POSITIVE, (value -- res)) { + res = PyNumber_Positive(value); Py_DECREF(value); - SET_TOP(res); - if (res == NULL) - goto error; + ERROR_IF(res == NULL, error); } - // stack effect: ( -- ) - inst(UNARY_NEGATIVE) { - PyObject *value = TOP(); - PyObject *res = PyNumber_Negative(value); + inst(UNARY_NEGATIVE, (value -- res)) { + res = PyNumber_Negative(value); Py_DECREF(value); - SET_TOP(res); - if (res == NULL) - goto error; + ERROR_IF(res == NULL, error); } - // stack effect: ( -- ) - inst(UNARY_NOT) { - PyObject *value = TOP(); + inst(UNARY_NOT, (value -- res)) { int err = PyObject_IsTrue(value); Py_DECREF(value); + ERROR_IF(err < 0, error); if (err == 0) { - Py_INCREF(Py_True); - SET_TOP(Py_True); - DISPATCH(); + res = Py_True; } - else if (err > 0) { - Py_INCREF(Py_False); - SET_TOP(Py_False); - DISPATCH(); + else { + res = Py_False; } - STACK_SHRINK(1); - goto error; + Py_INCREF(res); } - // stack effect: ( -- ) - inst(UNARY_INVERT) { - PyObject *value = TOP(); - PyObject *res = PyNumber_Invert(value); + inst(UNARY_INVERT, (value -- res)) { + res = PyNumber_Invert(value); Py_DECREF(value); - SET_TOP(res); - if (res == NULL) - goto error; + ERROR_IF(res == NULL, error); } - // stack effect: (__0 -- ) - inst(BINARY_OP_MULTIPLY_INT) { + inst(BINARY_OP_MULTIPLY_INT, (left, right -- prod)) { assert(cframe.use_tracing == 0); - PyObject *left = SECOND(); - PyObject *right = TOP(); DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); - PyObject *prod = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); - SET_SECOND(prod); + prod = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - STACK_SHRINK(1); - if (prod == NULL) { - goto error; - } + ERROR_IF(prod == NULL, error); JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP); } - // stack effect: (__0 -- ) - inst(BINARY_OP_MULTIPLY_FLOAT) { + inst(BINARY_OP_MULTIPLY_FLOAT, (left, right -- prod)) { assert(cframe.use_tracing == 0); - PyObject *left = SECOND(); - PyObject *right = TOP(); DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); double dprod = ((PyFloatObject *)left)->ob_fval * ((PyFloatObject *)right)->ob_fval; - PyObject *prod = PyFloat_FromDouble(dprod); - SET_SECOND(prod); + prod = PyFloat_FromDouble(dprod); _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); - STACK_SHRINK(1); - if (prod == NULL) { - goto error; - } + ERROR_IF(prod == NULL, error); JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP); } - // stack effect: (__0 -- ) - inst(BINARY_OP_SUBTRACT_INT) { + inst(BINARY_OP_SUBTRACT_INT, (left, right -- sub)) { assert(cframe.use_tracing == 0); - PyObject *left = SECOND(); - PyObject *right = TOP(); DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); - PyObject *sub = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); - SET_SECOND(sub); + sub = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - STACK_SHRINK(1); - if (sub == NULL) { - goto error; - } + ERROR_IF(sub == NULL, error); JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP); } - // stack effect: (__0 -- ) - inst(BINARY_OP_SUBTRACT_FLOAT) { + inst(BINARY_OP_SUBTRACT_FLOAT, (left, right -- sub)) { assert(cframe.use_tracing == 0); - PyObject *left = SECOND(); - PyObject *right = TOP(); DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); double dsub = ((PyFloatObject *)left)->ob_fval - ((PyFloatObject *)right)->ob_fval; - PyObject *sub = PyFloat_FromDouble(dsub); - SET_SECOND(sub); + sub = PyFloat_FromDouble(dsub); _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); - STACK_SHRINK(1); - if (sub == NULL) { - goto error; - } + ERROR_IF(sub == NULL, error); JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP); } - // stack effect: (__0 -- ) - inst(BINARY_OP_ADD_UNICODE) { + inst(BINARY_OP_ADD_UNICODE, (left, right -- res)) { assert(cframe.use_tracing == 0); - PyObject *left = SECOND(); - PyObject *right = TOP(); DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); - PyObject *res = PyUnicode_Concat(left, right); - STACK_SHRINK(1); - SET_TOP(res); + res = PyUnicode_Concat(left, right); _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - if (TOP() == NULL) { - goto error; - } + ERROR_IF(res == NULL, error); JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP); } - // stack effect: (__0 -- ) - inst(BINARY_OP_INPLACE_ADD_UNICODE) { + // This is a subtle one. It's a super-instruction for + // BINARY_OP_ADD_UNICODE followed by STORE_FAST + // where the store goes into the left argument. + // So the inputs are the same as for all BINARY_OP + // specializations, but there is no output. + // At the end we just skip over the STORE_FAST. + inst(BINARY_OP_INPLACE_ADD_UNICODE, (left, right --)) { assert(cframe.use_tracing == 0); - PyObject *left = SECOND(); - PyObject *right = TOP(); DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); _Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP]; @@ -358,107 +293,75 @@ dummy_func( */ assert(Py_REFCNT(left) >= 2); _Py_DECREF_NO_DEALLOC(left); - STACK_SHRINK(2); PyUnicode_Append(target_local, right); _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - if (*target_local == NULL) { - goto error; - } + ERROR_IF(*target_local == NULL, error); // The STORE_FAST is already done. JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP + 1); } - // stack effect: (__0 -- ) - inst(BINARY_OP_ADD_FLOAT) { + inst(BINARY_OP_ADD_FLOAT, (left, right -- sum)) { assert(cframe.use_tracing == 0); - PyObject *left = SECOND(); - PyObject *right = TOP(); DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); double dsum = ((PyFloatObject *)left)->ob_fval + ((PyFloatObject *)right)->ob_fval; - PyObject *sum = PyFloat_FromDouble(dsum); - SET_SECOND(sum); + sum = PyFloat_FromDouble(dsum); _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); - STACK_SHRINK(1); - if (sum == NULL) { - goto error; - } + ERROR_IF(sum == NULL, error); JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP); } - // stack effect: (__0 -- ) - inst(BINARY_OP_ADD_INT) { + inst(BINARY_OP_ADD_INT, (left, right -- sum)) { assert(cframe.use_tracing == 0); - PyObject *left = SECOND(); - PyObject *right = TOP(); DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); - PyObject *sum = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); - SET_SECOND(sum); + sum = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - STACK_SHRINK(1); - if (sum == NULL) { - goto error; - } + ERROR_IF(sum == NULL, error); JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP); } - // stack effect: (__0 -- ) - inst(BINARY_SUBSCR) { - PyObject *sub = POP(); - PyObject *container = TOP(); - PyObject *res = PyObject_GetItem(container, sub); + inst(BINARY_SUBSCR, (container, sub -- res)) { + res = PyObject_GetItem(container, sub); Py_DECREF(container); Py_DECREF(sub); - SET_TOP(res); - if (res == NULL) - goto error; + ERROR_IF(res == NULL, error); JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); } - // stack effect: (__0, __1 -- ) - inst(BINARY_SLICE) { - PyObject *stop = POP(); - PyObject *start = POP(); - PyObject *container = TOP(); - + inst(BINARY_SLICE, (container, start, stop -- res)) { PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); + // Can't use ERROR_IF() here, because we haven't + // DECREF'ed container yet, and we still own slice. if (slice == NULL) { - goto error; + res = NULL; } - PyObject *res = PyObject_GetItem(container, slice); - Py_DECREF(slice); - if (res == NULL) { - goto error; + else { + res = PyObject_GetItem(container, slice); + Py_DECREF(slice); } - SET_TOP(res); Py_DECREF(container); + ERROR_IF(res == NULL, error); } - // stack effect: (__0, __1, __2, __3 -- ) - inst(STORE_SLICE) { - PyObject *stop = POP(); - PyObject *start = POP(); - PyObject *container = TOP(); - PyObject *v = SECOND(); - + inst(STORE_SLICE, (v, container, start, stop -- )) { PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); + int err; if (slice == NULL) { - goto error; + err = 1; } - int err = PyObject_SetItem(container, slice, v); - Py_DECREF(slice); - if (err) { - goto error; + else { + err = PyObject_SetItem(container, slice, v); + Py_DECREF(slice); } - STACK_SHRINK(2); Py_DECREF(v); Py_DECREF(container); + ERROR_IF(err, error); } // stack effect: (__0 -- ) @@ -607,21 +510,14 @@ dummy_func( PREDICT(JUMP_BACKWARD); } - // stack effect: (__0, __1, __2 -- ) - inst(STORE_SUBSCR) { - PyObject *sub = TOP(); - PyObject *container = SECOND(); - PyObject *v = THIRD(); + inst(STORE_SUBSCR, (v, container, sub -- )) { int err; - STACK_SHRINK(3); /* container[sub] = v */ err = PyObject_SetItem(container, sub, v); Py_DECREF(v); Py_DECREF(container); Py_DECREF(sub); - if (err != 0) { - goto error; - } + ERROR_IF(err != 0, error); JUMPBY(INLINE_CACHE_ENTRIES_STORE_SUBSCR); } diff --git a/Python/ceval.c b/Python/ceval.c index 46fad972b06ec4..4828adb576fc86 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -806,6 +806,7 @@ GETITEM(PyObject *v, Py_ssize_t i) { #define THIRD() (stack_pointer[-3]) #define FOURTH() (stack_pointer[-4]) #define PEEK(n) (stack_pointer[-(n)]) +#define POKE(n, v) (stack_pointer[-(n)] = (v)) #define SET_TOP(v) (stack_pointer[-1] = (v)) #define SET_SECOND(v) (stack_pointer[-2] = (v)) #define BASIC_STACKADJ(n) (stack_pointer += n) @@ -1274,6 +1275,14 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int goto error; } +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); error: call_shape.kwnames = NULL; /* Double-check exception status. */ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 6ea1f48e629ee8..4e9b665db1a2eb 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -15,51 +15,57 @@ } TARGET(LOAD_CLOSURE) { + PyObject *value; /* We keep LOAD_CLOSURE so that the bytecode stays more readable. */ - PyObject *value = GETLOCAL(oparg); - if (value == NULL) { - goto unbound_local_error; - } + value = GETLOCAL(oparg); + if (value == NULL) goto unbound_local_error; Py_INCREF(value); - PUSH(value); + STACK_GROW(1); + POKE(1, value); DISPATCH(); } TARGET(LOAD_FAST_CHECK) { - PyObject *value = GETLOCAL(oparg); - if (value == NULL) { - goto unbound_local_error; - } + PyObject *value; + value = GETLOCAL(oparg); + if (value == NULL) goto unbound_local_error; Py_INCREF(value); - PUSH(value); + STACK_GROW(1); + POKE(1, value); DISPATCH(); } TARGET(LOAD_FAST) { - PyObject *value = GETLOCAL(oparg); + PyObject *value; + value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - PUSH(value); + STACK_GROW(1); + POKE(1, value); DISPATCH(); } TARGET(LOAD_CONST) { PREDICTED(LOAD_CONST); - PyObject *value = GETITEM(consts, oparg); + PyObject *value; + value = GETITEM(consts, oparg); Py_INCREF(value); - PUSH(value); + STACK_GROW(1); + POKE(1, value); DISPATCH(); } TARGET(STORE_FAST) { - PyObject *value = POP(); + PyObject *value = PEEK(1); SETLOCAL(oparg, value); + STACK_SHRINK(1); DISPATCH(); } TARGET(POP_TOP) { - PyObject *value = POP(); + PyObject *value = PEEK(1); Py_DECREF(value); + STACK_SHRINK(1); DISPATCH(); } @@ -70,163 +76,158 @@ } TARGET(END_FOR) { - PyObject *value = POP(); - Py_DECREF(value); - value = POP(); - Py_DECREF(value); + PyObject *value2 = PEEK(1); + PyObject *value1 = PEEK(2); + Py_DECREF(value1); + Py_DECREF(value2); + STACK_SHRINK(2); DISPATCH(); } TARGET(UNARY_POSITIVE) { - PyObject *value = TOP(); - PyObject *res = PyNumber_Positive(value); + PyObject *value = PEEK(1); + PyObject *res; + res = PyNumber_Positive(value); Py_DECREF(value); - SET_TOP(res); - if (res == NULL) - goto error; + if (res == NULL) goto pop_1_error; + POKE(1, res); DISPATCH(); } TARGET(UNARY_NEGATIVE) { - PyObject *value = TOP(); - PyObject *res = PyNumber_Negative(value); + PyObject *value = PEEK(1); + PyObject *res; + res = PyNumber_Negative(value); Py_DECREF(value); - SET_TOP(res); - if (res == NULL) - goto error; + if (res == NULL) goto pop_1_error; + POKE(1, res); DISPATCH(); } TARGET(UNARY_NOT) { - PyObject *value = TOP(); + PyObject *value = PEEK(1); + PyObject *res; int err = PyObject_IsTrue(value); Py_DECREF(value); + if (err < 0) goto pop_1_error; if (err == 0) { - Py_INCREF(Py_True); - SET_TOP(Py_True); - DISPATCH(); + res = Py_True; } - else if (err > 0) { - Py_INCREF(Py_False); - SET_TOP(Py_False); - DISPATCH(); + else { + res = Py_False; } - STACK_SHRINK(1); - goto error; + Py_INCREF(res); + POKE(1, res); + DISPATCH(); } TARGET(UNARY_INVERT) { - PyObject *value = TOP(); - PyObject *res = PyNumber_Invert(value); + PyObject *value = PEEK(1); + PyObject *res; + res = PyNumber_Invert(value); Py_DECREF(value); - SET_TOP(res); - if (res == NULL) - goto error; + if (res == NULL) goto pop_1_error; + POKE(1, res); DISPATCH(); } TARGET(BINARY_OP_MULTIPLY_INT) { + PyObject *right = PEEK(1); + PyObject *left = PEEK(2); + PyObject *prod; assert(cframe.use_tracing == 0); - PyObject *left = SECOND(); - PyObject *right = TOP(); DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); - PyObject *prod = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); - SET_SECOND(prod); + prod = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - STACK_SHRINK(1); - if (prod == NULL) { - goto error; - } + if (prod == NULL) goto pop_2_error; JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP); + STACK_SHRINK(1); + POKE(1, prod); DISPATCH(); } TARGET(BINARY_OP_MULTIPLY_FLOAT) { + PyObject *right = PEEK(1); + PyObject *left = PEEK(2); + PyObject *prod; assert(cframe.use_tracing == 0); - PyObject *left = SECOND(); - PyObject *right = TOP(); DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); double dprod = ((PyFloatObject *)left)->ob_fval * ((PyFloatObject *)right)->ob_fval; - PyObject *prod = PyFloat_FromDouble(dprod); - SET_SECOND(prod); + prod = PyFloat_FromDouble(dprod); _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); - STACK_SHRINK(1); - if (prod == NULL) { - goto error; - } + if (prod == NULL) goto pop_2_error; JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP); + STACK_SHRINK(1); + POKE(1, prod); DISPATCH(); } TARGET(BINARY_OP_SUBTRACT_INT) { + PyObject *right = PEEK(1); + PyObject *left = PEEK(2); + PyObject *sub; assert(cframe.use_tracing == 0); - PyObject *left = SECOND(); - PyObject *right = TOP(); DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); - PyObject *sub = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); - SET_SECOND(sub); + sub = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - STACK_SHRINK(1); - if (sub == NULL) { - goto error; - } + if (sub == NULL) goto pop_2_error; JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP); + STACK_SHRINK(1); + POKE(1, sub); DISPATCH(); } TARGET(BINARY_OP_SUBTRACT_FLOAT) { + PyObject *right = PEEK(1); + PyObject *left = PEEK(2); + PyObject *sub; assert(cframe.use_tracing == 0); - PyObject *left = SECOND(); - PyObject *right = TOP(); DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); double dsub = ((PyFloatObject *)left)->ob_fval - ((PyFloatObject *)right)->ob_fval; - PyObject *sub = PyFloat_FromDouble(dsub); - SET_SECOND(sub); + sub = PyFloat_FromDouble(dsub); _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); - STACK_SHRINK(1); - if (sub == NULL) { - goto error; - } + if (sub == NULL) goto pop_2_error; JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP); + STACK_SHRINK(1); + POKE(1, sub); DISPATCH(); } TARGET(BINARY_OP_ADD_UNICODE) { + PyObject *right = PEEK(1); + PyObject *left = PEEK(2); + PyObject *res; assert(cframe.use_tracing == 0); - PyObject *left = SECOND(); - PyObject *right = TOP(); DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); - PyObject *res = PyUnicode_Concat(left, right); - STACK_SHRINK(1); - SET_TOP(res); + res = PyUnicode_Concat(left, right); _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - if (TOP() == NULL) { - goto error; - } + if (res == NULL) goto pop_2_error; JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP); + STACK_SHRINK(1); + POKE(1, res); DISPATCH(); } TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { + PyObject *right = PEEK(1); + PyObject *left = PEEK(2); assert(cframe.use_tracing == 0); - PyObject *left = SECOND(); - PyObject *right = TOP(); DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); _Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP]; @@ -248,108 +249,108 @@ */ assert(Py_REFCNT(left) >= 2); _Py_DECREF_NO_DEALLOC(left); - STACK_SHRINK(2); PyUnicode_Append(target_local, right); _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - if (*target_local == NULL) { - goto error; - } + if (*target_local == NULL) goto pop_2_error; // The STORE_FAST is already done. JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP + 1); + STACK_SHRINK(2); DISPATCH(); } TARGET(BINARY_OP_ADD_FLOAT) { + PyObject *right = PEEK(1); + PyObject *left = PEEK(2); + PyObject *sum; assert(cframe.use_tracing == 0); - PyObject *left = SECOND(); - PyObject *right = TOP(); DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); double dsum = ((PyFloatObject *)left)->ob_fval + ((PyFloatObject *)right)->ob_fval; - PyObject *sum = PyFloat_FromDouble(dsum); - SET_SECOND(sum); + sum = PyFloat_FromDouble(dsum); _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); - STACK_SHRINK(1); - if (sum == NULL) { - goto error; - } + if (sum == NULL) goto pop_2_error; JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP); + STACK_SHRINK(1); + POKE(1, sum); DISPATCH(); } TARGET(BINARY_OP_ADD_INT) { + PyObject *right = PEEK(1); + PyObject *left = PEEK(2); + PyObject *sum; assert(cframe.use_tracing == 0); - PyObject *left = SECOND(); - PyObject *right = TOP(); DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); - PyObject *sum = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); - SET_SECOND(sum); + sum = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - STACK_SHRINK(1); - if (sum == NULL) { - goto error; - } + if (sum == NULL) goto pop_2_error; JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP); + STACK_SHRINK(1); + POKE(1, sum); DISPATCH(); } TARGET(BINARY_SUBSCR) { PREDICTED(BINARY_SUBSCR); - PyObject *sub = POP(); - PyObject *container = TOP(); - PyObject *res = PyObject_GetItem(container, sub); + PyObject *sub = PEEK(1); + PyObject *container = PEEK(2); + PyObject *res; + res = PyObject_GetItem(container, sub); Py_DECREF(container); Py_DECREF(sub); - SET_TOP(res); - if (res == NULL) - goto error; + if (res == NULL) goto pop_2_error; JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + STACK_SHRINK(1); + POKE(1, res); DISPATCH(); } TARGET(BINARY_SLICE) { - PyObject *stop = POP(); - PyObject *start = POP(); - PyObject *container = TOP(); - + PyObject *stop = PEEK(1); + PyObject *start = PEEK(2); + PyObject *container = PEEK(3); + PyObject *res; PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); + // Can't use ERROR_IF() here, because we haven't + // DECREF'ed container yet, and we still own slice. if (slice == NULL) { - goto error; + res = NULL; } - PyObject *res = PyObject_GetItem(container, slice); - Py_DECREF(slice); - if (res == NULL) { - goto error; + else { + res = PyObject_GetItem(container, slice); + Py_DECREF(slice); } - SET_TOP(res); Py_DECREF(container); + if (res == NULL) goto pop_3_error; + STACK_SHRINK(2); + POKE(1, res); DISPATCH(); } TARGET(STORE_SLICE) { - PyObject *stop = POP(); - PyObject *start = POP(); - PyObject *container = TOP(); - PyObject *v = SECOND(); - + PyObject *stop = PEEK(1); + PyObject *start = PEEK(2); + PyObject *container = PEEK(3); + PyObject *v = PEEK(4); PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); + int err; if (slice == NULL) { - goto error; + err = 1; } - int err = PyObject_SetItem(container, slice, v); - Py_DECREF(slice); - if (err) { - goto error; + else { + err = PyObject_SetItem(container, slice, v); + Py_DECREF(slice); } - STACK_SHRINK(2); Py_DECREF(v); Py_DECREF(container); + if (err) goto pop_4_error; + STACK_SHRINK(4); DISPATCH(); } @@ -500,20 +501,18 @@ TARGET(STORE_SUBSCR) { PREDICTED(STORE_SUBSCR); - PyObject *sub = TOP(); - PyObject *container = SECOND(); - PyObject *v = THIRD(); + PyObject *sub = PEEK(1); + PyObject *container = PEEK(2); + PyObject *v = PEEK(3); int err; - STACK_SHRINK(3); /* container[sub] = v */ err = PyObject_SetItem(container, sub, v); Py_DECREF(v); Py_DECREF(container); Py_DECREF(sub); - if (err != 0) { - goto error; - } + if (err != 0) goto pop_3_error; JUMPBY(INLINE_CACHE_ENTRIES_STORE_SUBSCR); + STACK_SHRINK(3); DISPATCH(); } @@ -3876,82 +3875,99 @@ TARGET(LOAD_FAST__LOAD_FAST) { { - PyObject *value = GETLOCAL(oparg); + PyObject *value; + value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - PUSH(value); + STACK_GROW(1); + POKE(1, value); } NEXTOPARG(); next_instr++; { - PyObject *value = GETLOCAL(oparg); + PyObject *value; + value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - PUSH(value); + STACK_GROW(1); + POKE(1, value); } DISPATCH(); } TARGET(LOAD_FAST__LOAD_CONST) { { - PyObject *value = GETLOCAL(oparg); + PyObject *value; + value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - PUSH(value); + STACK_GROW(1); + POKE(1, value); } NEXTOPARG(); next_instr++; { - PyObject *value = GETITEM(consts, oparg); + PyObject *value; + value = GETITEM(consts, oparg); Py_INCREF(value); - PUSH(value); + STACK_GROW(1); + POKE(1, value); } DISPATCH(); } TARGET(STORE_FAST__LOAD_FAST) { { - PyObject *value = POP(); + PyObject *value = PEEK(1); SETLOCAL(oparg, value); + STACK_SHRINK(1); } NEXTOPARG(); next_instr++; { - PyObject *value = GETLOCAL(oparg); + PyObject *value; + value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - PUSH(value); + STACK_GROW(1); + POKE(1, value); } DISPATCH(); } TARGET(STORE_FAST__STORE_FAST) { { - PyObject *value = POP(); + PyObject *value = PEEK(1); SETLOCAL(oparg, value); + STACK_SHRINK(1); } NEXTOPARG(); next_instr++; { - PyObject *value = POP(); + PyObject *value = PEEK(1); SETLOCAL(oparg, value); + STACK_SHRINK(1); } DISPATCH(); } TARGET(LOAD_CONST__LOAD_FAST) { { - PyObject *value = GETITEM(consts, oparg); + PyObject *value; + value = GETITEM(consts, oparg); Py_INCREF(value); - PUSH(value); + STACK_GROW(1); + POKE(1, value); } NEXTOPARG(); next_instr++; { - PyObject *value = GETLOCAL(oparg); + PyObject *value; + value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - PUSH(value); + STACK_GROW(1); + POKE(1, value); } DISPATCH(); } diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index c5a436d08aa96b..e5610343880f86 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -5,10 +5,10 @@ # TODO: Reuse C generation framework from deepfreeze.py? import argparse -import io import os import re import sys +from typing import TextIO, cast import parser from parser import InstDef # TODO: Use parser.InstDef @@ -20,13 +20,13 @@ arg_parser.add_argument("-q", "--quiet", action="store_true") -def eopen(filename: str, mode: str = "r"): +def eopen(filename: str, mode: str = "r") -> TextIO: if filename == "-": if "r" in mode: return sys.stdin else: return sys.stdout - return open(filename, mode) + return cast(TextIO, open(filename, mode)) def parse_cases( @@ -67,42 +67,72 @@ def always_exits(block: parser.Block) -> bool: return line.startswith(("goto ", "return ", "DISPATCH", "GO_TO_", "Py_UNREACHABLE()")) -def write_cases(f: io.TextIOBase, instrs: list[InstDef], supers: list[parser.Super]): - predictions = set() - for inst in instrs: - for target in re.findall(r"(?:PREDICT|GO_TO_INSTRUCTION)\((\w+)\)", inst.block.text): +def write_instr(instr: InstDef, predictions: set[str], indent: str, f: TextIO, dedent: int = 0): + assert instr.block + if dedent < 0: + indent += " " * -dedent + # TODO: Is it better to count forward or backward? + for i, input in enumerate(reversed(instr.inputs), 1): + f.write(f"{indent} PyObject *{input} = PEEK({i});\n") + for output in instr.outputs: + if output not in instr.inputs: + f.write(f"{indent} PyObject *{output};\n") + assert instr.block is not None + blocklines = instr.block.to_text(dedent=dedent).splitlines(True) + # Remove blank lines from ends + while blocklines and not blocklines[0].strip(): + blocklines.pop(0) + while blocklines and not blocklines[-1].strip(): + blocklines.pop() + # Remove leading '{' and trailing '}' + assert blocklines and blocklines[0].strip() == "{" + assert blocklines and blocklines[-1].strip() == "}" + blocklines.pop() + blocklines.pop(0) + # Remove trailing blank lines + while blocklines and not blocklines[-1].strip(): + blocklines.pop() + # Write the body + ninputs = len(instr.inputs or ()) + for line in blocklines: + if m := re.match(r"(\s*)ERROR_IF\((.+), (\w+)\);\s*$", line): + space, cond, label = m.groups() + # ERROR_IF() must remove the inputs from the stack. + # The code block is responsible for DECREF()ing them. + if ninputs: + f.write(f"{space}if ({cond}) goto pop_{ninputs}_{label};\n") + else: + f.write(f"{space}if ({cond}) goto {label};\n") + else: + f.write(line) + noutputs = len(instr.outputs or ()) + diff = noutputs - ninputs + if diff > 0: + f.write(f"{indent} STACK_GROW({diff});\n") + elif diff < 0: + f.write(f"{indent} STACK_SHRINK({-diff});\n") + for i, output in enumerate(reversed(instr.outputs or ()), 1): + if output not in (instr.inputs or ()): + f.write(f"{indent} POKE({i}, {output});\n") + assert instr.block + +def write_cases(f: TextIO, instrs: list[InstDef], supers: list[parser.Super]): + predictions: set[str] = set() + for instr in instrs: + assert isinstance(instr, InstDef) + assert instr.block is not None + for target in re.findall(r"(?:PREDICT|GO_TO_INSTRUCTION)\((\w+)\)", instr.block.text): predictions.add(target) indent = " " f.write(f"// This file is generated by {os.path.relpath(__file__)}\n") - f.write("// Do not edit!\n") + f.write(f"// Do not edit!\n") instr_index: dict[str, InstDef] = {} for instr in instrs: - assert isinstance(instr, InstDef) instr_index[instr.name] = instr f.write(f"\n{indent}TARGET({instr.name}) {{\n") if instr.name in predictions: f.write(f"{indent} PREDICTED({instr.name});\n") - # input = ", ".join(instr.inputs) - # output = ", ".join(instr.outputs) - # f.write(f"{indent} // {input} -- {output}\n") - assert instr.block - blocklines = instr.block.text.splitlines(True) - # Remove blank lines from ends - while blocklines and not blocklines[0].strip(): - blocklines.pop(0) - while blocklines and not blocklines[-1].strip(): - blocklines.pop() - # Remove leading '{' and trailing '}' - assert blocklines and blocklines[0].strip() == "{" - assert blocklines and blocklines[-1].strip() == "}" - blocklines.pop() - blocklines.pop(0) - # Remove trailing blank lines - while blocklines and not blocklines[-1].strip(): - blocklines.pop() - # Write the body - for line in blocklines: - f.write(line) + write_instr(instr, predictions, indent, f) assert instr.block if not always_exits(instr.block): f.write(f"{indent} DISPATCH();\n") @@ -114,14 +144,13 @@ def write_cases(f: io.TextIOBase, instrs: list[InstDef], supers: list[parser.Sup components = [instr_index[name] for name in sup.ops] f.write(f"\n{indent}TARGET({sup.name}) {{\n") for i, instr in enumerate(components): + assert instr.block if i > 0: f.write(f"{indent} NEXTOPARG();\n") f.write(f"{indent} next_instr++;\n") - text = instr.block.to_text(-4) - textlines = text.splitlines(True) - textlines = [line for line in textlines if not line.strip().startswith("PREDICTED(")] - text = "".join(textlines) - f.write(f"{indent} {text.strip()}\n") + f.write(f"{indent} {{\n") + write_instr(instr, predictions, indent, f, dedent=-4) + f.write(f" {indent}}}\n") f.write(f"{indent} DISPATCH();\n") f.write(f"{indent}}}\n") diff --git a/Tools/cases_generator/parser.py b/Tools/cases_generator/parser.py index f603bc6a9868e1..9e95cdb42d42db 100644 --- a/Tools/cases_generator/parser.py +++ b/Tools/cases_generator/parser.py @@ -57,11 +57,28 @@ class Block(Node): @dataclass -class InstDef(Node): +class InstHeader(Node): name: str - inputs: list[str] | None - outputs: list[str] | None - block: Block | None + inputs: list[str] + outputs: list[str] + + +@dataclass +class InstDef(Node): + header: InstHeader + block: Block + + @property + def name(self): + return self.header.name + + @property + def inputs(self): + return self.header.inputs + + @property + def outputs(self): + return self.header.outputs @dataclass @@ -82,30 +99,42 @@ class Parser(PLexer): def inst_def(self) -> InstDef | None: if header := self.inst_header(): if block := self.block(): - header.block = block - return header + return InstDef(header, block) raise self.make_syntax_error("Expected block") return None @contextual - def inst_header(self): + def inst_header(self) -> InstHeader | None: # inst(NAME) | inst(NAME, (inputs -- outputs)) # TODO: Error out when there is something unexpected. - # TODO: Make INST a keyword in the lexer. + # TODO: Make INST a keyword in the lexer.`` if (tkn := self.expect(lx.IDENTIFIER)) and tkn.text == "inst": if (self.expect(lx.LPAREN) and (tkn := self.expect(lx.IDENTIFIER))): name = tkn.text if self.expect(lx.COMMA): inp, outp = self.stack_effect() - if (self.expect(lx.RPAREN) - and self.peek().kind == lx.LBRACE): - return InstDef(name, inp, outp, []) + if self.expect(lx.RPAREN): + if ((tkn := self.peek()) + and tkn.kind == lx.LBRACE): + self.check_overlaps(inp, outp) + return InstHeader(name, inp, outp) elif self.expect(lx.RPAREN): - return InstDef(name, None, None, []) + return InstHeader(name, [], []) return None - def stack_effect(self): + def check_overlaps(self, inp: list[str], outp: list[str]): + for i, name in enumerate(inp): + try: + j = outp.index(name) + except ValueError: + continue + else: + if i != j: + raise self.make_syntax_error( + f"Input {name!r} at pos {i} repeated in output at different pos {j}") + + def stack_effect(self) -> tuple[list[str], list[str]]: # '(' [inputs] '--' [outputs] ')' if self.expect(lx.LPAREN): inp = self.inputs() or [] @@ -115,7 +144,7 @@ def stack_effect(self): return inp, outp raise self.make_syntax_error("Expected stack effect") - def inputs(self): + def inputs(self) -> list[str] | None: # input (, input)* here = self.getpos() if inp := self.input(): @@ -128,7 +157,7 @@ def inputs(self): self.setpos(here) return None - def input(self): + def input(self) -> str | None: # IDENTIFIER if (tkn := self.expect(lx.IDENTIFIER)): if self.expect(lx.LBRACKET): @@ -148,7 +177,7 @@ def input(self): return "??" return None - def outputs(self): + def outputs(self) -> list[str] | None: # output (, output)* here = self.getpos() if outp := self.output(): @@ -161,7 +190,7 @@ def outputs(self): self.setpos(here) return None - def output(self): + def output(self) -> str | None: return self.input() # TODO: They're not quite the same. @contextual @@ -176,7 +205,6 @@ def super_def(self) -> Super | None: return res def ops(self) -> list[str] | None: - here = self.getpos() if tkn := self.expect(lx.IDENTIFIER): ops = [tkn.text] while self.expect(lx.PLUS): @@ -197,7 +225,7 @@ def family_def(self) -> Family | None: return Family(tkn.text, members) return None - def members(self): + def members(self) -> list[str] | None: here = self.getpos() if tkn := self.expect(lx.IDENTIFIER): near = self.getpos() @@ -214,8 +242,8 @@ def block(self) -> Block: tokens = self.c_blob() return Block(tokens) - def c_blob(self): - tokens = [] + def c_blob(self) -> list[lx.Token]: + tokens: list[lx.Token] = [] level = 0 while tkn := self.next(raw=True): if tkn.kind in (lx.LBRACE, lx.LPAREN, lx.LBRACKET):