-
Notifications
You must be signed in to change notification settings - Fork 1.1k
/
interpreter.py
3424 lines (3013 loc) · 134 KB
/
interpreter.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
import builtins
import collections
import dis
import operator
import logging
import textwrap
from numba.core import errors, ir, config
from numba.core.errors import (
NotDefinedError,
UnsupportedBytecodeError,
error_extras,
)
from numba.core.ir_utils import get_definition, guard
from numba.core.utils import (PYVERSION, BINOPS_TO_OPERATORS,
INPLACE_BINOPS_TO_OPERATORS,)
from numba.core.byteflow import Flow, AdaptDFA, AdaptCFA, BlockKind
from numba.core.unsafe import eh
from numba.cpython.unsafe.tuple import unpack_single_tuple
if PYVERSION in ((3, 12), (3, 13)):
# Operands for CALL_INTRINSIC_1
from numba.core.byteflow import CALL_INTRINSIC_1_Operand as ci1op
elif PYVERSION in ((3, 10), (3, 11)):
pass
else:
raise NotImplementedError(PYVERSION)
class _UNKNOWN_VALUE(object):
"""Represents an unknown value, this is for ease of debugging purposes only.
"""
def __init__(self, varname):
self._varname = varname
def __repr__(self):
return "_UNKNOWN_VALUE({})".format(self._varname)
_logger = logging.getLogger(__name__)
class Assigner(object):
"""
This object keeps track of potential assignment simplifications
inside a code block.
For example `$O.1 = x` followed by `y = $0.1` can be simplified
into `y = x`, but it's not possible anymore if we have `x = z`
in-between those two instructions.
NOTE: this is not only an optimization, but is actually necessary
due to certain limitations of Numba - such as only accepting the
returning of an array passed as function argument.
"""
def __init__(self):
# { destination variable name -> source Var object }
self.dest_to_src = {}
# Basically a reverse mapping of dest_to_src:
# { source variable name -> all destination names in dest_to_src }
self.src_invalidate = collections.defaultdict(list)
self.unused_dests = set()
def assign(self, srcvar, destvar):
"""
Assign *srcvar* to *destvar*. Return either *srcvar* or a possible
simplified assignment source (earlier assigned to *srcvar*).
"""
srcname = srcvar.name
destname = destvar.name
if destname in self.src_invalidate:
# destvar will change, invalidate all previously known
# simplifications
for d in self.src_invalidate.pop(destname):
self.dest_to_src.pop(d)
if srcname in self.dest_to_src:
srcvar = self.dest_to_src[srcname]
if destvar.is_temp:
self.dest_to_src[destname] = srcvar
self.src_invalidate[srcname].append(destname)
self.unused_dests.add(destname)
return srcvar
def get_assignment_source(self, destname):
"""
Get a possible assignment source (a ir.Var instance) to replace
*destname*, otherwise None.
"""
if destname in self.dest_to_src:
return self.dest_to_src[destname]
self.unused_dests.discard(destname)
return None
def _remove_assignment_definition(old_body, idx, func_ir, already_deleted_defs):
"""
Deletes the definition defined for old_body at index idx
from func_ir. We assume this stmt will be deleted from
new_body.
In some optimizations we may update the same variable multiple times.
In this situation, we only need to delete a particular definition once,
this is tracked in already_deleted_def, which is a map from
assignment name to the set of values that have already been
deleted.
"""
lhs = old_body[idx].target.name
rhs = old_body[idx].value
if rhs in func_ir._definitions[lhs]:
func_ir._definitions[lhs].remove(rhs)
already_deleted_defs[lhs].add(rhs)
elif rhs not in already_deleted_defs[lhs]:
raise UnsupportedBytecodeError(
"Inconsistency found in the definitions while executing"
" a peephole optimization. This suggests an internal"
" error or inconsistency elsewhere in the compiler."
)
def _call_function_ex_replace_kws_small(
old_body,
keyword_expr,
new_body,
buildmap_idx,
func_ir,
already_deleted_defs
):
"""
Extracts the kws args passed as varkwarg
for CALL_FUNCTION_EX. This pass is taken when
n_kws <= 15 and the bytecode looks like:
# Start for each argument
LOAD_FAST # Load each argument.
# End for each argument
...
BUILD_CONST_KEY_MAP # Build a map
In the generated IR, the varkwarg refers
to a single build_map that contains all of the
kws. In addition to returning the kws, this
function updates new_body to remove all usage
of the map.
"""
kws = keyword_expr.items.copy()
# kws are required to have constant keys.
# We update these with the value_indexes
value_indexes = keyword_expr.value_indexes
for key, index in value_indexes.items():
kws[index] = (key, kws[index][1])
# Remove the build_map by setting the list
# index to None. Nones will be removed later.
new_body[buildmap_idx] = None
# Remove the definition.
_remove_assignment_definition(
old_body, buildmap_idx, func_ir, already_deleted_defs
)
return kws
def _call_function_ex_replace_kws_large(
old_body,
buildmap_name,
buildmap_idx,
search_end,
new_body,
func_ir,
errmsg,
already_deleted_defs
):
"""
Extracts the kws args passed as varkwarg
for CALL_FUNCTION_EX. This pass is taken when
n_kws > 15 and the bytecode looks like:
BUILD_MAP # Construct the map
# Start for each argument
LOAD_CONST # Load a constant for the name of the argument
LOAD_FAST # Load each argument.
MAP_ADD # Append the (key, value) pair to the map
# End for each argument
In the IR generated, the initial build map is empty and a series
of setitems are applied afterwards. THE IR looks like:
$build_map_var = build_map(items=[])
$constvar = const(str, ...) # create the const key
# CREATE THE ARGUMENT, This may take multiple lines.
$created_arg = ...
$var = getattr(
value=$build_map_var,
attr=__setitem__,
)
$unused_var = call $var($constvar, $created_arg)
We iterate through the IR, deleting all usages of the buildmap
from the new_body, and adds the kws to a new kws list.
"""
# Remove the build_map from the body.
new_body[buildmap_idx] = None
# Remove the definition.
_remove_assignment_definition(
old_body, buildmap_idx, func_ir, already_deleted_defs
)
kws = []
search_start = buildmap_idx + 1
while search_start <= search_end:
# The first value must be a constant.
const_stmt = old_body[search_start]
if not (
isinstance(const_stmt, ir.Assign)
and isinstance(const_stmt.value, ir.Const)
):
# We cannot handle this format so raise the
# original error message.
raise UnsupportedBytecodeError(errmsg)
key_var_name = const_stmt.target.name
key_val = const_stmt.value.value
search_start += 1
# Now we need to search for a getattr with setitem
found_getattr = False
while (
search_start <= search_end
and not found_getattr
):
getattr_stmt = old_body[search_start]
if (
isinstance(getattr_stmt, ir.Assign)
and isinstance(getattr_stmt.value, ir.Expr)
and getattr_stmt.value.op == "getattr"
and (
getattr_stmt.value.value.name
== buildmap_name
)
and getattr_stmt.value.attr == "__setitem__"
):
found_getattr = True
else:
# If the argument is "created" in JIT, then there
# will be intermediate operations in between setitems.
# For example we have arg5=pow(arg5, 2),
# then the IR would look like:
#
# # Creation of the constant key.
# $const44.26 = const(str, arg5)
#
# # Argument creation. This is the section we are skipping
# $46load_global.27 = global(pow: <built-in function pow>)
# $const50.29 = const(int, 2)
# $call.30 = call $46load_global.27(arg5, $const50.29)
#
# # Setitem with arg5
# $54map_add.31 = getattr(value=$map.2, attr=__setitem__)
# $54map_add.32 = call $54map_add.31($const44.26, $call.30)
search_start += 1
if (
not found_getattr
or search_start == search_end
):
# We cannot handle this format so raise the
# original error message.
raise UnsupportedBytecodeError(errmsg)
setitem_stmt = old_body[search_start + 1]
if not (
isinstance(setitem_stmt, ir.Assign)
and isinstance(setitem_stmt.value, ir.Expr)
and setitem_stmt.value.op == "call"
and (
setitem_stmt.value.func.name
== getattr_stmt.target.name
)
and len(setitem_stmt.value.args) == 2
and (
setitem_stmt.value.args[0].name
== key_var_name
)
):
# A call statement should always immediately follow the
# getattr. If for some reason this doesn't match the code
# format, we raise the original error message. This check
# is meant as a precaution.
raise UnsupportedBytecodeError(errmsg)
arg_var = setitem_stmt.value.args[1]
# Append the (key, value) pair.
kws.append((key_val, arg_var))
# Remove the __setitem__ getattr and call
new_body[search_start] = None
new_body[search_start + 1] = None
# Remove the definitions.
_remove_assignment_definition(
old_body, search_start, func_ir, already_deleted_defs
)
_remove_assignment_definition(
old_body, search_start + 1, func_ir, already_deleted_defs
)
search_start += 2
return kws
def _call_function_ex_replace_args_small(
old_body,
tuple_expr,
new_body,
buildtuple_idx,
func_ir,
already_deleted_defs
):
"""
Extracts the args passed as vararg
for CALL_FUNCTION_EX. This pass is taken when
n_args <= 30 and the bytecode looks like:
# Start for each argument
LOAD_FAST # Load each argument.
# End for each argument
...
BUILD_TUPLE # Create a tuple of the arguments
In the IR generated, the vararg refer
to a single build_tuple that contains all of the
args. In addition to returning the args, this
function updates new_body to remove all usage
of the tuple.
"""
# Delete the build tuple
new_body[buildtuple_idx] = None
# Remove the definition.
_remove_assignment_definition(
old_body, buildtuple_idx, func_ir, already_deleted_defs
)
# Return the args.
return tuple_expr.items
def _call_function_ex_replace_args_large(
old_body,
vararg_stmt,
new_body,
search_end,
func_ir,
errmsg,
already_deleted_defs
):
"""
Extracts the args passed as vararg
for CALL_FUNCTION_EX. This pass is taken when
n_args > 30 and the bytecode looks like:
BUILD_TUPLE # Create a list to append to
# Start for each argument
LOAD_FAST # Load each argument.
LIST_APPEND # Add the argument to the list
# End for each argument
...
LIST_TO_TUPLE # Convert the args to a tuple.
In the IR generated, the tuple is created by concatenating
together several 1 element tuples to an initial empty tuple.
We traverse backwards in the IR, collecting args, until we
find the original empty tuple. For example, the IR might
look like:
$orig_tuple = build_tuple(items=[])
$first_var = build_tuple(items=[Var(arg0, test.py:6)])
$next_tuple = $orig_tuple + $first_var
...
$final_var = build_tuple(items=[Var(argn, test.py:6)])
$final_tuple = $prev_tuple + $final_var
$varargs_var = $final_tuple
"""
# We traverse to the front of the block to look for the original
# tuple.
search_start = 0
total_args = []
if (
isinstance(vararg_stmt, ir.Assign)
and isinstance(vararg_stmt.value, ir.Var)
):
target_name = vararg_stmt.value.name
# If there is an initial assignment, delete it
new_body[search_end] = None
# Remove the definition.
_remove_assignment_definition(
old_body, search_end, func_ir, already_deleted_defs
)
search_end -= 1
else:
# There must always be an initial assignment
# https://github.com/numba/numba/blob/59fa2e335be68148b3bd72a29de3ff011430038d/numba/core/interpreter.py#L259-L260
# If this changes we may need to support this branch.
raise AssertionError("unreachable")
# Traverse backwards to find all concatenations
# until eventually reaching the original empty tuple.
while search_end >= search_start:
concat_stmt = old_body[search_end]
if (
isinstance(concat_stmt, ir.Assign)
and concat_stmt.target.name == target_name
and isinstance(concat_stmt.value, ir.Expr)
and concat_stmt.value.op == "build_tuple"
and not concat_stmt.value.items
):
new_body[search_end] = None
# Remove the definition.
_remove_assignment_definition(
old_body, search_end, func_ir, already_deleted_defs
)
# If we have reached the build_tuple we exit.
break
else:
# We expect to find another arg to append.
# The first stmt must be a binop "add"
if (search_end == search_start) or not (
isinstance(concat_stmt, ir.Assign)
and (
concat_stmt.target.name
== target_name
)
and isinstance(
concat_stmt.value, ir.Expr
)
and concat_stmt.value.op == "binop"
and concat_stmt.value.fn == operator.add
):
# We cannot handle this format.
raise UnsupportedBytecodeError(errmsg)
lhs_name = concat_stmt.value.lhs.name
rhs_name = concat_stmt.value.rhs.name
# The previous statement should be a
# build_tuple containing the arg.
arg_tuple_stmt = old_body[search_end - 1]
if not (
isinstance(arg_tuple_stmt, ir.Assign)
and isinstance(
arg_tuple_stmt.value, ir.Expr
)
and (
arg_tuple_stmt.value.op
== "build_tuple"
)
and len(arg_tuple_stmt.value.items) == 1
):
# We cannot handle this format.
raise UnsupportedBytecodeError(errmsg)
if arg_tuple_stmt.target.name == lhs_name:
# The tuple should always be generated on the RHS.
raise AssertionError("unreachable")
elif arg_tuple_stmt.target.name == rhs_name:
target_name = lhs_name
else:
# We cannot handle this format.
raise UnsupportedBytecodeError(errmsg)
total_args.append(
arg_tuple_stmt.value.items[0]
)
new_body[search_end] = None
new_body[search_end - 1] = None
# Remove the definitions.
_remove_assignment_definition(
old_body, search_end, func_ir, already_deleted_defs
)
_remove_assignment_definition(
old_body, search_end - 1, func_ir, already_deleted_defs
)
search_end -= 2
# Avoid any space between appends
keep_looking = True
while search_end >= search_start and keep_looking:
next_stmt = old_body[search_end]
if (
isinstance(next_stmt, ir.Assign)
and (
next_stmt.target.name
== target_name
)
):
keep_looking = False
else:
# If the argument is "created" in JIT, then there
# will be intermediate operations in between appends.
# For example if the next arg after arg4 is pow(arg5, 2),
# then the IR would look like:
#
# # Appending arg4
# $arg4_tup = build_tuple(items=[arg4])
# $append_var.5 = $append_var.4 + $arg4_tup
#
# # Creation of arg5.
# # This is the section that we are skipping.
# $32load_global.20 = global(pow: <built-in function pow>)
# $const36.22 = const(int, 2)
# $call.23 = call $32load_global.20(arg5, $const36.22)
#
# # Appending arg5
# $arg5_tup = build_tuple(items=[$call.23])
# $append_var.6 = $append_var.5 + $arg5_tup
search_end -= 1
if search_end == search_start:
# If we reached the start we never found the build_tuple.
# We cannot handle this format so raise the
# original error message.
raise UnsupportedBytecodeError(errmsg)
# Reverse the arguments so we get the correct order.
return total_args[::-1]
def peep_hole_call_function_ex_to_call_function_kw(func_ir):
"""
This peephole rewrites a bytecode sequence unique to Python 3.10
where CALL_FUNCTION_EX is used instead of CALL_FUNCTION_KW because of
stack limitations set by CPython. This limitation is imposed whenever
a function call has too many arguments or keyword arguments.
https://github.com/python/cpython/blob/a58ebcc701dd6c43630df941481475ff0f615a81/Python/compile.c#L55
https://github.com/python/cpython/blob/a58ebcc701dd6c43630df941481475ff0f615a81/Python/compile.c#L4442
In particular, this change is imposed whenever (n_args / 2) + n_kws > 15.
Different bytecode is generated for args depending on if n_args > 30
or n_args <= 30 and similarly if n_kws > 15 or n_kws <= 15.
This function unwraps the *args and **kwargs in the function call
and places these values directly into the args and kwargs of the call.
"""
# All changes are local to the a single block
# so it can be traversed in any order.
errmsg = textwrap.dedent("""
CALL_FUNCTION_EX with **kwargs not supported.
If you are not using **kwargs this may indicate that
you have a large number of kwargs and are using inlined control
flow. You can resolve this issue by moving the control flow out of
the function call. For example, if you have
f(a=1 if flag else 0, ...)
Replace that with:
a_val = 1 if flag else 0
f(a=a_val, ...)""")
# Track which definitions have already been deleted
already_deleted_defs = collections.defaultdict(set)
for blk in func_ir.blocks.values():
blk_changed = False
new_body = []
for i, stmt in enumerate(blk.body):
if (
isinstance(stmt, ir.Assign)
and isinstance(stmt.value, ir.Expr)
and stmt.value.op == "call"
and stmt.value.varkwarg is not None
):
blk_changed = True
call = stmt.value
args = call.args
kws = call.kws
# We need to check the call expression contents if
# it contains either vararg or varkwarg. If it contains
# varkwarg we need to update the IR. If it just contains
# vararg we don't need to update the IR, but we need to
# check if peep_hole_list_to_tuple failed to replace the
# vararg list with a tuple. If so, we output an error
# message with suggested code changes.
vararg = call.vararg
varkwarg = call.varkwarg
start_search = i - 1
# varkwarg should be defined second so we start there.
varkwarg_loc = start_search
keyword_def = None
found = False
while varkwarg_loc >= 0 and not found:
keyword_def = blk.body[varkwarg_loc]
if (
isinstance(keyword_def, ir.Assign)
and keyword_def.target.name == varkwarg.name
):
found = True
else:
varkwarg_loc -= 1
if (
kws
or not found
or not (
isinstance(keyword_def.value, ir.Expr)
and keyword_def.value.op == "build_map"
)
):
# If we couldn't find where the kwargs are created
# then it should be a normal **kwargs call
# so we produce an unsupported message.
raise UnsupportedBytecodeError(errmsg)
# Determine the kws
if keyword_def.value.items:
# n_kws <= 15 case.
# Here the IR looks like a series of
# constants, then the arguments and finally
# a build_map that contains all of the pairs.
# For Example:
#
# $const_n = const("arg_name")
# $arg_n = ...
# $kwargs_var = build_map(items=[
# ($const_0, $arg_0),
# ...,
# ($const_n, $arg_n),])
kws = _call_function_ex_replace_kws_small(
blk.body,
keyword_def.value,
new_body,
varkwarg_loc,
func_ir,
already_deleted_defs,
)
else:
# n_kws > 15 case.
# Here the IR is an initial empty build_map
# followed by a series of setitems with a constant
# key and then the argument.
# For example:
#
# $kwargs_var = build_map(items=[])
# $const_0 = const("arg_name")
# $arg_0 = ...
# $my_attr = getattr(const_0, attr=__setitem__)
# $unused_var = call $my_attr($const_0, $arg_0)
# ...
kws = _call_function_ex_replace_kws_large(
blk.body,
varkwarg.name,
varkwarg_loc,
i - 1,
new_body,
func_ir,
errmsg,
already_deleted_defs,
)
start_search = varkwarg_loc
# Vararg isn't required to be provided.
if vararg is not None:
if args:
# If we have vararg then args is expected to
# be an empty list.
raise UnsupportedBytecodeError(errmsg)
vararg_loc = start_search
args_def = None
found = False
while vararg_loc >= 0 and not found:
args_def = blk.body[vararg_loc]
if (
isinstance(args_def, ir.Assign)
and args_def.target.name == vararg.name
):
found = True
else:
vararg_loc -= 1
if not found:
# If we couldn't find where the args are created
# then we can't handle this format.
raise UnsupportedBytecodeError(errmsg)
if (
isinstance(args_def.value, ir.Expr)
and args_def.value.op == "build_tuple"
):
# n_args <= 30 case.
# Here the IR is a simple build_tuple containing
# all of the args.
# For example:
#
# $arg_n = ...
# $varargs = build_tuple(
# items=[$arg_0, ..., $arg_n]
# )
args = _call_function_ex_replace_args_small(
blk.body,
args_def.value,
new_body,
vararg_loc,
func_ir,
already_deleted_defs,
)
elif (
isinstance(args_def.value, ir.Expr)
and args_def.value.op == "list_to_tuple"
):
# If there is a call with vararg we need to check
# if the list -> tuple conversion failed and if so
# throw an error.
raise UnsupportedBytecodeError(errmsg)
else:
# Here the IR is an initial empty build_tuple.
# Then for each arg, a new tuple with a single
# element is created and one by one these are
# added to a growing tuple.
# For example:
#
# $combo_tup_0 = build_tuple(items=[])
# $arg0 = ...
# $arg0_tup = build_tuple(items=[$arg0])
# $combo_tup_1 = $combo_tup_0 + $arg0_tup
# $arg1 = ...
# $arg1_tup = build_tuple(items=[$arg1])
# $combo_tup_2 = $combo_tup_1 + $arg1_tup
# ...
# $combo_tup_n = $combo_tup_{n-1} + $argn_tup
#
# In addition, the IR contains a final
# assignment for the varargs that looks like:
#
# $varargs_var = $combo_tup_n
#
# Here args_def is expected to be a simple assignment.
args = _call_function_ex_replace_args_large(
blk.body,
args_def,
new_body,
vararg_loc,
func_ir,
errmsg,
already_deleted_defs,
)
# Create a new call updating the args and kws
new_call = ir.Expr.call(
call.func, args, kws, call.loc, target=call.target
)
# Drop the existing definition for this stmt.
_remove_assignment_definition(
blk.body, i, func_ir, already_deleted_defs
)
# Update the statement
stmt = ir.Assign(new_call, stmt.target, stmt.loc)
# Update the definition
func_ir._definitions[stmt.target.name].append(new_call)
elif (
isinstance(stmt, ir.Assign)
and isinstance(stmt.value, ir.Expr)
and stmt.value.op == "call"
and stmt.value.vararg is not None
):
# If there is a call with vararg we need to check
# if the list -> tuple conversion failed and if so
# throw an error.
call = stmt.value
vararg_name = call.vararg.name
if (
vararg_name in func_ir._definitions
and len(func_ir._definitions[vararg_name]) == 1
):
# If this value is still a list to tuple raise the
# exception.
expr = func_ir._definitions[vararg_name][0]
if isinstance(expr, ir.Expr) and expr.op == "list_to_tuple":
raise UnsupportedBytecodeError(errmsg)
new_body.append(stmt)
# Replace the block body if we changed the IR
if blk_changed:
blk.body.clear()
blk.body.extend([x for x in new_body if x is not None])
return func_ir
def peep_hole_list_to_tuple(func_ir):
"""
This peephole rewrites a bytecode sequence new to Python 3.9 that looks
like e.g.:
def foo(a):
return (*a,)
41 0 BUILD_LIST 0
2 LOAD_FAST 0 (a)
4 LIST_EXTEND 1
6 LIST_TO_TUPLE
8 RETURN_VAL
essentially, the unpacking of tuples is written as a list which is appended
to/extended and then "magicked" into a tuple by the new LIST_TO_TUPLE
opcode.
This peephole repeatedly analyses the bytecode in a block looking for a
window between a `LIST_TO_TUPLE` and `BUILD_LIST` and...
1. Turns the BUILD_LIST into a BUILD_TUPLE
2. Sets an accumulator's initial value as the target of the BUILD_TUPLE
3. Searches for 'extend' on the original list and turns these into binary
additions on the accumulator.
4. Searches for 'append' on the original list and turns these into a
`BUILD_TUPLE` which is then appended via binary addition to the
accumulator.
5. Assigns the accumulator to the variable that exits the peephole and the
rest of the block/code refers to as the result of the unpack operation.
6. Patches up
"""
_DEBUG = False
# For all blocks
for offset, blk in func_ir.blocks.items():
# keep doing the peephole rewrite until nothing is left that matches
while True:
# first try and find a matching region
# i.e. BUILD_LIST...<stuff>...LIST_TO_TUPLE
def find_postive_region():
found = False
for idx in reversed(range(len(blk.body))):
stmt = blk.body[idx]
if isinstance(stmt, ir.Assign):
value = stmt.value
if (isinstance(value, ir.Expr) and
value.op == 'list_to_tuple'):
target_list = value.info[0]
found = True
bt = (idx, stmt)
if found:
if isinstance(stmt, ir.Assign):
if stmt.target.name == target_list:
region = (bt, (idx, stmt))
return region
region = find_postive_region()
# if there's a peep hole region then do something with it
if region is not None:
peep_hole = blk.body[region[1][0] : region[0][0]]
if _DEBUG:
print("\nWINDOW:")
for x in peep_hole:
print(x)
print("")
appends = []
extends = []
init = region[1][1]
const_list = init.target.name
# Walk through the peep_hole and find things that are being
# "extend"ed and "append"ed to the BUILD_LIST
for x in peep_hole:
if isinstance(x, ir.Assign):
if isinstance(x.value, ir.Expr):
expr = x.value
if (expr.op == 'getattr' and
expr.value.name == const_list):
# it's not strictly necessary to split out
# extends and appends, but it helps with
# debugging to do so!
if expr.attr == 'extend':
extends.append(x.target.name)
elif expr.attr == 'append':
appends.append(x.target.name)
else:
assert 0
# go back through the peep hole build new IR based on it.
new_hole = []
def append_and_fix(x):
""" Adds to the new_hole and fixes up definitions"""
new_hole.append(x)
if x.target.name in func_ir._definitions:
# if there's already a definition, drop it, should only
# be 1 as the way cpython emits the sequence for
# `list_to_tuple` should ensure this.
assert len(func_ir._definitions[x.target.name]) == 1
func_ir._definitions[x.target.name].clear()
func_ir._definitions[x.target.name].append(x.value)
the_build_list = init.target
# Do the transform on the peep hole
if _DEBUG:
print("\nBLOCK:")
blk.dump()
# This section basically accumulates list appends and extends
# as binop(+) on tuples, it drops all the getattr() for extend
# and append as they are now dead and replaced with binop(+).
# It also switches out the build_list for a build_tuple and then
# ensures everything is wired up and defined ok.
t2l_agn = region[0][1]
acc = the_build_list
for x in peep_hole:
if isinstance(x, ir.Assign):
if isinstance(x.value, ir.Expr):
expr = x.value
if expr.op == 'getattr':
if (x.target.name in extends or
x.target.name in appends):
# drop definition, it's being wholesale
# replaced.
func_ir._definitions.pop(x.target.name)
continue
else:
# a getattr on something we're not
# interested in
new_hole.append(x)
elif expr.op == 'call':
fname = expr.func.name
if fname in extends or fname in appends:
arg = expr.args[0]
if isinstance(arg, ir.Var):
tmp_name = "%s_var_%s" % (fname,
arg.name)
if fname in appends:
bt = ir.Expr.build_tuple([arg,],
expr.loc)
else:
# Extend as tuple
gv_tuple = ir.Global(
name="tuple", value=tuple,
loc=expr.loc,
)
tuple_var = arg.scope.redefine(
"$_list_extend_gv_tuple",
loc=expr.loc,
)
new_hole.append(
ir.Assign(
target=tuple_var,
value=gv_tuple,
loc=expr.loc,
),
)
bt = ir.Expr.call(
tuple_var, (arg,), (),
loc=expr.loc,
)
var = ir.Var(arg.scope, tmp_name,
expr.loc)
asgn = ir.Assign(bt, var, expr.loc)
append_and_fix(asgn)
arg = var
# this needs to be a binary add
new = ir.Expr.binop(fn=operator.add,
lhs=acc,
rhs=arg,
loc=x.loc)
asgn = ir.Assign(new, x.target, expr.loc)
append_and_fix(asgn)
acc = asgn.target
else:
# there could be a call in the unpack, like
# *(a, x.append(y))
new_hole.append(x)
elif (expr.op == 'build_list' and
x.target.name == const_list):
new = ir.Expr.build_tuple(expr.items, expr.loc)
asgn = ir.Assign(new, x.target, expr.loc)
# Not a temporary any more
append_and_fix(asgn)
else:
new_hole.append(x)
else:
new_hole.append(x)
else:
# stick everything else in as-is
new_hole.append(x)
# Finally write the result back into the original build list as
# everything refers to it.
append_and_fix(ir.Assign(acc, t2l_agn.target,
the_build_list.loc))
if _DEBUG:
print("\nNEW HOLE:")
for x in new_hole:
print(x)
# and then update the block body with the modified region
cpy = blk.body[:]
head = cpy[:region[1][0]]
tail = blk.body[region[0][0] + 1:]
tmp = head + new_hole + tail
blk.body.clear()
blk.body.extend(tmp)
if _DEBUG:
print("\nDUMP post hole:")
blk.dump()
else:
# else escape
break
return func_ir
def peep_hole_delete_with_exit(func_ir):
"""
This rewrite removes variables used to store the `__exit__` function
loaded by SETUP_WITH.
"""
dead_vars = set()
for blk in func_ir.blocks.values():
for stmt in blk.body:
# Any statement that uses a variable with the '$setup_with_exitfn'
# prefix is considered dead.
used = set(stmt.list_vars())
for v in used:
if v.name.startswith('$setup_with_exitfn'):
dead_vars.add(v)